一个手贱,把我本地的快照全给清空了。
想了想感觉还是需要下载回来一部分,不然还是有些风险。
但是一个个翻页点击,体感还是太恐怖了。
看了一下,发现有提供快照相关的 API,所以写了一个脚本批量处理,以下的代码依赖于 RunJs
插件,请放到代码块里面,然后使用插件运行。
"use strict";
const request = globalThis.runJs.api.request;
class SnapshotScheduler {
constructor() {
this.now = new Date();
}
isInCurrentWeek(date) {
const startOfWeek = new Date(this.now);
startOfWeek.setDate(this.now.getDate() - this.now.getDay());
return date >= startOfWeek;
}
getTimeDifferenceInDays(date) {
return Math.floor((this.now.getTime() - date.getTime()) / (1000 * 3600 * 24));
}
isSameDay(date1, date2) {
return date1.getFullYear() === date2.getFullYear() &&
date1.getMonth() === date2.getMonth() &&
date1.getDate() === date2.getDate();
}
isHourDifferent(date1, date2) {
return date1.getHours() !== date2.getHours();
}
shouldDownloadSnapshot(last, current, next) {
const currentDate = new Date(current.created);
const daysDifference = this.getTimeDifferenceInDays(currentDate);
if (this.isInCurrentWeek(currentDate)) {
return true; // Download all snapshots from the current week
}
const lastDate = last ? new Date(last.created) : null;
const nextDate = next ? new Date(next.created) : null;
if (daysDifference <= 30) {
// Every hour, plus start and end of day
if (!lastDate || !this.isSameDay(currentDate, lastDate)) {
return true; // First snapshot of the day
}
if (!nextDate || !this.isSameDay(currentDate, nextDate)) {
return true; // Last snapshot of the day
}
return this.isHourDifferent(currentDate, lastDate);
}
if (daysDifference <= 90) {
// First and last snapshot of each day
return (!lastDate || !this.isSameDay(currentDate, lastDate)) ||
(!nextDate || !this.isSameDay(currentDate, nextDate));
}
if (daysDifference <= 180) {
// Last snapshot of each day
return !nextDate || !this.isSameDay(currentDate, nextDate);
}
if (daysDifference <= 365) {
// Every third day, last snapshot
if (daysDifference % 3 !== 0)
return false;
return !nextDate || !this.isSameDay(currentDate, nextDate);
}
// Over a year: weekly, last snapshot
const weekNumber = Math.floor(daysDifference / 7);
if (daysDifference % 7 !== 0)
return false;
return !nextDate || Math.floor(this.getTimeDifferenceInDays(nextDate) / 7) !== weekNumber;
}
}
class SnapshotManager {
constructor() {
this.scheduler = new SnapshotScheduler();
}
async getAllSnapshots(startFrom, endTo, startPage) {
let page = startPage !== null && startPage !== void 0 ? startPage : 0;
let allSnapshots = [];
let continueFetching = true;
while (continueFetching) {
const snaps = await this.getCloudRepoSnapshots(page);
const filteredSnapshots = snaps.snapshots.filter(snapshot => {
const snapshotDate = new Date(snapshot.created);
return snapshotDate <= startFrom && snapshotDate >= endTo;
});
console.debug(`Fetched page ${page + 1}, total snapshots: ${filteredSnapshots.length}: ${snaps.snapshots[0].hCreated} ~ ${snaps.snapshots[snaps.snapshots.length - 1].hCreated}`);
allSnapshots = allSnapshots.concat(filteredSnapshots);
page++;
if (page >= snaps.pageCount || new Date(snaps.snapshots[snaps.snapshots.length - 1].created) < endTo) {
continueFetching = false;
}
}
return allSnapshots.sort((a, b) => b.created - a.created); // Sort descending
}
async downloadSnapshots(startFrom, cutoff, startPage) {
console.log("Fetching all snapshots...");
const allSnapshots = await this.getAllSnapshots(startFrom, cutoff, startPage);
console.log(`Total snapshots fetched: ${allSnapshots.length}`);
let totalDownloaded = 0;
for (let i = 0; i < allSnapshots.length; i++) {
const currentSnapshot = allSnapshots[i];
const lastSnapshot = i > 0 ? allSnapshots[i - 1] : null;
const nextSnapshot = i < allSnapshots.length - 1 ? allSnapshots[i + 1] : null;
if (this.scheduler.shouldDownloadSnapshot(lastSnapshot, currentSnapshot, nextSnapshot)) {
await this.downloadSnapshot(currentSnapshot);
totalDownloaded++;
}
}
return totalDownloaded;
}
async getCloudRepoSnapshots(page) {
const response = await request('/api/repo/getCloudRepoSnapshots', {
page: page
});
return response;
}
async downloadSnapshot(snapshot) {
const snapshotId = snapshot.id;
console.debug(`[${snapshot.hCreated}] Downloaded snapshot ${snapshotId}`);
await request('/api/repo/downloadCloudSnapshot', {
id: snapshotId,
tag: ''
});
}
}
async function runMainWithTiming(startFrom, endTo, startPage) {
console.log("Starting snapshot download process...");
console.log(`Start date: ${startFrom.toISOString()}`);
console.log(`Cutoff date: ${endTo.toISOString()}`);
const startTime = Date.now();
const snapshotManager = new SnapshotManager();
const totalDownloaded = await snapshotManager.downloadSnapshots(startFrom, endTo, startPage);
const endTime = Date.now();
const executionTime = (endTime - startTime) / 1000; // Convert to seconds
console.log(`Download process completed.`);
console.log(`Total snapshots downloaded: ${totalDownloaded}`);
console.log(`Total execution time: ${executionTime.toFixed(2)} seconds`);
}
let startFrom = new Date('2024-09-01');
let endTo = new Date('2023-12-01');
runMainWithTiming(startFrom, endTo, 74);
下载快照的时候,遵循以下策略:
- 对于当前周的快照,全部下载
- 对于最近一个月内的快照,确保每小时至少下载一个,并且总是下载每天的第一个和最后一个快照
- 对于 1-3 个月内的快照,只保留每天的第一个和最后一个快照
- 对于 3-6 个月内的快照,只保留每天的最后一个快照
- 对于 6 个月到 1 年内的快照,每三天保留最后一个快照。
- 对于超过 1 年的快照,每周保留最后一个快照
运行时间还是相当长的,我大概跑了半个小时的样子,主要问题在于为了谨慎期间没有做并行处理,而是在 for loop 里面一个个 await Promise。
最后效果如下,原本 387 页,下载之后保留了 22 页。临近的一段时间,快照保存的都比较密集;而更为久远的时间段,快照的保存就稀疏一些。
欢迎来到这里!
我们正在构建一个小众社区,大家在这里相互信任,以平等 • 自由 • 奔放的价值观进行分享交流。最终,希望大家能够找到与自己志同道合的伙伴,共同成长。
注册 关于