diff --git a/docker-compose.yaml b/docker-compose.yaml index bfeb4d13..64b04863 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -92,6 +92,8 @@ services: - CHUNK_POST_URLS=${CHUNK_POST_URLS:-} - CHUNK_POST_RESPONSE_TIMEOUT_MS=${CHUNK_POST_RESPONSE_TIMEOUT_MS:-} - CHUNK_POST_ABORT_TIMEOUT_MS=${CHUNK_POST_ABORT_TIMEOUT_MS:-} + - ENABLE_FS_CHUNK_CACHE_CLEANUP=${ENABLE_FS_CHUNK_CACHE_CLEANUP:-} + - CHUNK_DATA_CACHE_CLEANUP_THRESHOLD=${CHUNK_DATA_CACHE_CLEANUP_THRESHOLD:-} - AO_CU_URL=${AO_CU_URL:-} - AO_MU_URL=${AO_MU_URL:-} - AO_GATEWAY_URL=${AO_GATEWAY_URL:-} diff --git a/src/app.ts b/src/app.ts index e94273bf..a1ccd08a 100644 --- a/src/app.ts +++ b/src/app.ts @@ -36,6 +36,8 @@ system.headerFsCacheCleanupWorker?.start(); system.contiguousDataFsCacheCleanupWorker?.start(); +system.chunkDataFsCacheCleanupWorker?.start(); + // Allow starting without writers to support SQLite replication if (config.START_WRITERS) { system.blockImporter.start(); diff --git a/src/config.ts b/src/config.ts index 5b29bc19..2b60164f 100644 --- a/src/config.ts +++ b/src/config.ts @@ -233,6 +233,15 @@ export const CONTIGUOUS_DATA_CACHE_CLEANUP_THRESHOLD = env.varOrDefault( '', ); +export const ENABLE_FS_CHUNK_CACHE_CLEANUP = + env.varOrDefault('ENABLE_FS_CHUNK_CACHE_CLEANUP', 'false') === 'true'; + +// The threshold in seconds to cleanup the filesystem chunk data cache +export const CHUNK_DATA_CACHE_CLEANUP_THRESHOLD = +env.varOrDefault( + 'CHUNK_DATA_CACHE_CLEANUP_THRESHOLD', + `${60 * 60 * 24}`, // 1 day +); + // // Webhooks // diff --git a/src/system.ts b/src/system.ts index 9f19d1af..b5daf485 100644 --- a/src/system.ts +++ b/src/system.ts @@ -224,6 +224,34 @@ export const contiguousDataFsCacheCleanupWorker = !isNaN( }) : undefined; +export const chunkDataFsCacheCleanupWorker = + config.ENABLE_FS_CHUNK_CACHE_CLEANUP && + !isNaN(config.CHUNK_DATA_CACHE_CLEANUP_THRESHOLD) + ? new FsCleanupWorker({ + log, + basePath: 'data/chunks', + shouldDelete: async (path) => { + try { + const stats = await fs.promises.stat(path); + const mostRecentTime = + stats.atime > stats.mtime ? stats.atime : stats.mtime; + + const currentTimestamp = Date.now(); + + const thresholdDate = new Date( + currentTimestamp - + config.CHUNK_DATA_CACHE_CLEANUP_THRESHOLD * 1000, + ); + + return mostRecentTime <= thresholdDate; + } catch (err) { + log.error(`Error getting file stats for ${path}`, err); + return false; + } + }, + }) + : undefined; + const ans104TxMatcher = new MatchTags([ { name: 'Bundle-Format', value: 'binary' }, { name: 'Bundle-Version', valueStartsWith: '2.' }, @@ -671,6 +699,7 @@ export const shutdown = async (express: Server) => { await db.stop(); await headerFsCacheCleanupWorker?.stop(); await contiguousDataFsCacheCleanupWorker?.stop(); + await chunkDataFsCacheCleanupWorker?.stop(); process.exit(0); }); diff --git a/src/workers/fs-cleanup-worker.ts b/src/workers/fs-cleanup-worker.ts index a4b3bbf9..06ac6f9c 100644 --- a/src/workers/fs-cleanup-worker.ts +++ b/src/workers/fs-cleanup-worker.ts @@ -54,7 +54,7 @@ export class FsCleanupWorker { pauseDuration?: number; restartPauseDuration?: number; }) { - this.log = log.child({ class: this.constructor.name }); + this.log = log.child({ class: this.constructor.name, basePath }); this.shouldDelete = shouldDelete ?? (() => Promise.resolve(true)); this.deleteCallback = deleteCallback ?? ((file: string) => fs.promises.unlink(file));