feat: add cache TTL configuration and implement cache eviction logic

This commit is contained in:
hexboy 2026-07-01 02:03:44 +03:30
parent ccac511e4d
commit db6c6bd46d
10 changed files with 139 additions and 34 deletions

View file

@ -4,6 +4,10 @@ PORT: 8008
CACHE_DIR: ./local-cache
LOG_REQUESTS: false
# Delete cached artifacts that haven't been re-downloaded in this many days.
# Comment out (or remove) to keep cached artifacts forever.
# CACHE_TTL_DAYS: 365
IGNORE_FILES:
- react-native-0.61.5.pom
- -sources.jar

View file

@ -21,6 +21,8 @@ const CACHE_DIR = path.resolve(config.CACHE_DIR, '__MMT_CACHE__');
const TMP_DIR = path.resolve(config.CACHE_DIR, '__MMT_TMP__');
const DEFAULT_PATH = config.DEFAULT_PATH ?? 'v1';
const VERBOSE = config.LOG_REQUESTS ?? false;
// Unset (undefined) means cached artifacts are kept forever, matching prior behavior.
const CACHE_TTL_DAYS = config.CACHE_TTL_DAYS;
export {
PORT,
@ -31,5 +33,6 @@ export {
DEFAULT_PATH,
IGNORE_FILES,
REPOSITORIES,
CACHE_TTL_DAYS,
VALID_FILE_TYPES,
};

View file

@ -7,6 +7,7 @@ import { PROXIES, CACHE_DIR, TMP_DIR, REPOSITORIES } from '../config';
import { ProxyAgent } from 'proxy-agent';
import { TServer } from 'app/types';
import { extractFileInfo } from '../utils';
import { logger } from '../logger';
interface DownloadEntry {
responses: Set<Response>;
@ -172,7 +173,7 @@ export class GotDownloader {
addToResponses();
}
console.log(`🔗 [${srv.name}]`, url);
logger.info(`🔗 [${srv.name}] ${url}`);
return;
}
@ -210,13 +211,13 @@ export class GotDownloader {
stream.once('downloadProgress', ({ total }) => {
if (total) {
console.log(`⏳ [${srv.name}]`, url);
logger.info(`⏳ [${srv.name}] ${url}`);
}
});
stream.on('error', (err) => {
console.log('❌', srv.url + url);
console.log('⛔️', err.message);
logger.error(`${srv.url + url}`);
logger.error(`⛔️ ${err.message}`);
// Destroy all waiting responses
for (const res of entry.responses) {
if (!res.destroyed) {
@ -258,7 +259,7 @@ export class GotDownloader {
gotResponse.statusCode >= 200 &&
gotResponse.statusCode < 300
) {
console.log(`✅ [${srv.name}]`, url);
logger.info(`✅ [${srv.name}] ${url}`);
const destPath = path.join(
CACHE_DIR,
srv.name,

View file

@ -3,7 +3,8 @@ import { RequestHandler } from 'express';
import { GotDownloader } from '../downloader/got';
import { CACHE_DIR } from '../config';
import { getCachedServer } from '../utils';
import { getCachedServer, touchFile } from '../utils';
import { logger } from '../logger';
const downloader = new GotDownloader();
@ -13,7 +14,8 @@ export const CacheRequestHandler: RequestHandler = (request, response) => {
const server = getCachedServer(url);
if (server) {
const cachedPath = path.join(CACHE_DIR, server.name, url);
console.log(`📦 [${server.name}]`, url);
logger.info(`📦 [${server.name}] ${url}`);
touchFile(cachedPath);
return response.sendFile(cachedPath);
}

View file

@ -1,6 +1,7 @@
import { RequestHandler } from 'express';
import got from 'got';
import { extractFileInfo, getCachedServer } from '../utils';
import { logger } from '../logger';
const gradleApi = 'https://plugins.gradle.org/api/gradle/4.10/plugin/use';
@ -40,7 +41,7 @@ export const LegacyGradlePluginsHandler: RequestHandler = (req, res, next) => {
url.replaceAll(`${info.id}.gradle.plugin`, newId)
)?.[0];
const newUrl = `${basePath}/${/^[\w.]+(?=:)/.exec(info.implementation.gav)?.[0]?.replaceAll('.', '/')}/${newId}/${version}/${newFileName}`;
console.log('🔀 [301]', url);
logger.info(`🔀 [301] ${url}`);
req.headers['alias-url'] = url.replace(/^\/\w+\//, '/');
req.url = newUrl;
next();
@ -49,8 +50,7 @@ export const LegacyGradlePluginsHandler: RequestHandler = (req, res, next) => {
next();
});
} catch (error) {
// eslint-disable-next-line @typescript-eslint/no-unused-expressions
error;
logger.warn(`⚠️ failed to resolve legacy gradle plugin ${url}`, error);
next();
}
};

View file

@ -2,6 +2,7 @@ import { RequestHandler } from 'express';
import { IGNORE_FILES, VALID_FILE_TYPES } from '../config';
import { extractFileInfo } from '../utils';
import { logger } from '../logger';
export const ValidateRequestHandler: RequestHandler = (req, res, next) => {
const url = req.url.replace(/^\/\w+\//, '/');
@ -12,16 +13,16 @@ export const ValidateRequestHandler: RequestHandler = (req, res, next) => {
const { fileExtension } = extractFileInfo(url);
if (!VALID_FILE_TYPES.includes('.' + fileExtension)) {
console.log('♻️', url);
logger.info(`♻️ [404] ${url}`);
return res.status(404);
}
if (IGNORE_FILES.find((str) => url.includes(str))) {
console.log('❌ [404]', url);
logger.info(`❌ [404] ${url}`);
return res.status(404);
}
console.log(req.method, url);
logger.info(`${req.method} ${url}`);
next();
};

View file

@ -4,8 +4,15 @@ import chalk from 'chalk';
import morgan from 'morgan';
import express from 'express';
import { PORT, TMP_DIR, VERBOSE, CACHE_DIR, DEFAULT_PATH } from './config';
import { printServedEndpoints } from './utils';
import {
PORT,
TMP_DIR,
VERBOSE,
CACHE_DIR,
DEFAULT_PATH,
CACHE_TTL_DAYS,
} from './config';
import { printServedEndpoints, evictExpiredCacheFiles } from './utils';
import { CacheRequestHandler } from './handlers/cache-handler';
import { ValidateRequestHandler } from './handlers/validate-request-handler';
@ -21,6 +28,16 @@ if (!fs.existsSync(path.resolve(TMP_DIR))) {
fs.mkdirSync(path.resolve(TMP_DIR), { recursive: true });
}
// evict stale cache entries on startup, then on a recurring sweep
if (CACHE_TTL_DAYS) {
const CACHE_SWEEP_INTERVAL_MS = 24 * 60 * 60 * 1000;
evictExpiredCacheFiles(CACHE_DIR, CACHE_TTL_DAYS);
setInterval(
() => evictExpiredCacheFiles(CACHE_DIR, CACHE_TTL_DAYS!),
CACHE_SWEEP_INTERVAL_MS
);
}
const app = express();
if (VERBOSE) {
app.use(morgan('combined'));

35
src/logger.ts Normal file
View file

@ -0,0 +1,35 @@
enum Level {
Warn = 'w',
Info = 'i',
Error = 'e',
}
const pad = (n: number, width = 2) => String(n).padStart(width, '0');
// Local wall-clock time with its UTC offset, e.g. 2026/07/01 14:32:05.123
const formatLocalTimestamp = (date: Date) => {
return (
`${date.getFullYear()}/${pad(date.getMonth() + 1)}/${pad(date.getDate())} ` +
`${pad(date.getHours())}:${pad(date.getMinutes())}:${pad(date.getSeconds())}.${pad(date.getMilliseconds(), 3)}`
);
};
const write = (level: Level, message: string, ...args: unknown[]) => {
const line = `${formatLocalTimestamp(new Date())} [${level.toUpperCase()}] ${message}`;
if (level === Level.Error) {
console.error(line, ...args);
} else if (level === Level.Warn) {
console.warn(line, ...args);
} else {
console.log(line, ...args);
}
};
export const logger = {
info: (message: string, ...args: unknown[]) =>
write(Level.Info, message, ...args),
warn: (message: string, ...args: unknown[]) =>
write(Level.Warn, message, ...args),
error: (message: string, ...args: unknown[]) =>
write(Level.Error, message, ...args),
};

View file

@ -3,6 +3,7 @@ import os from 'os';
import path from 'path';
import { CACHE_DIR, REPOSITORIES } from './config';
import { logger } from './logger';
export const getCachedServer = (filePath: string) => {
const srv = REPOSITORIES.find((s) => {
@ -52,29 +53,69 @@ export const printServedEndpoints = (
}
};
export const extractFileInfo = (url: string) => {
// Split the pathname into segments
const segments = url.split('/');
// Eviction only cares about day-level age, so skip the metadata write
// when the file was already touched recently in this window.
const TOUCH_THRESHOLD_MS = 24 * 60 * 60 * 1000;
// Get the last segment (filename)
const filename = segments[segments.length - 1].replace(/\?.*$/, '');
// Bump mtime so eviction tracks last-served time, not last-downloaded time
// (sendFile() never touches the file, so a frequently-served cache hit would
// otherwise look just as stale as one nobody has asked for in months).
export const touchFile = (filePath: string) => {
try {
const { mtimeMs } = fs.statSync(filePath);
if (Date.now() - mtimeMs < TOUCH_THRESHOLD_MS) {
return;
}
const now = new Date();
fs.utimesSync(filePath, now, now);
} catch {
// best-effort; a missed touch just makes the file evict a bit early
}
};
// Extract the file extension
const fileExtension = filename.split('.').pop();
export const evictExpiredCacheFiles = (
cacheDir: string,
ttlDays: number
): void => {
const maxAgeMs = ttlDays * 24 * 60 * 60 * 1000;
const cutoff = Date.now() - maxAgeMs;
let removed = 0;
// Construct the relative path (excluding the filename)
const relativePath = segments.slice(0, -1).join('/');
const walk = (dir: string) => {
let entries: fs.Dirent[];
try {
entries = fs.readdirSync(dir, { withFileTypes: true });
} catch {
return;
}
// Create the output object
const fileInfo = {
fileName: filename,
relativePath: relativePath,
fileExtension: fileExtension,
} as {
fileName: string;
relativePath: string;
fileExtension: string;
for (const entry of entries) {
const entryPath = path.join(dir, entry.name);
if (entry.isDirectory()) {
walk(entryPath);
continue;
}
const { mtimeMs } = fs.statSync(entryPath);
if (mtimeMs < cutoff) {
fs.unlinkSync(entryPath);
removed += 1;
}
}
};
return fileInfo;
walk(cacheDir);
if (removed > 0) {
logger.info(`🧹 evicted ${removed} cached file(s) older than ${ttlDays}d`);
}
};
export const extractFileInfo = (url: string) => {
// Use a dummy base so relative/absolute paths both parse, then strip the query string
const pathname = new URL(url, 'http://localhost').pathname;
const fileName = path.basename(pathname);
const relativePath = path.dirname(pathname);
const fileExtension = path.extname(fileName).replace(/^\./, '');
return { fileName, relativePath, fileExtension };
};

1
types.d.ts vendored
View file

@ -27,5 +27,6 @@ export interface IConfig {
LOG_REQUESTS?: boolean;
IGNORE_FILES?: string[];
VALID_FILE_TYPES?: string[];
CACHE_TTL_DAYS?: number;
PROXIES: Record<string, TProxy>;
}