mirror of
https://github.com/hexboy/maven-mirror-tool.git
synced 2026-07-09 17:28:25 +00:00
feat: add cache TTL configuration and implement cache eviction logic
This commit is contained in:
parent
ccac511e4d
commit
db6c6bd46d
10 changed files with 139 additions and 34 deletions
|
|
@ -4,6 +4,10 @@ PORT: 8008
|
|||
CACHE_DIR: ./local-cache
|
||||
LOG_REQUESTS: false
|
||||
|
||||
# Delete cached artifacts that haven't been re-downloaded in this many days.
|
||||
# Comment out (or remove) to keep cached artifacts forever.
|
||||
# CACHE_TTL_DAYS: 365
|
||||
|
||||
IGNORE_FILES:
|
||||
- react-native-0.61.5.pom
|
||||
- -sources.jar
|
||||
|
|
|
|||
|
|
@ -21,6 +21,8 @@ const CACHE_DIR = path.resolve(config.CACHE_DIR, '__MMT_CACHE__');
|
|||
const TMP_DIR = path.resolve(config.CACHE_DIR, '__MMT_TMP__');
|
||||
const DEFAULT_PATH = config.DEFAULT_PATH ?? 'v1';
|
||||
const VERBOSE = config.LOG_REQUESTS ?? false;
|
||||
// Unset (undefined) means cached artifacts are kept forever, matching prior behavior.
|
||||
const CACHE_TTL_DAYS = config.CACHE_TTL_DAYS;
|
||||
|
||||
export {
|
||||
PORT,
|
||||
|
|
@ -31,5 +33,6 @@ export {
|
|||
DEFAULT_PATH,
|
||||
IGNORE_FILES,
|
||||
REPOSITORIES,
|
||||
CACHE_TTL_DAYS,
|
||||
VALID_FILE_TYPES,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import { PROXIES, CACHE_DIR, TMP_DIR, REPOSITORIES } from '../config';
|
|||
import { ProxyAgent } from 'proxy-agent';
|
||||
import { TServer } from 'app/types';
|
||||
import { extractFileInfo } from '../utils';
|
||||
import { logger } from '../logger';
|
||||
|
||||
interface DownloadEntry {
|
||||
responses: Set<Response>;
|
||||
|
|
@ -172,7 +173,7 @@ export class GotDownloader {
|
|||
addToResponses();
|
||||
}
|
||||
|
||||
console.log(`🔗 [${srv.name}]`, url);
|
||||
logger.info(`🔗 [${srv.name}] ${url}`);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -210,13 +211,13 @@ export class GotDownloader {
|
|||
|
||||
stream.once('downloadProgress', ({ total }) => {
|
||||
if (total) {
|
||||
console.log(`⏳ [${srv.name}]`, url);
|
||||
logger.info(`⏳ [${srv.name}] ${url}`);
|
||||
}
|
||||
});
|
||||
|
||||
stream.on('error', (err) => {
|
||||
console.log('❌', srv.url + url);
|
||||
console.log('⛔️', err.message);
|
||||
logger.error(`❌ ${srv.url + url}`);
|
||||
logger.error(`⛔️ ${err.message}`);
|
||||
// Destroy all waiting responses
|
||||
for (const res of entry.responses) {
|
||||
if (!res.destroyed) {
|
||||
|
|
@ -258,7 +259,7 @@ export class GotDownloader {
|
|||
gotResponse.statusCode >= 200 &&
|
||||
gotResponse.statusCode < 300
|
||||
) {
|
||||
console.log(`✅ [${srv.name}]`, url);
|
||||
logger.info(`✅ [${srv.name}] ${url}`);
|
||||
const destPath = path.join(
|
||||
CACHE_DIR,
|
||||
srv.name,
|
||||
|
|
|
|||
|
|
@ -3,7 +3,8 @@ import { RequestHandler } from 'express';
|
|||
|
||||
import { GotDownloader } from '../downloader/got';
|
||||
import { CACHE_DIR } from '../config';
|
||||
import { getCachedServer } from '../utils';
|
||||
import { getCachedServer, touchFile } from '../utils';
|
||||
import { logger } from '../logger';
|
||||
|
||||
const downloader = new GotDownloader();
|
||||
|
||||
|
|
@ -13,7 +14,8 @@ export const CacheRequestHandler: RequestHandler = (request, response) => {
|
|||
const server = getCachedServer(url);
|
||||
if (server) {
|
||||
const cachedPath = path.join(CACHE_DIR, server.name, url);
|
||||
console.log(`📦 [${server.name}]`, url);
|
||||
logger.info(`📦 [${server.name}] ${url}`);
|
||||
touchFile(cachedPath);
|
||||
return response.sendFile(cachedPath);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { RequestHandler } from 'express';
|
||||
import got from 'got';
|
||||
import { extractFileInfo, getCachedServer } from '../utils';
|
||||
import { logger } from '../logger';
|
||||
|
||||
const gradleApi = 'https://plugins.gradle.org/api/gradle/4.10/plugin/use';
|
||||
|
||||
|
|
@ -40,7 +41,7 @@ export const LegacyGradlePluginsHandler: RequestHandler = (req, res, next) => {
|
|||
url.replaceAll(`${info.id}.gradle.plugin`, newId)
|
||||
)?.[0];
|
||||
const newUrl = `${basePath}/${/^[\w.]+(?=:)/.exec(info.implementation.gav)?.[0]?.replaceAll('.', '/')}/${newId}/${version}/${newFileName}`;
|
||||
console.log('🔀 [301]', url);
|
||||
logger.info(`🔀 [301] ${url}`);
|
||||
req.headers['alias-url'] = url.replace(/^\/\w+\//, '/');
|
||||
req.url = newUrl;
|
||||
next();
|
||||
|
|
@ -49,8 +50,7 @@ export const LegacyGradlePluginsHandler: RequestHandler = (req, res, next) => {
|
|||
next();
|
||||
});
|
||||
} catch (error) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-expressions
|
||||
error;
|
||||
logger.warn(`⚠️ failed to resolve legacy gradle plugin ${url}`, error);
|
||||
next();
|
||||
}
|
||||
};
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import { RequestHandler } from 'express';
|
|||
|
||||
import { IGNORE_FILES, VALID_FILE_TYPES } from '../config';
|
||||
import { extractFileInfo } from '../utils';
|
||||
import { logger } from '../logger';
|
||||
|
||||
export const ValidateRequestHandler: RequestHandler = (req, res, next) => {
|
||||
const url = req.url.replace(/^\/\w+\//, '/');
|
||||
|
|
@ -12,16 +13,16 @@ export const ValidateRequestHandler: RequestHandler = (req, res, next) => {
|
|||
const { fileExtension } = extractFileInfo(url);
|
||||
|
||||
if (!VALID_FILE_TYPES.includes('.' + fileExtension)) {
|
||||
console.log('♻️', url);
|
||||
logger.info(`♻️ [404] ${url}`);
|
||||
return res.status(404);
|
||||
}
|
||||
|
||||
if (IGNORE_FILES.find((str) => url.includes(str))) {
|
||||
console.log('❌ [404]', url);
|
||||
logger.info(`❌ [404] ${url}`);
|
||||
return res.status(404);
|
||||
}
|
||||
|
||||
console.log(req.method, url);
|
||||
logger.info(`${req.method} ${url}`);
|
||||
|
||||
next();
|
||||
};
|
||||
|
|
|
|||
21
src/index.ts
21
src/index.ts
|
|
@ -4,8 +4,15 @@ import chalk from 'chalk';
|
|||
import morgan from 'morgan';
|
||||
import express from 'express';
|
||||
|
||||
import { PORT, TMP_DIR, VERBOSE, CACHE_DIR, DEFAULT_PATH } from './config';
|
||||
import { printServedEndpoints } from './utils';
|
||||
import {
|
||||
PORT,
|
||||
TMP_DIR,
|
||||
VERBOSE,
|
||||
CACHE_DIR,
|
||||
DEFAULT_PATH,
|
||||
CACHE_TTL_DAYS,
|
||||
} from './config';
|
||||
import { printServedEndpoints, evictExpiredCacheFiles } from './utils';
|
||||
|
||||
import { CacheRequestHandler } from './handlers/cache-handler';
|
||||
import { ValidateRequestHandler } from './handlers/validate-request-handler';
|
||||
|
|
@ -21,6 +28,16 @@ if (!fs.existsSync(path.resolve(TMP_DIR))) {
|
|||
fs.mkdirSync(path.resolve(TMP_DIR), { recursive: true });
|
||||
}
|
||||
|
||||
// evict stale cache entries on startup, then on a recurring sweep
|
||||
if (CACHE_TTL_DAYS) {
|
||||
const CACHE_SWEEP_INTERVAL_MS = 24 * 60 * 60 * 1000;
|
||||
evictExpiredCacheFiles(CACHE_DIR, CACHE_TTL_DAYS);
|
||||
setInterval(
|
||||
() => evictExpiredCacheFiles(CACHE_DIR, CACHE_TTL_DAYS!),
|
||||
CACHE_SWEEP_INTERVAL_MS
|
||||
);
|
||||
}
|
||||
|
||||
const app = express();
|
||||
if (VERBOSE) {
|
||||
app.use(morgan('combined'));
|
||||
|
|
|
|||
35
src/logger.ts
Normal file
35
src/logger.ts
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
enum Level {
|
||||
Warn = 'w',
|
||||
Info = 'i',
|
||||
Error = 'e',
|
||||
}
|
||||
|
||||
const pad = (n: number, width = 2) => String(n).padStart(width, '0');
|
||||
|
||||
// Local wall-clock time with its UTC offset, e.g. 2026/07/01 14:32:05.123
|
||||
const formatLocalTimestamp = (date: Date) => {
|
||||
return (
|
||||
`${date.getFullYear()}/${pad(date.getMonth() + 1)}/${pad(date.getDate())} ` +
|
||||
`${pad(date.getHours())}:${pad(date.getMinutes())}:${pad(date.getSeconds())}.${pad(date.getMilliseconds(), 3)}`
|
||||
);
|
||||
};
|
||||
|
||||
const write = (level: Level, message: string, ...args: unknown[]) => {
|
||||
const line = `${formatLocalTimestamp(new Date())} [${level.toUpperCase()}] ${message}`;
|
||||
if (level === Level.Error) {
|
||||
console.error(line, ...args);
|
||||
} else if (level === Level.Warn) {
|
||||
console.warn(line, ...args);
|
||||
} else {
|
||||
console.log(line, ...args);
|
||||
}
|
||||
};
|
||||
|
||||
export const logger = {
|
||||
info: (message: string, ...args: unknown[]) =>
|
||||
write(Level.Info, message, ...args),
|
||||
warn: (message: string, ...args: unknown[]) =>
|
||||
write(Level.Warn, message, ...args),
|
||||
error: (message: string, ...args: unknown[]) =>
|
||||
write(Level.Error, message, ...args),
|
||||
};
|
||||
79
src/utils.ts
79
src/utils.ts
|
|
@ -3,6 +3,7 @@ import os from 'os';
|
|||
import path from 'path';
|
||||
|
||||
import { CACHE_DIR, REPOSITORIES } from './config';
|
||||
import { logger } from './logger';
|
||||
|
||||
export const getCachedServer = (filePath: string) => {
|
||||
const srv = REPOSITORIES.find((s) => {
|
||||
|
|
@ -52,29 +53,69 @@ export const printServedEndpoints = (
|
|||
}
|
||||
};
|
||||
|
||||
export const extractFileInfo = (url: string) => {
|
||||
// Split the pathname into segments
|
||||
const segments = url.split('/');
|
||||
// Eviction only cares about day-level age, so skip the metadata write
|
||||
// when the file was already touched recently in this window.
|
||||
const TOUCH_THRESHOLD_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
// Get the last segment (filename)
|
||||
const filename = segments[segments.length - 1].replace(/\?.*$/, '');
|
||||
// Bump mtime so eviction tracks last-served time, not last-downloaded time
|
||||
// (sendFile() never touches the file, so a frequently-served cache hit would
|
||||
// otherwise look just as stale as one nobody has asked for in months).
|
||||
export const touchFile = (filePath: string) => {
|
||||
try {
|
||||
const { mtimeMs } = fs.statSync(filePath);
|
||||
if (Date.now() - mtimeMs < TOUCH_THRESHOLD_MS) {
|
||||
return;
|
||||
}
|
||||
const now = new Date();
|
||||
fs.utimesSync(filePath, now, now);
|
||||
} catch {
|
||||
// best-effort; a missed touch just makes the file evict a bit early
|
||||
}
|
||||
};
|
||||
|
||||
// Extract the file extension
|
||||
const fileExtension = filename.split('.').pop();
|
||||
export const evictExpiredCacheFiles = (
|
||||
cacheDir: string,
|
||||
ttlDays: number
|
||||
): void => {
|
||||
const maxAgeMs = ttlDays * 24 * 60 * 60 * 1000;
|
||||
const cutoff = Date.now() - maxAgeMs;
|
||||
let removed = 0;
|
||||
|
||||
// Construct the relative path (excluding the filename)
|
||||
const relativePath = segments.slice(0, -1).join('/');
|
||||
const walk = (dir: string) => {
|
||||
let entries: fs.Dirent[];
|
||||
try {
|
||||
entries = fs.readdirSync(dir, { withFileTypes: true });
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
|
||||
// Create the output object
|
||||
const fileInfo = {
|
||||
fileName: filename,
|
||||
relativePath: relativePath,
|
||||
fileExtension: fileExtension,
|
||||
} as {
|
||||
fileName: string;
|
||||
relativePath: string;
|
||||
fileExtension: string;
|
||||
for (const entry of entries) {
|
||||
const entryPath = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
walk(entryPath);
|
||||
continue;
|
||||
}
|
||||
const { mtimeMs } = fs.statSync(entryPath);
|
||||
if (mtimeMs < cutoff) {
|
||||
fs.unlinkSync(entryPath);
|
||||
removed += 1;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
return fileInfo;
|
||||
walk(cacheDir);
|
||||
if (removed > 0) {
|
||||
logger.info(`🧹 evicted ${removed} cached file(s) older than ${ttlDays}d`);
|
||||
}
|
||||
};
|
||||
|
||||
export const extractFileInfo = (url: string) => {
|
||||
// Use a dummy base so relative/absolute paths both parse, then strip the query string
|
||||
const pathname = new URL(url, 'http://localhost').pathname;
|
||||
|
||||
const fileName = path.basename(pathname);
|
||||
const relativePath = path.dirname(pathname);
|
||||
const fileExtension = path.extname(fileName).replace(/^\./, '');
|
||||
|
||||
return { fileName, relativePath, fileExtension };
|
||||
};
|
||||
|
|
|
|||
1
types.d.ts
vendored
1
types.d.ts
vendored
|
|
@ -27,5 +27,6 @@ export interface IConfig {
|
|||
LOG_REQUESTS?: boolean;
|
||||
IGNORE_FILES?: string[];
|
||||
VALID_FILE_TYPES?: string[];
|
||||
CACHE_TTL_DAYS?: number;
|
||||
PROXIES: Record<string, TProxy>;
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue