vikarti.anatra/maven-mirror-tool
1
0
Fork 0
mirror of https://github.com/hexboy/maven-mirror-tool.git synced 2025-09-02 02:29:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

create single instance downloader for same url

Browse source
This commit is contained in:
Hassan Yousefi 2023-09-04 17:23:05 +03:30
parent bcf376bbe7
commit f457ad8e55
1 changed files with 72 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

110
src/utils.ts
View file

@ -1,9 +1,16 @@
import fs from 'fs'; import fs from 'fs';
import path from 'path'; import path from 'path';
import { exec } from 'child_process'; import { promisify } from 'util';
const exec = promisify(require('child_process').exec);
import { PROXIES, CACHE_DIR, IGNORE_FILES, REPOSITORIES } from './config'; import { PROXIES, CACHE_DIR, IGNORE_FILES, REPOSITORIES } from './config';
type TDownloadRes = null | {
fileName: string;
serverName: string;
filePath: string;
};
export const getCachedPath = (filePath: string) => { export const getCachedPath = (filePath: string) => {
const srv = REPOSITORIES.find((s) => { const srv = REPOSITORIES.find((s) => {
const fPath = path.join(CACHE_DIR, s.name, filePath); const fPath = path.join(CACHE_DIR, s.name, filePath);
@ -16,40 +23,68 @@ export const getCachedPath = (filePath: string) => {
return srv ? path.join(CACHE_DIR, srv.name, filePath) : null; return srv ? path.join(CACHE_DIR, srv.name, filePath) : null;
}; };
export const download = async ( const activeDownloads: {
url: string, [K: string]: {
outputDir: string, subscribers: ((value: TDownloadRes | PromiseLike<TDownloadRes>) => void)[];
srv: TServer };
) => { } = {};
const download = async (url: string, srv: TServer): Promise<TDownloadRes> => {
const fileName = url.split('/').pop() || ''; const fileName = url.split('/').pop() || '';
const outputDir = path.join(CACHE_DIR, srv.name, url).replace(fileName, '');
const fileType = fileName.slice(fileName.lastIndexOf('.')); const fileType = fileName.slice(fileName.lastIndexOf('.'));
const tmpPath = path.resolve(CACHE_DIR, '_tmp_', fileName); const tmpPath = path.resolve(CACHE_DIR, '_tmp_', fileName);
return new Promise((resolve, reject) => { if (srv.fileTypes && !srv.fileTypes.includes(fileType)) {
if (srv.fileTypes && !srv.fileTypes.includes(fileType)) { return null;
return resolve(null); }
} const proxy = srv.proxy && srv.proxy in PROXIES ? PROXIES[srv.proxy] : null;
const proxy = srv.proxy && srv.proxy in PROXIES ? PROXIES[srv.proxy] : null; const cmd = `curl ${
const cmd = `curl ${ proxy ? `-x "${proxy.protocol}://${proxy.host}:${proxy.port}" ` : ''
proxy ? `-x "${proxy.protocol}://${proxy.host}:${proxy.port}" ` : '' } ${
} ${ srv.auth ? `-u "${srv.auth.username}:${srv.auth.password}" ` : ''
srv.auth ? `-u "${srv.auth.username}:${srv.auth.password}" ` : '' }--connect-timeout 3 ${
}--connect-timeout 3 ${ fileType === '.jar' || fileType === '.aar' ? '--max-time 1000 ' : ''
fileType === '.jar' || fileType === '.aar' ? '--max-time 1000 ' : '' }--max-redirs 3 -L -f -o ${tmpPath} ${srv.url + url}`;
}--max-redirs 3 -L -f -o ${tmpPath} ${srv.url + url}`;
exec(cmd, (error: any, stdout: any, stderr: any) => { try {
if (error) { await exec(cmd);
return resolve(null); if (fs.existsSync(tmpPath) ? fs.statSync(tmpPath).size : 0) {
} else if (fs.existsSync(tmpPath) ? fs.statSync(tmpPath).size : 0) { if (!fs.existsSync(outputDir)) {
if (!fs.existsSync(outputDir)) { fs.mkdirSync(outputDir, { recursive: true });
fs.mkdirSync(outputDir, { recursive: true });
}
fs.renameSync(tmpPath, path.join(outputDir, fileName));
return resolve(srv.name);
} else {
return resolve(null);
} }
}); fs.renameSync(tmpPath, path.join(outputDir, fileName));
return {
fileName,
serverName: srv.name,
filePath: path.join(outputDir, fileName),
};
} else {
return null;
}
} catch (error) {
return null;
}
};
const attachToDownloader = async (url: string): Promise<TDownloadRes> =>
new Promise((resolve, reject) => {
activeDownloads[url].subscribers.push(resolve);
}); });
const downloadMultiServer = async (url: string) => {
if (!(url in activeDownloads)) {
activeDownloads[url] = { subscribers: [] };
for await (const srv of REPOSITORIES) {
const result = await download(url, srv);
if (result) {
activeDownloads[url].subscribers.forEach((sub) => sub(result));
delete activeDownloads[url];
return;
}
}
activeDownloads[url].subscribers.forEach((sub) => sub(null));
delete activeDownloads[url];
}
}; };
export const downloadFile = async (url: string, res: any) => { export const downloadFile = async (url: string, res: any) => {
@ -57,14 +92,13 @@ export const downloadFile = async (url: string, res: any) => {
console.log('❌ [404]', url); console.log('❌ [404]', url);
return res.sendStatus(404); return res.sendStatus(404);
} }
for await (const srv of REPOSITORIES) {
const fileName = url.split('/').pop() || ''; downloadMultiServer(url);
const outputDir = path.join(CACHE_DIR, srv.name, url).replace(fileName, '');
const downloadedFrom = await download(url, outputDir, srv); const downloadRes = await attachToDownloader(url);
if (downloadedFrom) { if (downloadRes) {
console.log(`✅ [${downloadedFrom}]`, url); console.log(`✅ [${downloadRes.serverName}]`, url);
return res.sendFile(path.join(outputDir, fileName)); return res.sendFile(downloadRes.filePath);
}
} }
console.log('❌ [403]', url); console.log('❌ [403]', url);
return res.sendStatus(403); return res.sendStatus(403);