vikarti.anatra/maven-mirror-tool
1
0
Fork 0
mirror of https://github.com/hexboy/maven-mirror-tool.git synced 2025-09-01 18:19:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

create single instance downloader for same url

Browse source
This commit is contained in:
Hassan Yousefi 2023-09-04 17:23:05 +03:30
parent bcf376bbe7
commit f457ad8e55
1 changed files with 72 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

110
src/utils.ts
View file

@ -1,9 +1,16 @@
import fs from 'fs';
import path from 'path';
import { exec } from 'child_process';
import { promisify } from 'util';
const exec = promisify(require('child_process').exec);
import { PROXIES, CACHE_DIR, IGNORE_FILES, REPOSITORIES } from './config';
type TDownloadRes = null | {
fileName: string;
serverName: string;
filePath: string;
};
export const getCachedPath = (filePath: string) => {
const srv = REPOSITORIES.find((s) => {
const fPath = path.join(CACHE_DIR, s.name, filePath);
@ -16,40 +23,68 @@ export const getCachedPath = (filePath: string) => {
return srv ? path.join(CACHE_DIR, srv.name, filePath) : null;
};
export const download = async (
url: string,
outputDir: string,
srv: TServer
) => {
const activeDownloads: {
[K: string]: {
subscribers: ((value: TDownloadRes | PromiseLike<TDownloadRes>) => void)[];
};
} = {};
const download = async (url: string, srv: TServer): Promise<TDownloadRes> => {
const fileName = url.split('/').pop() || '';
const outputDir = path.join(CACHE_DIR, srv.name, url).replace(fileName, '');
const fileType = fileName.slice(fileName.lastIndexOf('.'));
const tmpPath = path.resolve(CACHE_DIR, '_tmp_', fileName);
return new Promise((resolve, reject) => {
if (srv.fileTypes && !srv.fileTypes.includes(fileType)) {
return resolve(null);
}
const proxy = srv.proxy && srv.proxy in PROXIES ? PROXIES[srv.proxy] : null;
const cmd = `curl ${
proxy ? `-x "${proxy.protocol}://${proxy.host}:${proxy.port}" ` : ''
} ${
srv.auth ? `-u "${srv.auth.username}:${srv.auth.password}" ` : ''
}--connect-timeout 3 ${
fileType === '.jar' || fileType === '.aar' ? '--max-time 1000 ' : ''
}--max-redirs 3 -L -f -o ${tmpPath} ${srv.url + url}`;
exec(cmd, (error: any, stdout: any, stderr: any) => {
if (error) {
return resolve(null);
} else if (fs.existsSync(tmpPath) ? fs.statSync(tmpPath).size : 0) {
if (!fs.existsSync(outputDir)) {
fs.mkdirSync(outputDir, { recursive: true });
}
fs.renameSync(tmpPath, path.join(outputDir, fileName));
return resolve(srv.name);
} else {
return resolve(null);
if (srv.fileTypes && !srv.fileTypes.includes(fileType)) {
return null;
}
const proxy = srv.proxy && srv.proxy in PROXIES ? PROXIES[srv.proxy] : null;
const cmd = `curl ${
proxy ? `-x "${proxy.protocol}://${proxy.host}:${proxy.port}" ` : ''
} ${
srv.auth ? `-u "${srv.auth.username}:${srv.auth.password}" ` : ''
}--connect-timeout 3 ${
fileType === '.jar' || fileType === '.aar' ? '--max-time 1000 ' : ''
}--max-redirs 3 -L -f -o ${tmpPath} ${srv.url + url}`;
try {
await exec(cmd);
if (fs.existsSync(tmpPath) ? fs.statSync(tmpPath).size : 0) {
if (!fs.existsSync(outputDir)) {
fs.mkdirSync(outputDir, { recursive: true });
}
});
fs.renameSync(tmpPath, path.join(outputDir, fileName));
return {
fileName,
serverName: srv.name,
filePath: path.join(outputDir, fileName),
};
} else {
return null;
}
} catch (error) {
return null;
}
};
const attachToDownloader = async (url: string): Promise<TDownloadRes> =>
new Promise((resolve, reject) => {
activeDownloads[url].subscribers.push(resolve);
});
const downloadMultiServer = async (url: string) => {
if (!(url in activeDownloads)) {
activeDownloads[url] = { subscribers: [] };
for await (const srv of REPOSITORIES) {
const result = await download(url, srv);
if (result) {
activeDownloads[url].subscribers.forEach((sub) => sub(result));
delete activeDownloads[url];
return;
}
}
activeDownloads[url].subscribers.forEach((sub) => sub(null));
delete activeDownloads[url];
}
};
export const downloadFile = async (url: string, res: any) => {
@ -57,14 +92,13 @@ export const downloadFile = async (url: string, res: any) => {
console.log('❌ [404]', url);
return res.sendStatus(404);
}
for await (const srv of REPOSITORIES) {
const fileName = url.split('/').pop() || '';
const outputDir = path.join(CACHE_DIR, srv.name, url).replace(fileName, '');
const downloadedFrom = await download(url, outputDir, srv);
if (downloadedFrom) {
console.log(`✅ [${downloadedFrom}]`, url);
return res.sendFile(path.join(outputDir, fileName));
}
downloadMultiServer(url);
const downloadRes = await attachToDownloader(url);
if (downloadRes) {
console.log(`✅ [${downloadRes.serverName}]`, url);
return res.sendFile(downloadRes.filePath);
}
console.log('❌ [403]', url);
return res.sendStatus(403);