init

.vscode/launch.json

@@ -0,0 +1,56 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Launch Program",
+            "type": "node",
+            "request": "launch",
+            "skipFiles": [
+                "<node_internals>/**"
+            ],
+            // "runtimeExecutable": "${workspaceFolder}/node_modules/.bin/ts-node",
+            "runtimeArgs": [
+                "-r",
+                "ts-node/register"
+                // "--transpile-only",
+                // "--esm"
+            ],
+            "program": "${workspaceFolder}/src/index.ts",
+            // "outFiles": [
+            //     "${workspaceFolder}/**/*.js"
+            // ]
+        },
+        {
+            "name": "tsx",
+            "type": "node",
+            "request": "launch",
+            // Debug current file in VSCode
+            "program": "src/index.ts",
+            /*
+            Path to tsx binary
+            Assuming locally installed
+            */
+            "runtimeExecutable": "tsx",
+            /*
+            Open terminal when debugging starts (Optional)
+            Useful to see console.logs
+            */
+            "console": "integratedTerminal",
+            "internalConsoleOptions": "neverOpen",
+            "runtimeArgs": [
+                "-e"
+            ],
+            // Files to exclude from debugger (e.g. call stack)
+            "skipFiles": [
+                // Node.js internal core modules
+                "<node_internals>/**",
+                // Ignore all dependencies (optional)
+                "${workspaceFolder}/node_modules/**",
+            ],
+            "sourceMaps": true
+        }
+    ]
+}

docker-compose.yaml

@@ -5,8 +5,10 @@ services:
     build:
       context: .
       dockerfile: Dockerfile
-    
+
+    restart: unless-stopped
     volumes:
       - $PWD/local-cache:/home/node/app/local-cache
+      - $PWD/config.local.yml:/home/node/app/config.local.yml
     ports:
       - 0.0.0.0:8008:8008

package.json

@@ -25,13 +25,16 @@
     "pm2": "^5.3.0",
     "proxy-agent": "^6.3.1",
     "tsx": "^4.7.0",
-    "typescript": "^5.3.3"
+    "winston": "^3.13.1"
   },
   "devDependencies": {
     "@types/express": "^4.17.21",
     "@types/got": "^9.6.12",
     "@types/js-yaml": "^4.0.9",
     "@types/minimist": "^1.2.5",
-    "@types/morgan": "^1.9.9"
+    "@types/morgan": "^1.9.9",
+    "@types/node": "^20.14.11",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.5.3"
   }
 }

src/downloader/got.ts

@@ -1,11 +1,13 @@
-import path from 'path';
+import path from "path";
-import got, { GotOptions } from 'got';
+import got, { GotOptions, HTTPError } from "got";
-import type { Response } from 'express';
+import type { Response } from "express";
-import fs, { createWriteStream } from 'fs';
+import fs, { createWriteStream } from "fs";
 
-import { PROXIES, CACHE_DIR, TMP_DIR, REPOSITORIES } from '../config';
+import { PROXIES, CACHE_DIR, TMP_DIR, REPOSITORIES } from "../config";
-import { ProxyAgent } from 'proxy-agent';
+import { ProxyAgent } from "proxy-agent";
+import { logger } from "../utils";
 
+console.log("fdsfsfd");
 export class GotDownloader {
   db: {
     [K: string]: {
@@ -26,7 +28,7 @@ export class GotDownloader {
     return null;
   };
 
-  getOptions = (srv: TServer, method: 'get' | 'head' = 'get') => {
+  getOptions = (srv: TServer, method: "get" | "head" = "get") => {
     const options: GotOptions<typeof method> = { method };
     const agent = this.getAgent(srv);
     if (agent) {
@@ -40,10 +42,10 @@ export class GotDownloader {
       options.headers = {};
       options.headers.authorization = `Basic ${Buffer.from(
         `${srv.auth.username}:${srv.auth.password}`
-      ).toString('base64')}`;
+      ).toString("base64")}`;
     }
 
-    if (method === 'head') {
+    if (method === "head") {
       options.timeout = { request: 5000 };
     }
 
@@ -51,52 +53,77 @@ export class GotDownloader {
   };
 
   checkServer = (url: string, srv: TServer) => {
-    const options = this.getOptions(srv, 'head');
+    const options = this.getOptions(srv, "head");
-    return got.head(srv.url + url, options);
+
+    return got.head(srv.url + url, options).catch((error) => {
+      if (error instanceof HTTPError && error.response.statusCode == 404)
+        return null;
+      logger.error(`hit ${url} with ${srv.url + url} ${error}`);
+      return null;
+    });
+  };
+
+  getMatchUrlWithServer = (url: string): TServer | null => {
+    for (const element of REPOSITORIES) {
+      if (element.paths == null || element.paths.length == 0) continue;
+      for (const path of element.paths) {
+        if (url.includes(path)) return element;
+      }
+    }
+    return null;
   };
 
   getSupportedServer = async (url: string) => {
+    var matched: TServer | null = this.getMatchUrlWithServer(url);
+    if (matched != null) {
+      return matched;
+    }
     if (this.db[url]?.serverIndex) {
       return REPOSITORIES[this.db[url].serverIndex];
     }
     const gotPromises = REPOSITORIES.map((srv) => this.checkServer(url, srv));
-    return Promise.any(gotPromises.map((req, index) => req.then(() => index)))
+    return Promise.any(
+      gotPromises.map((req, index) =>
+        req.then((resp) => {
+          if (resp == null) return Promise.reject();
+          return index;
+        })
+      )
+    )
       .then((index) => {
-        // cancel all got requests
-        gotPromises.forEach((req) => req.cancel());
-        this.db[url] = {
-          serverIndex: index,
-        };
         return REPOSITORIES[index];
       })
-      .catch(() => null);
+      .catch((e) => {
+        logger.error(`hit exception ${e}`);
+      });
   };
 
   head = (url: string, srv: TServer, res: Response) => {
     return got
-      .head(srv.url + url, this.getOptions(srv, 'head'))
+      .head(srv.url + url, this.getOptions(srv, "head"))
       .then((r) => {
         res.set(r.headers);
         res.sendStatus(r.statusCode);
       })
       .catch((r) => {
+        console.log(`✅ [${r}]`, url);
         res.sendStatus(r.statusCode);
       });
   };
 
   download = (url: string, srv: TServer, res: Response) => {
-    const fileName = url.split('/').pop() || '';
+    const fileName = url.split("/").pop() || "";
     const tmpPath = path.resolve(TMP_DIR, fileName);
-    const outputDir = path.join(CACHE_DIR, srv.name, url).replace(fileName, '');
+    const outputDir = path.join(CACHE_DIR, srv.name, url).replace(fileName, "");
     const stream = got.stream(srv.url + url, this.getOptions(srv));
     const fileWriterStream = createWriteStream(tmpPath);
-    stream.pipe(fileWriterStream).on('finish', () => {
+    stream.pipe(fileWriterStream).on("finish", () => {
       console.log(`✅ [${srv.name}]`, url);
       this.moveToCache(fileName, outputDir);
       delete this.db[url];
     });
 
-    stream.pipe(res).on('error', console.error);
+    stream.pipe(res).on("error", console.error);
   };
 
   moveToCache = (fileName: string, outputDir: string) => {

src/index.ts

@@ -13,7 +13,7 @@ import {
   IGNORE_FILES,
   VALID_FILE_TYPES,
 } from './config';
-import { getCachedServer, printServedEndpoints } from './utils';
+import { getCachedServer, printServedEndpoints, logger } from './utils';
 import { GotDownloader } from './downloader/got';
 
 const downloader = new GotDownloader();
@@ -59,7 +59,10 @@ const cacheRequestHandler: RequestHandler = (req, res, next) => {
         res.sendStatus(403);
       }
     })
-    .catch(() => res.sendStatus(403));
+    .catch((e) => {
+      logger.info(e)
+      return res.sendStatus(403);
+    });
 };
 
 // init cache dir
@@ -76,7 +79,9 @@ const app = express();
 if (VERBOSE) {
   app.use(morgan('combined'));
 }
-app.get('*', cacheRequestHandler);
+app.get('*', async (req, res, next) => {
+  await cacheRequestHandler(req, res, next);
+});
 app.listen(PORT, () => {
   console.log('add this ⬇️  in build.gradle');
   console.log(

src/utils.ts

@@ -46,3 +46,14 @@ export const printServedEndpoints = (port: number | string, urlPath: string) =>
     console.log('--------------------------------------------');
   }
 };
+
+
+const winston = require("winston");
+export const logger = winston.createLogger({
+  level: "info",
+  format: winston.format.json(),
+  transports: [
+    new winston.transports.Console(),
+    new winston.transports.File({ filename: "logs/app.log" }),
+  ],
+});

tsconfig.json

@@ -47,7 +47,7 @@
     // "declaration": true,                              /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
     // "declarationMap": true,                           /* Create sourcemaps for d.ts files. */
     // "emitDeclarationOnly": true,                      /* Only output d.ts files and not JavaScript files. */
-    // "sourceMap": true,                                /* Create source map files for emitted JavaScript files. */
+    "sourceMap": true,                                /* Create source map files for emitted JavaScript files. */
     // "outFile": "./",                                  /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
     // "outDir": "./",                                   /* Specify an output folder for all emitted files. */
     // "removeComments": true,                           /* Disable emitting comments. */

types.ts

@@ -12,6 +12,7 @@ type TServer = {
   name: string;
   url: string;
   fileTypes?: string[];
+  paths?: string[];
   proxy?: string;
   auth?: {
     username: string;