// ========= Copyright 2025-2026 @ Eigent.ai All Rights Reserved. ========= // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // ========= Copyright 2025-2026 @ Eigent.ai All Rights Reserved. ========= /* global console, process */ // @ts-check import fs from 'fs'; import http from 'http'; import https from 'https'; import tls from 'tls'; import { URL } from 'url'; /** * Check if the target URL should bypass proxy based on NO_PROXY env var. * @param {string} targetUrl - The target URL * @returns {boolean} True if proxy should be bypassed */ function shouldBypassProxy(targetUrl) { const noProxy = process.env.NO_PROXY || process.env.no_proxy; if (!noProxy) return false; try { const targetHost = new URL(targetUrl).hostname.toLowerCase(); const noProxyList = noProxy.split(',').map((s) => s.trim().toLowerCase()); for (const pattern of noProxyList) { if (!pattern) continue; if (pattern === '*') return true; if (targetHost === pattern) return true; if (pattern.startsWith('.') && targetHost.endsWith(pattern)) return true; if (targetHost.endsWith('.' + pattern)) return true; } } catch (error) { console.warn(`Warning: Failed to parse NO_PROXY: ${error.message}`); } return false; } /** * Get proxy URL from environment variables. * @param {string} targetUrl - The target URL to determine which proxy to use * @returns {string | null} The proxy URL or null if not configured */ function getProxyUrl(targetUrl) { // Check NO_PROXY first if (shouldBypassProxy(targetUrl)) { return null; } const isHttps = targetUrl.startsWith('https://'); // Priority order for proxy env vars (check both uppercase and lowercase) const envVars = isHttps ? ['HTTPS_PROXY', 'https_proxy', 'HTTP_PROXY', 'http_proxy'] : ['HTTP_PROXY', 'http_proxy']; for (const envVar of envVars) { const value = process.env[envVar]; if (value && value.trim()) { return value.trim(); } } return null; } /** * Mask credentials in a proxy URL for safe logging. * @param {string} url - The URL to mask * @returns {string} The masked URL */ function maskProxyUrl(url) { try { const parsed = new URL(url); if (parsed.username || parsed.password) { parsed.username = parsed.username ? '***' : ''; parsed.password = parsed.password ? '***' : ''; return parsed.toString(); } } catch (error) { // Not a valid URL, return as-is console.warn(`Warning: Failed to parse proxy URL: ${error.message}`); } return url; } /** * Make an HTTP GET request with optional proxy support. * For HTTPS URLs through HTTP proxy, uses CONNECT tunnel with TLS. * @param {string} url - The URL to request * @param {(response: http.IncomingMessage) => void} callback - Response callback * @param {(error: Error) => void} onError - Error callback */ function makeRequest(url, callback, onError) { const proxyUrl = getProxyUrl(url); const isHttps = url.startsWith('https://'); const targetUrl = new URL(url); const targetPort = parseInt(targetUrl.port, 10) || (isHttps ? 443 : 80); if (!proxyUrl) { // Direct connection (no proxy) const httpModule = isHttps ? https : http; const req = httpModule.get(url, callback); req.on('error', onError); return; } console.log(`Using proxy: ${maskProxyUrl(proxyUrl)}`); const proxy = new URL(proxyUrl); const proxyPort = parseInt(proxy.port, 10) || 80; // Build proxy auth header if credentials provided const proxyAuthHeader = proxy.username || proxy.password ? { 'Proxy-Authorization': `Basic ${Buffer.from( `${decodeURIComponent(proxy.username || '')}:${decodeURIComponent(proxy.password || '')}` ).toString('base64')}`, } : {}; if (isHttps) { // HTTPS through HTTP proxy: Use CONNECT tunnel const connectReq = http.request({ host: proxy.hostname, port: proxyPort, method: 'CONNECT', path: `${targetUrl.hostname}:${targetPort}`, headers: { Host: `${targetUrl.hostname}:${targetPort}`, ...proxyAuthHeader, }, }); // Track resources for cleanup let tlsSocket = null; let httpsReq = null; // Cleanup function to destroy all connections const cleanup = () => { if (httpsReq && !httpsReq.destroyed) { httpsReq.destroy(); } if (tlsSocket && !tlsSocket.destroyed) { tlsSocket.destroy(); } if (connectReq && !connectReq.destroyed) { connectReq.destroy(); } }; connectReq.on('connect', (res, socket) => { if (res.statusCode !== 200) { socket.destroy(); cleanup(); onError( new Error( `Proxy CONNECT failed with status ${res.statusCode}: ${res.statusMessage}` ) ); return; } // Upgrade socket to TLS tlsSocket = tls.connect( { host: targetUrl.hostname, port: targetPort, socket: socket, servername: targetUrl.hostname, // SNI }, () => { // Make HTTPS request over TLS socket httpsReq = https.request( { hostname: targetUrl.hostname, port: targetPort, path: targetUrl.pathname + targetUrl.search, method: 'GET', headers: { Host: targetUrl.host }, agent: false, // Use createConnection to provide the pre-established TLS socket createConnection: () => tlsSocket, }, (response) => { // Cleanup connections when response ends response.on('end', cleanup); response.on('close', cleanup); callback(response); } ); httpsReq.on('error', (err) => { cleanup(); onError(new Error(`HTTPS request error: ${err.message}`)); }); httpsReq.end(); } ); tlsSocket.on('error', (err) => { cleanup(); onError(new Error(`TLS connection error: ${err.message}`)); }); }); connectReq.on('error', (err) => { cleanup(); onError(new Error(`Proxy connection error: ${err.message}`)); }); connectReq.setTimeout(30000, () => { cleanup(); onError(new Error('Proxy connection timeout after 30 seconds')); }); connectReq.end(); } else { // HTTP through HTTP proxy: Use proxy as target with full URL as path const req = http.request( { host: proxy.hostname, port: proxyPort, path: url, // Full URL for HTTP proxy method: 'GET', headers: { Host: targetUrl.host, ...proxyAuthHeader, }, }, callback ); req.on('error', (err) => { onError(new Error(`HTTP proxy request error: ${err.message}`)); }); req.end(); } } /** * Downloads a file from a URL with redirect handling and proxy support. * Proxy is automatically detected from environment variables: * - HTTPS_PROXY / https_proxy (for HTTPS URLs) * - HTTP_PROXY / http_proxy (for HTTP URLs, or as fallback for HTTPS) * - NO_PROXY / no_proxy (to bypass proxy for specific hosts) * * @param {string} url The URL to download from * @param {string} destinationPath The path to save the file to * @returns {Promise} Promise that resolves when download is complete */ export async function downloadWithRedirects(url, destinationPath) { return new Promise((resolve, reject) => { const timeoutMs = 10 * 60 * 1000; // 10 minutes let timeoutId = null; // Use flag to prevent multiple resolve/reject calls let settled = false; const safeReject = (error) => { if (!settled) { settled = true; if (timeoutId) clearTimeout(timeoutId); reject(error); } }; const safeResolve = () => { if (!settled) { settled = true; if (timeoutId) clearTimeout(timeoutId); resolve(); } }; timeoutId = setTimeout(() => { safeReject(new Error(`Download timeout after ${timeoutMs / 1000} seconds`)); }, timeoutMs); const request = (requestUrl) => { makeRequest( requestUrl, (response) => { const statusCode = response.statusCode || 0; // Handle redirects (301, 302, 307, 308) if ( statusCode >= 301 && statusCode <= 308 && response.headers.location ) { let redirectUrl = response.headers.location; // Handle relative redirects if (redirectUrl.startsWith('/')) { try { const originalUrl = new URL(requestUrl); redirectUrl = `${originalUrl.protocol}//${originalUrl.host}${redirectUrl}`; } catch (error) { safeReject(new Error(`Failed to parse redirect URL: ${error.message}`)); return; } } console.log(`Following redirect to: ${redirectUrl}`); request(redirectUrl); return; } if (statusCode !== 200) { safeReject( new Error( `Download failed with status ${statusCode}: ${response.statusMessage || 'Unknown error'}` ) ); return; } const file = fs.createWriteStream(destinationPath); let downloadedBytes = 0; const expectedBytes = parseInt( response.headers['content-length'] || '0', 10 ); const startTime = Date.now(); let lastProgressTime = Date.now(); if (expectedBytes > 0) { console.log( `Downloading ${(expectedBytes / 1024 / 1024).toFixed(2)} MB...` ); } else { console.log('Downloading...'); } response.on('data', (chunk) => { downloadedBytes += chunk.length; // Show progress every 1 second const now = Date.now(); if (now - lastProgressTime >= 1000) { if (expectedBytes > 0) { const percent = ( (downloadedBytes / expectedBytes) * 100 ).toFixed(1); const speed = downloadedBytes / ((now - startTime) / 1000) / 1024 / 1024; console.log( `Progress: ${percent}% (${(downloadedBytes / 1024 / 1024).toFixed(2)} MB) - ${speed.toFixed(2)} MB/s` ); } else { console.log( `Downloaded: ${(downloadedBytes / 1024 / 1024).toFixed(2)} MB` ); } lastProgressTime = now; } }); response.pipe(file); file.on('finish', () => { file.close(() => { // Don't proceed if already rejected (e.g., by error handler) if (settled) return; // Verify the download is complete if (expectedBytes > 0 && downloadedBytes !== expectedBytes) { try { if (fs.existsSync(destinationPath)) { fs.unlinkSync(destinationPath); } } catch (cleanupError) { console.warn( `Warning: Failed to delete incomplete file: ${cleanupError.message}` ); } safeReject( new Error( `Download incomplete: received ${downloadedBytes} bytes, expected ${expectedBytes}` ) ); return; } // Check if file exists and has size > 0 try { if (fs.existsSync(destinationPath)) { const stats = fs.statSync(destinationPath); if (stats.size === 0) { fs.unlinkSync(destinationPath); safeReject(new Error('Downloaded file is empty')); return; } safeResolve(); } else { safeReject(new Error('Downloaded file does not exist')); } } catch (verifyError) { safeReject( new Error(`Failed to verify download: ${verifyError.message}`) ); } }); }); file.on('error', (err) => { try { if (fs.existsSync(destinationPath)) { fs.unlinkSync(destinationPath); } } catch (cleanupError) { console.warn( `Warning: Failed to delete file after error: ${cleanupError.message}` ); } safeReject(new Error(`File write error: ${err.message}`)); }); }, safeReject ); }; request(url); }); }