diff --git a/backend/app/utils/toolkit/terminal_toolkit.py b/backend/app/utils/toolkit/terminal_toolkit.py index d3ca259de..1199ec4aa 100644 --- a/backend/app/utils/toolkit/terminal_toolkit.py +++ b/backend/app/utils/toolkit/terminal_toolkit.py @@ -1,7 +1,9 @@ import asyncio import logging import os +import platform import shutil +import subprocess import threading import time from concurrent.futures import ThreadPoolExecutor @@ -17,6 +19,20 @@ from utils import traceroot_wrapper as traceroot logger = traceroot.get_logger("terminal_toolkit") +# App version - should match electron app version +# TODO: Consider getting this from a shared config +APP_VERSION = "0.0.80" + + +def get_terminal_base_venv_path() -> str: + """Get the path to the terminal base venv created during app installation.""" + return os.path.join( + os.path.expanduser("~"), + ".eigent", + "venvs", + f"terminal_base-{APP_VERSION}" + ) + @auto_listen_toolkit(BaseTerminalToolkit) class TerminalToolkit(BaseTerminalToolkit, AbstractToolkit): @@ -41,12 +57,14 @@ class TerminalToolkit(BaseTerminalToolkit, AbstractToolkit): if agent_name is not None: self.agent_name = agent_name if working_directory is None: - working_directory = env("file_save_path", os.path.expanduser("~/.eigent/terminal/")) + base_dir = env("file_save_path", os.path.expanduser("~/.eigent/terminal/")) + # Each agent gets its own subdirectory to avoid race conditions when + # multiple agents create .venv simultaneously during parallel initialization + working_directory = os.path.join(base_dir, self.agent_name) logger.debug(f"Initializing TerminalToolkit for agent={self.agent_name}", extra={ "api_task_id": api_task_id, "working_directory": working_directory, - "clone_current_env": clone_current_env }) if TerminalToolkit._thread_pool is None: @@ -63,16 +81,10 @@ class TerminalToolkit(BaseTerminalToolkit, AbstractToolkit): session_logs_dir=session_logs_dir, safe_mode=safe_mode, allowed_commands=allowed_commands, - clone_current_env=clone_current_env, - install_dependencies=[ - "pandas", - "numpy", - "matplotlib", - "requests", - "openpyxl", - ], + clone_current_env=True, + install_dependencies=[], ) - + # Auto-register with TaskLock for cleanup when task ends from app.service.task import get_task_lock_if_exists task_lock = get_task_lock_if_exists(api_task_id) @@ -83,6 +95,113 @@ class TerminalToolkit(BaseTerminalToolkit, AbstractToolkit): "working_directory": working_directory }) + def _setup_cloned_environment(self): + """Override to clone from terminal_base venv instead of current process venv. + + Creates a lightweight clone using symlinks to the terminal_base venv, + which contains pre-installed packages (pandas, numpy, matplotlib, etc.). + """ + self.cloned_env_path = os.path.join(self.working_dir, ".venv") + terminal_base_path = get_terminal_base_venv_path() + + # Check if terminal_base exists + if platform.system() == 'Windows': + base_python = os.path.join(terminal_base_path, "Scripts", "python.exe") + else: + base_python = os.path.join(terminal_base_path, "bin", "python") + + if not os.path.exists(base_python): + logger.warning( + f"Terminal base venv not found at {terminal_base_path}, " + "falling back to system Python" + ) + return + + # Check if cloned env already exists + if platform.system() == 'Windows': + cloned_python = os.path.join(self.cloned_env_path, "Scripts", "python.exe") + else: + cloned_python = os.path.join(self.cloned_env_path, "bin", "python") + + if os.path.exists(cloned_python): + logger.info(f"Using existing cloned environment: {self.cloned_env_path}") + self.python_executable = cloned_python + return + + logger.info(f"Cloning terminal_base venv to: {self.cloned_env_path}") + + try: + # Create the cloned venv directory + os.makedirs(self.cloned_env_path, exist_ok=True) + + # Clone using symlinks for efficiency + # We need to create proper venv structure with symlinks to terminal_base + self._clone_venv_with_symlinks(terminal_base_path, self.cloned_env_path) + + self.python_executable = cloned_python + logger.info(f"Successfully cloned environment to: {self.cloned_env_path}") + + except Exception as e: + logger.error(f"Failed to clone terminal_base venv: {e}", exc_info=True) + # Cleanup partial clone + if os.path.exists(self.cloned_env_path): + shutil.rmtree(self.cloned_env_path, ignore_errors=True) + logger.warning("Falling back to system Python") + + def _clone_venv_with_symlinks(self, source_venv: str, target_venv: str): + """Clone a venv using symlinks for efficiency. + + Only creates the minimum structure needed: pyvenv.cfg, bin/python, and lib symlink. + Activation scripts are not needed since we use python_executable directly. + """ + is_windows = platform.system() == 'Windows' + + # Read source pyvenv.cfg to get Python home + source_cfg = os.path.join(source_venv, "pyvenv.cfg") + python_home = None + + with open(source_cfg, 'r') as f: + for line in f: + if line.startswith('home = '): + python_home = line.split('=', 1)[1].strip() + break + + if not python_home: + raise RuntimeError(f"Could not determine Python home from {source_cfg}") + + # Copy pyvenv.cfg (simpler than recreating) + shutil.copy2(source_cfg, os.path.join(target_venv, "pyvenv.cfg")) + + if is_windows: + # Windows: copy executables from source + target_bin = os.path.join(target_venv, "Scripts") + os.makedirs(target_bin, exist_ok=True) + source_scripts = os.path.join(source_venv, "Scripts") + for exe in ["python.exe", "pythonw.exe"]: + src = os.path.join(source_scripts, exe) + if os.path.exists(src): + shutil.copy2(src, os.path.join(target_bin, exe)) + # Use directory junction for Lib (no admin rights needed, unlike symlink) + source_lib = os.path.join(source_venv, "Lib") + target_lib = os.path.join(target_venv, "Lib") + subprocess.run(["cmd", "/c", "mklink", "/J", target_lib, source_lib], + check=True, capture_output=True) + else: + # Unix: symlink python executable and lib directory + target_bin = os.path.join(target_venv, "bin") + os.makedirs(target_bin, exist_ok=True) + + # Symlink python to the base Python + python_exe = os.path.join(python_home, "python3") + if not os.path.exists(python_exe): + python_exe = os.path.join(python_home, "python") + os.symlink(python_exe, os.path.join(target_bin, "python")) + os.symlink("python", os.path.join(target_bin, "python3")) + + # Symlink lib directory + source_lib = os.path.join(source_venv, "lib") + os.symlink(source_lib, os.path.join(target_venv, "lib")) + def _write_to_log(self, log_file: str, content: str) -> None: r"""Write content to log file with optional ANSI stripping. @@ -198,32 +317,34 @@ class TerminalToolkit(BaseTerminalToolkit, AbstractToolkit): return # Remove cloned env (.venv) if it exists - if self.cloned_env_path and os.path.exists(self.cloned_env_path): + cloned_env_path = getattr(self, 'cloned_env_path', None) + if cloned_env_path and os.path.exists(cloned_env_path): try: - shutil.rmtree(self.cloned_env_path) + shutil.rmtree(cloned_env_path) logger.info("Removed cloned venv", extra={ "api_task_id": self.api_task_id, - "path": self.cloned_env_path + "path": cloned_env_path }) except Exception as e: logger.warning("Failed to remove cloned venv", extra={ "api_task_id": self.api_task_id, - "path": self.cloned_env_path, + "path": cloned_env_path, "error": str(e) }) # Remove initial env (.initial_env) if it exists - if self.initial_env_path and os.path.exists(self.initial_env_path): + initial_env_path = getattr(self, 'initial_env_path', None) + if initial_env_path and os.path.exists(initial_env_path): try: - shutil.rmtree(self.initial_env_path) + shutil.rmtree(initial_env_path) logger.info("Removed initial env", extra={ "api_task_id": self.api_task_id, - "path": self.initial_env_path + "path": initial_env_path }) except Exception as e: logger.warning("Failed to remove initial env", extra={ "api_task_id": self.api_task_id, - "path": self.initial_env_path, + "path": initial_env_path, "error": str(e) }) diff --git a/electron/main/install-deps.ts b/electron/main/install-deps.ts index daaebb0fa..e5babcc71 100644 --- a/electron/main/install-deps.ts +++ b/electron/main/install-deps.ts @@ -8,10 +8,12 @@ import { getBinaryPath, getCachePath, getVenvPath, + getTerminalVenvPath, getUvEnv, cleanupOldVenvs, isBinaryExists, runInstallScript, + TERMINAL_BASE_PACKAGES, } from './utils/process'; import { spawn } from 'child_process'; import { safeMainWindowSend } from './utils/safeWebContentsSend'; @@ -482,6 +484,124 @@ const runInstall = (extraArgs: string[], version: string) => { }); }; +/** + * Install terminal base venv with common packages for terminal tasks. + * This is a lightweight venv separate from the backend venv. + */ +async function installTerminalBaseVenv(version: string): Promise { + const terminalVenvPath = getTerminalVenvPath(version); + const pythonPath = process.platform === 'win32' + ? path.join(terminalVenvPath, 'Scripts', 'python.exe') + : path.join(terminalVenvPath, 'bin', 'python'); + + // Check if terminal base venv already exists and is valid + if (fs.existsSync(pythonPath)) { + log.info('[DEPS INSTALL] Terminal base venv already exists, skipping creation'); + return { message: 'Terminal base venv already exists', success: true }; + } + + log.info('[DEPS INSTALL] Creating terminal base venv...'); + safeMainWindowSend('install-dependencies-log', { + type: 'stdout', + data: 'Creating terminal base environment...\n', + }); + + try { + // Create the venv using uv + await new Promise((resolve, reject) => { + const createVenv = spawn( + uv_path, + ['venv', '--python', '3.10', terminalVenvPath], + { + env: { + ...process.env, + UV_PYTHON_INSTALL_DIR: getCachePath('uv_python'), + }, + } + ); + + createVenv.stdout.on('data', (data) => { + log.info(`[DEPS INSTALL] terminal venv: ${data}`); + }); + + createVenv.stderr.on('data', (data) => { + log.info(`[DEPS INSTALL] terminal venv: ${data}`); + }); + + createVenv.on('close', (code) => { + if (code === 0) { + resolve(); + } else { + reject(new Error(`Failed to create terminal venv, exit code: ${code}`)); + } + }); + + createVenv.on('error', reject); + }); + + // Install base packages + log.info('[DEPS INSTALL] Installing terminal base packages...'); + safeMainWindowSend('install-dependencies-log', { + type: 'stdout', + data: `Installing packages: ${TERMINAL_BASE_PACKAGES.join(', ')}...\n`, + }); + + await new Promise((resolve, reject) => { + const installPkgs = spawn( + uv_path, + [ + 'pip', + 'install', + '--python', + pythonPath, + ...TERMINAL_BASE_PACKAGES, + ], + { + env: { + ...process.env, + UV_PYTHON_INSTALL_DIR: getCachePath('uv_python'), + }, + } + ); + + installPkgs.stdout.on('data', (data) => { + log.info(`[DEPS INSTALL] terminal packages: ${data}`); + safeMainWindowSend('install-dependencies-log', { + type: 'stdout', + data: data.toString(), + }); + }); + + installPkgs.stderr.on('data', (data) => { + log.info(`[DEPS INSTALL] terminal packages: ${data}`); + safeMainWindowSend('install-dependencies-log', { + type: 'stdout', + data: data.toString(), + }); + }); + + installPkgs.on('close', (code) => { + if (code === 0) { + resolve(); + } else { + reject(new Error(`Failed to install terminal packages, exit code: ${code}`)); + } + }); + + installPkgs.on('error', reject); + }); + + log.info('[DEPS INSTALL] Terminal base venv created successfully'); + return { message: 'Terminal base venv created successfully', success: true }; + } catch (error) { + log.error('[DEPS INSTALL] Failed to create terminal base venv:', error); + return { + message: `Failed to create terminal base venv: ${error}`, + success: false, + }; + } +} + export async function installDependencies( version: string ): Promise { @@ -890,6 +1010,13 @@ export async function installDependencies( // try default install const installSuccess = await runInstall([], version); if (installSuccess.success) { + // Install terminal base venv (lightweight venv for terminal tasks) + log.info('[DEPS INSTALL] Installing terminal base venv...'); + const terminalResult = await installTerminalBaseVenv(version); + if (!terminalResult.success) { + log.warn('[DEPS INSTALL] Terminal base venv installation failed, but continuing...', terminalResult.message); + } + // Install hybrid_browser_toolkit npm dependencies after Python packages are installed log.info( '[DEPS INSTALL] Installing hybrid_browser_toolkit dependencies...' @@ -922,6 +1049,13 @@ export async function installDependencies( : await runInstall([], version); if (mirrorInstallSuccess.success) { + // Install terminal base venv (lightweight venv for terminal tasks) + log.info('[DEPS INSTALL] Installing terminal base venv...'); + const terminalResult = await installTerminalBaseVenv(version); + if (!terminalResult.success) { + log.warn('[DEPS INSTALL] Terminal base venv installation failed, but continuing...', terminalResult.message); + } + // Install hybrid_browser_toolkit npm dependencies after Python packages are installed log.info( '[DEPS INSTALL] Installing hybrid_browser_toolkit dependencies...' diff --git a/electron/main/utils/process.ts b/electron/main/utils/process.ts index c8647e686..c6d861e3d 100644 --- a/electron/main/utils/process.ts +++ b/electron/main/utils/process.ts @@ -185,6 +185,43 @@ export function getVenvsBaseDir(): string { return path.join(os.homedir(), '.eigent', 'venvs'); } +/** + * Packages to install in the terminal base venv. + * These are commonly used packages for terminal tasks (data processing, visualization, etc.) + * Keep this list minimal - users can install additional packages as needed. + */ +export const TERMINAL_BASE_PACKAGES = [ + 'pandas', + 'numpy', + 'matplotlib', + 'requests', + 'openpyxl', + 'beautifulsoup4', + 'pillow', +]; + +/** + * Get path to the terminal base venv. + * This is a lightweight venv with common packages for terminal tasks, + * separate from the backend venv. + */ +export function getTerminalVenvPath(version: string): string { + const venvDir = path.join( + os.homedir(), + '.eigent', + 'venvs', + `terminal_base-${version}` + ); + + // Ensure venvs directory exists + const venvsBaseDir = path.dirname(venvDir); + if (!fs.existsSync(venvsBaseDir)) { + fs.mkdirSync(venvsBaseDir, { recursive: true }); + } + + return venvDir; +} + export async function cleanupOldVenvs(currentVersion: string): Promise { const venvsBaseDir = getVenvsBaseDir(); @@ -193,23 +230,34 @@ export async function cleanupOldVenvs(currentVersion: string): Promise { return; } + // Patterns to match: backend-{version} and terminal_base-{version} + const venvPatterns = [ + { prefix: 'backend-', regex: /^backend-(.+)$/ }, + { prefix: 'terminal_base-', regex: /^terminal_base-(.+)$/ }, + ]; + try { const entries = fs.readdirSync(venvsBaseDir, { withFileTypes: true }); for (const entry of entries) { - if (entry.isDirectory() && entry.name.startsWith('backend-')) { - const versionMatch = entry.name.match(/^backend-(.+)$/); - if (versionMatch && versionMatch[1] !== currentVersion) { - const oldVenvPath = path.join(venvsBaseDir, entry.name); - console.log(`Cleaning up old venv: ${oldVenvPath}`); + if (!entry.isDirectory()) continue; - try { - // Remove old venv directory recursively - fs.rmSync(oldVenvPath, { recursive: true, force: true }); - console.log(`Successfully removed old venv: ${entry.name}`); - } catch (err) { - console.error(`Failed to remove old venv ${entry.name}:`, err); + for (const pattern of venvPatterns) { + if (entry.name.startsWith(pattern.prefix)) { + const versionMatch = entry.name.match(pattern.regex); + if (versionMatch && versionMatch[1] !== currentVersion) { + const oldVenvPath = path.join(venvsBaseDir, entry.name); + console.log(`Cleaning up old venv: ${oldVenvPath}`); + + try { + // Remove old venv directory recursively + fs.rmSync(oldVenvPath, { recursive: true, force: true }); + console.log(`Successfully removed old venv: ${entry.name}`); + } catch (err) { + console.error(`Failed to remove old venv ${entry.name}:`, err); + } } + break; // Found matching pattern, no need to check others } } }