Remove open-model-gym eval suite (#10174)

AGENTS.md

@@ -48,7 +48,6 @@ crates/
 ├── goose-test         # test utilities
 └── goose-test-support # test helpers
 
-evals/open-model-gym/  # benchmarking / evals
 ui/desktop/            # Electron app
 ```
 

evals/open-model-gym/.gitignore

@@ -1,72 +0,0 @@
-# Dependencies
-node_modules/
-.pnpm-store/
-.workdir/
-report.html
-
-# Build outputs
-dist/
-build/
-out/
-.next/
-.nuxt/
-.output/
-.opencode-root
-suite/.pi-root
-
-# TypeScript
-*.tsbuildinfo
-*.d.ts.map
-.g3/
-# Logs
-logs/
-*.log
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-pnpm-debug.log*
-
-# Runtime data
-pids/
-*.pid
-*.seed
-*.pid.lock
-
-# Coverage & testing
-coverage/
-.nyc_output/
-.jest/
-
-# Caches
-.cache/
-.parcel-cache/
-.turbo/
-.eslintcache
-.stylelintcache
-*.swp
-*.swo
-
-# IDE & editors
-.idea/
-.vscode/
-*.sublime-project
-*.sublime-workspace
-
-# OS files
-.DS_Store
-Thumbs.db
-
-# Environment variables
-.env
-.env.local
-.env.*.local
-
-# Lock files (optional - uncomment if you don't want to track)
-# package-lock.json
-# yarn.lock
-# pnpm-lock.yaml
-
-# Temporary files
-tmp/
-temp/
-*.tmp

evals/open-model-gym/Justfile

@@ -1,71 +0,0 @@
-# Agent Runner - Test Suite (supports goose and opencode)
-
-# Default recipe
-default: run
-
-# Full test run - all scenarios, all agents, 3 repetitions (worst kept)
-run: _install
-    cd suite && npm run test
-
-# Full run with artifacts isolated under ~/.goose/gym-runs/<YYYYDDMMHHMMSS> (keeps the repo clean)
-run-clean: _install
-    #!/usr/bin/env bash
-    set -euo pipefail
-    export GYM_OUTPUT_DIR="$HOME/.goose/gym-runs/$(date +%Y%d%m%H%M%S)"
-    echo "Artifacts → $GYM_OUTPUT_DIR"
-    cd suite && npm run test
-
-# Quick test - file-editing + everyday-app-automation, single run each (no repetition)
-test: _install
-    cd suite && npx tsx src/runner.ts --scenario=file-editing,everyday-app-automation --run-count=1
-
-# Run a specific scenario (all agents, 3 reps)
-scenario name: _install
-    cd suite && npx tsx src/runner.ts --scenario={{name}}
-
-# Run against a specific agent (all scenarios, 3 reps)
-agent name: _install
-    cd suite && npx tsx src/runner.ts --agent={{name}}
-
-# Open report in browser (honors GYM_OUTPUT_DIR if set)
-report:
-    #!/usr/bin/env bash
-    set -euo pipefail
-    dir="${GYM_OUTPUT_DIR:-.}"
-    open "${dir/#\~/$HOME}/report.html"
-
-# Install all dependencies
-install:
-    cd suite && npm install
-    cd mcp-harness && npm install && npm run build
-    @# Install pi-mcp-adapter for Pi runner MCP support
-    @pi list 2>/dev/null | grep -q "pi-mcp-adapter" || pi install npm:pi-mcp-adapter
-
-# Build TypeScript
-build: _install
-    cd suite && npm run build
-
-# Clear the test cache
-clear-cache:
-    cd suite && npx tsx src/runner.ts --clear-cache
-
-# Run tests ignoring cache (force fresh runs)
-run-fresh: _install
-    cd suite && npx tsx src/runner.ts --no-cache
-
-# Show cache stats
-cache-stats:
-    @if [ -f suite/.cache/index.json ]; then \
-        echo "Cache entries: $$(cat suite/.cache/index.json | grep -o '"[a-f0-9]\{16\}":' | wc -l | tr -d ' ')"; \
-        echo "Cache size: $$(du -sh suite/.cache 2>/dev/null | cut -f1 || echo '0')"; \
-    else \
-        echo "No cache found"; \
-    fi
-
-# Internal: install if node_modules missing, always rebuild mcp-harness
-_install:
-    @[ -d suite/node_modules ] || (cd suite && npm install)
-    @[ -d mcp-harness/node_modules ] || (cd mcp-harness && npm install)
-    @cd mcp-harness && npm run build
-    @# Ensure pi-mcp-adapter is installed for Pi runner
-    @pi list 2>/dev/null | grep -q "pi-mcp-adapter" || pi install npm:pi-mcp-adapter

evals/open-model-gym/README.md

@@ -1,323 +0,0 @@
-# Open Model Gym
-
-Run agent tests across a matrix of **models × runners × scenarios**. 
-
-It isn't hard for any agent to do ok with opus, but lets scale things in the other direction. What do we have to break things down to.
-
-<img width="1768" height="1133" alt="image" src="https://github.com/user-attachments/assets/29915659-ee6b-4a8b-ba5e-58420b168b43" />
-
-## Quick Start
-
-```bash
-just install   # one-time setup
-just run       # run full matrix (3 reps each)
-just report    # view results
-```
-
-## How It Works
-
-The test harness runs every combination of models, runners, and scenarios defined in your matrix. Each test runs multiple times (default 3) and keeps the **worst result** — if a test fails even once, it's marked failed. This catches flaky passes.
-
-## Configuration
-
-Edit `config.yaml` to define your test matrix:
-
-### Models
-
-LLMs to test against. Supports any provider (Anthropic, OpenAI, Ollama, etc.):
-
-```yaml
-models:
-  - name: opus
-    provider: anthropic
-    model: claude-opus-4-5-20251101
-
-  - name: qwen3-coder
-    provider: ollama
-    model: qwen3-coder:64k
-
-  - name: gpt4
-    provider: openai
-    model: gpt-4-turbo
-```
-
-### Runners
-
-Agent frameworks that execute the tests. Each runner has its own binary, type, and configuration:
-
-```yaml
-runners:
-  # Goose agent with extensions
-  - name: goose-full
-    type: goose
-    bin: goose                    # path to binary (can be absolute)
-    extensions: [developer, todo, skills]
-    stdio:
-      - node mcp-harness/dist/index.js
-
-  # OpenCode agent
-  - name: opencode
-    type: opencode
-    bin: opencode                 # path to binary
-    stdio:
-      - node mcp-harness/dist/index.js
-
-  # Custom goose binary path
-  - name: goose-dev
-    type: goose
-    bin: /path/to/my/goose-dev
-    extensions: [developer]
-```
-
-**Supported runner types:**
-- `goose` — [Goose](https://github.com/aaif-goose/goose) agent framework
-- `opencode` — [OpenCode](https://opencode.ai) agent framework
-- `pi` — [Pi](https://github.com/badlogic/pi-mono) coding agent
-
-## Runner Details
-
-Each runner has different setup requirements, MCP integration methods, and session handling.
-
-### Goose
-
-[Goose](https://github.com/aaif-goose/goose) is an open-source coding agent with built-in MCP support.
-
-**Setup:** Install via `brew install goose` or from source.
-
-**MCP Integration:** Native support. The harness writes a `config.yaml` to an isolated `.goose-root/` directory with extensions and MCP servers:
-
-```yaml
-extensions:
-  developer:
-    enabled: true
-  mcp_harness:
-    type: stdio
-    enabled: true
-    cmd: node
-    args: [mcp-harness/dist/index.js]
-```
-
-**Session Handling:** Uses `--name <session>` for named sessions, `--resume` to continue:
-- Turn 1: `goose run -i <prompt> --name <session>`
-- Turn 2+: `goose run -i <prompt> --name <session> --resume`
-- Single-turn: `goose run -i <prompt> --no-session`
-
-### OpenCode
-
-[OpenCode](https://opencode.ai) is a terminal-based coding agent.
-
-**Setup:** Install via their website or package manager.
-
-**MCP Integration:** Native support. The harness writes an `opencode.json` config to the workdir:
-
-```json
-{
-  "mcp": {
-    "harness": {
-      "type": "local",
-      "command": ["node", "mcp-harness/dist/index.js"],
-      "enabled": true
-    }
-  },
-  "model": "anthropic/claude-opus-4-5-20251101"
-}
-```
-
-**Session Handling:** Uses `--continue` to resume the last session in the working directory:
-- Turn 1: `opencode run "<prompt>"`
-- Turn 2+: `opencode run --continue "<prompt>"`
-
-⚠️ OpenCode doesn't support named sessions, so multi-turn scenarios exclude it.
-
-### Pi
-
-[Pi](https://github.com/badlogic/pi-mono) is a lightweight coding agent that requires an adapter for MCP support.
-
-**Setup:**
-```bash
-# Install Pi
-npm install -g @anthropic/pi   # or from source
-
-# Install the MCP adapter (required for MCP tools)
-pi install npm:pi-mcp-adapter
-```
-
-The `just install` recipe auto-installs pi-mcp-adapter if missing.
-
-**MCP Integration:** Via [pi-mcp-adapter](https://github.com/nicobailon/pi-mcp-adapter). The harness dynamically writes a `.pi-mcp.json` config to the workdir:
-
-```json
-{
-  "mcpServers": {
-    "harness": {
-      "command": "node",
-      "args": ["mcp-harness/dist/index.js"],
-      "lifecycle": "eager",
-      "env": { "MCP_HARNESS_LOG": "<workdir>/tool-calls.log" }
-    }
-  },
-  "settings": { "directTools": true }
-}
-```
-
-Key settings:
-- `directTools: true` — Registers MCP tools directly in Pi's tool list (no wrapper)
-- `lifecycle: "eager"` — Connects to MCP servers at startup
-
-**Model Configuration:** Pi requires custom models (like Ollama) to be defined in `models.json`. The harness automatically generates this config in an isolated `.pi-root/` directory and sets `PI_CODING_AGENT_DIR` to use it:
-
-```json
-{
-  "providers": {
-    "ollama": {
-      "baseUrl": "http://localhost:11434/v1",
-      "api": "openai-completions",
-      "apiKey": "ollama",
-      "models": [{ "id": "model-name", "name": "Model Name", ... }]
-    }
-  }
-}
-```
-
-The harness copies `auth.json` from your real Pi config (`~/.pi/agent/`) so API keys work.
-
-**Session Handling:** Uses `--session <path>` for file-based sessions, `--continue` to resume:
-- Turn 1: `pi -p --session <path> "<prompt>"`
-- Turn 2+: `pi -p --continue --session <path> "<prompt>"`
-- Single-turn: `pi -p --no-session "<prompt>"`
-
-The `-p` flag runs Pi in non-interactive "print" mode for automation
-
-### Matrix
-
-Define which scenarios run against which models/runners:
-
-```yaml
-matrix:
-  - scenario: file-editing
-    models: [opus, qwen3-coder]      # omit to run all models
-    runners: [goose-full, opencode]  # omit to run all runners
-
-  - scenario: everyday-app-automation
-    # runs against ALL models and ALL runners
-```
-
-## Scenarios
-
-Scenarios live in `suite/scenarios/` as YAML files:
-
-```yaml
-name: file-editing
-description: Create and edit files
-prompt: |
-  1. Create joke.md containing a short joke
-  2. Edit hello.rs to add a debug function
-
-setup:
-  hello.rs: |
-    fn main() { println!("Hello!"); }
-
-validate:
-  - type: file_exists
-    path: joke.md
-  - type: file_matches
-    path: hello.rs
-    regex: "fn\\s+debug"
-```
-
-### Validation Rules
-
-| Rule | Description |
-|------|-------------|
-| `file_exists` | File exists at path |
-| `file_not_empty` | File exists and has content |
-| `file_contains` | File contains literal string |
-| `file_matches` | File matches regex pattern |
-| `command_succeeds` | Shell command exits 0 |
-| `tool_called` | MCP tool was called with matching args (regex supported) |
-
-**Tool call validation example:**
-```yaml
-validate:
-  - type: tool_called
-    tool: slack_search_messages
-    args:
-      query: /quarterly.?review/    # regex pattern
-  - type: tool_called
-    tool: jira_create_issue
-    args:
-      summary: /Q1.*Review/
-      description: /David Brown/
-```
-
-## MCP Harness
-
-Mock MCP server providing simulated tools for testing agent tool-use without hitting real APIs.
-
-```bash
-cd mcp-harness && npm install && npm run build
-```
-
-**Available tools:** gdrive, sheets, salesforce, slack, calendar, gmail, jira, github
-
-Each tool returns realistic mock data. Tool calls are logged to `tool-calls.log` in the workdir for validation.
-
-## Commands
-
-| Command | Description |
-|---------|-------------|
-| `just run` | Full test run (3 reps each, worst kept) |
-| `just run-clean` | Full run with artifacts isolated under `~/.goose/gym-runs/<YYYYDDMMHHMMSS>` |
-| `just test` | Quick run (1 rep each) |
-| `just scenario <name>` | Run specific scenario |
-| `just agent <name>` | Run specific agent |
-| `just report` | Open HTML results |
-
-### CLI Flags
-
-```bash
-# Filter by scenario, model, or runner
-npx tsx src/runner.ts --scenario=file-editing --model=opus --runner=goose
-
-# Control repetition count
-npx tsx src/runner.ts --run-count=5
-
-# Don't auto-open browser
-npx tsx src/runner.ts --no-open
-
-# Redirect all run artifacts outside the repo (see Output below)
-npx tsx src/runner.ts --output-dir=~/.goose/gym-runs/latest
-
-# Raise the per-agent timeout (seconds) for slow local models on heavy
-# scenarios. Default 300s; also settable via GYM_AGENT_TIMEOUT.
-npx tsx src/runner.ts --agent-timeout=1200
-```
-
-## Output
-
-- `report.html` — Live-updating HTML matrix showing pass/fail status, duration, and validation details
-- `logs/` — Full agent output logs for each run
-
-By default these (plus the cache, scratch workdir, and isolated agent config
-roots `.goose-root/` / `.opencode-root/` / `.pi-root/`) are written inside the
-gym directory. They're gitignored, but still pile up in your checkout — awkward
-if you want to run the bench regularly or from a worktree.
-
-To keep the repo clean, redirect **all** run artifacts to a single base
-directory with the `GYM_OUTPUT_DIR` env var (or the `--output-dir=` flag).
-`config.yaml` and `scenarios/` are still read from the repo.
-
-```bash
-# Everything lands under a timestamped dir outside the repo (YYYYDDMMHHMMSS)
-GYM_OUTPUT_DIR=~/.goose/gym-runs/$(date +%Y%d%m%H%M%S) just run
-
-# Convenience recipe that does the timestamping for you
-just run-clean
-
-# View the report from a redirected run
-GYM_OUTPUT_DIR=~/.goose/gym-runs/20261406101500 just report
-```
-
-> Note: the run cache lives under the output dir too, so a fresh timestamped
-> dir means a fresh (cold) cache. Point `GYM_OUTPUT_DIR` at a stable directory
-> if you want cache reuse across runs.

evals/open-model-gym/config.yaml

@@ -1,85 +0,0 @@
-# =============================================================================
-# Models - the LLMs to test
-# =============================================================================
-models:
-  - name: opus
-    provider: anthropic
-    model: claude-opus-4-5-20251101
-
-  - name: glm-4.7-flash
-    provider: ollama
-    model: glm-4.7-flash:latest
-
-  # too slow on 64g:
-  #- name: frob/qwen3-coder-next:latest
-  #  provider: ollama
-  #  model: frob/qwen3-coder-next:latest
-
-  - name: kimi-k2.5
-    provider: ollama
-    model: kimi-k2.5:cloud
-
-  - name: gpt-oss-120b
-    provider: ollama
-    model: gpt-oss:120b-cloud
-
-  - name: gpt-oss-20b
-    provider: ollama
-    model: gpt-oss:20b
-
-  - name: qwen3-coder:latest
-    provider: ollama
-    model: qwen3-coder:latest
-
-  # good but too slow on 64G
-  #- name: nemotron-3-nano
-  #  provider: ollama
-  #  model: nemotron-3-nano:latest
-
-# =============================================================================
-# Runners - agent frameworks with their specific configurations
-# =============================================================================
-# Each runner has its own binary, extensions/config, and isolated config directory
-runners:
-  # - name: goose
-  #   type: goose
-  #   bin: goose
-  #   extensions: [developer]
-  #   stdio:
-  #     - node mcp-harness/dist/index.js
-
-  - name: goose-full
-    type: goose
-    bin: goose
-    extensions: [developer, todo, skills, code_execution, extensionmanager]
-    stdio:
-      - node mcp-harness/dist/index.js
-
-  - name: opencode
-    type: opencode
-    bin: opencode
-    stdio:
-      - node mcp-harness/dist/index.js
-
-  - name: pi
-    type: pi
-    bin: pi
-    # Pi takes provider/model from the test matrix, not config
-    # MCP support via pi-mcp-adapter: `pi install npm:pi-mcp-adapter`
-    stdio:
-      - node mcp-harness/dist/index.js
-
-# =============================================================================
-# Test Matrix
-# =============================================================================
-# scenarios × models × runners
-# - Omit 'models' to run against ALL models
-# - Omit 'runners' to run against ALL runners
-matrix:
-  # Single-turn scenarios: all models × all runners
-  - scenario: everyday-app-automation
-  - scenario: file-editing
-
-  # Multi-turn: goose and pi only (opencode doesn't support session continuation)
-  - scenario: multi-turn-edit
-    runners: [goose-full]

evals/open-model-gym/mcp-harness/README.md

@@ -1,87 +0,0 @@
-# MCP Harness
-
-A simulated MCP server with realistic fake tools for testing. Provides mock implementations of common business integrations without requiring actual API credentials.
-
-## Tools Included (35 tools)
-
-### Google Drive
-- `gdrive_search` - Search files by name, content, or type
-- `gdrive_read_file` - Read file contents
-- `gdrive_create_file` - Create new files
-- `gdrive_share_file` - Share files with users
-
-### Google Sheets
-- `sheets_read` - Read spreadsheet data
-- `sheets_write` - Write/update cells
-- `sheets_append` - Append rows
-- `sheets_create` - Create new spreadsheets
-
-### Salesforce
-- `salesforce_query` - Execute SOQL queries
-- `salesforce_get_record` - Get record by ID
-- `salesforce_create_record` - Create records
-- `salesforce_update_record` - Update records
-- `salesforce_search` - SOSL search
-
-### Slack
-- `slack_send_message` - Send messages
-- `slack_get_messages` - Get channel messages
-- `slack_search_messages` - Search messages
-- `slack_list_channels` - List channels
-- `slack_get_user_info` - Get user info
-- `slack_set_status` - Set user status
-
-### Google Calendar
-- `calendar_list_events` - List events
-- `calendar_create_event` - Create events
-- `calendar_update_event` - Update events
-- `calendar_delete_event` - Delete events
-
-### Gmail
-- `gmail_search` - Search emails
-- `gmail_read_message` - Read email content
-- `gmail_send` - Send emails
-- `gmail_create_draft` - Create drafts
-
-### Jira
-- `jira_search_issues` - Search with JQL
-- `jira_get_issue` - Get issue details
-- `jira_create_issue` - Create issues
-- `jira_update_issue` - Update issues
-- `jira_add_comment` - Add comments
-
-### GitHub
-- `github_search_repos` - Search repositories
-- `github_list_issues` - List issues
-- `github_create_issue` - Create issues
-- `github_list_prs` - List pull requests
-
-## Setup
-
-```bash
-npm install
-npm run build
-```
-
-## Run
-
-```bash
-npm run start
-# or
-./run.sh
-```
-
-## MCP Config
-
-Add to your MCP client config:
-
-```json
-{
-  "mcpServers": {
-    "harness": {
-      "command": "node",
-      "args": ["/path/to/mcp-harness/dist/index.js"]
-    }
-  }
-}
-```

evals/open-model-gym/mcp-harness/package.json

@@ -1,18 +0,0 @@
-{
-  "name": "mcp-harness",
-  "version": "1.0.0",
-  "description": "Simulated real-world MCP tools for testing - Google Drive, Sheets, Salesforce, Slack, and more",
-  "private": true,
-  "type": "module",
-  "scripts": {
-    "build": "tsc -p tsconfig.json",
-    "start": "node dist/index.js"
-  },
-  "dependencies": {
-    "@modelcontextprotocol/sdk": "^1.26.0"
-  },
-  "devDependencies": {
-    "@types/node": "^25.2.0",
-    "typescript": "^5.6.3"
-  }
-}

evals/open-model-gym/mcp-harness/run.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-cd "$(dirname "$0")"
-npm run build && npm run start

evals/open-model-gym/mcp-harness/tsconfig.json

@@ -1,14 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "module": "NodeNext",
-    "moduleResolution": "NodeNext",
-    "outDir": "dist",
-    "rootDir": "src",
-    "strict": true,
-    "esModuleInterop": true,
-    "skipLibCheck": true,
-    "declaration": true
-  },
-  "include": ["src/**/*"]
-}

evals/open-model-gym/suite/.gitignore

@@ -1,4 +0,0 @@
-node_modules/
-.workdir/
-.goose-root/
-.cache/

evals/open-model-gym/suite/package.json

@@ -1,19 +0,0 @@
-{
-  "name": "agent-runner",
-  "version": "0.1.0",
-  "type": "module",
-  "scripts": {
-    "build": "tsc",
-    "test": "tsx src/runner.ts",
-    "test:scenario": "tsx src/runner.ts --scenario"
-  },
-  "dependencies": {
-    "glob": "^11.0.0",
-    "yaml": "^2.8.3"
-  },
-  "devDependencies": {
-    "@types/node": "^22.0.0",
-    "tsx": "^4.22.4",
-    "typescript": "^5.5.0"
-  }
-}

evals/open-model-gym/suite/scenarios/everyday-app-automation.yaml

@@ -1,47 +0,0 @@
-name: everyday-app-automation
-description: Multi-step workflow using everyday app tools (Slack, Jira, Calendar) with data dependencies
-prompt: |
-  Using the available tools, complete these tasks:
-  1. Search Slack for messages mentioning "quarterly review"
-  2. Look up the user who posted the message about quarterly review to get their full name
-  3. Create a Jira issue titled "Q1 Review Follow-ups" with a description that includes the name of the person who posted the Slack message
-  4. Create a calendar event for next Monday at 2pm called "Review Discussion"
-  5. Write a summary of what you did to a file called workflow-log.md
-
-tags:
-  - complex
-  - multi-step
-  - mcp-harness
-  - data-flow
-
-validate:
-  # Check workflow summary was written
-  - type: file_exists
-    path: workflow-log.md
-  - type: file_not_empty
-    path: workflow-log.md
-
-  # Check Slack search was called with the right query
-  - type: tool_called
-    tool: slack_search_messages
-    args:
-      query: /quarterly.?review/
-
-  # Check that the agent looked up the user info (data dependency: requires reading user ID from search results)
-  - type: tool_called
-    tool: slack_get_user_info
-    args:
-      userId: /U004/
-
-  # Check Jira issue was created with expected title and includes the user's name (data dependency: requires reading name from user info)
-  - type: tool_called
-    tool: jira_create_issue
-    args:
-      summary: /q1.?review|follow.?up/
-      description: /David.?Brown/
-
-  # Check calendar event was created with expected title
-  - type: tool_called
-    tool: calendar_create_event
-    args:
-      summary: /review.?discussion/

evals/open-model-gym/suite/scenarios/file-editing.yaml

@@ -1,110 +0,0 @@
-name: file-editing
-description: Navigate a small codebase and make a targeted edit
-prompt: |
-  The User struct in user.rs is missing a display_name() method.
-  Add a method that returns the full name formatted as "first_name last_name".
-
-tags:
-  - file-editing
-  - code-navigation
-
-setup:
-  Cargo.toml: |
-    [package]
-    name = "user-service"
-    version = "0.1.0"
-    edition = "2021"
-
-    [workspace]
-
-  src/main.rs: |
-    mod models;
-    mod utils;
-
-    use models::user::User;
-
-    fn main() {
-        let user = User::new("Alice", "Smith", "alice@example.com");
-        println!("Created user: {}", user.email());
-    }
-
-  src/models/mod.rs: |
-    pub mod user;
-
-  src/models/user.rs: |
-    pub struct User {
-        first_name: String,
-        last_name: String,
-        email: String,
-    }
-
-    impl User {
-        pub fn new(first_name: &str, last_name: &str, email: &str) -> Self {
-            Self {
-                first_name: first_name.to_string(),
-                last_name: last_name.to_string(),
-                email: email.to_string(),
-            }
-        }
-
-        pub fn email(&self) -> &str {
-            &self.email
-        }
-
-        pub fn first_name(&self) -> &str {
-            &self.first_name
-        }
-
-        pub fn last_name(&self) -> &str {
-            &self.last_name
-        }
-    }
-
-  src/utils/mod.rs: |
-    pub mod formatting;
-
-  src/utils/formatting.rs: |
-    pub fn capitalize(s: &str) -> String {
-        let mut chars = s.chars();
-        match chars.next() {
-            None => String::new(),
-            Some(c) => c.to_uppercase().collect::<String>() + chars.as_str(),
-        }
-    }
-
-validate:
-  # The edit was made to the correct file
-  - type: file_exists
-    path: src/models/user.rs
-    name: user.rs exists
-  # Method was added
-  - type: file_matches
-    path: src/models/user.rs
-    regex: "fn\\s+display_name"
-    name: display_name() added
-  # Method returns a String or &str
-  - type: file_matches
-    path: src/models/user.rs
-    regex: "display_name.*->.*String|display_name.*->.*str"
-    name: has return type
-  # Original code preserved
-  - type: file_contains
-    path: src/models/user.rs
-    pattern: "pub fn email"
-    name: email() preserved
-  - type: file_contains
-    path: src/models/user.rs
-    pattern: "pub fn first_name"
-    name: first_name() preserved
-  # Other files untouched
-  - type: file_exists
-    path: src/main.rs
-    name: main.rs exists
-  - type: file_contains
-    path: src/main.rs
-    pattern: "mod models"
-    name: main.rs unchanged
-  # Code compiles
-  - type: command_succeeds
-    command: "cargo build"
-    name: cargo build

evals/open-model-gym/suite/scenarios/multi-turn-edit.yaml

@@ -1,80 +0,0 @@
-name: multi-turn-edit
-description: Multi-turn conversation - add a method, then rename it
-
-tags:
-  - file-editing
-  - multi-turn
-
-setup:
-  Cargo.toml: |
-    [package]
-    name = "user-service"
-    version = "0.1.0"
-    edition = "2021"
-
-    [workspace]
-
-  src/main.rs: |
-    mod models;
-
-    use models::user::User;
-
-    fn main() {
-        let user = User::new("Alice", "Smith");
-        println!("User: {} {}", user.first_name(), user.last_name());
-    }
-
-  src/models/mod.rs: |
-    pub mod user;
-
-  src/models/user.rs: |
-    pub struct User {
-        first_name: String,
-        last_name: String,
-    }
-
-    impl User {
-        pub fn new(first_name: &str, last_name: &str) -> Self {
-            Self {
-                first_name: first_name.to_string(),
-                last_name: last_name.to_string(),
-            }
-        }
-
-        pub fn first_name(&self) -> &str {
-            &self.first_name
-        }
-
-        pub fn last_name(&self) -> &str {
-            &self.last_name
-        }
-    }
-
-turns:
-  - prompt: |
-      Add an email() method to the User struct in src/models/user.rs.
-      It should return a generated email in the format "first_name.last_name@example.com" (lowercase).
-    validate:
-      - type: file_matches
-        path: src/models/user.rs
-        regex: "fn\\s+email"
-        name: email() added
-      - type: command_succeeds
-        command: "cargo build"
-        name: compiles after turn 1
-
-  - prompt: |
-      Actually, can you rename the email() method to generated_email() instead?
-      Make sure to update any references.
-    validate:
-      - type: file_matches
-        path: src/models/user.rs
-        regex: "fn\\s+generated_email"
-        name: renamed to generated_email()
-      - type: file_not_matches
-        path: src/models/user.rs
-        regex: "fn\\s+email\\s*\\("
-        name: old email() removed
-      - type: command_succeeds
-        command: "cargo build"
-        name: compiles after turn 2

evals/open-model-gym/suite/scenarios/remove-feature.yaml

@@ -1,926 +0,0 @@
-name: remove-feature
-description: Remove a feature (insert_image) from a document editor tool module
-prompt: |
-  Remove the `insert_image` feature entirely from this codebase.
-  It should be removed from docs.py and any references in doc_utils.py.
-  Make sure no traces remain: the operation literal, the ImageParams class,
-  the image_params parameter, the insert_image handler function, and any
-  validation/error handling specific to insert_image should all be removed.
-
-tags:
-  - file-editing
-  - feature-removal
-  - python
-
-setup:
-  docs.py: |
-    """Tools for interacting with documents."""
-
-    import json
-    from dataclasses import dataclass
-    from functools import partial
-    from typing import Annotated, Any, Literal
-
-    from pydantic import Field
-
-    from doc_utils import execute_request, execute_requests
-
-    DOC_OPERATIONS = Literal[
-        "get_document",
-        "insert_text",
-        "append_text",
-        "replace_text",
-        "delete_content",
-        "insert_table",
-        "update_table_cell",
-        "insert_table_row",
-        "insert_table_column",
-        "delete_table_row",
-        "delete_table_column",
-        "insert_image",
-        "format_existing_text",
-    ]
-
-    RESPONSE_CHAR_LIMIT = 400000
-
-
-    @dataclass
-    class FormatTextParams:
-        """Parameters for format_existing_text operation."""
-
-        search_text: str
-        foreground_color: str | None = None
-        background_color: str | None = None
-        bold: bool | None = None
-        italic: bool | None = None
-        underline: bool | None = None
-        strikethrough: bool | None = None
-        font_size: int | None = None
-        font_family: str | None = None
-        heading_level: int | None = None
-        link_url: str | None = None
-        list_type: str | None = None
-
-
-    @dataclass
-    class TableParams:
-        """Parameters for table operations.
-
-        Used by operations: insert_table, update_table_cell, insert_table_row,
-        insert_table_column, delete_table_row, delete_table_column.
-        """
-
-        rows: int | None = Field(None, description="Number of rows for insert_table operation")
-        columns: int | None = Field(None, description="Number of columns for insert_table operation")
-        row_index: int | None = Field(
-            None, description="Row index (0-based) for update_table_cell, insert_table_row, and delete_table_row"
-        )
-        column_index: int | None = Field(
-            None, description="Column index (0-based) for update_table_cell, insert_table_column, and delete_table_column"
-        )
-        insert_below: bool = Field(
-            False, description="For insert_table_row: True to insert below the specified row, False to insert above"
-        )
-        insert_right: bool = Field(
-            False, description="For insert_table_column: True to insert right of column, False to insert left"
-        )
-
-
-    @dataclass
-    class ImageParams:
-        """Parameters for image insertion."""
-
-        image_url: str = Field(..., description="URL of the image to insert (must be publicly accessible)")
-        width: int | None = Field(None, description="Width of the image in points (PT)")
-        height: int | None = Field(None, description="Height of the image in points (PT)")
-
-
-    async def doc_tool(
-        document_id: str,
-        operation: DOC_OPERATIONS = "get_document",
-        text: Annotated[
-            str,
-            Field(
-                description=(
-                    "Text content to insert, append, or match for replacement. "
-                    "For insert_text and append_text, Markdown formatting is supported. "
-                    "For replace_text, this should be unformatted plain text."
-                )
-            ),
-        ] = "",
-        replace_text: Annotated[
-            str,
-            Field(
-                description=(
-                    "New plain text that will replace all occurrences of the original text. "
-                    "Only used for the replace_text operation."
-                )
-            ),
-        ] = "",
-        start_position: Annotated[
-            int | None,
-            Field(
-                description="Document index (1-based) for insert or delete operations.",
-                ge=1,
-            ),
-        ] = None,
-        end_position: Annotated[
-            int | None,
-            Field(
-                description="Document index (1-based, exclusive) for delete_content",
-                ge=1,
-            ),
-        ] = None,
-        table_params: Annotated[
-            TableParams | None,
-            Field(
-                None,
-                description="Parameters for table operations",
-            ),
-        ] = None,
-        image_params: ImageParams | None = None,
-        format_params: FormatTextParams | None = None,
-    ) -> str:
-        """Perform operations on an existing document.
-
-        Supported operations:
-            - get_document: Returns document content
-            - insert_text: Inserts text at a specific position
-            - append_text: Appends text at the end of the document
-            - replace_text: Replaces all instances of text with replace_text
-            - delete_content: Deletes content between two positions
-            - insert_table: Creates a table with specified rows and columns
-            - update_table_cell: Updates content in a specific table cell
-            - insert_table_row: Inserts a row above or below the specified row
-            - insert_table_column: Inserts a column left or right of the specified column
-            - delete_table_row: Deletes the specified row from a table
-            - delete_table_column: Deletes the specified column from a table
-            - insert_image: Inserts an image from a URL at the specified position
-            - format_existing_text: Finds and applies formatting to text
-        """
-        if not text and operation in ["insert_text", "append_text", "replace_text"]:
-            raise ValueError(f"text is required for {operation} operation")
-
-        table_operations = [
-            "insert_table",
-            "update_table_cell",
-            "insert_table_row",
-            "insert_table_column",
-            "delete_table_row",
-            "delete_table_column",
-        ]
-        if operation in table_operations and not table_params:
-            raise ValueError(f"table_params is required for {operation} operation")
-
-        if operation == "insert_image" and not image_params:
-            raise ValueError("image_params is required for insert_image operation")
-
-        if operation == "format_existing_text" and not format_params:
-            raise ValueError("format_params is required for format_existing_text operation")
-
-        operation_handlers = {
-            "get_document": partial(read_document, document_id),
-            "insert_text": partial(insert_text, document_id, text, start_position),
-            "append_text": partial(append_text, document_id, text),
-            "replace_text": partial(replace_all_text, document_id, text, replace_text),
-            "delete_content": partial(delete_content, document_id, start_position, end_position),
-            "insert_table": partial(
-                insert_table,
-                document_id,
-                table_params.rows if table_params else None,
-                table_params.columns if table_params else None,
-                start_position,
-            ),
-            "update_table_cell": partial(
-                update_table_cell,
-                document_id,
-                table_params.row_index if table_params else None,
-                table_params.column_index if table_params else None,
-                text,
-                start_position,
-            ),
-            "insert_table_row": partial(
-                modify_table_structure,
-                document_id,
-                "insert_row",
-                table_params.row_index if table_params else None,
-                None,
-                table_params.insert_below if table_params else False,
-                False,
-                start_position,
-            ),
-            "insert_table_column": partial(
-                modify_table_structure,
-                document_id,
-                "insert_column",
-                None,
-                table_params.column_index if table_params else None,
-                False,
-                table_params.insert_right if table_params else False,
-                start_position,
-            ),
-            "delete_table_row": partial(
-                modify_table_structure,
-                document_id,
-                "delete_row",
-                table_params.row_index if table_params else None,
-                None,
-                False,
-                False,
-                start_position,
-            ),
-            "delete_table_column": partial(
-                modify_table_structure,
-                document_id,
-                "delete_column",
-                None,
-                table_params.column_index if table_params else None,
-                False,
-                False,
-                start_position,
-            ),
-            "insert_image": partial(
-                insert_image,
-                document_id,
-                image_params.image_url if image_params else None,
-                start_position,
-                image_params.width if image_params else None,
-                image_params.height if image_params else None,
-            ),
-            "format_existing_text": partial(
-                format_existing_text,
-                document_id,
-                format_params.search_text if format_params else None,
-                format_params.foreground_color if format_params else None,
-                format_params.background_color if format_params else None,
-                format_params.font_size if format_params else None,
-                format_params.font_family if format_params else None,
-                format_params.bold if format_params else None,
-                format_params.italic if format_params else None,
-                format_params.underline if format_params else None,
-                format_params.strikethrough if format_params else None,
-                format_params.heading_level if format_params else None,
-                format_params.link_url if format_params else None,
-                format_params.list_type if format_params else None,
-            ),
-        }
-
-        if operation not in operation_handlers:
-            raise ValueError(f"Invalid operation: {operation}")
-
-        response = await operation_handlers[operation]()
-        return json.dumps(response, indent=2)
-
-
-    def _extract_text_from_element(element: dict) -> str:
-        """Recursively pull text from paragraphs, tables, etc."""
-        text_parts = ""
-        if "paragraph" in element:
-            paragraph_elements = element["paragraph"].get("elements", [])
-            for el in paragraph_elements:
-                if "textRun" in el:
-                    content = el["textRun"]["content"]
-                    url = el["textRun"].get("textStyle", {}).get("link", {}).get("url")
-                    if url:
-                        text_parts += f"[{content}]({url})"
-                    else:
-                        text_parts += content
-        elif "table" in element:
-            table_rows = element["table"].get("tableRows", [])
-            for row in table_rows:
-                for cell in row["tableCells"]:
-                    for cell_content in cell["content"]:
-                        text_parts += _extract_text_from_element(cell_content)
-            text_parts += "\n"
-        return text_parts
-
-
-    async def read_document(document_id: str) -> dict[str, Any]:
-        """Returns document content."""
-        document = await execute_request("get", document_id, {})
-        content = document.get("body", {}).get("content", [])
-        text = "".join(_extract_text_from_element(e) for e in content)
-
-        result = {"content": text, "document_id": document_id}
-
-        if len(json.dumps(result)) > RESPONSE_CHAR_LIMIT:
-            raise ValueError(f"Document {document_id} is too large to read.")
-
-        return result
-
-
-    async def insert_text(
-        document_id: str, text: str, start_position: int | None
-    ) -> dict[str, Any]:
-        """Insert text at a specific index in a document."""
-        if start_position is None:
-            raise ValueError("start_position is required for insert_text operation")
-
-        request = {"insertText": {"location": {"index": start_position}, "text": text}}
-        await execute_request("update", document_id, request)
-
-        from doc_utils import calculate_utf16_length
-
-        inserted_length = calculate_utf16_length(text)
-
-        return {
-            "message": f"Inserted text at position {start_position}",
-            "text": text,
-            "start_position": start_position,
-            "end_position": start_position + inserted_length,
-        }
-
-
-    async def append_text(document_id: str, text: str) -> dict[str, Any]:
-        """Append text to the end of a document."""
-        end_index = await get_document_last_index(document_id)
-        if text[0] != "\n":
-            text = "\n" + text
-        response = await insert_text(document_id, text, end_index - 1)
-        response["message"] = "Appended text to the end of the document"
-        return response
-
-
-    async def replace_all_text(
-        document_id: str, text: str, replace_text: str
-    ) -> dict[str, str]:
-        """Replace all instances of a string in a document."""
-        if not replace_text:
-            raise ValueError("replace_text parameter is required for replace_text operation")
-
-        request = {
-            "replaceAllText": {
-                "containsText": {"text": text, "matchCase": True},
-                "replaceText": replace_text,
-            }
-        }
-
-        response = await execute_request("update", document_id, request)
-        occurrences = response.get("occurrencesChanged", 0)
-        if occurrences == 0:
-            return {
-                "message": f"No occurrences of text '{text}' found in the document.",
-                "text": text,
-                "replace_text": replace_text,
-            }
-
-        return {
-            "message": f"Replaced {occurrences} occurrences of '{text}' with '{replace_text}'",
-            "text": text,
-            "replace_text": replace_text,
-        }
-
-
-    async def delete_content(
-        document_id: str, start_index: int | None, end_index: int | None
-    ) -> dict[str, Any]:
-        """Delete content between two positions."""
-        if start_index is None or end_index is None:
-            raise ValueError("Both start_index and end_index are required for delete_content")
-
-        request = {
-            "deleteContentRange": {
-                "range": {"startIndex": start_index, "endIndex": end_index}
-            }
-        }
-        await execute_request("update", document_id, request)
-        return {
-            "message": f"Deleted content between index {start_index} and {end_index}",
-            "start_index": start_index,
-            "end_index": end_index,
-        }
-
-
-    async def get_document_last_index(document_id: str) -> int:
-        """Get the last index of the document."""
-        document = await execute_request("get", document_id, {})
-        return document.get("body", {}).get("content", [])[-1].get("endIndex", 1)
-
-
-    async def insert_table(
-        document_id: str, rows: int | None, columns: int | None, start_position: int | None
-    ) -> str:
-        """Insert a table at a specific position."""
-        if not rows or not columns:
-            raise ValueError("rows and columns are required for insert_table operation")
-        if start_position is None:
-            raise ValueError("start_position is required for insert_table operation")
-
-        request = {
-            "insertTable": {
-                "rows": rows,
-                "columns": columns,
-                "location": {"index": start_position},
-            }
-        }
-        await execute_request("update", document_id, request)
-        return f"Inserted {rows}x{columns} table at position {start_position}"
-
-
-    def _table_matches_position(element: dict, start_position: int | None) -> bool:
-        """Check if a table element contains the specified position."""
-        if start_position is None:
-            return True
-        table_start = element.get("startIndex")
-        table_end = element.get("endIndex")
-        return table_start <= start_position < table_end
-
-
-    def _find_table_cell_range(
-        document: dict, row_index: int, column_index: int, start_position: int | None = None
-    ) -> tuple[int, int] | None:
-        """Find the content range within a table cell."""
-        content = document.get("body", {}).get("content", [])
-        for element in content:
-            if "table" in element:
-                if not _table_matches_position(element, start_position):
-                    continue
-                table = element["table"]
-                if row_index < len(table.get("tableRows", [])):
-                    row = table["tableRows"][row_index]
-                    if column_index < len(row.get("tableCells", [])):
-                        cell = row["tableCells"][column_index]
-                        cell_content = cell.get("content", [])
-                        for cell_element in cell_content:
-                            if "paragraph" in cell_element:
-                                para_start = cell_element.get("startIndex")
-                                para_end = cell_element.get("endIndex")
-                                return (para_start, para_end)
-                        cell_start = cell.get("startIndex")
-                        cell_end = cell.get("endIndex")
-                        if cell_start is not None and cell_end is not None:
-                            return (cell_start + 1, cell_end - 1)
-        return None
-
-
-    def _find_table_start_index(
-        document: dict, row_index: int, column_index: int, start_position: int | None = None
-    ) -> int | None:
-        """Find the start index of a table element."""
-        content = document.get("body", {}).get("content", [])
-        for element in content:
-            if "table" in element:
-                if not _table_matches_position(element, start_position):
-                    continue
-                table = element["table"]
-                if row_index < len(table.get("tableRows", [])):
-                    row = table["tableRows"][row_index]
-                    if column_index < len(row.get("tableCells", [])):
-                        return element.get("startIndex")
-        return None
-
-
-    async def update_table_cell(
-        document_id: str,
-        row_index: int | None,
-        column_index: int | None,
-        text: str,
-        start_position: int | None = None,
-    ) -> str:
-        """Update content in a specific table cell."""
-        if row_index is None or column_index is None:
-            raise ValueError("row_index and column_index are required for update_table_cell")
-
-        document = await execute_request("get", document_id, {})
-        cell_range = _find_table_cell_range(document, row_index, column_index, start_position)
-        if not cell_range:
-            raise ValueError(f"Table cell at row {row_index}, col {column_index} not found")
-
-        cell_start, cell_end = cell_range
-
-        requests = []
-        if cell_end > cell_start + 1:
-            requests.append(
-                {"deleteContentRange": {"range": {"startIndex": cell_start, "endIndex": cell_end - 1}}}
-            )
-
-        requests.append({"insertText": {"location": {"index": cell_start}, "text": text}})
-
-        await execute_requests(document_id, requests)
-        return f"Updated cell at row {row_index}, column {column_index}"
-
-
-    async def modify_table_structure(
-        document_id: str,
-        operation: str,
-        row_index: int | None = None,
-        column_index: int | None = None,
-        insert_below: bool = False,
-        insert_right: bool = False,
-        start_position: int | None = None,
-    ) -> str:
-        """Modify table structure by inserting or deleting rows/columns."""
-        if operation in ["insert_row", "delete_row"] and row_index is None:
-            raise ValueError(f"row_index is required for {operation} operation")
-        if operation in ["insert_column", "delete_column"] and column_index is None:
-            raise ValueError(f"column_index is required for {operation} operation")
-
-        document = await execute_request("get", document_id, {})
-
-        if operation in ["insert_row", "delete_row"]:
-            table_start = _find_table_start_index(document, row_index, 0, start_position)
-            if not table_start:
-                raise ValueError(f"Table row {row_index} not found")
-            location = {"tableStartLocation": {"index": table_start}, "rowIndex": row_index}
-        else:
-            table_start = _find_table_start_index(document, 0, column_index, start_position)
-            if not table_start:
-                raise ValueError(f"Table column {column_index} not found")
-            location = {"tableStartLocation": {"index": table_start}, "columnIndex": column_index}
-
-        operation_map = {
-            "insert_row": "insertTableRow",
-            "insert_column": "insertTableColumn",
-            "delete_row": "deleteTableRow",
-            "delete_column": "deleteTableColumn",
-        }
-        request_key = operation_map[operation]
-
-        if operation == "insert_row":
-            request = {request_key: {"tableCellLocation": location, "insertBelow": insert_below}}
-            message = f"Inserted row {'below' if insert_below else 'above'} row {row_index}"
-        elif operation == "insert_column":
-            request = {request_key: {"tableCellLocation": location, "insertRight": insert_right}}
-            message = f"Inserted column {'right of' if insert_right else 'left of'} column {column_index}"
-        elif operation == "delete_row":
-            request = {request_key: {"tableCellLocation": location}}
-            message = f"Deleted row {row_index}"
-        else:
-            request = {request_key: {"tableCellLocation": location}}
-            message = f"Deleted column {column_index}"
-
-        await execute_request("update", document_id, request)
-        return message
-
-
-    async def insert_image(
-        document_id: str,
-        image_url: str | None,
-        start_position: int | None,
-        width: int | None,
-        height: int | None,
-    ) -> str:
-        """Insert an image from a URL at the specified position."""
-        if not image_url:
-            raise ValueError("image_url is required for insert_image operation")
-        if start_position is None:
-            raise ValueError("start_position is required for insert_image operation")
-
-        request = {
-            "insertInlineImage": {
-                "uri": image_url,
-                "location": {"index": start_position},
-            }
-        }
-
-        if width or height:
-            object_size = {}
-            if width:
-                object_size["width"] = {"magnitude": width, "unit": "PT"}
-            if height:
-                object_size["height"] = {"magnitude": height, "unit": "PT"}
-            request["insertInlineImage"]["objectSize"] = object_size
-
-        await execute_request("update", document_id, request)
-        return f"Inserted image from {image_url} at position {start_position}"
-
-
-    async def format_existing_text(
-        document_id: str,
-        search_text: str | None,
-        foreground_color: str | None,
-        background_color: str | None,
-        font_size: int | None,
-        font_family: str | None,
-        bold: bool | None,
-        italic: bool | None,
-        underline: bool | None,
-        strikethrough: bool | None,
-        heading_level: int | None,
-        link_url: str | None,
-        list_type: str | None,
-    ) -> str:
-        """Find and apply formatting to all occurrences of text."""
-        if not search_text:
-            raise ValueError("search_text is required for format_existing_text operation")
-
-        document = await execute_request("get", document_id, {})
-
-        from doc_utils import build_text_style, find_text_positions
-
-        positions = find_text_positions(document, search_text)
-
-        if not positions:
-            return f"No occurrences of '{search_text}' found"
-
-        text_style, fields = build_text_style(
-            foreground_color=foreground_color,
-            background_color=background_color,
-            font_size=font_size,
-            font_family=font_family,
-            bold=bold,
-            italic=italic,
-            underline=underline,
-            strikethrough=strikethrough,
-            link_url=link_url,
-        )
-
-        if not fields and not heading_level and not list_type:
-            raise ValueError("At least one formatting option must be specified")
-
-        positions.sort(reverse=True)
-
-        requests = []
-        for start, end in positions:
-            range_dict = {"startIndex": start, "endIndex": end}
-
-            if fields:
-                requests.append(
-                    {"updateTextStyle": {"range": range_dict, "textStyle": text_style, "fields": fields}}
-                )
-
-            if heading_level:
-                if not (1 <= heading_level <= 6):
-                    raise ValueError("heading_level must be between 1 and 6")
-                requests.append(
-                    {
-                        "updateParagraphStyle": {
-                            "range": range_dict,
-                            "paragraphStyle": {"namedStyleType": f"HEADING_{heading_level}"},
-                            "fields": "namedStyleType",
-                        }
-                    }
-                )
-
-            if list_type:
-                if list_type == "bullet":
-                    preset = "BULLET_DISC_CIRCLE_SQUARE"
-                elif list_type == "numbered":
-                    preset = "NUMBERED_DECIMAL_ALPHA_ROMAN"
-                else:
-                    raise ValueError("list_type must be 'bullet' or 'numbered'")
-                requests.append({"createParagraphBullets": {"range": range_dict, "bulletPreset": preset}})
-
-        if not requests:
-            raise ValueError(f"No formatting requests generated for '{search_text}'.")
-
-        await execute_requests(document_id, requests)
-        return f"Formatted {len(positions)} occurrences of '{search_text}'"
-
-  doc_utils.py: |
-    """Utility functions for document operations."""
-
-    import re
-    from typing import Any
-
-
-    async def execute_request(
-        method: str, document_id: str, request: dict, **kwargs
-    ) -> dict[str, Any]:
-        """Execute a single document API request."""
-        raise NotImplementedError("Stub: would call document backend")
-
-
-    async def execute_requests(
-        document_id: str, requests: list[dict], **kwargs
-    ) -> dict[str, Any]:
-        """Execute a batch of document API requests."""
-        raise NotImplementedError("Stub: would call document backend")
-
-
-    def calculate_utf16_length(text: str) -> int:
-        """Calculate text length in UTF-16 code units."""
-        return len(text.encode("utf-16-le")) // 2
-
-
-    def hex_to_rgb(hex_color: str) -> dict:
-        """Convert hex color to RGB format (0.0-1.0 range)."""
-        hex_color = hex_color.lstrip("#")
-
-        if len(hex_color) == 3:
-            hex_color = "".join([c * 2 for c in hex_color])
-
-        if not re.match(r"^[0-9A-Fa-f]{6}$", hex_color):
-            raise ValueError(f"Invalid hex color: {hex_color}")
-
-        return {
-            "color": {
-                "rgbColor": {
-                    "red": int(hex_color[0:2], 16) / 255.0,
-                    "green": int(hex_color[2:4], 16) / 255.0,
-                    "blue": int(hex_color[4:6], 16) / 255.0,
-                }
-            }
-        }
-
-
-    def build_text_style(
-        foreground_color: str | None = None,
-        background_color: str | None = None,
-        font_size: int | None = None,
-        font_family: str | None = None,
-        bold: bool | None = None,
-        italic: bool | None = None,
-        underline: bool | None = None,
-        strikethrough: bool | None = None,
-        link_url: str | None = None,
-    ) -> tuple[dict, str]:
-        """Build text style object and field mask from provided formatting options."""
-        style = {}
-        fields = []
-
-        if foreground_color:
-            style["foregroundColor"] = hex_to_rgb(foreground_color)
-            fields.append("foregroundColor")
-
-        if background_color:
-            style["backgroundColor"] = hex_to_rgb(background_color)
-            fields.append("backgroundColor")
-
-        if font_size is not None:
-            style["fontSize"] = {"magnitude": font_size, "unit": "PT"}
-            fields.append("fontSize")
-
-        if font_family:
-            style["weightedFontFamily"] = {"fontFamily": font_family}
-            fields.append("weightedFontFamily")
-
-        if bold is not None:
-            style["bold"] = bold
-            fields.append("bold")
-
-        if italic is not None:
-            style["italic"] = italic
-            fields.append("italic")
-
-        if underline is not None:
-            style["underline"] = underline
-            fields.append("underline")
-
-        if strikethrough is not None:
-            style["strikethrough"] = strikethrough
-            fields.append("strikethrough")
-
-        if link_url:
-            style["link"] = {"url": link_url}
-            fields.append("link")
-
-        return style, ",".join(fields)
-
-
-    def find_text_positions(
-        document: dict, search_text: str
-    ) -> list[tuple[int, int]]:
-        """Find all occurrences of text in a document with UTF-16 positions."""
-        positions = []
-        search_len = calculate_utf16_length(search_text)
-
-        content = document.get("body", {}).get("content", [])
-        _find_text_in_elements(content, search_text, search_len, positions)
-
-        return positions
-
-
-    def _find_text_in_elements(
-        elements: list[dict], search_text: str, search_len: int, positions: list
-    ):
-        """Recursively find text in document elements."""
-        for element in elements:
-            if "paragraph" in element:
-                para_elements = element["paragraph"].get("elements", [])
-                for elem in para_elements:
-                    if "textRun" in elem:
-                        text_run = elem["textRun"]
-                        content = text_run.get("content", "")
-                        start_index = elem.get("startIndex", 0)
-
-                        idx = 0
-                        while True:
-                            pos = content.find(search_text, idx)
-                            if pos == -1:
-                                break
-                            prefix_len = calculate_utf16_length(content[:pos])
-                            match_start = start_index + prefix_len
-                            match_end = match_start + search_len
-                            positions.append((match_start, match_end))
-                            idx = pos + len(search_text)
-
-            elif "table" in element:
-                table_rows = element["table"].get("tableRows", [])
-                for row in table_rows:
-                    for cell in row.get("tableCells", []):
-                        cell_content = cell.get("content", [])
-                        _find_text_in_elements(cell_content, search_text, search_len, positions)
-
-validate:
-  # docs.py must still exist
-  - type: file_exists
-    path: docs.py
-    name: docs.py exists
-
-  # doc_utils.py must still exist
-  - type: file_exists
-    path: doc_utils.py
-    name: doc_utils.py exists
-
-  # insert_image removed from the operations literal
-  - type: file_not_matches
-    path: docs.py
-    regex: "insert_image"
-    name: insert_image removed from operations literal
-
-  # ImageParams class removed
-  - type: file_not_matches
-    path: docs.py
-    regex: "ImageParams"
-    name: ImageParams class removed
-
-  # image_params parameter removed
-  - type: file_not_matches
-    path: docs.py
-    regex: "image_params"
-    name: image_params parameter removed
-
-  # insert_image function removed
-  - type: file_not_matches
-    path: docs.py
-    regex: "async def insert_image"
-    name: insert_image function removed
-
-  # image_url reference removed
-  - type: file_not_matches
-    path: docs.py
-    regex: "image_url"
-    name: image_url references removed
-
-  # insertInlineImage reference removed
-  - type: file_not_matches
-    path: docs.py
-    regex: "insertInlineImage"
-    name: insertInlineImage reference removed
-
-  # Other operations still present
-  - type: file_contains
-    path: docs.py
-    pattern: "get_document"
-    name: get_document preserved
-
-  - type: file_contains
-    path: docs.py
-    pattern: "insert_text"
-    name: insert_text preserved
-
-  - type: file_contains
-    path: docs.py
-    pattern: "replace_text"
-    name: replace_text preserved
-
-  - type: file_contains
-    path: docs.py
-    pattern: "insert_table"
-    name: insert_table preserved
-
-  - type: file_contains
-    path: docs.py
-    pattern: "format_existing_text"
-    name: format_existing_text preserved
-
-  - type: file_contains
-    path: docs.py
-    pattern: "async def doc_tool"
-    name: doc_tool function preserved
-
-  - type: file_contains
-    path: docs.py
-    pattern: "FormatTextParams"
-    name: FormatTextParams preserved
-
-  - type: file_contains
-    path: docs.py
-    pattern: "TableParams"
-    name: TableParams preserved
-
-  # doc_utils.py should be untouched (no image references existed there)
-  - type: file_contains
-    path: doc_utils.py
-    pattern: "def build_text_style"
-    name: doc_utils build_text_style preserved
-
-  - type: file_contains
-    path: doc_utils.py
-    pattern: "def find_text_positions"
-    name: doc_utils find_text_positions preserved
-
-  - type: file_contains
-    path: doc_utils.py
-    pattern: "def calculate_utf16_length"
-    name: doc_utils calculate_utf16_length preserved
-
-  # Python syntax check
-  - type: command_succeeds
-    command: "python3 -c \"import ast; ast.parse(open('docs.py').read())\""
-    name: docs.py valid python syntax
-
-  - type: command_succeeds
-    command: "python3 -c \"import ast; ast.parse(open('doc_utils.py').read())\""
-    name: doc_utils.py valid python syntax

evals/open-model-gym/suite/src/types.ts

@@ -1,74 +0,0 @@
-export interface AgentConfig {
-  model: string;
-  provider: string;
-  /** Extensions (runner knows which are platform vs builtin) */
-  extensions?: string[];
-  /** Stdio extension commands (for custom MCP servers) */
-  stdio?: string[];
-  /** Path to goose binary (default: "goose") */
-  "goose-bin"?: string;
-  temperature?: number;
-  maxTokens?: number;
-}
-
-export interface Scenario {
-  name: string;
-  description: string;
-  prompt?: string;
-  /** Files to create before running (relative paths) */
-  setup?: Record<string, string>;
-  /** Validation rules to check after agent completes (single-turn) */
-  validate?: ValidationRule[];
-  /** Multi-turn conversation (alternative to single prompt+validate) */
-  turns?: Turn[];
-  /** Tags for filtering scenarios */
-  tags?: string[];
-}
-
-/** A single turn in a multi-turn conversation */
-export interface Turn {
-  /** The prompt for this turn */
-  prompt: string;
-  /** Validation rules to check after this turn completes */
-  validate: ValidationRule[];
-}
-
-export type ValidationRule =
-  | { type: "file_exists"; path: string; name?: string }
-  | { type: "file_contains"; path: string; pattern: string; name?: string }
-  | { type: "file_matches"; path: string; regex: string; name?: string }
-  | { type: "file_not_matches"; path: string; regex: string; name?: string }
-  | { type: "file_not_empty"; path: string; name?: string }
-  | { type: "command_succeeds"; command: string; name?: string }
-  | { type: "tool_called"; tool: string; args?: Record<string, string | RegExp>; name?: string }
-  | { type: "custom"; fn: string; name?: string };
-
-export interface TestRun {
-  scenario: Scenario;
-  config: AgentConfig;
-  workdir: string;
-  startTime: Date;
-  endTime?: Date;
-  status: "pending" | "running" | "passed" | "failed";
-  errors?: string[];
-}
-
-export interface TestResult {
-  run: TestRun;
-  validations: Array<{
-    rule: ValidationRule;
-    passed: boolean;
-    message?: string;
-  }>;
-}
-
-export interface SuiteConfig {
-  /** Agent configurations to permute */
-  agents: AgentConfig[];
-  /** Scenarios to run */
-  scenarios: string[];
-  /** Base directory for test workspaces */
-  workdir: string;
-  /** Parallel execution count */
-  parallel?: number;
-}

evals/open-model-gym/suite/src/validator.ts

@@ -1,184 +0,0 @@
-import { existsSync, readFileSync, statSync } from "node:fs";
-import { execSync } from "node:child_process";
-import { join } from "node:path";
-import type { ValidationRule } from "./types.js";
-
-export interface ValidationResult {
-  passed: boolean;
-  message?: string;
-}
-
-export function validateRule(
-  rule: ValidationRule,
-  workdir: string
-): ValidationResult {
-  switch (rule.type) {
-    case "file_exists": {
-      const fullPath = join(workdir, rule.path);
-      const exists = existsSync(fullPath);
-      return {
-        passed: exists,
-        message: exists ? undefined : `File not found: ${rule.path}`,
-      };
-    }
-
-    case "file_not_empty": {
-      const fullPath = join(workdir, rule.path);
-      if (!existsSync(fullPath)) {
-        return { passed: false, message: `File not found: ${rule.path}` };
-      }
-      const stat = statSync(fullPath);
-      return {
-        passed: stat.size > 0,
-        message: stat.size > 0 ? undefined : `File is empty: ${rule.path}`,
-      };
-    }
-
-    case "file_contains": {
-      const fullPath = join(workdir, rule.path);
-      if (!existsSync(fullPath)) {
-        return { passed: false, message: `File not found: ${rule.path}` };
-      }
-      const content = readFileSync(fullPath, "utf-8");
-      const contains = content.includes(rule.pattern);
-      return {
-        passed: contains,
-        message: contains
-          ? undefined
-          : `File ${rule.path} does not contain: ${rule.pattern}`,
-      };
-    }
-
-    case "file_matches": {
-      const fullPath = join(workdir, rule.path);
-      if (!existsSync(fullPath)) {
-        return { passed: false, message: `File not found: ${rule.path}` };
-      }
-      const content = readFileSync(fullPath, "utf-8");
-      const regex = new RegExp(rule.regex);
-      const matches = regex.test(content);
-      return {
-        passed: matches,
-        message: matches
-          ? undefined
-          : `File ${rule.path} does not match regex: ${rule.regex}`,
-      };
-    }
-
-    case "file_not_matches": {
-      const fullPath = join(workdir, rule.path);
-      if (!existsSync(fullPath)) {
-        return { passed: false, message: `File not found: ${rule.path}` };
-      }
-      const content = readFileSync(fullPath, "utf-8");
-      const regex = new RegExp(rule.regex);
-      const matches = regex.test(content);
-      return {
-        passed: !matches,
-        message: !matches
-          ? undefined
-          : `File ${rule.path} should not match regex: ${rule.regex}`,
-      };
-    }
-
-    case "command_succeeds": {
-      try {
-        execSync(rule.command, { cwd: workdir, stdio: "pipe" });
-        return { passed: true };
-      } catch (err) {
-        return {
-          passed: false,
-          message: `Command failed: ${rule.command}`,
-        };
-      }
-    }
-
-    case "tool_called": {
-      const logPath = join(workdir, "tool-calls.log");
-      if (!existsSync(logPath)) {
-        return { passed: false, message: "tool-calls.log not found" };
-      }
-
-      const content = readFileSync(logPath, "utf-8");
-      const lines = content.trim().split("\n").filter(Boolean);
-
-      // Find all calls to the specified tool
-      const matchingCalls = lines
-        .map((line) => {
-          try {
-            return JSON.parse(line);
-          } catch {
-            return null;
-          }
-        })
-        .filter((entry) => entry?.tool === rule.tool);
-
-      if (matchingCalls.length === 0) {
-        return { passed: false, message: `Tool not called: ${rule.tool}` };
-      }
-
-      // If no arg requirements, just check tool was called
-      if (!rule.args) {
-        return { passed: true };
-      }
-
-      // Check if any call matches the arg requirements
-      for (const call of matchingCalls) {
-        const args = call.arguments || {};
-        let allMatch = true;
-
-        for (const [key, expected] of Object.entries(rule.args)) {
-          const actual = args[key];
-          if (actual === undefined) {
-            allMatch = false;
-            break;
-          }
-
-          // If expected starts/ends with /, treat as regex pattern
-          if (typeof expected === "string" && expected.startsWith("/") && expected.endsWith("/")) {
-            const pattern = new RegExp(expected.slice(1, -1), "i");
-            if (!pattern.test(String(actual))) {
-              allMatch = false;
-              break;
-            }
-          } else {
-            // Exact match (case-insensitive for strings)
-            const actualStr = String(actual).toLowerCase();
-            const expectedStr = String(expected).toLowerCase();
-            if (!actualStr.includes(expectedStr)) {
-              allMatch = false;
-              break;
-            }
-          }
-        }
-
-        if (allMatch) {
-          return { passed: true };
-        }
-      }
-
-      return {
-        passed: false,
-        message: `Tool ${rule.tool} called but args didn't match: expected ${JSON.stringify(rule.args)}`,
-      };
-    }
-
-    case "custom": {
-      // Custom validators loaded dynamically
-      return { passed: false, message: "Custom validators not yet implemented" };
-    }
-
-    default:
-      return { passed: false, message: `Unknown rule type` };
-  }
-}
-
-export function validateAll(
-  rules: ValidationRule[],
-  workdir: string
-): Array<{ rule: ValidationRule; result: ValidationResult }> {
-  return rules.map((rule) => ({
-    rule,
-    result: validateRule(rule, workdir),
-  }));
-}

evals/open-model-gym/suite/tsconfig.json

@@ -1,12 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "module": "NodeNext",
-    "moduleResolution": "NodeNext",
-    "strict": true,
-    "esModuleInterop": true,
-    "skipLibCheck": true,
-    "outDir": "dist"
-  },
-  "include": ["src"]
-}