mirror of
https://github.com/block/goose.git
synced 2026-07-09 16:09:22 +00:00
Remove open-model-gym eval suite (#10174)
Some checks failed
Canary / Prepare Version (push) Waiting to run
Canary / build-cli (push) Blocked by required conditions
Canary / Upload Install Script (push) Blocked by required conditions
Canary / bundle-desktop (push) Blocked by required conditions
Canary / bundle-desktop-intel (push) Blocked by required conditions
Canary / bundle-desktop-linux (push) Blocked by required conditions
Canary / bundle-desktop-windows (push) Blocked by required conditions
Canary / bundle-desktop-windows-cuda (push) Blocked by required conditions
Canary / Release (push) Blocked by required conditions
Unused Dependencies / machete (push) Waiting to run
CI / changes (push) Waiting to run
CI / Check Rust Code Format (push) Blocked by required conditions
CI / Build and Test Rust Project (push) Blocked by required conditions
CI / Build Rust Project on Windows (push) Waiting to run
CI / Check MSRV (push) Blocked by required conditions
CI / Lint Rust Code (push) Blocked by required conditions
CI / Check Generated Schemas are Up-to-Date (push) Blocked by required conditions
CI / Test and Lint Electron Desktop App (push) Blocked by required conditions
Create Minor Release PR / check-version-bump-pr (push) Waiting to run
Create Minor Release PR / release (push) Blocked by required conditions
Live Provider Tests / check-fork (push) Waiting to run
Live Provider Tests / changes (push) Blocked by required conditions
Live Provider Tests / Build Binary (push) Blocked by required conditions
Live Provider Tests / Smoke Tests (push) Blocked by required conditions
Live Provider Tests / Smoke Tests (Code Execution) (push) Blocked by required conditions
Live Provider Tests / Compaction Tests (push) Blocked by required conditions
Publish Docker Image / docker (push) Waiting to run
Scorecard supply-chain security / Scorecard analysis (push) Waiting to run
Deploy Documentation / deploy (push) Has been cancelled
Publish Ask AI Bot Docker Image / docker (push) Has been cancelled
Some checks failed
Canary / Prepare Version (push) Waiting to run
Canary / build-cli (push) Blocked by required conditions
Canary / Upload Install Script (push) Blocked by required conditions
Canary / bundle-desktop (push) Blocked by required conditions
Canary / bundle-desktop-intel (push) Blocked by required conditions
Canary / bundle-desktop-linux (push) Blocked by required conditions
Canary / bundle-desktop-windows (push) Blocked by required conditions
Canary / bundle-desktop-windows-cuda (push) Blocked by required conditions
Canary / Release (push) Blocked by required conditions
Unused Dependencies / machete (push) Waiting to run
CI / changes (push) Waiting to run
CI / Check Rust Code Format (push) Blocked by required conditions
CI / Build and Test Rust Project (push) Blocked by required conditions
CI / Build Rust Project on Windows (push) Waiting to run
CI / Check MSRV (push) Blocked by required conditions
CI / Lint Rust Code (push) Blocked by required conditions
CI / Check Generated Schemas are Up-to-Date (push) Blocked by required conditions
CI / Test and Lint Electron Desktop App (push) Blocked by required conditions
Create Minor Release PR / check-version-bump-pr (push) Waiting to run
Create Minor Release PR / release (push) Blocked by required conditions
Live Provider Tests / check-fork (push) Waiting to run
Live Provider Tests / changes (push) Blocked by required conditions
Live Provider Tests / Build Binary (push) Blocked by required conditions
Live Provider Tests / Smoke Tests (push) Blocked by required conditions
Live Provider Tests / Smoke Tests (Code Execution) (push) Blocked by required conditions
Live Provider Tests / Compaction Tests (push) Blocked by required conditions
Publish Docker Image / docker (push) Waiting to run
Scorecard supply-chain security / Scorecard analysis (push) Waiting to run
Deploy Documentation / deploy (push) Has been cancelled
Publish Ask AI Bot Docker Image / docker (push) Has been cancelled
This commit is contained in:
Michael Neale
GitHub
parent
b83b194dad
commit
27958c4c29
24 changed files with 0 additions and 6967 deletions
|
|
@ -48,7 +48,6 @@ crates/
|
||||||
├── goose-test # test utilities
|
├── goose-test # test utilities
|
||||||
└── goose-test-support # test helpers
|
└── goose-test-support # test helpers
|
||||||
|
|
||||||
evals/open-model-gym/ # benchmarking / evals
|
|
||||||
ui/desktop/ # Electron app
|
ui/desktop/ # Electron app
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
|
||||||
72
evals/open-model-gym/.gitignore
vendored
72
evals/open-model-gym/.gitignore
vendored
|
|
@ -1,72 +0,0 @@
|
||||||
# Dependencies
|
|
||||||
node_modules/
|
|
||||||
.pnpm-store/
|
|
||||||
.workdir/
|
|
||||||
report.html
|
|
||||||
|
|
||||||
# Build outputs
|
|
||||||
dist/
|
|
||||||
build/
|
|
||||||
out/
|
|
||||||
.next/
|
|
||||||
.nuxt/
|
|
||||||
.output/
|
|
||||||
.opencode-root
|
|
||||||
suite/.pi-root
|
|
||||||
|
|
||||||
# TypeScript
|
|
||||||
*.tsbuildinfo
|
|
||||||
*.d.ts.map
|
|
||||||
.g3/
|
|
||||||
# Logs
|
|
||||||
logs/
|
|
||||||
*.log
|
|
||||||
npm-debug.log*
|
|
||||||
yarn-debug.log*
|
|
||||||
yarn-error.log*
|
|
||||||
pnpm-debug.log*
|
|
||||||
|
|
||||||
# Runtime data
|
|
||||||
pids/
|
|
||||||
*.pid
|
|
||||||
*.seed
|
|
||||||
*.pid.lock
|
|
||||||
|
|
||||||
# Coverage & testing
|
|
||||||
coverage/
|
|
||||||
.nyc_output/
|
|
||||||
.jest/
|
|
||||||
|
|
||||||
# Caches
|
|
||||||
.cache/
|
|
||||||
.parcel-cache/
|
|
||||||
.turbo/
|
|
||||||
.eslintcache
|
|
||||||
.stylelintcache
|
|
||||||
*.swp
|
|
||||||
*.swo
|
|
||||||
|
|
||||||
# IDE & editors
|
|
||||||
.idea/
|
|
||||||
.vscode/
|
|
||||||
*.sublime-project
|
|
||||||
*.sublime-workspace
|
|
||||||
|
|
||||||
# OS files
|
|
||||||
.DS_Store
|
|
||||||
Thumbs.db
|
|
||||||
|
|
||||||
# Environment variables
|
|
||||||
.env
|
|
||||||
.env.local
|
|
||||||
.env.*.local
|
|
||||||
|
|
||||||
# Lock files (optional - uncomment if you don't want to track)
|
|
||||||
# package-lock.json
|
|
||||||
# yarn.lock
|
|
||||||
# pnpm-lock.yaml
|
|
||||||
|
|
||||||
# Temporary files
|
|
||||||
tmp/
|
|
||||||
temp/
|
|
||||||
*.tmp
|
|
||||||
|
|
@ -1,71 +0,0 @@
|
||||||
# Agent Runner - Test Suite (supports goose and opencode)
|
|
||||||
|
|
||||||
# Default recipe
|
|
||||||
default: run
|
|
||||||
|
|
||||||
# Full test run - all scenarios, all agents, 3 repetitions (worst kept)
|
|
||||||
run: _install
|
|
||||||
cd suite && npm run test
|
|
||||||
|
|
||||||
# Full run with artifacts isolated under ~/.goose/gym-runs/<YYYYDDMMHHMMSS> (keeps the repo clean)
|
|
||||||
run-clean: _install
|
|
||||||
#!/usr/bin/env bash
|
|
||||||
set -euo pipefail
|
|
||||||
export GYM_OUTPUT_DIR="$HOME/.goose/gym-runs/$(date +%Y%d%m%H%M%S)"
|
|
||||||
echo "Artifacts → $GYM_OUTPUT_DIR"
|
|
||||||
cd suite && npm run test
|
|
||||||
|
|
||||||
# Quick test - file-editing + everyday-app-automation, single run each (no repetition)
|
|
||||||
test: _install
|
|
||||||
cd suite && npx tsx src/runner.ts --scenario=file-editing,everyday-app-automation --run-count=1
|
|
||||||
|
|
||||||
# Run a specific scenario (all agents, 3 reps)
|
|
||||||
scenario name: _install
|
|
||||||
cd suite && npx tsx src/runner.ts --scenario={{name}}
|
|
||||||
|
|
||||||
# Run against a specific agent (all scenarios, 3 reps)
|
|
||||||
agent name: _install
|
|
||||||
cd suite && npx tsx src/runner.ts --agent={{name}}
|
|
||||||
|
|
||||||
# Open report in browser (honors GYM_OUTPUT_DIR if set)
|
|
||||||
report:
|
|
||||||
#!/usr/bin/env bash
|
|
||||||
set -euo pipefail
|
|
||||||
dir="${GYM_OUTPUT_DIR:-.}"
|
|
||||||
open "${dir/#\~/$HOME}/report.html"
|
|
||||||
|
|
||||||
# Install all dependencies
|
|
||||||
install:
|
|
||||||
cd suite && npm install
|
|
||||||
cd mcp-harness && npm install && npm run build
|
|
||||||
@# Install pi-mcp-adapter for Pi runner MCP support
|
|
||||||
@pi list 2>/dev/null | grep -q "pi-mcp-adapter" || pi install npm:pi-mcp-adapter
|
|
||||||
|
|
||||||
# Build TypeScript
|
|
||||||
build: _install
|
|
||||||
cd suite && npm run build
|
|
||||||
|
|
||||||
# Clear the test cache
|
|
||||||
clear-cache:
|
|
||||||
cd suite && npx tsx src/runner.ts --clear-cache
|
|
||||||
|
|
||||||
# Run tests ignoring cache (force fresh runs)
|
|
||||||
run-fresh: _install
|
|
||||||
cd suite && npx tsx src/runner.ts --no-cache
|
|
||||||
|
|
||||||
# Show cache stats
|
|
||||||
cache-stats:
|
|
||||||
@if [ -f suite/.cache/index.json ]; then \
|
|
||||||
echo "Cache entries: $$(cat suite/.cache/index.json | grep -o '"[a-f0-9]\{16\}":' | wc -l | tr -d ' ')"; \
|
|
||||||
echo "Cache size: $$(du -sh suite/.cache 2>/dev/null | cut -f1 || echo '0')"; \
|
|
||||||
else \
|
|
||||||
echo "No cache found"; \
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Internal: install if node_modules missing, always rebuild mcp-harness
|
|
||||||
_install:
|
|
||||||
@[ -d suite/node_modules ] || (cd suite && npm install)
|
|
||||||
@[ -d mcp-harness/node_modules ] || (cd mcp-harness && npm install)
|
|
||||||
@cd mcp-harness && npm run build
|
|
||||||
@# Ensure pi-mcp-adapter is installed for Pi runner
|
|
||||||
@pi list 2>/dev/null | grep -q "pi-mcp-adapter" || pi install npm:pi-mcp-adapter
|
|
||||||
|
|
@ -1,323 +0,0 @@
|
||||||
# Open Model Gym
|
|
||||||
|
|
||||||
Run agent tests across a matrix of **models × runners × scenarios**.
|
|
||||||
|
|
||||||
It isn't hard for any agent to do ok with opus, but lets scale things in the other direction. What do we have to break things down to.
|
|
||||||
|
|
||||||
<img width="1768" height="1133" alt="image" src="https://github.com/user-attachments/assets/29915659-ee6b-4a8b-ba5e-58420b168b43" />
|
|
||||||
|
|
||||||
## Quick Start
|
|
||||||
|
|
||||||
```bash
|
|
||||||
just install # one-time setup
|
|
||||||
just run # run full matrix (3 reps each)
|
|
||||||
just report # view results
|
|
||||||
```
|
|
||||||
|
|
||||||
## How It Works
|
|
||||||
|
|
||||||
The test harness runs every combination of models, runners, and scenarios defined in your matrix. Each test runs multiple times (default 3) and keeps the **worst result** — if a test fails even once, it's marked failed. This catches flaky passes.
|
|
||||||
|
|
||||||
## Configuration
|
|
||||||
|
|
||||||
Edit `config.yaml` to define your test matrix:
|
|
||||||
|
|
||||||
### Models
|
|
||||||
|
|
||||||
LLMs to test against. Supports any provider (Anthropic, OpenAI, Ollama, etc.):
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
models:
|
|
||||||
- name: opus
|
|
||||||
provider: anthropic
|
|
||||||
model: claude-opus-4-5-20251101
|
|
||||||
|
|
||||||
- name: qwen3-coder
|
|
||||||
provider: ollama
|
|
||||||
model: qwen3-coder:64k
|
|
||||||
|
|
||||||
- name: gpt4
|
|
||||||
provider: openai
|
|
||||||
model: gpt-4-turbo
|
|
||||||
```
|
|
||||||
|
|
||||||
### Runners
|
|
||||||
|
|
||||||
Agent frameworks that execute the tests. Each runner has its own binary, type, and configuration:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
runners:
|
|
||||||
# Goose agent with extensions
|
|
||||||
- name: goose-full
|
|
||||||
type: goose
|
|
||||||
bin: goose # path to binary (can be absolute)
|
|
||||||
extensions: [developer, todo, skills]
|
|
||||||
stdio:
|
|
||||||
- node mcp-harness/dist/index.js
|
|
||||||
|
|
||||||
# OpenCode agent
|
|
||||||
- name: opencode
|
|
||||||
type: opencode
|
|
||||||
bin: opencode # path to binary
|
|
||||||
stdio:
|
|
||||||
- node mcp-harness/dist/index.js
|
|
||||||
|
|
||||||
# Custom goose binary path
|
|
||||||
- name: goose-dev
|
|
||||||
type: goose
|
|
||||||
bin: /path/to/my/goose-dev
|
|
||||||
extensions: [developer]
|
|
||||||
```
|
|
||||||
|
|
||||||
**Supported runner types:**
|
|
||||||
- `goose` — [Goose](https://github.com/aaif-goose/goose) agent framework
|
|
||||||
- `opencode` — [OpenCode](https://opencode.ai) agent framework
|
|
||||||
- `pi` — [Pi](https://github.com/badlogic/pi-mono) coding agent
|
|
||||||
|
|
||||||
## Runner Details
|
|
||||||
|
|
||||||
Each runner has different setup requirements, MCP integration methods, and session handling.
|
|
||||||
|
|
||||||
### Goose
|
|
||||||
|
|
||||||
[Goose](https://github.com/aaif-goose/goose) is an open-source coding agent with built-in MCP support.
|
|
||||||
|
|
||||||
**Setup:** Install via `brew install goose` or from source.
|
|
||||||
|
|
||||||
**MCP Integration:** Native support. The harness writes a `config.yaml` to an isolated `.goose-root/` directory with extensions and MCP servers:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
extensions:
|
|
||||||
developer:
|
|
||||||
enabled: true
|
|
||||||
mcp_harness:
|
|
||||||
type: stdio
|
|
||||||
enabled: true
|
|
||||||
cmd: node
|
|
||||||
args: [mcp-harness/dist/index.js]
|
|
||||||
```
|
|
||||||
|
|
||||||
**Session Handling:** Uses `--name <session>` for named sessions, `--resume` to continue:
|
|
||||||
- Turn 1: `goose run -i <prompt> --name <session>`
|
|
||||||
- Turn 2+: `goose run -i <prompt> --name <session> --resume`
|
|
||||||
- Single-turn: `goose run -i <prompt> --no-session`
|
|
||||||
|
|
||||||
### OpenCode
|
|
||||||
|
|
||||||
[OpenCode](https://opencode.ai) is a terminal-based coding agent.
|
|
||||||
|
|
||||||
**Setup:** Install via their website or package manager.
|
|
||||||
|
|
||||||
**MCP Integration:** Native support. The harness writes an `opencode.json` config to the workdir:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"mcp": {
|
|
||||||
"harness": {
|
|
||||||
"type": "local",
|
|
||||||
"command": ["node", "mcp-harness/dist/index.js"],
|
|
||||||
"enabled": true
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"model": "anthropic/claude-opus-4-5-20251101"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
**Session Handling:** Uses `--continue` to resume the last session in the working directory:
|
|
||||||
- Turn 1: `opencode run "<prompt>"`
|
|
||||||
- Turn 2+: `opencode run --continue "<prompt>"`
|
|
||||||
|
|
||||||
⚠️ OpenCode doesn't support named sessions, so multi-turn scenarios exclude it.
|
|
||||||
|
|
||||||
### Pi
|
|
||||||
|
|
||||||
[Pi](https://github.com/badlogic/pi-mono) is a lightweight coding agent that requires an adapter for MCP support.
|
|
||||||
|
|
||||||
**Setup:**
|
|
||||||
```bash
|
|
||||||
# Install Pi
|
|
||||||
npm install -g @anthropic/pi # or from source
|
|
||||||
|
|
||||||
# Install the MCP adapter (required for MCP tools)
|
|
||||||
pi install npm:pi-mcp-adapter
|
|
||||||
```
|
|
||||||
|
|
||||||
The `just install` recipe auto-installs pi-mcp-adapter if missing.
|
|
||||||
|
|
||||||
**MCP Integration:** Via [pi-mcp-adapter](https://github.com/nicobailon/pi-mcp-adapter). The harness dynamically writes a `.pi-mcp.json` config to the workdir:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"mcpServers": {
|
|
||||||
"harness": {
|
|
||||||
"command": "node",
|
|
||||||
"args": ["mcp-harness/dist/index.js"],
|
|
||||||
"lifecycle": "eager",
|
|
||||||
"env": { "MCP_HARNESS_LOG": "<workdir>/tool-calls.log" }
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"settings": { "directTools": true }
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Key settings:
|
|
||||||
- `directTools: true` — Registers MCP tools directly in Pi's tool list (no wrapper)
|
|
||||||
- `lifecycle: "eager"` — Connects to MCP servers at startup
|
|
||||||
|
|
||||||
**Model Configuration:** Pi requires custom models (like Ollama) to be defined in `models.json`. The harness automatically generates this config in an isolated `.pi-root/` directory and sets `PI_CODING_AGENT_DIR` to use it:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"providers": {
|
|
||||||
"ollama": {
|
|
||||||
"baseUrl": "http://localhost:11434/v1",
|
|
||||||
"api": "openai-completions",
|
|
||||||
"apiKey": "ollama",
|
|
||||||
"models": [{ "id": "model-name", "name": "Model Name", ... }]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
The harness copies `auth.json` from your real Pi config (`~/.pi/agent/`) so API keys work.
|
|
||||||
|
|
||||||
**Session Handling:** Uses `--session <path>` for file-based sessions, `--continue` to resume:
|
|
||||||
- Turn 1: `pi -p --session <path> "<prompt>"`
|
|
||||||
- Turn 2+: `pi -p --continue --session <path> "<prompt>"`
|
|
||||||
- Single-turn: `pi -p --no-session "<prompt>"`
|
|
||||||
|
|
||||||
The `-p` flag runs Pi in non-interactive "print" mode for automation
|
|
||||||
|
|
||||||
### Matrix
|
|
||||||
|
|
||||||
Define which scenarios run against which models/runners:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
matrix:
|
|
||||||
- scenario: file-editing
|
|
||||||
models: [opus, qwen3-coder] # omit to run all models
|
|
||||||
runners: [goose-full, opencode] # omit to run all runners
|
|
||||||
|
|
||||||
- scenario: everyday-app-automation
|
|
||||||
# runs against ALL models and ALL runners
|
|
||||||
```
|
|
||||||
|
|
||||||
## Scenarios
|
|
||||||
|
|
||||||
Scenarios live in `suite/scenarios/` as YAML files:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
name: file-editing
|
|
||||||
description: Create and edit files
|
|
||||||
prompt: |
|
|
||||||
1. Create joke.md containing a short joke
|
|
||||||
2. Edit hello.rs to add a debug function
|
|
||||||
|
|
||||||
setup:
|
|
||||||
hello.rs: |
|
|
||||||
fn main() { println!("Hello!"); }
|
|
||||||
|
|
||||||
validate:
|
|
||||||
- type: file_exists
|
|
||||||
path: joke.md
|
|
||||||
- type: file_matches
|
|
||||||
path: hello.rs
|
|
||||||
regex: "fn\\s+debug"
|
|
||||||
```
|
|
||||||
|
|
||||||
### Validation Rules
|
|
||||||
|
|
||||||
| Rule | Description |
|
|
||||||
|------|-------------|
|
|
||||||
| `file_exists` | File exists at path |
|
|
||||||
| `file_not_empty` | File exists and has content |
|
|
||||||
| `file_contains` | File contains literal string |
|
|
||||||
| `file_matches` | File matches regex pattern |
|
|
||||||
| `command_succeeds` | Shell command exits 0 |
|
|
||||||
| `tool_called` | MCP tool was called with matching args (regex supported) |
|
|
||||||
|
|
||||||
**Tool call validation example:**
|
|
||||||
```yaml
|
|
||||||
validate:
|
|
||||||
- type: tool_called
|
|
||||||
tool: slack_search_messages
|
|
||||||
args:
|
|
||||||
query: /quarterly.?review/ # regex pattern
|
|
||||||
- type: tool_called
|
|
||||||
tool: jira_create_issue
|
|
||||||
args:
|
|
||||||
summary: /Q1.*Review/
|
|
||||||
description: /David Brown/
|
|
||||||
```
|
|
||||||
|
|
||||||
## MCP Harness
|
|
||||||
|
|
||||||
Mock MCP server providing simulated tools for testing agent tool-use without hitting real APIs.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd mcp-harness && npm install && npm run build
|
|
||||||
```
|
|
||||||
|
|
||||||
**Available tools:** gdrive, sheets, salesforce, slack, calendar, gmail, jira, github
|
|
||||||
|
|
||||||
Each tool returns realistic mock data. Tool calls are logged to `tool-calls.log` in the workdir for validation.
|
|
||||||
|
|
||||||
## Commands
|
|
||||||
|
|
||||||
| Command | Description |
|
|
||||||
|---------|-------------|
|
|
||||||
| `just run` | Full test run (3 reps each, worst kept) |
|
|
||||||
| `just run-clean` | Full run with artifacts isolated under `~/.goose/gym-runs/<YYYYDDMMHHMMSS>` |
|
|
||||||
| `just test` | Quick run (1 rep each) |
|
|
||||||
| `just scenario <name>` | Run specific scenario |
|
|
||||||
| `just agent <name>` | Run specific agent |
|
|
||||||
| `just report` | Open HTML results |
|
|
||||||
|
|
||||||
### CLI Flags
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Filter by scenario, model, or runner
|
|
||||||
npx tsx src/runner.ts --scenario=file-editing --model=opus --runner=goose
|
|
||||||
|
|
||||||
# Control repetition count
|
|
||||||
npx tsx src/runner.ts --run-count=5
|
|
||||||
|
|
||||||
# Don't auto-open browser
|
|
||||||
npx tsx src/runner.ts --no-open
|
|
||||||
|
|
||||||
# Redirect all run artifacts outside the repo (see Output below)
|
|
||||||
npx tsx src/runner.ts --output-dir=~/.goose/gym-runs/latest
|
|
||||||
|
|
||||||
# Raise the per-agent timeout (seconds) for slow local models on heavy
|
|
||||||
# scenarios. Default 300s; also settable via GYM_AGENT_TIMEOUT.
|
|
||||||
npx tsx src/runner.ts --agent-timeout=1200
|
|
||||||
```
|
|
||||||
|
|
||||||
## Output
|
|
||||||
|
|
||||||
- `report.html` — Live-updating HTML matrix showing pass/fail status, duration, and validation details
|
|
||||||
- `logs/` — Full agent output logs for each run
|
|
||||||
|
|
||||||
By default these (plus the cache, scratch workdir, and isolated agent config
|
|
||||||
roots `.goose-root/` / `.opencode-root/` / `.pi-root/`) are written inside the
|
|
||||||
gym directory. They're gitignored, but still pile up in your checkout — awkward
|
|
||||||
if you want to run the bench regularly or from a worktree.
|
|
||||||
|
|
||||||
To keep the repo clean, redirect **all** run artifacts to a single base
|
|
||||||
directory with the `GYM_OUTPUT_DIR` env var (or the `--output-dir=` flag).
|
|
||||||
`config.yaml` and `scenarios/` are still read from the repo.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Everything lands under a timestamped dir outside the repo (YYYYDDMMHHMMSS)
|
|
||||||
GYM_OUTPUT_DIR=~/.goose/gym-runs/$(date +%Y%d%m%H%M%S) just run
|
|
||||||
|
|
||||||
# Convenience recipe that does the timestamping for you
|
|
||||||
just run-clean
|
|
||||||
|
|
||||||
# View the report from a redirected run
|
|
||||||
GYM_OUTPUT_DIR=~/.goose/gym-runs/20261406101500 just report
|
|
||||||
```
|
|
||||||
|
|
||||||
> Note: the run cache lives under the output dir too, so a fresh timestamped
|
|
||||||
> dir means a fresh (cold) cache. Point `GYM_OUTPUT_DIR` at a stable directory
|
|
||||||
> if you want cache reuse across runs.
|
|
||||||
|
|
@ -1,85 +0,0 @@
|
||||||
# =============================================================================
|
|
||||||
# Models - the LLMs to test
|
|
||||||
# =============================================================================
|
|
||||||
models:
|
|
||||||
- name: opus
|
|
||||||
provider: anthropic
|
|
||||||
model: claude-opus-4-5-20251101
|
|
||||||
|
|
||||||
- name: glm-4.7-flash
|
|
||||||
provider: ollama
|
|
||||||
model: glm-4.7-flash:latest
|
|
||||||
|
|
||||||
# too slow on 64g:
|
|
||||||
#- name: frob/qwen3-coder-next:latest
|
|
||||||
# provider: ollama
|
|
||||||
# model: frob/qwen3-coder-next:latest
|
|
||||||
|
|
||||||
- name: kimi-k2.5
|
|
||||||
provider: ollama
|
|
||||||
model: kimi-k2.5:cloud
|
|
||||||
|
|
||||||
- name: gpt-oss-120b
|
|
||||||
provider: ollama
|
|
||||||
model: gpt-oss:120b-cloud
|
|
||||||
|
|
||||||
- name: gpt-oss-20b
|
|
||||||
provider: ollama
|
|
||||||
model: gpt-oss:20b
|
|
||||||
|
|
||||||
- name: qwen3-coder:latest
|
|
||||||
provider: ollama
|
|
||||||
model: qwen3-coder:latest
|
|
||||||
|
|
||||||
# good but too slow on 64G
|
|
||||||
#- name: nemotron-3-nano
|
|
||||||
# provider: ollama
|
|
||||||
# model: nemotron-3-nano:latest
|
|
||||||
|
|
||||||
# =============================================================================
|
|
||||||
# Runners - agent frameworks with their specific configurations
|
|
||||||
# =============================================================================
|
|
||||||
# Each runner has its own binary, extensions/config, and isolated config directory
|
|
||||||
runners:
|
|
||||||
# - name: goose
|
|
||||||
# type: goose
|
|
||||||
# bin: goose
|
|
||||||
# extensions: [developer]
|
|
||||||
# stdio:
|
|
||||||
# - node mcp-harness/dist/index.js
|
|
||||||
|
|
||||||
- name: goose-full
|
|
||||||
type: goose
|
|
||||||
bin: goose
|
|
||||||
extensions: [developer, todo, skills, code_execution, extensionmanager]
|
|
||||||
stdio:
|
|
||||||
- node mcp-harness/dist/index.js
|
|
||||||
|
|
||||||
- name: opencode
|
|
||||||
type: opencode
|
|
||||||
bin: opencode
|
|
||||||
stdio:
|
|
||||||
- node mcp-harness/dist/index.js
|
|
||||||
|
|
||||||
- name: pi
|
|
||||||
type: pi
|
|
||||||
bin: pi
|
|
||||||
# Pi takes provider/model from the test matrix, not config
|
|
||||||
# MCP support via pi-mcp-adapter: `pi install npm:pi-mcp-adapter`
|
|
||||||
stdio:
|
|
||||||
- node mcp-harness/dist/index.js
|
|
||||||
|
|
||||||
# =============================================================================
|
|
||||||
# Test Matrix
|
|
||||||
# =============================================================================
|
|
||||||
# scenarios × models × runners
|
|
||||||
# - Omit 'models' to run against ALL models
|
|
||||||
# - Omit 'runners' to run against ALL runners
|
|
||||||
matrix:
|
|
||||||
# Single-turn scenarios: all models × all runners
|
|
||||||
- scenario: everyday-app-automation
|
|
||||||
- scenario: file-editing
|
|
||||||
|
|
||||||
# Multi-turn: goose and pi only (opencode doesn't support session continuation)
|
|
||||||
- scenario: multi-turn-edit
|
|
||||||
runners: [goose-full]
|
|
||||||
Binary file not shown.
|
Before Width: | Height: | Size: 2.3 MiB |
|
|
@ -1,87 +0,0 @@
|
||||||
# MCP Harness
|
|
||||||
|
|
||||||
A simulated MCP server with realistic fake tools for testing. Provides mock implementations of common business integrations without requiring actual API credentials.
|
|
||||||
|
|
||||||
## Tools Included (35 tools)
|
|
||||||
|
|
||||||
### Google Drive
|
|
||||||
- `gdrive_search` - Search files by name, content, or type
|
|
||||||
- `gdrive_read_file` - Read file contents
|
|
||||||
- `gdrive_create_file` - Create new files
|
|
||||||
- `gdrive_share_file` - Share files with users
|
|
||||||
|
|
||||||
### Google Sheets
|
|
||||||
- `sheets_read` - Read spreadsheet data
|
|
||||||
- `sheets_write` - Write/update cells
|
|
||||||
- `sheets_append` - Append rows
|
|
||||||
- `sheets_create` - Create new spreadsheets
|
|
||||||
|
|
||||||
### Salesforce
|
|
||||||
- `salesforce_query` - Execute SOQL queries
|
|
||||||
- `salesforce_get_record` - Get record by ID
|
|
||||||
- `salesforce_create_record` - Create records
|
|
||||||
- `salesforce_update_record` - Update records
|
|
||||||
- `salesforce_search` - SOSL search
|
|
||||||
|
|
||||||
### Slack
|
|
||||||
- `slack_send_message` - Send messages
|
|
||||||
- `slack_get_messages` - Get channel messages
|
|
||||||
- `slack_search_messages` - Search messages
|
|
||||||
- `slack_list_channels` - List channels
|
|
||||||
- `slack_get_user_info` - Get user info
|
|
||||||
- `slack_set_status` - Set user status
|
|
||||||
|
|
||||||
### Google Calendar
|
|
||||||
- `calendar_list_events` - List events
|
|
||||||
- `calendar_create_event` - Create events
|
|
||||||
- `calendar_update_event` - Update events
|
|
||||||
- `calendar_delete_event` - Delete events
|
|
||||||
|
|
||||||
### Gmail
|
|
||||||
- `gmail_search` - Search emails
|
|
||||||
- `gmail_read_message` - Read email content
|
|
||||||
- `gmail_send` - Send emails
|
|
||||||
- `gmail_create_draft` - Create drafts
|
|
||||||
|
|
||||||
### Jira
|
|
||||||
- `jira_search_issues` - Search with JQL
|
|
||||||
- `jira_get_issue` - Get issue details
|
|
||||||
- `jira_create_issue` - Create issues
|
|
||||||
- `jira_update_issue` - Update issues
|
|
||||||
- `jira_add_comment` - Add comments
|
|
||||||
|
|
||||||
### GitHub
|
|
||||||
- `github_search_repos` - Search repositories
|
|
||||||
- `github_list_issues` - List issues
|
|
||||||
- `github_create_issue` - Create issues
|
|
||||||
- `github_list_prs` - List pull requests
|
|
||||||
|
|
||||||
## Setup
|
|
||||||
|
|
||||||
```bash
|
|
||||||
npm install
|
|
||||||
npm run build
|
|
||||||
```
|
|
||||||
|
|
||||||
## Run
|
|
||||||
|
|
||||||
```bash
|
|
||||||
npm run start
|
|
||||||
# or
|
|
||||||
./run.sh
|
|
||||||
```
|
|
||||||
|
|
||||||
## MCP Config
|
|
||||||
|
|
||||||
Add to your MCP client config:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"mcpServers": {
|
|
||||||
"harness": {
|
|
||||||
"command": "node",
|
|
||||||
"args": ["/path/to/mcp-harness/dist/index.js"]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
1173
evals/open-model-gym/mcp-harness/package-lock.json
generated
1173
evals/open-model-gym/mcp-harness/package-lock.json
generated
File diff suppressed because it is too large
Load diff
|
|
@ -1,18 +0,0 @@
|
||||||
{
|
|
||||||
"name": "mcp-harness",
|
|
||||||
"version": "1.0.0",
|
|
||||||
"description": "Simulated real-world MCP tools for testing - Google Drive, Sheets, Salesforce, Slack, and more",
|
|
||||||
"private": true,
|
|
||||||
"type": "module",
|
|
||||||
"scripts": {
|
|
||||||
"build": "tsc -p tsconfig.json",
|
|
||||||
"start": "node dist/index.js"
|
|
||||||
},
|
|
||||||
"dependencies": {
|
|
||||||
"@modelcontextprotocol/sdk": "^1.26.0"
|
|
||||||
},
|
|
||||||
"devDependencies": {
|
|
||||||
"@types/node": "^25.2.0",
|
|
||||||
"typescript": "^5.6.3"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,3 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
cd "$(dirname "$0")"
|
|
||||||
npm run build && npm run start
|
|
||||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,14 +0,0 @@
|
||||||
{
|
|
||||||
"compilerOptions": {
|
|
||||||
"target": "ES2022",
|
|
||||||
"module": "NodeNext",
|
|
||||||
"moduleResolution": "NodeNext",
|
|
||||||
"outDir": "dist",
|
|
||||||
"rootDir": "src",
|
|
||||||
"strict": true,
|
|
||||||
"esModuleInterop": true,
|
|
||||||
"skipLibCheck": true,
|
|
||||||
"declaration": true
|
|
||||||
},
|
|
||||||
"include": ["src/**/*"]
|
|
||||||
}
|
|
||||||
4
evals/open-model-gym/suite/.gitignore
vendored
4
evals/open-model-gym/suite/.gitignore
vendored
|
|
@ -1,4 +0,0 @@
|
||||||
node_modules/
|
|
||||||
.workdir/
|
|
||||||
.goose-root/
|
|
||||||
.cache/
|
|
||||||
1059
evals/open-model-gym/suite/package-lock.json
generated
1059
evals/open-model-gym/suite/package-lock.json
generated
File diff suppressed because it is too large
Load diff
|
|
@ -1,19 +0,0 @@
|
||||||
{
|
|
||||||
"name": "agent-runner",
|
|
||||||
"version": "0.1.0",
|
|
||||||
"type": "module",
|
|
||||||
"scripts": {
|
|
||||||
"build": "tsc",
|
|
||||||
"test": "tsx src/runner.ts",
|
|
||||||
"test:scenario": "tsx src/runner.ts --scenario"
|
|
||||||
},
|
|
||||||
"dependencies": {
|
|
||||||
"glob": "^11.0.0",
|
|
||||||
"yaml": "^2.8.3"
|
|
||||||
},
|
|
||||||
"devDependencies": {
|
|
||||||
"@types/node": "^22.0.0",
|
|
||||||
"tsx": "^4.22.4",
|
|
||||||
"typescript": "^5.5.0"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,47 +0,0 @@
|
||||||
name: everyday-app-automation
|
|
||||||
description: Multi-step workflow using everyday app tools (Slack, Jira, Calendar) with data dependencies
|
|
||||||
prompt: |
|
|
||||||
Using the available tools, complete these tasks:
|
|
||||||
1. Search Slack for messages mentioning "quarterly review"
|
|
||||||
2. Look up the user who posted the message about quarterly review to get their full name
|
|
||||||
3. Create a Jira issue titled "Q1 Review Follow-ups" with a description that includes the name of the person who posted the Slack message
|
|
||||||
4. Create a calendar event for next Monday at 2pm called "Review Discussion"
|
|
||||||
5. Write a summary of what you did to a file called workflow-log.md
|
|
||||||
|
|
||||||
tags:
|
|
||||||
- complex
|
|
||||||
- multi-step
|
|
||||||
- mcp-harness
|
|
||||||
- data-flow
|
|
||||||
|
|
||||||
validate:
|
|
||||||
# Check workflow summary was written
|
|
||||||
- type: file_exists
|
|
||||||
path: workflow-log.md
|
|
||||||
- type: file_not_empty
|
|
||||||
path: workflow-log.md
|
|
||||||
|
|
||||||
# Check Slack search was called with the right query
|
|
||||||
- type: tool_called
|
|
||||||
tool: slack_search_messages
|
|
||||||
args:
|
|
||||||
query: /quarterly.?review/
|
|
||||||
|
|
||||||
# Check that the agent looked up the user info (data dependency: requires reading user ID from search results)
|
|
||||||
- type: tool_called
|
|
||||||
tool: slack_get_user_info
|
|
||||||
args:
|
|
||||||
userId: /U004/
|
|
||||||
|
|
||||||
# Check Jira issue was created with expected title and includes the user's name (data dependency: requires reading name from user info)
|
|
||||||
- type: tool_called
|
|
||||||
tool: jira_create_issue
|
|
||||||
args:
|
|
||||||
summary: /q1.?review|follow.?up/
|
|
||||||
description: /David.?Brown/
|
|
||||||
|
|
||||||
# Check calendar event was created with expected title
|
|
||||||
- type: tool_called
|
|
||||||
tool: calendar_create_event
|
|
||||||
args:
|
|
||||||
summary: /review.?discussion/
|
|
||||||
|
|
@ -1,110 +0,0 @@
|
||||||
name: file-editing
|
|
||||||
description: Navigate a small codebase and make a targeted edit
|
|
||||||
prompt: |
|
|
||||||
The User struct in user.rs is missing a display_name() method.
|
|
||||||
Add a method that returns the full name formatted as "first_name last_name".
|
|
||||||
|
|
||||||
tags:
|
|
||||||
- file-editing
|
|
||||||
- code-navigation
|
|
||||||
|
|
||||||
setup:
|
|
||||||
Cargo.toml: |
|
|
||||||
[package]
|
|
||||||
name = "user-service"
|
|
||||||
version = "0.1.0"
|
|
||||||
edition = "2021"
|
|
||||||
|
|
||||||
[workspace]
|
|
||||||
|
|
||||||
src/main.rs: |
|
|
||||||
mod models;
|
|
||||||
mod utils;
|
|
||||||
|
|
||||||
use models::user::User;
|
|
||||||
|
|
||||||
fn main() {
|
|
||||||
let user = User::new("Alice", "Smith", "alice@example.com");
|
|
||||||
println!("Created user: {}", user.email());
|
|
||||||
}
|
|
||||||
|
|
||||||
src/models/mod.rs: |
|
|
||||||
pub mod user;
|
|
||||||
|
|
||||||
src/models/user.rs: |
|
|
||||||
pub struct User {
|
|
||||||
first_name: String,
|
|
||||||
last_name: String,
|
|
||||||
email: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl User {
|
|
||||||
pub fn new(first_name: &str, last_name: &str, email: &str) -> Self {
|
|
||||||
Self {
|
|
||||||
first_name: first_name.to_string(),
|
|
||||||
last_name: last_name.to_string(),
|
|
||||||
email: email.to_string(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn email(&self) -> &str {
|
|
||||||
&self.email
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn first_name(&self) -> &str {
|
|
||||||
&self.first_name
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn last_name(&self) -> &str {
|
|
||||||
&self.last_name
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
src/utils/mod.rs: |
|
|
||||||
pub mod formatting;
|
|
||||||
|
|
||||||
src/utils/formatting.rs: |
|
|
||||||
pub fn capitalize(s: &str) -> String {
|
|
||||||
let mut chars = s.chars();
|
|
||||||
match chars.next() {
|
|
||||||
None => String::new(),
|
|
||||||
Some(c) => c.to_uppercase().collect::<String>() + chars.as_str(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
validate:
|
|
||||||
# The edit was made to the correct file
|
|
||||||
- type: file_exists
|
|
||||||
path: src/models/user.rs
|
|
||||||
name: user.rs exists
|
|
||||||
# Method was added
|
|
||||||
- type: file_matches
|
|
||||||
path: src/models/user.rs
|
|
||||||
regex: "fn\\s+display_name"
|
|
||||||
name: display_name() added
|
|
||||||
# Method returns a String or &str
|
|
||||||
- type: file_matches
|
|
||||||
path: src/models/user.rs
|
|
||||||
regex: "display_name.*->.*String|display_name.*->.*str"
|
|
||||||
name: has return type
|
|
||||||
# Original code preserved
|
|
||||||
- type: file_contains
|
|
||||||
path: src/models/user.rs
|
|
||||||
pattern: "pub fn email"
|
|
||||||
name: email() preserved
|
|
||||||
- type: file_contains
|
|
||||||
path: src/models/user.rs
|
|
||||||
pattern: "pub fn first_name"
|
|
||||||
name: first_name() preserved
|
|
||||||
# Other files untouched
|
|
||||||
- type: file_exists
|
|
||||||
path: src/main.rs
|
|
||||||
name: main.rs exists
|
|
||||||
- type: file_contains
|
|
||||||
path: src/main.rs
|
|
||||||
pattern: "mod models"
|
|
||||||
name: main.rs unchanged
|
|
||||||
# Code compiles
|
|
||||||
- type: command_succeeds
|
|
||||||
command: "cargo build"
|
|
||||||
name: cargo build
|
|
||||||
|
|
@ -1,80 +0,0 @@
|
||||||
name: multi-turn-edit
|
|
||||||
description: Multi-turn conversation - add a method, then rename it
|
|
||||||
|
|
||||||
tags:
|
|
||||||
- file-editing
|
|
||||||
- multi-turn
|
|
||||||
|
|
||||||
setup:
|
|
||||||
Cargo.toml: |
|
|
||||||
[package]
|
|
||||||
name = "user-service"
|
|
||||||
version = "0.1.0"
|
|
||||||
edition = "2021"
|
|
||||||
|
|
||||||
[workspace]
|
|
||||||
|
|
||||||
src/main.rs: |
|
|
||||||
mod models;
|
|
||||||
|
|
||||||
use models::user::User;
|
|
||||||
|
|
||||||
fn main() {
|
|
||||||
let user = User::new("Alice", "Smith");
|
|
||||||
println!("User: {} {}", user.first_name(), user.last_name());
|
|
||||||
}
|
|
||||||
|
|
||||||
src/models/mod.rs: |
|
|
||||||
pub mod user;
|
|
||||||
|
|
||||||
src/models/user.rs: |
|
|
||||||
pub struct User {
|
|
||||||
first_name: String,
|
|
||||||
last_name: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl User {
|
|
||||||
pub fn new(first_name: &str, last_name: &str) -> Self {
|
|
||||||
Self {
|
|
||||||
first_name: first_name.to_string(),
|
|
||||||
last_name: last_name.to_string(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn first_name(&self) -> &str {
|
|
||||||
&self.first_name
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn last_name(&self) -> &str {
|
|
||||||
&self.last_name
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
turns:
|
|
||||||
- prompt: |
|
|
||||||
Add an email() method to the User struct in src/models/user.rs.
|
|
||||||
It should return a generated email in the format "first_name.last_name@example.com" (lowercase).
|
|
||||||
validate:
|
|
||||||
- type: file_matches
|
|
||||||
path: src/models/user.rs
|
|
||||||
regex: "fn\\s+email"
|
|
||||||
name: email() added
|
|
||||||
- type: command_succeeds
|
|
||||||
command: "cargo build"
|
|
||||||
name: compiles after turn 1
|
|
||||||
|
|
||||||
- prompt: |
|
|
||||||
Actually, can you rename the email() method to generated_email() instead?
|
|
||||||
Make sure to update any references.
|
|
||||||
validate:
|
|
||||||
- type: file_matches
|
|
||||||
path: src/models/user.rs
|
|
||||||
regex: "fn\\s+generated_email"
|
|
||||||
name: renamed to generated_email()
|
|
||||||
- type: file_not_matches
|
|
||||||
path: src/models/user.rs
|
|
||||||
regex: "fn\\s+email\\s*\\("
|
|
||||||
name: old email() removed
|
|
||||||
- type: command_succeeds
|
|
||||||
command: "cargo build"
|
|
||||||
name: compiles after turn 2
|
|
||||||
|
|
@ -1,926 +0,0 @@
|
||||||
name: remove-feature
|
|
||||||
description: Remove a feature (insert_image) from a document editor tool module
|
|
||||||
prompt: |
|
|
||||||
Remove the `insert_image` feature entirely from this codebase.
|
|
||||||
It should be removed from docs.py and any references in doc_utils.py.
|
|
||||||
Make sure no traces remain: the operation literal, the ImageParams class,
|
|
||||||
the image_params parameter, the insert_image handler function, and any
|
|
||||||
validation/error handling specific to insert_image should all be removed.
|
|
||||||
|
|
||||||
tags:
|
|
||||||
- file-editing
|
|
||||||
- feature-removal
|
|
||||||
- python
|
|
||||||
|
|
||||||
setup:
|
|
||||||
docs.py: |
|
|
||||||
"""Tools for interacting with documents."""
|
|
||||||
|
|
||||||
import json
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from functools import partial
|
|
||||||
from typing import Annotated, Any, Literal
|
|
||||||
|
|
||||||
from pydantic import Field
|
|
||||||
|
|
||||||
from doc_utils import execute_request, execute_requests
|
|
||||||
|
|
||||||
DOC_OPERATIONS = Literal[
|
|
||||||
"get_document",
|
|
||||||
"insert_text",
|
|
||||||
"append_text",
|
|
||||||
"replace_text",
|
|
||||||
"delete_content",
|
|
||||||
"insert_table",
|
|
||||||
"update_table_cell",
|
|
||||||
"insert_table_row",
|
|
||||||
"insert_table_column",
|
|
||||||
"delete_table_row",
|
|
||||||
"delete_table_column",
|
|
||||||
"insert_image",
|
|
||||||
"format_existing_text",
|
|
||||||
]
|
|
||||||
|
|
||||||
RESPONSE_CHAR_LIMIT = 400000
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class FormatTextParams:
|
|
||||||
"""Parameters for format_existing_text operation."""
|
|
||||||
|
|
||||||
search_text: str
|
|
||||||
foreground_color: str | None = None
|
|
||||||
background_color: str | None = None
|
|
||||||
bold: bool | None = None
|
|
||||||
italic: bool | None = None
|
|
||||||
underline: bool | None = None
|
|
||||||
strikethrough: bool | None = None
|
|
||||||
font_size: int | None = None
|
|
||||||
font_family: str | None = None
|
|
||||||
heading_level: int | None = None
|
|
||||||
link_url: str | None = None
|
|
||||||
list_type: str | None = None
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class TableParams:
|
|
||||||
"""Parameters for table operations.
|
|
||||||
|
|
||||||
Used by operations: insert_table, update_table_cell, insert_table_row,
|
|
||||||
insert_table_column, delete_table_row, delete_table_column.
|
|
||||||
"""
|
|
||||||
|
|
||||||
rows: int | None = Field(None, description="Number of rows for insert_table operation")
|
|
||||||
columns: int | None = Field(None, description="Number of columns for insert_table operation")
|
|
||||||
row_index: int | None = Field(
|
|
||||||
None, description="Row index (0-based) for update_table_cell, insert_table_row, and delete_table_row"
|
|
||||||
)
|
|
||||||
column_index: int | None = Field(
|
|
||||||
None, description="Column index (0-based) for update_table_cell, insert_table_column, and delete_table_column"
|
|
||||||
)
|
|
||||||
insert_below: bool = Field(
|
|
||||||
False, description="For insert_table_row: True to insert below the specified row, False to insert above"
|
|
||||||
)
|
|
||||||
insert_right: bool = Field(
|
|
||||||
False, description="For insert_table_column: True to insert right of column, False to insert left"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class ImageParams:
|
|
||||||
"""Parameters for image insertion."""
|
|
||||||
|
|
||||||
image_url: str = Field(..., description="URL of the image to insert (must be publicly accessible)")
|
|
||||||
width: int | None = Field(None, description="Width of the image in points (PT)")
|
|
||||||
height: int | None = Field(None, description="Height of the image in points (PT)")
|
|
||||||
|
|
||||||
|
|
||||||
async def doc_tool(
|
|
||||||
document_id: str,
|
|
||||||
operation: DOC_OPERATIONS = "get_document",
|
|
||||||
text: Annotated[
|
|
||||||
str,
|
|
||||||
Field(
|
|
||||||
description=(
|
|
||||||
"Text content to insert, append, or match for replacement. "
|
|
||||||
"For insert_text and append_text, Markdown formatting is supported. "
|
|
||||||
"For replace_text, this should be unformatted plain text."
|
|
||||||
)
|
|
||||||
),
|
|
||||||
] = "",
|
|
||||||
replace_text: Annotated[
|
|
||||||
str,
|
|
||||||
Field(
|
|
||||||
description=(
|
|
||||||
"New plain text that will replace all occurrences of the original text. "
|
|
||||||
"Only used for the replace_text operation."
|
|
||||||
)
|
|
||||||
),
|
|
||||||
] = "",
|
|
||||||
start_position: Annotated[
|
|
||||||
int | None,
|
|
||||||
Field(
|
|
||||||
description="Document index (1-based) for insert or delete operations.",
|
|
||||||
ge=1,
|
|
||||||
),
|
|
||||||
] = None,
|
|
||||||
end_position: Annotated[
|
|
||||||
int | None,
|
|
||||||
Field(
|
|
||||||
description="Document index (1-based, exclusive) for delete_content",
|
|
||||||
ge=1,
|
|
||||||
),
|
|
||||||
] = None,
|
|
||||||
table_params: Annotated[
|
|
||||||
TableParams | None,
|
|
||||||
Field(
|
|
||||||
None,
|
|
||||||
description="Parameters for table operations",
|
|
||||||
),
|
|
||||||
] = None,
|
|
||||||
image_params: ImageParams | None = None,
|
|
||||||
format_params: FormatTextParams | None = None,
|
|
||||||
) -> str:
|
|
||||||
"""Perform operations on an existing document.
|
|
||||||
|
|
||||||
Supported operations:
|
|
||||||
- get_document: Returns document content
|
|
||||||
- insert_text: Inserts text at a specific position
|
|
||||||
- append_text: Appends text at the end of the document
|
|
||||||
- replace_text: Replaces all instances of text with replace_text
|
|
||||||
- delete_content: Deletes content between two positions
|
|
||||||
- insert_table: Creates a table with specified rows and columns
|
|
||||||
- update_table_cell: Updates content in a specific table cell
|
|
||||||
- insert_table_row: Inserts a row above or below the specified row
|
|
||||||
- insert_table_column: Inserts a column left or right of the specified column
|
|
||||||
- delete_table_row: Deletes the specified row from a table
|
|
||||||
- delete_table_column: Deletes the specified column from a table
|
|
||||||
- insert_image: Inserts an image from a URL at the specified position
|
|
||||||
- format_existing_text: Finds and applies formatting to text
|
|
||||||
"""
|
|
||||||
if not text and operation in ["insert_text", "append_text", "replace_text"]:
|
|
||||||
raise ValueError(f"text is required for {operation} operation")
|
|
||||||
|
|
||||||
table_operations = [
|
|
||||||
"insert_table",
|
|
||||||
"update_table_cell",
|
|
||||||
"insert_table_row",
|
|
||||||
"insert_table_column",
|
|
||||||
"delete_table_row",
|
|
||||||
"delete_table_column",
|
|
||||||
]
|
|
||||||
if operation in table_operations and not table_params:
|
|
||||||
raise ValueError(f"table_params is required for {operation} operation")
|
|
||||||
|
|
||||||
if operation == "insert_image" and not image_params:
|
|
||||||
raise ValueError("image_params is required for insert_image operation")
|
|
||||||
|
|
||||||
if operation == "format_existing_text" and not format_params:
|
|
||||||
raise ValueError("format_params is required for format_existing_text operation")
|
|
||||||
|
|
||||||
operation_handlers = {
|
|
||||||
"get_document": partial(read_document, document_id),
|
|
||||||
"insert_text": partial(insert_text, document_id, text, start_position),
|
|
||||||
"append_text": partial(append_text, document_id, text),
|
|
||||||
"replace_text": partial(replace_all_text, document_id, text, replace_text),
|
|
||||||
"delete_content": partial(delete_content, document_id, start_position, end_position),
|
|
||||||
"insert_table": partial(
|
|
||||||
insert_table,
|
|
||||||
document_id,
|
|
||||||
table_params.rows if table_params else None,
|
|
||||||
table_params.columns if table_params else None,
|
|
||||||
start_position,
|
|
||||||
),
|
|
||||||
"update_table_cell": partial(
|
|
||||||
update_table_cell,
|
|
||||||
document_id,
|
|
||||||
table_params.row_index if table_params else None,
|
|
||||||
table_params.column_index if table_params else None,
|
|
||||||
text,
|
|
||||||
start_position,
|
|
||||||
),
|
|
||||||
"insert_table_row": partial(
|
|
||||||
modify_table_structure,
|
|
||||||
document_id,
|
|
||||||
"insert_row",
|
|
||||||
table_params.row_index if table_params else None,
|
|
||||||
None,
|
|
||||||
table_params.insert_below if table_params else False,
|
|
||||||
False,
|
|
||||||
start_position,
|
|
||||||
),
|
|
||||||
"insert_table_column": partial(
|
|
||||||
modify_table_structure,
|
|
||||||
document_id,
|
|
||||||
"insert_column",
|
|
||||||
None,
|
|
||||||
table_params.column_index if table_params else None,
|
|
||||||
False,
|
|
||||||
table_params.insert_right if table_params else False,
|
|
||||||
start_position,
|
|
||||||
),
|
|
||||||
"delete_table_row": partial(
|
|
||||||
modify_table_structure,
|
|
||||||
document_id,
|
|
||||||
"delete_row",
|
|
||||||
table_params.row_index if table_params else None,
|
|
||||||
None,
|
|
||||||
False,
|
|
||||||
False,
|
|
||||||
start_position,
|
|
||||||
),
|
|
||||||
"delete_table_column": partial(
|
|
||||||
modify_table_structure,
|
|
||||||
document_id,
|
|
||||||
"delete_column",
|
|
||||||
None,
|
|
||||||
table_params.column_index if table_params else None,
|
|
||||||
False,
|
|
||||||
False,
|
|
||||||
start_position,
|
|
||||||
),
|
|
||||||
"insert_image": partial(
|
|
||||||
insert_image,
|
|
||||||
document_id,
|
|
||||||
image_params.image_url if image_params else None,
|
|
||||||
start_position,
|
|
||||||
image_params.width if image_params else None,
|
|
||||||
image_params.height if image_params else None,
|
|
||||||
),
|
|
||||||
"format_existing_text": partial(
|
|
||||||
format_existing_text,
|
|
||||||
document_id,
|
|
||||||
format_params.search_text if format_params else None,
|
|
||||||
format_params.foreground_color if format_params else None,
|
|
||||||
format_params.background_color if format_params else None,
|
|
||||||
format_params.font_size if format_params else None,
|
|
||||||
format_params.font_family if format_params else None,
|
|
||||||
format_params.bold if format_params else None,
|
|
||||||
format_params.italic if format_params else None,
|
|
||||||
format_params.underline if format_params else None,
|
|
||||||
format_params.strikethrough if format_params else None,
|
|
||||||
format_params.heading_level if format_params else None,
|
|
||||||
format_params.link_url if format_params else None,
|
|
||||||
format_params.list_type if format_params else None,
|
|
||||||
),
|
|
||||||
}
|
|
||||||
|
|
||||||
if operation not in operation_handlers:
|
|
||||||
raise ValueError(f"Invalid operation: {operation}")
|
|
||||||
|
|
||||||
response = await operation_handlers[operation]()
|
|
||||||
return json.dumps(response, indent=2)
|
|
||||||
|
|
||||||
|
|
||||||
def _extract_text_from_element(element: dict) -> str:
|
|
||||||
"""Recursively pull text from paragraphs, tables, etc."""
|
|
||||||
text_parts = ""
|
|
||||||
if "paragraph" in element:
|
|
||||||
paragraph_elements = element["paragraph"].get("elements", [])
|
|
||||||
for el in paragraph_elements:
|
|
||||||
if "textRun" in el:
|
|
||||||
content = el["textRun"]["content"]
|
|
||||||
url = el["textRun"].get("textStyle", {}).get("link", {}).get("url")
|
|
||||||
if url:
|
|
||||||
text_parts += f"[{content}]({url})"
|
|
||||||
else:
|
|
||||||
text_parts += content
|
|
||||||
elif "table" in element:
|
|
||||||
table_rows = element["table"].get("tableRows", [])
|
|
||||||
for row in table_rows:
|
|
||||||
for cell in row["tableCells"]:
|
|
||||||
for cell_content in cell["content"]:
|
|
||||||
text_parts += _extract_text_from_element(cell_content)
|
|
||||||
text_parts += "\n"
|
|
||||||
return text_parts
|
|
||||||
|
|
||||||
|
|
||||||
async def read_document(document_id: str) -> dict[str, Any]:
|
|
||||||
"""Returns document content."""
|
|
||||||
document = await execute_request("get", document_id, {})
|
|
||||||
content = document.get("body", {}).get("content", [])
|
|
||||||
text = "".join(_extract_text_from_element(e) for e in content)
|
|
||||||
|
|
||||||
result = {"content": text, "document_id": document_id}
|
|
||||||
|
|
||||||
if len(json.dumps(result)) > RESPONSE_CHAR_LIMIT:
|
|
||||||
raise ValueError(f"Document {document_id} is too large to read.")
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
async def insert_text(
|
|
||||||
document_id: str, text: str, start_position: int | None
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
"""Insert text at a specific index in a document."""
|
|
||||||
if start_position is None:
|
|
||||||
raise ValueError("start_position is required for insert_text operation")
|
|
||||||
|
|
||||||
request = {"insertText": {"location": {"index": start_position}, "text": text}}
|
|
||||||
await execute_request("update", document_id, request)
|
|
||||||
|
|
||||||
from doc_utils import calculate_utf16_length
|
|
||||||
|
|
||||||
inserted_length = calculate_utf16_length(text)
|
|
||||||
|
|
||||||
return {
|
|
||||||
"message": f"Inserted text at position {start_position}",
|
|
||||||
"text": text,
|
|
||||||
"start_position": start_position,
|
|
||||||
"end_position": start_position + inserted_length,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
async def append_text(document_id: str, text: str) -> dict[str, Any]:
|
|
||||||
"""Append text to the end of a document."""
|
|
||||||
end_index = await get_document_last_index(document_id)
|
|
||||||
if text[0] != "\n":
|
|
||||||
text = "\n" + text
|
|
||||||
response = await insert_text(document_id, text, end_index - 1)
|
|
||||||
response["message"] = "Appended text to the end of the document"
|
|
||||||
return response
|
|
||||||
|
|
||||||
|
|
||||||
async def replace_all_text(
|
|
||||||
document_id: str, text: str, replace_text: str
|
|
||||||
) -> dict[str, str]:
|
|
||||||
"""Replace all instances of a string in a document."""
|
|
||||||
if not replace_text:
|
|
||||||
raise ValueError("replace_text parameter is required for replace_text operation")
|
|
||||||
|
|
||||||
request = {
|
|
||||||
"replaceAllText": {
|
|
||||||
"containsText": {"text": text, "matchCase": True},
|
|
||||||
"replaceText": replace_text,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
response = await execute_request("update", document_id, request)
|
|
||||||
occurrences = response.get("occurrencesChanged", 0)
|
|
||||||
if occurrences == 0:
|
|
||||||
return {
|
|
||||||
"message": f"No occurrences of text '{text}' found in the document.",
|
|
||||||
"text": text,
|
|
||||||
"replace_text": replace_text,
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
"message": f"Replaced {occurrences} occurrences of '{text}' with '{replace_text}'",
|
|
||||||
"text": text,
|
|
||||||
"replace_text": replace_text,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
async def delete_content(
|
|
||||||
document_id: str, start_index: int | None, end_index: int | None
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
"""Delete content between two positions."""
|
|
||||||
if start_index is None or end_index is None:
|
|
||||||
raise ValueError("Both start_index and end_index are required for delete_content")
|
|
||||||
|
|
||||||
request = {
|
|
||||||
"deleteContentRange": {
|
|
||||||
"range": {"startIndex": start_index, "endIndex": end_index}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await execute_request("update", document_id, request)
|
|
||||||
return {
|
|
||||||
"message": f"Deleted content between index {start_index} and {end_index}",
|
|
||||||
"start_index": start_index,
|
|
||||||
"end_index": end_index,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
async def get_document_last_index(document_id: str) -> int:
|
|
||||||
"""Get the last index of the document."""
|
|
||||||
document = await execute_request("get", document_id, {})
|
|
||||||
return document.get("body", {}).get("content", [])[-1].get("endIndex", 1)
|
|
||||||
|
|
||||||
|
|
||||||
async def insert_table(
|
|
||||||
document_id: str, rows: int | None, columns: int | None, start_position: int | None
|
|
||||||
) -> str:
|
|
||||||
"""Insert a table at a specific position."""
|
|
||||||
if not rows or not columns:
|
|
||||||
raise ValueError("rows and columns are required for insert_table operation")
|
|
||||||
if start_position is None:
|
|
||||||
raise ValueError("start_position is required for insert_table operation")
|
|
||||||
|
|
||||||
request = {
|
|
||||||
"insertTable": {
|
|
||||||
"rows": rows,
|
|
||||||
"columns": columns,
|
|
||||||
"location": {"index": start_position},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
await execute_request("update", document_id, request)
|
|
||||||
return f"Inserted {rows}x{columns} table at position {start_position}"
|
|
||||||
|
|
||||||
|
|
||||||
def _table_matches_position(element: dict, start_position: int | None) -> bool:
|
|
||||||
"""Check if a table element contains the specified position."""
|
|
||||||
if start_position is None:
|
|
||||||
return True
|
|
||||||
table_start = element.get("startIndex")
|
|
||||||
table_end = element.get("endIndex")
|
|
||||||
return table_start <= start_position < table_end
|
|
||||||
|
|
||||||
|
|
||||||
def _find_table_cell_range(
|
|
||||||
document: dict, row_index: int, column_index: int, start_position: int | None = None
|
|
||||||
) -> tuple[int, int] | None:
|
|
||||||
"""Find the content range within a table cell."""
|
|
||||||
content = document.get("body", {}).get("content", [])
|
|
||||||
for element in content:
|
|
||||||
if "table" in element:
|
|
||||||
if not _table_matches_position(element, start_position):
|
|
||||||
continue
|
|
||||||
table = element["table"]
|
|
||||||
if row_index < len(table.get("tableRows", [])):
|
|
||||||
row = table["tableRows"][row_index]
|
|
||||||
if column_index < len(row.get("tableCells", [])):
|
|
||||||
cell = row["tableCells"][column_index]
|
|
||||||
cell_content = cell.get("content", [])
|
|
||||||
for cell_element in cell_content:
|
|
||||||
if "paragraph" in cell_element:
|
|
||||||
para_start = cell_element.get("startIndex")
|
|
||||||
para_end = cell_element.get("endIndex")
|
|
||||||
return (para_start, para_end)
|
|
||||||
cell_start = cell.get("startIndex")
|
|
||||||
cell_end = cell.get("endIndex")
|
|
||||||
if cell_start is not None and cell_end is not None:
|
|
||||||
return (cell_start + 1, cell_end - 1)
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _find_table_start_index(
|
|
||||||
document: dict, row_index: int, column_index: int, start_position: int | None = None
|
|
||||||
) -> int | None:
|
|
||||||
"""Find the start index of a table element."""
|
|
||||||
content = document.get("body", {}).get("content", [])
|
|
||||||
for element in content:
|
|
||||||
if "table" in element:
|
|
||||||
if not _table_matches_position(element, start_position):
|
|
||||||
continue
|
|
||||||
table = element["table"]
|
|
||||||
if row_index < len(table.get("tableRows", [])):
|
|
||||||
row = table["tableRows"][row_index]
|
|
||||||
if column_index < len(row.get("tableCells", [])):
|
|
||||||
return element.get("startIndex")
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
async def update_table_cell(
|
|
||||||
document_id: str,
|
|
||||||
row_index: int | None,
|
|
||||||
column_index: int | None,
|
|
||||||
text: str,
|
|
||||||
start_position: int | None = None,
|
|
||||||
) -> str:
|
|
||||||
"""Update content in a specific table cell."""
|
|
||||||
if row_index is None or column_index is None:
|
|
||||||
raise ValueError("row_index and column_index are required for update_table_cell")
|
|
||||||
|
|
||||||
document = await execute_request("get", document_id, {})
|
|
||||||
cell_range = _find_table_cell_range(document, row_index, column_index, start_position)
|
|
||||||
if not cell_range:
|
|
||||||
raise ValueError(f"Table cell at row {row_index}, col {column_index} not found")
|
|
||||||
|
|
||||||
cell_start, cell_end = cell_range
|
|
||||||
|
|
||||||
requests = []
|
|
||||||
if cell_end > cell_start + 1:
|
|
||||||
requests.append(
|
|
||||||
{"deleteContentRange": {"range": {"startIndex": cell_start, "endIndex": cell_end - 1}}}
|
|
||||||
)
|
|
||||||
|
|
||||||
requests.append({"insertText": {"location": {"index": cell_start}, "text": text}})
|
|
||||||
|
|
||||||
await execute_requests(document_id, requests)
|
|
||||||
return f"Updated cell at row {row_index}, column {column_index}"
|
|
||||||
|
|
||||||
|
|
||||||
async def modify_table_structure(
|
|
||||||
document_id: str,
|
|
||||||
operation: str,
|
|
||||||
row_index: int | None = None,
|
|
||||||
column_index: int | None = None,
|
|
||||||
insert_below: bool = False,
|
|
||||||
insert_right: bool = False,
|
|
||||||
start_position: int | None = None,
|
|
||||||
) -> str:
|
|
||||||
"""Modify table structure by inserting or deleting rows/columns."""
|
|
||||||
if operation in ["insert_row", "delete_row"] and row_index is None:
|
|
||||||
raise ValueError(f"row_index is required for {operation} operation")
|
|
||||||
if operation in ["insert_column", "delete_column"] and column_index is None:
|
|
||||||
raise ValueError(f"column_index is required for {operation} operation")
|
|
||||||
|
|
||||||
document = await execute_request("get", document_id, {})
|
|
||||||
|
|
||||||
if operation in ["insert_row", "delete_row"]:
|
|
||||||
table_start = _find_table_start_index(document, row_index, 0, start_position)
|
|
||||||
if not table_start:
|
|
||||||
raise ValueError(f"Table row {row_index} not found")
|
|
||||||
location = {"tableStartLocation": {"index": table_start}, "rowIndex": row_index}
|
|
||||||
else:
|
|
||||||
table_start = _find_table_start_index(document, 0, column_index, start_position)
|
|
||||||
if not table_start:
|
|
||||||
raise ValueError(f"Table column {column_index} not found")
|
|
||||||
location = {"tableStartLocation": {"index": table_start}, "columnIndex": column_index}
|
|
||||||
|
|
||||||
operation_map = {
|
|
||||||
"insert_row": "insertTableRow",
|
|
||||||
"insert_column": "insertTableColumn",
|
|
||||||
"delete_row": "deleteTableRow",
|
|
||||||
"delete_column": "deleteTableColumn",
|
|
||||||
}
|
|
||||||
request_key = operation_map[operation]
|
|
||||||
|
|
||||||
if operation == "insert_row":
|
|
||||||
request = {request_key: {"tableCellLocation": location, "insertBelow": insert_below}}
|
|
||||||
message = f"Inserted row {'below' if insert_below else 'above'} row {row_index}"
|
|
||||||
elif operation == "insert_column":
|
|
||||||
request = {request_key: {"tableCellLocation": location, "insertRight": insert_right}}
|
|
||||||
message = f"Inserted column {'right of' if insert_right else 'left of'} column {column_index}"
|
|
||||||
elif operation == "delete_row":
|
|
||||||
request = {request_key: {"tableCellLocation": location}}
|
|
||||||
message = f"Deleted row {row_index}"
|
|
||||||
else:
|
|
||||||
request = {request_key: {"tableCellLocation": location}}
|
|
||||||
message = f"Deleted column {column_index}"
|
|
||||||
|
|
||||||
await execute_request("update", document_id, request)
|
|
||||||
return message
|
|
||||||
|
|
||||||
|
|
||||||
async def insert_image(
|
|
||||||
document_id: str,
|
|
||||||
image_url: str | None,
|
|
||||||
start_position: int | None,
|
|
||||||
width: int | None,
|
|
||||||
height: int | None,
|
|
||||||
) -> str:
|
|
||||||
"""Insert an image from a URL at the specified position."""
|
|
||||||
if not image_url:
|
|
||||||
raise ValueError("image_url is required for insert_image operation")
|
|
||||||
if start_position is None:
|
|
||||||
raise ValueError("start_position is required for insert_image operation")
|
|
||||||
|
|
||||||
request = {
|
|
||||||
"insertInlineImage": {
|
|
||||||
"uri": image_url,
|
|
||||||
"location": {"index": start_position},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if width or height:
|
|
||||||
object_size = {}
|
|
||||||
if width:
|
|
||||||
object_size["width"] = {"magnitude": width, "unit": "PT"}
|
|
||||||
if height:
|
|
||||||
object_size["height"] = {"magnitude": height, "unit": "PT"}
|
|
||||||
request["insertInlineImage"]["objectSize"] = object_size
|
|
||||||
|
|
||||||
await execute_request("update", document_id, request)
|
|
||||||
return f"Inserted image from {image_url} at position {start_position}"
|
|
||||||
|
|
||||||
|
|
||||||
async def format_existing_text(
|
|
||||||
document_id: str,
|
|
||||||
search_text: str | None,
|
|
||||||
foreground_color: str | None,
|
|
||||||
background_color: str | None,
|
|
||||||
font_size: int | None,
|
|
||||||
font_family: str | None,
|
|
||||||
bold: bool | None,
|
|
||||||
italic: bool | None,
|
|
||||||
underline: bool | None,
|
|
||||||
strikethrough: bool | None,
|
|
||||||
heading_level: int | None,
|
|
||||||
link_url: str | None,
|
|
||||||
list_type: str | None,
|
|
||||||
) -> str:
|
|
||||||
"""Find and apply formatting to all occurrences of text."""
|
|
||||||
if not search_text:
|
|
||||||
raise ValueError("search_text is required for format_existing_text operation")
|
|
||||||
|
|
||||||
document = await execute_request("get", document_id, {})
|
|
||||||
|
|
||||||
from doc_utils import build_text_style, find_text_positions
|
|
||||||
|
|
||||||
positions = find_text_positions(document, search_text)
|
|
||||||
|
|
||||||
if not positions:
|
|
||||||
return f"No occurrences of '{search_text}' found"
|
|
||||||
|
|
||||||
text_style, fields = build_text_style(
|
|
||||||
foreground_color=foreground_color,
|
|
||||||
background_color=background_color,
|
|
||||||
font_size=font_size,
|
|
||||||
font_family=font_family,
|
|
||||||
bold=bold,
|
|
||||||
italic=italic,
|
|
||||||
underline=underline,
|
|
||||||
strikethrough=strikethrough,
|
|
||||||
link_url=link_url,
|
|
||||||
)
|
|
||||||
|
|
||||||
if not fields and not heading_level and not list_type:
|
|
||||||
raise ValueError("At least one formatting option must be specified")
|
|
||||||
|
|
||||||
positions.sort(reverse=True)
|
|
||||||
|
|
||||||
requests = []
|
|
||||||
for start, end in positions:
|
|
||||||
range_dict = {"startIndex": start, "endIndex": end}
|
|
||||||
|
|
||||||
if fields:
|
|
||||||
requests.append(
|
|
||||||
{"updateTextStyle": {"range": range_dict, "textStyle": text_style, "fields": fields}}
|
|
||||||
)
|
|
||||||
|
|
||||||
if heading_level:
|
|
||||||
if not (1 <= heading_level <= 6):
|
|
||||||
raise ValueError("heading_level must be between 1 and 6")
|
|
||||||
requests.append(
|
|
||||||
{
|
|
||||||
"updateParagraphStyle": {
|
|
||||||
"range": range_dict,
|
|
||||||
"paragraphStyle": {"namedStyleType": f"HEADING_{heading_level}"},
|
|
||||||
"fields": "namedStyleType",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
if list_type:
|
|
||||||
if list_type == "bullet":
|
|
||||||
preset = "BULLET_DISC_CIRCLE_SQUARE"
|
|
||||||
elif list_type == "numbered":
|
|
||||||
preset = "NUMBERED_DECIMAL_ALPHA_ROMAN"
|
|
||||||
else:
|
|
||||||
raise ValueError("list_type must be 'bullet' or 'numbered'")
|
|
||||||
requests.append({"createParagraphBullets": {"range": range_dict, "bulletPreset": preset}})
|
|
||||||
|
|
||||||
if not requests:
|
|
||||||
raise ValueError(f"No formatting requests generated for '{search_text}'.")
|
|
||||||
|
|
||||||
await execute_requests(document_id, requests)
|
|
||||||
return f"Formatted {len(positions)} occurrences of '{search_text}'"
|
|
||||||
|
|
||||||
doc_utils.py: |
|
|
||||||
"""Utility functions for document operations."""
|
|
||||||
|
|
||||||
import re
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
|
|
||||||
async def execute_request(
|
|
||||||
method: str, document_id: str, request: dict, **kwargs
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
"""Execute a single document API request."""
|
|
||||||
raise NotImplementedError("Stub: would call document backend")
|
|
||||||
|
|
||||||
|
|
||||||
async def execute_requests(
|
|
||||||
document_id: str, requests: list[dict], **kwargs
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
"""Execute a batch of document API requests."""
|
|
||||||
raise NotImplementedError("Stub: would call document backend")
|
|
||||||
|
|
||||||
|
|
||||||
def calculate_utf16_length(text: str) -> int:
|
|
||||||
"""Calculate text length in UTF-16 code units."""
|
|
||||||
return len(text.encode("utf-16-le")) // 2
|
|
||||||
|
|
||||||
|
|
||||||
def hex_to_rgb(hex_color: str) -> dict:
|
|
||||||
"""Convert hex color to RGB format (0.0-1.0 range)."""
|
|
||||||
hex_color = hex_color.lstrip("#")
|
|
||||||
|
|
||||||
if len(hex_color) == 3:
|
|
||||||
hex_color = "".join([c * 2 for c in hex_color])
|
|
||||||
|
|
||||||
if not re.match(r"^[0-9A-Fa-f]{6}$", hex_color):
|
|
||||||
raise ValueError(f"Invalid hex color: {hex_color}")
|
|
||||||
|
|
||||||
return {
|
|
||||||
"color": {
|
|
||||||
"rgbColor": {
|
|
||||||
"red": int(hex_color[0:2], 16) / 255.0,
|
|
||||||
"green": int(hex_color[2:4], 16) / 255.0,
|
|
||||||
"blue": int(hex_color[4:6], 16) / 255.0,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def build_text_style(
|
|
||||||
foreground_color: str | None = None,
|
|
||||||
background_color: str | None = None,
|
|
||||||
font_size: int | None = None,
|
|
||||||
font_family: str | None = None,
|
|
||||||
bold: bool | None = None,
|
|
||||||
italic: bool | None = None,
|
|
||||||
underline: bool | None = None,
|
|
||||||
strikethrough: bool | None = None,
|
|
||||||
link_url: str | None = None,
|
|
||||||
) -> tuple[dict, str]:
|
|
||||||
"""Build text style object and field mask from provided formatting options."""
|
|
||||||
style = {}
|
|
||||||
fields = []
|
|
||||||
|
|
||||||
if foreground_color:
|
|
||||||
style["foregroundColor"] = hex_to_rgb(foreground_color)
|
|
||||||
fields.append("foregroundColor")
|
|
||||||
|
|
||||||
if background_color:
|
|
||||||
style["backgroundColor"] = hex_to_rgb(background_color)
|
|
||||||
fields.append("backgroundColor")
|
|
||||||
|
|
||||||
if font_size is not None:
|
|
||||||
style["fontSize"] = {"magnitude": font_size, "unit": "PT"}
|
|
||||||
fields.append("fontSize")
|
|
||||||
|
|
||||||
if font_family:
|
|
||||||
style["weightedFontFamily"] = {"fontFamily": font_family}
|
|
||||||
fields.append("weightedFontFamily")
|
|
||||||
|
|
||||||
if bold is not None:
|
|
||||||
style["bold"] = bold
|
|
||||||
fields.append("bold")
|
|
||||||
|
|
||||||
if italic is not None:
|
|
||||||
style["italic"] = italic
|
|
||||||
fields.append("italic")
|
|
||||||
|
|
||||||
if underline is not None:
|
|
||||||
style["underline"] = underline
|
|
||||||
fields.append("underline")
|
|
||||||
|
|
||||||
if strikethrough is not None:
|
|
||||||
style["strikethrough"] = strikethrough
|
|
||||||
fields.append("strikethrough")
|
|
||||||
|
|
||||||
if link_url:
|
|
||||||
style["link"] = {"url": link_url}
|
|
||||||
fields.append("link")
|
|
||||||
|
|
||||||
return style, ",".join(fields)
|
|
||||||
|
|
||||||
|
|
||||||
def find_text_positions(
|
|
||||||
document: dict, search_text: str
|
|
||||||
) -> list[tuple[int, int]]:
|
|
||||||
"""Find all occurrences of text in a document with UTF-16 positions."""
|
|
||||||
positions = []
|
|
||||||
search_len = calculate_utf16_length(search_text)
|
|
||||||
|
|
||||||
content = document.get("body", {}).get("content", [])
|
|
||||||
_find_text_in_elements(content, search_text, search_len, positions)
|
|
||||||
|
|
||||||
return positions
|
|
||||||
|
|
||||||
|
|
||||||
def _find_text_in_elements(
|
|
||||||
elements: list[dict], search_text: str, search_len: int, positions: list
|
|
||||||
):
|
|
||||||
"""Recursively find text in document elements."""
|
|
||||||
for element in elements:
|
|
||||||
if "paragraph" in element:
|
|
||||||
para_elements = element["paragraph"].get("elements", [])
|
|
||||||
for elem in para_elements:
|
|
||||||
if "textRun" in elem:
|
|
||||||
text_run = elem["textRun"]
|
|
||||||
content = text_run.get("content", "")
|
|
||||||
start_index = elem.get("startIndex", 0)
|
|
||||||
|
|
||||||
idx = 0
|
|
||||||
while True:
|
|
||||||
pos = content.find(search_text, idx)
|
|
||||||
if pos == -1:
|
|
||||||
break
|
|
||||||
prefix_len = calculate_utf16_length(content[:pos])
|
|
||||||
match_start = start_index + prefix_len
|
|
||||||
match_end = match_start + search_len
|
|
||||||
positions.append((match_start, match_end))
|
|
||||||
idx = pos + len(search_text)
|
|
||||||
|
|
||||||
elif "table" in element:
|
|
||||||
table_rows = element["table"].get("tableRows", [])
|
|
||||||
for row in table_rows:
|
|
||||||
for cell in row.get("tableCells", []):
|
|
||||||
cell_content = cell.get("content", [])
|
|
||||||
_find_text_in_elements(cell_content, search_text, search_len, positions)
|
|
||||||
|
|
||||||
validate:
|
|
||||||
# docs.py must still exist
|
|
||||||
- type: file_exists
|
|
||||||
path: docs.py
|
|
||||||
name: docs.py exists
|
|
||||||
|
|
||||||
# doc_utils.py must still exist
|
|
||||||
- type: file_exists
|
|
||||||
path: doc_utils.py
|
|
||||||
name: doc_utils.py exists
|
|
||||||
|
|
||||||
# insert_image removed from the operations literal
|
|
||||||
- type: file_not_matches
|
|
||||||
path: docs.py
|
|
||||||
regex: "insert_image"
|
|
||||||
name: insert_image removed from operations literal
|
|
||||||
|
|
||||||
# ImageParams class removed
|
|
||||||
- type: file_not_matches
|
|
||||||
path: docs.py
|
|
||||||
regex: "ImageParams"
|
|
||||||
name: ImageParams class removed
|
|
||||||
|
|
||||||
# image_params parameter removed
|
|
||||||
- type: file_not_matches
|
|
||||||
path: docs.py
|
|
||||||
regex: "image_params"
|
|
||||||
name: image_params parameter removed
|
|
||||||
|
|
||||||
# insert_image function removed
|
|
||||||
- type: file_not_matches
|
|
||||||
path: docs.py
|
|
||||||
regex: "async def insert_image"
|
|
||||||
name: insert_image function removed
|
|
||||||
|
|
||||||
# image_url reference removed
|
|
||||||
- type: file_not_matches
|
|
||||||
path: docs.py
|
|
||||||
regex: "image_url"
|
|
||||||
name: image_url references removed
|
|
||||||
|
|
||||||
# insertInlineImage reference removed
|
|
||||||
- type: file_not_matches
|
|
||||||
path: docs.py
|
|
||||||
regex: "insertInlineImage"
|
|
||||||
name: insertInlineImage reference removed
|
|
||||||
|
|
||||||
# Other operations still present
|
|
||||||
- type: file_contains
|
|
||||||
path: docs.py
|
|
||||||
pattern: "get_document"
|
|
||||||
name: get_document preserved
|
|
||||||
|
|
||||||
- type: file_contains
|
|
||||||
path: docs.py
|
|
||||||
pattern: "insert_text"
|
|
||||||
name: insert_text preserved
|
|
||||||
|
|
||||||
- type: file_contains
|
|
||||||
path: docs.py
|
|
||||||
pattern: "replace_text"
|
|
||||||
name: replace_text preserved
|
|
||||||
|
|
||||||
- type: file_contains
|
|
||||||
path: docs.py
|
|
||||||
pattern: "insert_table"
|
|
||||||
name: insert_table preserved
|
|
||||||
|
|
||||||
- type: file_contains
|
|
||||||
path: docs.py
|
|
||||||
pattern: "format_existing_text"
|
|
||||||
name: format_existing_text preserved
|
|
||||||
|
|
||||||
- type: file_contains
|
|
||||||
path: docs.py
|
|
||||||
pattern: "async def doc_tool"
|
|
||||||
name: doc_tool function preserved
|
|
||||||
|
|
||||||
- type: file_contains
|
|
||||||
path: docs.py
|
|
||||||
pattern: "FormatTextParams"
|
|
||||||
name: FormatTextParams preserved
|
|
||||||
|
|
||||||
- type: file_contains
|
|
||||||
path: docs.py
|
|
||||||
pattern: "TableParams"
|
|
||||||
name: TableParams preserved
|
|
||||||
|
|
||||||
# doc_utils.py should be untouched (no image references existed there)
|
|
||||||
- type: file_contains
|
|
||||||
path: doc_utils.py
|
|
||||||
pattern: "def build_text_style"
|
|
||||||
name: doc_utils build_text_style preserved
|
|
||||||
|
|
||||||
- type: file_contains
|
|
||||||
path: doc_utils.py
|
|
||||||
pattern: "def find_text_positions"
|
|
||||||
name: doc_utils find_text_positions preserved
|
|
||||||
|
|
||||||
- type: file_contains
|
|
||||||
path: doc_utils.py
|
|
||||||
pattern: "def calculate_utf16_length"
|
|
||||||
name: doc_utils calculate_utf16_length preserved
|
|
||||||
|
|
||||||
# Python syntax check
|
|
||||||
- type: command_succeeds
|
|
||||||
command: "python3 -c \"import ast; ast.parse(open('docs.py').read())\""
|
|
||||||
name: docs.py valid python syntax
|
|
||||||
|
|
||||||
- type: command_succeeds
|
|
||||||
command: "python3 -c \"import ast; ast.parse(open('doc_utils.py').read())\""
|
|
||||||
name: doc_utils.py valid python syntax
|
|
||||||
Binary file not shown.
|
Before Width: | Height: | Size: 2.3 MiB |
File diff suppressed because it is too large
Load diff
|
|
@ -1,74 +0,0 @@
|
||||||
export interface AgentConfig {
|
|
||||||
model: string;
|
|
||||||
provider: string;
|
|
||||||
/** Extensions (runner knows which are platform vs builtin) */
|
|
||||||
extensions?: string[];
|
|
||||||
/** Stdio extension commands (for custom MCP servers) */
|
|
||||||
stdio?: string[];
|
|
||||||
/** Path to goose binary (default: "goose") */
|
|
||||||
"goose-bin"?: string;
|
|
||||||
temperature?: number;
|
|
||||||
maxTokens?: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface Scenario {
|
|
||||||
name: string;
|
|
||||||
description: string;
|
|
||||||
prompt?: string;
|
|
||||||
/** Files to create before running (relative paths) */
|
|
||||||
setup?: Record<string, string>;
|
|
||||||
/** Validation rules to check after agent completes (single-turn) */
|
|
||||||
validate?: ValidationRule[];
|
|
||||||
/** Multi-turn conversation (alternative to single prompt+validate) */
|
|
||||||
turns?: Turn[];
|
|
||||||
/** Tags for filtering scenarios */
|
|
||||||
tags?: string[];
|
|
||||||
}
|
|
||||||
|
|
||||||
/** A single turn in a multi-turn conversation */
|
|
||||||
export interface Turn {
|
|
||||||
/** The prompt for this turn */
|
|
||||||
prompt: string;
|
|
||||||
/** Validation rules to check after this turn completes */
|
|
||||||
validate: ValidationRule[];
|
|
||||||
}
|
|
||||||
|
|
||||||
export type ValidationRule =
|
|
||||||
| { type: "file_exists"; path: string; name?: string }
|
|
||||||
| { type: "file_contains"; path: string; pattern: string; name?: string }
|
|
||||||
| { type: "file_matches"; path: string; regex: string; name?: string }
|
|
||||||
| { type: "file_not_matches"; path: string; regex: string; name?: string }
|
|
||||||
| { type: "file_not_empty"; path: string; name?: string }
|
|
||||||
| { type: "command_succeeds"; command: string; name?: string }
|
|
||||||
| { type: "tool_called"; tool: string; args?: Record<string, string | RegExp>; name?: string }
|
|
||||||
| { type: "custom"; fn: string; name?: string };
|
|
||||||
|
|
||||||
export interface TestRun {
|
|
||||||
scenario: Scenario;
|
|
||||||
config: AgentConfig;
|
|
||||||
workdir: string;
|
|
||||||
startTime: Date;
|
|
||||||
endTime?: Date;
|
|
||||||
status: "pending" | "running" | "passed" | "failed";
|
|
||||||
errors?: string[];
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface TestResult {
|
|
||||||
run: TestRun;
|
|
||||||
validations: Array<{
|
|
||||||
rule: ValidationRule;
|
|
||||||
passed: boolean;
|
|
||||||
message?: string;
|
|
||||||
}>;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface SuiteConfig {
|
|
||||||
/** Agent configurations to permute */
|
|
||||||
agents: AgentConfig[];
|
|
||||||
/** Scenarios to run */
|
|
||||||
scenarios: string[];
|
|
||||||
/** Base directory for test workspaces */
|
|
||||||
workdir: string;
|
|
||||||
/** Parallel execution count */
|
|
||||||
parallel?: number;
|
|
||||||
}
|
|
||||||
|
|
@ -1,184 +0,0 @@
|
||||||
import { existsSync, readFileSync, statSync } from "node:fs";
|
|
||||||
import { execSync } from "node:child_process";
|
|
||||||
import { join } from "node:path";
|
|
||||||
import type { ValidationRule } from "./types.js";
|
|
||||||
|
|
||||||
export interface ValidationResult {
|
|
||||||
passed: boolean;
|
|
||||||
message?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function validateRule(
|
|
||||||
rule: ValidationRule,
|
|
||||||
workdir: string
|
|
||||||
): ValidationResult {
|
|
||||||
switch (rule.type) {
|
|
||||||
case "file_exists": {
|
|
||||||
const fullPath = join(workdir, rule.path);
|
|
||||||
const exists = existsSync(fullPath);
|
|
||||||
return {
|
|
||||||
passed: exists,
|
|
||||||
message: exists ? undefined : `File not found: ${rule.path}`,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
case "file_not_empty": {
|
|
||||||
const fullPath = join(workdir, rule.path);
|
|
||||||
if (!existsSync(fullPath)) {
|
|
||||||
return { passed: false, message: `File not found: ${rule.path}` };
|
|
||||||
}
|
|
||||||
const stat = statSync(fullPath);
|
|
||||||
return {
|
|
||||||
passed: stat.size > 0,
|
|
||||||
message: stat.size > 0 ? undefined : `File is empty: ${rule.path}`,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
case "file_contains": {
|
|
||||||
const fullPath = join(workdir, rule.path);
|
|
||||||
if (!existsSync(fullPath)) {
|
|
||||||
return { passed: false, message: `File not found: ${rule.path}` };
|
|
||||||
}
|
|
||||||
const content = readFileSync(fullPath, "utf-8");
|
|
||||||
const contains = content.includes(rule.pattern);
|
|
||||||
return {
|
|
||||||
passed: contains,
|
|
||||||
message: contains
|
|
||||||
? undefined
|
|
||||||
: `File ${rule.path} does not contain: ${rule.pattern}`,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
case "file_matches": {
|
|
||||||
const fullPath = join(workdir, rule.path);
|
|
||||||
if (!existsSync(fullPath)) {
|
|
||||||
return { passed: false, message: `File not found: ${rule.path}` };
|
|
||||||
}
|
|
||||||
const content = readFileSync(fullPath, "utf-8");
|
|
||||||
const regex = new RegExp(rule.regex);
|
|
||||||
const matches = regex.test(content);
|
|
||||||
return {
|
|
||||||
passed: matches,
|
|
||||||
message: matches
|
|
||||||
? undefined
|
|
||||||
: `File ${rule.path} does not match regex: ${rule.regex}`,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
case "file_not_matches": {
|
|
||||||
const fullPath = join(workdir, rule.path);
|
|
||||||
if (!existsSync(fullPath)) {
|
|
||||||
return { passed: false, message: `File not found: ${rule.path}` };
|
|
||||||
}
|
|
||||||
const content = readFileSync(fullPath, "utf-8");
|
|
||||||
const regex = new RegExp(rule.regex);
|
|
||||||
const matches = regex.test(content);
|
|
||||||
return {
|
|
||||||
passed: !matches,
|
|
||||||
message: !matches
|
|
||||||
? undefined
|
|
||||||
: `File ${rule.path} should not match regex: ${rule.regex}`,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
case "command_succeeds": {
|
|
||||||
try {
|
|
||||||
execSync(rule.command, { cwd: workdir, stdio: "pipe" });
|
|
||||||
return { passed: true };
|
|
||||||
} catch (err) {
|
|
||||||
return {
|
|
||||||
passed: false,
|
|
||||||
message: `Command failed: ${rule.command}`,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
case "tool_called": {
|
|
||||||
const logPath = join(workdir, "tool-calls.log");
|
|
||||||
if (!existsSync(logPath)) {
|
|
||||||
return { passed: false, message: "tool-calls.log not found" };
|
|
||||||
}
|
|
||||||
|
|
||||||
const content = readFileSync(logPath, "utf-8");
|
|
||||||
const lines = content.trim().split("\n").filter(Boolean);
|
|
||||||
|
|
||||||
// Find all calls to the specified tool
|
|
||||||
const matchingCalls = lines
|
|
||||||
.map((line) => {
|
|
||||||
try {
|
|
||||||
return JSON.parse(line);
|
|
||||||
} catch {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.filter((entry) => entry?.tool === rule.tool);
|
|
||||||
|
|
||||||
if (matchingCalls.length === 0) {
|
|
||||||
return { passed: false, message: `Tool not called: ${rule.tool}` };
|
|
||||||
}
|
|
||||||
|
|
||||||
// If no arg requirements, just check tool was called
|
|
||||||
if (!rule.args) {
|
|
||||||
return { passed: true };
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if any call matches the arg requirements
|
|
||||||
for (const call of matchingCalls) {
|
|
||||||
const args = call.arguments || {};
|
|
||||||
let allMatch = true;
|
|
||||||
|
|
||||||
for (const [key, expected] of Object.entries(rule.args)) {
|
|
||||||
const actual = args[key];
|
|
||||||
if (actual === undefined) {
|
|
||||||
allMatch = false;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
// If expected starts/ends with /, treat as regex pattern
|
|
||||||
if (typeof expected === "string" && expected.startsWith("/") && expected.endsWith("/")) {
|
|
||||||
const pattern = new RegExp(expected.slice(1, -1), "i");
|
|
||||||
if (!pattern.test(String(actual))) {
|
|
||||||
allMatch = false;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// Exact match (case-insensitive for strings)
|
|
||||||
const actualStr = String(actual).toLowerCase();
|
|
||||||
const expectedStr = String(expected).toLowerCase();
|
|
||||||
if (!actualStr.includes(expectedStr)) {
|
|
||||||
allMatch = false;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (allMatch) {
|
|
||||||
return { passed: true };
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
passed: false,
|
|
||||||
message: `Tool ${rule.tool} called but args didn't match: expected ${JSON.stringify(rule.args)}`,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
case "custom": {
|
|
||||||
// Custom validators loaded dynamically
|
|
||||||
return { passed: false, message: "Custom validators not yet implemented" };
|
|
||||||
}
|
|
||||||
|
|
||||||
default:
|
|
||||||
return { passed: false, message: `Unknown rule type` };
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export function validateAll(
|
|
||||||
rules: ValidationRule[],
|
|
||||||
workdir: string
|
|
||||||
): Array<{ rule: ValidationRule; result: ValidationResult }> {
|
|
||||||
return rules.map((rule) => ({
|
|
||||||
rule,
|
|
||||||
result: validateRule(rule, workdir),
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
@ -1,12 +0,0 @@
|
||||||
{
|
|
||||||
"compilerOptions": {
|
|
||||||
"target": "ES2022",
|
|
||||||
"module": "NodeNext",
|
|
||||||
"moduleResolution": "NodeNext",
|
|
||||||
"strict": true,
|
|
||||||
"esModuleInterop": true,
|
|
||||||
"skipLibCheck": true,
|
|
||||||
"outDir": "dist"
|
|
||||||
},
|
|
||||||
"include": ["src"]
|
|
||||||
}
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue