vrr/goose
2
0
Fork 0
mirror of https://github.com/block/goose.git synced 2026-07-09 16:09:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 22

tidy: clean up old benchmark and add gym (#7081)

Browse source
This commit is contained in:
Michael Neale 2026-02-09 17:08:46 +11:00 • committed by GitHub
parent d4865ae9fe
commit a3ba124178
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
76 changed files with 6448 additions and 91262 deletions
Download patch file Download diff file Expand all files Collapse all files

24
Cargo.lock generated
View file

@ -4239,29 +4239,6 @@ dependencies = [
"wiremock",
]
[[package]]
name = "goose-bench"
version = "1.23.0"
dependencies = [
"anyhow",
"async-trait",
"chrono",
"ctor",
"dotenvy",
"goose",
"include_dir",
"once_cell",
"paste",
"regex",
"rmcp 0.14.0",
"serde",
"serde_json",
"tokio",
"tracing",
"tracing-subscriber",
"winapi",
]
[[package]]
name = "goose-cli"
version = "1.23.0"
@ -4282,7 +4259,6 @@ dependencies = [
"futures",
"goose",
"goose-acp",
"goose-bench",
"goose-mcp",
"http 1.4.0",
"indicatif 0.18.3",

32
crates/goose-bench/Cargo.toml
View file

@ -1,32 +0,0 @@
[package]
name = "goose-bench"
version.workspace = true
edition.workspace = true
authors.workspace = true
license.workspace = true
repository.workspace = true
description.workspace = true
[lints]
workspace = true
[dependencies]
anyhow = { workspace = true }
paste = "1.0"
ctor = "0.2.7"
goose = { path = "../goose" }
rmcp = { workspace = true }
async-trait = "0.1.89"
chrono = { version = "0.4", features = ["serde"] }
serde_json = { workspace = true }
serde = { version = "1.0", features = ["derive"] }
tracing = { workspace = true }
tracing-subscriber = { version = "0.3", features = ["registry"] }
tokio = { workspace = true }
include_dir = "0.7.4"
once_cell = "1.19"
regex = { workspace = true }
dotenvy = "0.15.7"
[target.'cfg(target_os = "windows")'.dependencies]
winapi = { version = "0.3", features = ["wincred"] }

273
crates/goose-bench/README.md
View file

@ -1,273 +0,0 @@
# goose Benchmarking Framework
The `goose-bench` crate provides a framework for benchmarking and evaluating LLM models with the goose framework. This tool helps quantify model performance across various tasks and generate structured reports.
## Features
- Run benchmark suites across multiple LLM models
- Execute evaluations in parallel when supported
- Generate structured JSON and CSV reports
- Process evaluation results with custom scripts
- Calculate aggregate metrics across evaluations
- Support for tool-shim evaluation
- Generate leaderboards and comparative metrics
## Prerequisites
- **Python Environment**: The `generate-leaderboard` command executes Python scripts and requires a valid Python environment with necessary dependencies (pandas, etc.)
- **OpenAI API Key**: For evaluations using LLM-as-judge (like `blog_summary` and `restaurant_research`), you must have an `OPENAI_API_KEY` environment variable set, as the judge uses the OpenAI GPT-4o model
## Benchmark Workflow
Running benchmarks is a two-step process:
### Step 1: Run Benchmarks
First, run the benchmark evaluations with your configuration:
```bash
goose bench run --config /path/to/your-config.json
```
This will execute all evaluations for all models specified in your configuration and create a benchmark directory with results.
### Step 2: Generate Leaderboard
After the benchmarks complete, generate the leaderboard and aggregated metrics:
```bash
goose bench generate-leaderboard --benchmark-dir /path/to/benchmark-output-directory
```
The benchmark directory path will be shown in the output of the previous command, typically in the format `benchmark-YYYY-MM-DD-HH:MM:SS`.
**Note**: This command requires a valid Python environment as it executes Python scripts for data aggregation and leaderboard generation.
## Configuration
Benchmark configuration is provided through a JSON file. Here's a sample configuration file (leaderboard-config.json) that you can use as a template:
```json
{
"models": [
{
"provider": "databricks",
"name": "gpt-4-1-mini",
"parallel_safe": true,
"tool_shim": {
"use_tool_shim": false,
"tool_shim_model": null
}
},
{
"provider": "databricks",
"name": "claude-sonnet-4",
"parallel_safe": true,
"tool_shim": null
},
{
"provider": "databricks",
"name": "gpt-4o",
"parallel_safe": true,
"tool_shim": null
}
],
"evals": [
{
"selector": "core:developer",
"post_process_cmd": null,
"parallel_safe": true
},
{
"selector": "core:developer_search_replace",
"post_process_cmd": null,
"parallel_safe": true
},
{
"selector": "vibes:blog_summary",
"post_process_cmd": "/Users/ahau/Development/goose-1.0/goose/scripts/bench-postprocess-scripts/llm-judges/run_vibes_judge.sh",
"parallel_safe": true
},
{
"selector": "vibes:restaurant_research",
"post_process_cmd": "/Users/ahau/Development/goose-1.0/goose/scripts/bench-postprocess-scripts/llm-judges/run_vibes_judge.sh",
"parallel_safe": true
}
],
"include_dirs": [],
"repeat": 3,
"run_id": null,
"output_dir": "/path/to/output/directory",
"eval_result_filename": "eval-results.json",
"run_summary_filename": "run-results-summary.json",
"env_file": "/path/to/.goosebench.env"
}
```
## Configuration Options
### Models
- `provider`: The LLM provider (e.g., "databricks", "openai")
- `name`: The model name
- `parallel_safe`: Whether the model can be run in parallel
- `tool_shim`: Configuration for tool-shim support
- `use_tool_shim`: Whether to use tool-shim
- `tool_shim_model`: Optional custom model for tool-shim
### Evaluations
- `selector`: The evaluation selector in format `suite:evaluation`
- `post_process_cmd`: Optional path to a post-processing script
- `parallel_safe`: Whether the evaluation can be run in parallel
### Global Configuration
- `include_dirs`: Additional directories to include in the benchmark environment
- `repeat`: Number of times to repeat evaluations (for statistical significance)
- `run_id`: Optional identifier for the run (defaults to timestamp)
- `output_dir`: Directory to store benchmark results (must be absolute path)
- `eval_result_filename`: Filename for individual evaluation results
- `run_summary_filename`: Filename for run summary
- `env_file`: Optional path to environment variables file
## Environment Variables
You can provide environment variables through the `env_file` configuration option. This is useful for provider API keys and other sensitive information. Example `.goosebench.env` file:
```bash
OPENAI_API_KEY=your_openai_api_key_here
DATABRICKS_TOKEN=your_databricks_token_here
# Add other environment variables as needed
```
**Important**: For evaluations that use LLM-as-judge (like `blog_summary` and `restaurant_research`), you must set `OPENAI_API_KEY` as the judging system uses OpenAI's GPT-4o model.
## Post-Processing
You can specify post-processing commands for evaluations, which will be executed after each evaluation completes. The command receives the path to the evaluation results file as its first argument.
For example, the `run_vibes_judge.sh` script processes outputs from the `blog_summary` and `restaurant_research` evaluations, using LLM-based judging to assign scores.
## Output Structure
Results are organized in a directory structure that follows this pattern:
```
{benchmark_dir}/
├── config.cfg # Configuration used for the benchmark
├── {provider}-{model}/
│ ├── eval-results/
│ │ └── aggregate_metrics.csv # Aggregated metrics for this model
│ └── run-{run_id}/
│ ├── {suite}/
│ │ └── {evaluation}/
│ │ ├── eval-results.json # Individual evaluation results
│ │ ├── {eval_name}.jsonl # Session logs
│ │ └── work_dir.json # Info about evaluation working dir
│ └── run-results-summary.json # Summary of all evaluations in this run
├── leaderboard.csv # Final leaderboard comparing all models
└── all_metrics.csv # Union of all metrics across all models
```
### Output Files Explained
#### Per-Model Files
- **`eval-results/aggregate_metrics.csv`**: Contains aggregated metrics for each evaluation, averaged across all runs. Includes metrics like `score_mean`, `total_tokens_mean`, `prompt_execution_time_seconds_mean`, etc.
#### Global Output Files
- **`leaderboard.csv`**: Final leaderboard ranking all models by their average performance across evaluations. Contains columns like:
- `provider`, `model_name`: Model identification
- `avg_score_mean`: Average score across all evaluations
- `avg_prompt_execution_time_seconds_mean`: Average execution time
- `avg_total_tool_calls_mean`: Average number of tool calls
- `avg_total_tokens_mean`: Average token usage
- **`all_metrics.csv`**: Comprehensive dataset containing detailed metrics for every model-evaluation combination. This is a union of all individual model metrics, useful for detailed analysis and custom reporting.
Each model gets its own directory, containing run results and aggregated CSV files for analysis. The `generate-leaderboard` command processes all individual evaluation results and creates the comparative metrics files.
## Error Handling and Troubleshooting
**Important**: The current version of goose-bench does not have robust error handling for common issues that can occur during evaluation runs, such as:
- Rate limiting from inference providers
- Network timeouts or connection errors
- Provider API errors that cause early session termination
- Resource exhaustion or memory issues
### Checking for Failed Evaluations
After running benchmarks, you should inspect the generated metrics files to identify any evaluations that may have failed or terminated early:
1. **Check the `aggregate_metrics.csv` files** in each model's `eval-results/` directory for:
- Missing evaluations (fewer rows than expected)
- Unusually low scores or metrics
- Zero or near-zero execution times
- Missing or NaN values
2. **Look for `server_error_mean` column** in the aggregate metrics - values greater than 0 indicate server errors occurred during evaluation
3. **Review session logs** (`.jsonl` files) in individual evaluation directories for error messages like:
- "Server error"
- "Rate limit exceeded"
- "TEMPORARILY_UNAVAILABLE"
- Unexpected session terminations
### Re-running Failed Evaluations
If you identify failed evaluations, you may need to:
1. **Adjust rate limiting**: Add delays between requests or reduce parallel execution
2. **Update environment variables**: Ensure API keys and tokens are valid
3. **Re-run specific model/evaluation combinations**: Create a new config with only the failed combinations
4. **Check provider status**: Verify the inference provider is operational
Example of creating a config to re-run failed evaluations:
```json
{
"models": [
{
"provider": "databricks",
"name": "claude-sonnet-4",
"parallel_safe": false
}
],
"evals": [
{
"selector": "vibes:blog_summary",
"post_process_cmd": "/path/to/scripts/bench-postprocess-scripts/llm-judges/run_vibes_judge.sh",
"parallel_safe": false
}
],
"repeat": 1,
"output_dir": "/path/to/retry-benchmark"
}
```
We recommend monitoring evaluation progress and checking for errors regularly, especially when running large benchmark suites across multiple models.
## Available Commands
### List Evaluations
```bash
goose bench selectors --config /path/to/config.json
```
### Generate Initial Config
```bash
goose bench init-config --name my-benchmark-config.json
```
### Run Benchmarks
```bash
goose bench run --config /path/to/config.json
```
### Generate Leaderboard
```bash
goose bench generate-leaderboard --benchmark-dir /path/to/benchmark-output
```

15
crates/goose-bench/src/assets/kubernetes.patch
View file

@ -1,15 +0,0 @@
diff --git a/kubernetes_swagger.json b/kubernetes_swagger.json
index 3e11d92..859a63e 100644
--- a/kubernetes_swagger.json
+++ b/kubernetes_swagger.json
@@ -371,8 +371,8 @@
},
"type": "object"
},
- "io.k8s.api.admissionregistration.v1.ServiceReference": {
- "description": "ServiceReference holds a reference to Service.legacy.k8s.io",
+ "io.k8s.api.admissionregistration.v1.FakeServiceReference": {
+ "description": "FakeServiceReference simulates a reference to a fake service for testing purposes.",
"properties": {
"name": {
"description": "`name` is the name of the service. Required",

86037
crates/goose-bench/src/assets/kubernetes_swagger.json
View file

File diff suppressed because it is too large Load diff

1
crates/goose-bench/src/assets/squirrel-data.csv
View file

File diff suppressed because one or more lines are too long

458
crates/goose-bench/src/assets/vscode.patch
View file

@ -1,458 +0,0 @@
diff --git a/vscode_config_registry.ts b/vscode_config_registry.ts
index d2ba316..1834518 100644
--- a/vscode_config_registry.ts
+++ b/vscode_config_registry.ts
@@ -23,68 +23,68 @@ export const Extensions = {
Configuration: 'base.contributions.configuration'
};
-export interface IConfigurationDelta {
- removedDefaults?: IConfigurationDefaults[];
- removedConfigurations?: IConfigurationNode[];
- addedDefaults?: IConfigurationDefaults[];
- addedConfigurations?: IConfigurationNode[];
+export interface PConfigurationDelta {
+ removedDefaults?: PConfigurationDefaults[];
+ removedConfigurations?: PConfigurationNode[];
+ addedDefaults?: PConfigurationDefaults[];
+ addedConfigurations?: PConfigurationNode[];
}
-export interface IConfigurationRegistry {
+export interface PConfigurationRegistry {
/**
* Register a configuration to the registry.
*/
- registerConfiguration(configuration: IConfigurationNode): void;
+ registerConfiguration(configuration: PConfigurationNode): void;
/**
* Register multiple configurations to the registry.
*/
- registerConfigurations(configurations: IConfigurationNode[], validate?: boolean): void;
+ registerConfigurations(configurations: PConfigurationNode[], validate?: boolean): void;
/**
* Deregister multiple configurations from the registry.
*/
- deregisterConfigurations(configurations: IConfigurationNode[]): void;
+ deregisterConfigurations(configurations: PConfigurationNode[]): void;
/**
* update the configuration registry by
* - registering the configurations to add
* - dereigstering the configurations to remove
*/
- updateConfigurations(configurations: { add: IConfigurationNode[]; remove: IConfigurationNode[] }): void;
+ updateConfigurations(configurations: { add: PConfigurationNode[]; remove: PConfigurationNode[] }): void;
/**
* Register multiple default configurations to the registry.
*/
- registerDefaultConfigurations(defaultConfigurations: IConfigurationDefaults[]): void;
+ registerDefaultConfigurations(defaultConfigurations: PConfigurationDefaults[]): void;
/**
* Deregister multiple default configurations from the registry.
*/
- deregisterDefaultConfigurations(defaultConfigurations: IConfigurationDefaults[]): void;
+ deregisterDefaultConfigurations(defaultConfigurations: PConfigurationDefaults[]): void;
/**
* Bulk update of the configuration registry (default and configurations, remove and add)
* @param delta
*/
- deltaConfiguration(delta: IConfigurationDelta): void;
+ deltaConfiguration(delta: PConfigurationDelta): void;
/**
* Return the registered default configurations
*/
- getRegisteredDefaultConfigurations(): IConfigurationDefaults[];
+ getRegisteredDefaultConfigurations(): PConfigurationDefaults[];
/**
* Return the registered configuration defaults overrides
*/
- getConfigurationDefaultsOverrides(): Map<string, IConfigurationDefaultOverrideValue>;
+ getConfigurationDefaultsOverrides(): Map<string, PConfigurationDefaultOverrideValue>;
/**
* Signal that the schema of a configuration setting has changes. It is currently only supported to change enumeration values.
* Property or default value changes are not allowed.
*/
- notifyConfigurationSchemaUpdated(...configurations: IConfigurationNode[]): void;
+ notifyConfigurationSchemaUpdated(...configurations: PConfigurationNode[]): void;
/**
* Event that fires whenever a configuration has been
@@ -101,12 +101,12 @@ export interface IConfigurationRegistry {
/**
* Returns all configuration nodes contributed to this registry.
*/
- getConfigurations(): IConfigurationNode[];
+ getConfigurations(): PConfigurationNode[];
/**
* Returns all configurations settings of all configuration nodes contributed to this registry.
*/
- getConfigurationProperties(): IStringDictionary<IRegisteredConfigurationPropertySchema>;
+ getConfigurationProperties(): IStringDictionary<PRegisteredConfigurationPropertySchema>;
/**
* Return all configurations by policy name
@@ -116,7 +116,7 @@ export interface IConfigurationRegistry {
/**
* Returns all excluded configurations settings of all configuration nodes contributed to this registry.
*/
- getExcludedConfigurationProperties(): IStringDictionary<IRegisteredConfigurationPropertySchema>;
+ getExcludedConfigurationProperties(): IStringDictionary<PRegisteredConfigurationPropertySchema>;
/**
* Register the identifiers for editor configurations
@@ -168,7 +168,7 @@ export interface IPolicy {
readonly minimumVersion: `${number}.${number}`;
}
-export interface IConfigurationPropertySchema extends IJSONSchema {
+export interface PConfigurationPropertySchema extends IJSONSchema {
scope?: ConfigurationScope;
@@ -235,14 +235,14 @@ export interface IExtensionInfo {
displayName?: string;
}
-export interface IConfigurationNode {
+export interface PConfigurationNode {
id?: string;
order?: number;
type?: string | string[];
title?: string;
description?: string;
- properties?: IStringDictionary<IConfigurationPropertySchema>;
- allOf?: IConfigurationNode[];
+ properties?: IStringDictionary<PConfigurationPropertySchema>;
+ allOf?: PConfigurationNode[];
scope?: ConfigurationScope;
extensionInfo?: IExtensionInfo;
restrictedProperties?: string[];
@@ -250,49 +250,49 @@ export interface IConfigurationNode {
export type ConfigurationDefaultValueSource = IExtensionInfo | Map<string, IExtensionInfo>;
-export interface IConfigurationDefaults {
+export interface PConfigurationDefaults {
overrides: IStringDictionary<any>;
source?: IExtensionInfo;
}
-export type IRegisteredConfigurationPropertySchema = IConfigurationPropertySchema & {
+export type PRegisteredConfigurationPropertySchema = PConfigurationPropertySchema & {
defaultDefaultValue?: any;
source?: IExtensionInfo; // Source of the Property
defaultValueSource?: ConfigurationDefaultValueSource; // Source of the Default Value
};
-export interface IConfigurationDefaultOverride {
+export interface PConfigurationDefaultOverride {
readonly value: any;
readonly source?: IExtensionInfo; // Source of the default override
}
-export interface IConfigurationDefaultOverrideValue {
+export interface PConfigurationDefaultOverrideValue {
readonly value: any;
readonly source?: ConfigurationDefaultValueSource;
}
-export const allSettings: { properties: IStringDictionary<IConfigurationPropertySchema>; patternProperties: IStringDictionary<IConfigurationPropertySchema> } = { properties: {}, patternProperties: {} };
-export const applicationSettings: { properties: IStringDictionary<IConfigurationPropertySchema>; patternProperties: IStringDictionary<IConfigurationPropertySchema> } = { properties: {}, patternProperties: {} };
-export const applicationMachineSettings: { properties: IStringDictionary<IConfigurationPropertySchema>; patternProperties: IStringDictionary<IConfigurationPropertySchema> } = { properties: {}, patternProperties: {} };
-export const machineSettings: { properties: IStringDictionary<IConfigurationPropertySchema>; patternProperties: IStringDictionary<IConfigurationPropertySchema> } = { properties: {}, patternProperties: {} };
-export const machineOverridableSettings: { properties: IStringDictionary<IConfigurationPropertySchema>; patternProperties: IStringDictionary<IConfigurationPropertySchema> } = { properties: {}, patternProperties: {} };
-export const windowSettings: { properties: IStringDictionary<IConfigurationPropertySchema>; patternProperties: IStringDictionary<IConfigurationPropertySchema> } = { properties: {}, patternProperties: {} };
-export const resourceSettings: { properties: IStringDictionary<IConfigurationPropertySchema>; patternProperties: IStringDictionary<IConfigurationPropertySchema> } = { properties: {}, patternProperties: {} };
+export const allSettings: { properties: IStringDictionary<PConfigurationPropertySchema>; patternProperties: IStringDictionary<PConfigurationPropertySchema> } = { properties: {}, patternProperties: {} };
+export const applicationSettings: { properties: IStringDictionary<PConfigurationPropertySchema>; patternProperties: IStringDictionary<PConfigurationPropertySchema> } = { properties: {}, patternProperties: {} };
+export const applicationMachineSettings: { properties: IStringDictionary<PConfigurationPropertySchema>; patternProperties: IStringDictionary<PConfigurationPropertySchema> } = { properties: {}, patternProperties: {} };
+export const machineSettings: { properties: IStringDictionary<PConfigurationPropertySchema>; patternProperties: IStringDictionary<PConfigurationPropertySchema> } = { properties: {}, patternProperties: {} };
+export const machineOverridableSettings: { properties: IStringDictionary<PConfigurationPropertySchema>; patternProperties: IStringDictionary<PConfigurationPropertySchema> } = { properties: {}, patternProperties: {} };
+export const windowSettings: { properties: IStringDictionary<PConfigurationPropertySchema>; patternProperties: IStringDictionary<PConfigurationPropertySchema> } = { properties: {}, patternProperties: {} };
+export const resourceSettings: { properties: IStringDictionary<PConfigurationPropertySchema>; patternProperties: IStringDictionary<PConfigurationPropertySchema> } = { properties: {}, patternProperties: {} };
export const resourceLanguageSettingsSchemaId = 'vscode://schemas/settings/resourceLanguage';
export const configurationDefaultsSchemaId = 'vscode://schemas/settings/configurationDefaults';
const contributionRegistry = Registry.as<IJSONContributionRegistry>(JSONExtensions.JSONContribution);
-class ConfigurationRegistry implements IConfigurationRegistry {
+class ConfigurationRegistry implements PConfigurationRegistry {
- private readonly registeredConfigurationDefaults: IConfigurationDefaults[] = [];
- private readonly configurationDefaultsOverrides: Map<string, { configurationDefaultOverrides: IConfigurationDefaultOverride[]; configurationDefaultOverrideValue?: IConfigurationDefaultOverrideValue }>;
- private readonly defaultLanguageConfigurationOverridesNode: IConfigurationNode;
- private readonly configurationContributors: IConfigurationNode[];
- private readonly configurationProperties: IStringDictionary<IRegisteredConfigurationPropertySchema>;
+ private readonly registeredConfigurationDefaults: PConfigurationDefaults[] = [];
+ private readonly configurationDefaultsOverrides: Map<string, { configurationDefaultOverrides: PConfigurationDefaultOverride[]; configurationDefaultOverrideValue?: PConfigurationDefaultOverrideValue }>;
+ private readonly defaultLanguageConfigurationOverridesNode: PConfigurationNode;
+ private readonly configurationContributors: PConfigurationNode[];
+ private readonly configurationProperties: IStringDictionary<PRegisteredConfigurationPropertySchema>;
private readonly policyConfigurations: Map<PolicyName, string>;
- private readonly excludedConfigurationProperties: IStringDictionary<IRegisteredConfigurationPropertySchema>;
+ private readonly excludedConfigurationProperties: IStringDictionary<PRegisteredConfigurationPropertySchema>;
private readonly resourceLanguageSettingsSchema: IJSONSchema;
private readonly overrideIdentifiers = new Set<string>();
@@ -325,11 +325,11 @@ class ConfigurationRegistry implements IConfigurationRegistry {
this.registerOverridePropertyPatternKey();
}
- public registerConfiguration(configuration: IConfigurationNode, validate: boolean = true): void {
+ public registerConfiguration(configuration: PConfigurationNode, validate: boolean = true): void {
this.registerConfigurations([configuration], validate);
}
- public registerConfigurations(configurations: IConfigurationNode[], validate: boolean = true): void {
+ public registerConfigurations(configurations: PConfigurationNode[], validate: boolean = true): void {
const properties = new Set<string>();
this.doRegisterConfigurations(configurations, validate, properties);
@@ -338,7 +338,7 @@ class ConfigurationRegistry implements IConfigurationRegistry {
this._onDidUpdateConfiguration.fire({ properties });
}
- public deregisterConfigurations(configurations: IConfigurationNode[]): void {
+ public deregisterConfigurations(configurations: PConfigurationNode[]): void {
const properties = new Set<string>();
this.doDeregisterConfigurations(configurations, properties);
@@ -347,7 +347,7 @@ class ConfigurationRegistry implements IConfigurationRegistry {
this._onDidUpdateConfiguration.fire({ properties });
}
- public updateConfigurations({ add, remove }: { add: IConfigurationNode[]; remove: IConfigurationNode[] }): void {
+ public updateConfigurations({ add, remove }: { add: PConfigurationNode[]; remove: PConfigurationNode[] }): void {
const properties = new Set<string>();
this.doDeregisterConfigurations(remove, properties);
this.doRegisterConfigurations(add, false, properties);
@@ -357,14 +357,14 @@ class ConfigurationRegistry implements IConfigurationRegistry {
this._onDidUpdateConfiguration.fire({ properties });
}
- public registerDefaultConfigurations(configurationDefaults: IConfigurationDefaults[]): void {
+ public registerDefaultConfigurations(configurationDefaults: PConfigurationDefaults[]): void {
const properties = new Set<string>();
this.doRegisterDefaultConfigurations(configurationDefaults, properties);
this._onDidSchemaChange.fire();
this._onDidUpdateConfiguration.fire({ properties, defaultsOverrides: true });
}
- private doRegisterDefaultConfigurations(configurationDefaults: IConfigurationDefaults[], bucket: Set<string>) {
+ private doRegisterDefaultConfigurations(configurationDefaults: PConfigurationDefaults[], bucket: Set<string>) {
this.registeredConfigurationDefaults.push(...configurationDefaults);
@@ -413,14 +413,14 @@ class ConfigurationRegistry implements IConfigurationRegistry {
this.doRegisterOverrideIdentifiers(overrideIdentifiers);
}
- public deregisterDefaultConfigurations(defaultConfigurations: IConfigurationDefaults[]): void {
+ public deregisterDefaultConfigurations(defaultConfigurations: PConfigurationDefaults[]): void {
const properties = new Set<string>();
this.doDeregisterDefaultConfigurations(defaultConfigurations, properties);
this._onDidSchemaChange.fire();
this._onDidUpdateConfiguration.fire({ properties, defaultsOverrides: true });
}
- private doDeregisterDefaultConfigurations(defaultConfigurations: IConfigurationDefaults[], bucket: Set<string>): void {
+ private doDeregisterDefaultConfigurations(defaultConfigurations: PConfigurationDefaults[], bucket: Set<string>): void {
for (const defaultConfiguration of defaultConfigurations) {
const index = this.registeredConfigurationDefaults.indexOf(defaultConfiguration);
if (index !== -1) {
@@ -447,7 +447,7 @@ class ConfigurationRegistry implements IConfigurationRegistry {
}
if (OVERRIDE_PROPERTY_REGEX.test(key)) {
- let configurationDefaultOverrideValue: IConfigurationDefaultOverrideValue | undefined;
+ let configurationDefaultOverrideValue: PConfigurationDefaultOverrideValue | undefined;
for (const configurationDefaultOverride of configurationDefaultOverridesForKey.configurationDefaultOverrides) {
configurationDefaultOverrideValue = this.mergeDefaultConfigurationsForOverrideIdentifier(key, configurationDefaultOverride.value, configurationDefaultOverride.source, configurationDefaultOverrideValue);
}
@@ -460,7 +460,7 @@ class ConfigurationRegistry implements IConfigurationRegistry {
delete this.defaultLanguageConfigurationOverridesNode.properties![key];
}
} else {
- let configurationDefaultOverrideValue: IConfigurationDefaultOverrideValue | undefined;
+ let configurationDefaultOverrideValue: PConfigurationDefaultOverrideValue | undefined;
for (const configurationDefaultOverride of configurationDefaultOverridesForKey.configurationDefaultOverrides) {
configurationDefaultOverrideValue = this.mergeDefaultConfigurationsForConfigurationProperty(key, configurationDefaultOverride.value, configurationDefaultOverride.source, configurationDefaultOverrideValue);
}
@@ -477,8 +477,8 @@ class ConfigurationRegistry implements IConfigurationRegistry {
this.updateOverridePropertyPatternKey();
}
- private updateDefaultOverrideProperty(key: string, newDefaultOverride: IConfigurationDefaultOverrideValue, source: IExtensionInfo | undefined): void {
- const property: IRegisteredConfigurationPropertySchema = {
+ private updateDefaultOverrideProperty(key: string, newDefaultOverride: PConfigurationDefaultOverrideValue, source: IExtensionInfo | undefined): void {
+ const property: PRegisteredConfigurationPropertySchema = {
type: 'object',
default: newDefaultOverride.value,
description: nls.localize('defaultLanguageConfiguration.description', "Configure settings to be overridden for the {0} language.", getLanguageTagSettingPlainKey(key)),
@@ -491,7 +491,7 @@ class ConfigurationRegistry implements IConfigurationRegistry {
this.defaultLanguageConfigurationOverridesNode.properties![key] = property;
}
- private mergeDefaultConfigurationsForOverrideIdentifier(overrideIdentifier: string, configurationValueObject: IStringDictionary<any>, valueSource: IExtensionInfo | undefined, existingDefaultOverride: IConfigurationDefaultOverrideValue | undefined): IConfigurationDefaultOverrideValue | undefined {
+ private mergeDefaultConfigurationsForOverrideIdentifier(overrideIdentifier: string, configurationValueObject: IStringDictionary<any>, valueSource: IExtensionInfo | undefined, existingDefaultOverride: PConfigurationDefaultOverrideValue | undefined): PConfigurationDefaultOverrideValue | undefined {
const defaultValue = existingDefaultOverride?.value || {};
const source = existingDefaultOverride?.source ?? new Map<string, IExtensionInfo>();
@@ -532,7 +532,7 @@ class ConfigurationRegistry implements IConfigurationRegistry {
return { value: defaultValue, source };
}
- private mergeDefaultConfigurationsForConfigurationProperty(propertyKey: string, value: any, valuesSource: IExtensionInfo | undefined, existingDefaultOverride: IConfigurationDefaultOverrideValue | undefined): IConfigurationDefaultOverrideValue | undefined {
+ private mergeDefaultConfigurationsForConfigurationProperty(propertyKey: string, value: any, valuesSource: IExtensionInfo | undefined, existingDefaultOverride: PConfigurationDefaultOverrideValue | undefined): PConfigurationDefaultOverrideValue | undefined {
const property = this.configurationProperties[propertyKey];
const existingDefaultValue = existingDefaultOverride?.value ?? property?.defaultDefaultValue;
let source: ConfigurationDefaultValueSource | undefined = valuesSource;
@@ -564,7 +564,7 @@ class ConfigurationRegistry implements IConfigurationRegistry {
return { value, source };
}
- public deltaConfiguration(delta: IConfigurationDelta): void {
+ public deltaConfiguration(delta: PConfigurationDelta): void {
// defaults: remove
let defaultsOverrides = false;
const properties = new Set<string>();
@@ -589,7 +589,7 @@ class ConfigurationRegistry implements IConfigurationRegistry {
this._onDidUpdateConfiguration.fire({ properties, defaultsOverrides });
}
- public notifyConfigurationSchemaUpdated(...configurations: IConfigurationNode[]) {
+ public notifyConfigurationSchemaUpdated(...configurations: PConfigurationNode[]) {
this._onDidSchemaChange.fire();
}
@@ -605,7 +605,7 @@ class ConfigurationRegistry implements IConfigurationRegistry {
this.updateOverridePropertyPatternKey();
}
- private doRegisterConfigurations(configurations: IConfigurationNode[], validate: boolean, bucket: Set<string>): void {
+ private doRegisterConfigurations(configurations: PConfigurationNode[], validate: boolean, bucket: Set<string>): void {
configurations.forEach(configuration => {
@@ -616,9 +616,9 @@ class ConfigurationRegistry implements IConfigurationRegistry {
});
}
- private doDeregisterConfigurations(configurations: IConfigurationNode[], bucket: Set<string>): void {
+ private doDeregisterConfigurations(configurations: PConfigurationNode[], bucket: Set<string>): void {
- const deregisterConfiguration = (configuration: IConfigurationNode) => {
+ const deregisterConfiguration = (configuration: PConfigurationNode) => {
if (configuration.properties) {
for (const key in configuration.properties) {
bucket.add(key);
@@ -641,12 +641,12 @@ class ConfigurationRegistry implements IConfigurationRegistry {
}
}
- private validateAndRegisterProperties(configuration: IConfigurationNode, validate: boolean = true, extensionInfo: IExtensionInfo | undefined, restrictedProperties: string[] | undefined, scope: ConfigurationScope = ConfigurationScope.WINDOW, bucket: Set<string>): void {
+ private validateAndRegisterProperties(configuration: PConfigurationNode, validate: boolean = true, extensionInfo: IExtensionInfo | undefined, restrictedProperties: string[] | undefined, scope: ConfigurationScope = ConfigurationScope.WINDOW, bucket: Set<string>): void {
scope = types.isUndefinedOrNull(configuration.scope) ? scope : configuration.scope;
const properties = configuration.properties;
if (properties) {
for (const key in properties) {
- const property: IRegisteredConfigurationPropertySchema = properties[key];
+ const property: PRegisteredConfigurationPropertySchema = properties[key];
if (validate && validateProperty(key, property)) {
delete properties[key];
continue;
@@ -696,7 +696,7 @@ class ConfigurationRegistry implements IConfigurationRegistry {
}
// TODO: @sandy081 - Remove this method and include required info in getConfigurationProperties
- getConfigurations(): IConfigurationNode[] {
+ getConfigurations(): PConfigurationNode[] {
return this.configurationContributors;
}
@@ -712,12 +712,12 @@ class ConfigurationRegistry implements IConfigurationRegistry {
return this.excludedConfigurationProperties;
}
- getRegisteredDefaultConfigurations(): IConfigurationDefaults[] {
+ getRegisteredDefaultConfigurations(): PConfigurationDefaults[] {
return [...this.registeredConfigurationDefaults];
}
- getConfigurationDefaultsOverrides(): Map<string, IConfigurationDefaultOverrideValue> {
- const configurationDefaultsOverrides = new Map<string, IConfigurationDefaultOverrideValue>();
+ getConfigurationDefaultsOverrides(): Map<string, PConfigurationDefaultOverrideValue> {
+ const configurationDefaultsOverrides = new Map<string, PConfigurationDefaultOverrideValue>();
for (const [key, value] of this.configurationDefaultsOverrides) {
if (value.configurationDefaultOverrideValue) {
configurationDefaultsOverrides.set(key, value.configurationDefaultOverrideValue);
@@ -726,8 +726,8 @@ class ConfigurationRegistry implements IConfigurationRegistry {
return configurationDefaultsOverrides;
}
- private registerJSONConfiguration(configuration: IConfigurationNode) {
- const register = (configuration: IConfigurationNode) => {
+ private registerJSONConfiguration(configuration: PConfigurationNode) {
+ const register = (configuration: PConfigurationNode) => {
const properties = configuration.properties;
if (properties) {
for (const key in properties) {
@@ -740,7 +740,7 @@ class ConfigurationRegistry implements IConfigurationRegistry {
register(configuration);
}
- private updateSchema(key: string, property: IConfigurationPropertySchema): void {
+ private updateSchema(key: string, property: PConfigurationPropertySchema): void {
allSettings.properties[key] = property;
switch (property.scope) {
case ConfigurationScope.APPLICATION:
@@ -768,7 +768,7 @@ class ConfigurationRegistry implements IConfigurationRegistry {
}
}
- private removeFromSchema(key: string, property: IConfigurationPropertySchema): void {
+ private removeFromSchema(key: string, property: PConfigurationPropertySchema): void {
delete allSettings.properties[key];
switch (property.scope) {
case ConfigurationScope.APPLICATION:
@@ -831,7 +831,7 @@ class ConfigurationRegistry implements IConfigurationRegistry {
this._onDidSchemaChange.fire();
}
- private updatePropertyDefaultValue(key: string, property: IRegisteredConfigurationPropertySchema): void {
+ private updatePropertyDefaultValue(key: string, property: PRegisteredConfigurationPropertySchema): void {
const configurationdefaultOverride = this.configurationDefaultsOverrides.get(key)?.configurationDefaultOverrideValue;
let defaultValue = undefined;
let defaultSource = undefined;
@@ -899,7 +899,7 @@ export function getDefaultValue(type: string | string[] | undefined) {
const configurationRegistry = new ConfigurationRegistry();
Registry.add(Extensions.Configuration, configurationRegistry);
-export function validateProperty(property: string, schema: IRegisteredConfigurationPropertySchema): string | null {
+export function validateProperty(property: string, schema: PRegisteredConfigurationPropertySchema): string | null {
if (!property.trim()) {
return nls.localize('config.property.empty', "Cannot register an empty property");
}
@@ -926,8 +926,8 @@ export function getScopes(): [string, ConfigurationScope | undefined][] {
return scopes;
}
-export function getAllConfigurationProperties(configurationNode: IConfigurationNode[]): IStringDictionary<IRegisteredConfigurationPropertySchema> {
- const result: IStringDictionary<IRegisteredConfigurationPropertySchema> = {};
+export function getAllConfigurationProperties(configurationNode: PConfigurationNode[]): IStringDictionary<PRegisteredConfigurationPropertySchema> {
+ const result: IStringDictionary<PRegisteredConfigurationPropertySchema> = {};
for (const configuration of configurationNode) {
const properties = configuration.properties;
if (types.isObject(properties)) {

960
crates/goose-bench/src/assets/vscode_config_registry.ts
View file

@ -1,960 +0,0 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import { distinct } from '../../../base/common/arrays.js';
import { IStringDictionary } from '../../../base/common/collections.js';
import { Emitter, Event } from '../../../base/common/event.js';
import { IJSONSchema } from '../../../base/common/jsonSchema.js';
import * as types from '../../../base/common/types.js';
import * as nls from '../../../nls.js';
import { getLanguageTagSettingPlainKey } from './configuration.js';
import { Extensions as JSONExtensions, IJSONContributionRegistry } from '../../jsonschemas/common/jsonContributionRegistry.js';
import { PolicyName } from '../../policy/common/policy.js';
import { Registry } from '../../registry/common/platform.js';
export enum EditPresentationTypes {
Multiline = 'multilineText',
Singleline = 'singlelineText'
}
export const Extensions = {
Configuration: 'base.contributions.configuration'
};
export interface IConfigurationDelta {
removedDefaults?: IConfigurationDefaults[];
removedConfigurations?: IConfigurationNode[];
addedDefaults?: IConfigurationDefaults[];
addedConfigurations?: IConfigurationNode[];
}
export interface IConfigurationRegistry {
/**
* Register a configuration to the registry.
*/
registerConfiguration(configuration: IConfigurationNode): void;
/**
* Register multiple configurations to the registry.
*/
registerConfigurations(configurations: IConfigurationNode[], validate?: boolean): void;
/**
* Deregister multiple configurations from the registry.
*/
deregisterConfigurations(configurations: IConfigurationNode[]): void;
/**
* update the configuration registry by
* - registering the configurations to add
* - dereigstering the configurations to remove
*/
updateConfigurations(configurations: { add: IConfigurationNode[]; remove: IConfigurationNode[] }): void;
/**
* Register multiple default configurations to the registry.
*/
registerDefaultConfigurations(defaultConfigurations: IConfigurationDefaults[]): void;
/**
* Deregister multiple default configurations from the registry.
*/
deregisterDefaultConfigurations(defaultConfigurations: IConfigurationDefaults[]): void;
/**
* Bulk update of the configuration registry (default and configurations, remove and add)
* @param delta
*/
deltaConfiguration(delta: IConfigurationDelta): void;
/**
* Return the registered default configurations
*/
getRegisteredDefaultConfigurations(): IConfigurationDefaults[];
/**
* Return the registered configuration defaults overrides
*/
getConfigurationDefaultsOverrides(): Map<string, IConfigurationDefaultOverrideValue>;
/**
* Signal that the schema of a configuration setting has changes. It is currently only supported to change enumeration values.
* Property or default value changes are not allowed.
*/
notifyConfigurationSchemaUpdated(...configurations: IConfigurationNode[]): void;
/**
* Event that fires whenever a configuration has been
* registered.
*/
readonly onDidSchemaChange: Event<void>;
/**
* Event that fires whenever a configuration has been
* registered.
*/
readonly onDidUpdateConfiguration: Event<{ properties: ReadonlySet<string>; defaultsOverrides?: boolean }>;
/**
* Returns all configuration nodes contributed to this registry.
*/
getConfigurations(): IConfigurationNode[];
/**
* Returns all configurations settings of all configuration nodes contributed to this registry.
*/
getConfigurationProperties(): IStringDictionary<IRegisteredConfigurationPropertySchema>;
/**
* Return all configurations by policy name
*/
getPolicyConfigurations(): Map<PolicyName, string>;
/**
* Returns all excluded configurations settings of all configuration nodes contributed to this registry.
*/
getExcludedConfigurationProperties(): IStringDictionary<IRegisteredConfigurationPropertySchema>;
/**
* Register the identifiers for editor configurations
*/
registerOverrideIdentifiers(identifiers: string[]): void;
}
export const enum ConfigurationScope {
/**
* Application specific configuration, which can be configured only in default profile user settings.
*/
APPLICATION = 1,
/**
* Machine specific configuration, which can be configured only in local and remote user settings.
*/
MACHINE,
/**
* An application machine specific configuration, which can be configured only in default profile user settings and remote user settings.
*/
APPLICATION_MACHINE,
/**
* Window specific configuration, which can be configured in the user or workspace settings.
*/
WINDOW,
/**
* Resource specific configuration, which can be configured in the user, workspace or folder settings.
*/
RESOURCE,
/**
* Resource specific configuration that can be configured in language specific settings
*/
LANGUAGE_OVERRIDABLE,
/**
* Machine specific configuration that can also be configured in workspace or folder settings.
*/
MACHINE_OVERRIDABLE,
}
export interface IPolicy {
/**
* The policy name.
*/
readonly name: PolicyName;
/**
* The Code version in which this policy was introduced.
*/
readonly minimumVersion: `${number}.${number}`;
}
export interface IConfigurationPropertySchema extends IJSONSchema {
scope?: ConfigurationScope;
/**
* When restricted, value of this configuration will be read only from trusted sources.
* For eg., If the workspace is not trusted, then the value of this configuration is not read from workspace settings file.
*/
restricted?: boolean;
/**
* When `false` this property is excluded from the registry. Default is to include.
*/
included?: boolean;
/**
* List of tags associated to the property.
* - A tag can be used for filtering
* - Use `experimental` tag for marking the setting as experimental.
* - Use `onExP` tag for marking that the default of the setting can be changed by running experiments.
*/
tags?: string[];
/**
* When enabled this setting is ignored during sync and user can override this.
*/
ignoreSync?: boolean;
/**
* When enabled this setting is ignored during sync and user cannot override this.
*/
disallowSyncIgnore?: boolean;
/**
* Disallow extensions to contribute configuration default value for this setting.
*/
disallowConfigurationDefault?: boolean;
/**
* Labels for enumeration items
*/
enumItemLabels?: string[];
/**
* When specified, controls the presentation format of string settings.
* Otherwise, the presentation format defaults to `singleline`.
*/
editPresentation?: EditPresentationTypes;
/**
* When specified, gives an order number for the setting
* within the settings editor. Otherwise, the setting is placed at the end.
*/
order?: number;
/**
* When specified, this setting's value can always be overwritten by
* a system-wide policy.
*/
policy?: IPolicy;
}
export interface IExtensionInfo {
id: string;
displayName?: string;
}
export interface IConfigurationNode {
id?: string;
order?: number;
type?: string | string[];
title?: string;
description?: string;
properties?: IStringDictionary<IConfigurationPropertySchema>;
allOf?: IConfigurationNode[];
scope?: ConfigurationScope;
extensionInfo?: IExtensionInfo;
restrictedProperties?: string[];
}
export type ConfigurationDefaultValueSource = IExtensionInfo | Map<string, IExtensionInfo>;
export interface IConfigurationDefaults {
overrides: IStringDictionary<any>;
source?: IExtensionInfo;
}
export type IRegisteredConfigurationPropertySchema = IConfigurationPropertySchema & {
defaultDefaultValue?: any;
source?: IExtensionInfo; // Source of the Property
defaultValueSource?: ConfigurationDefaultValueSource; // Source of the Default Value
};
export interface IConfigurationDefaultOverride {
readonly value: any;
readonly source?: IExtensionInfo; // Source of the default override
}
export interface IConfigurationDefaultOverrideValue {
readonly value: any;
readonly source?: ConfigurationDefaultValueSource;
}
export const allSettings: { properties: IStringDictionary<IConfigurationPropertySchema>; patternProperties: IStringDictionary<IConfigurationPropertySchema> } = { properties: {}, patternProperties: {} };
export const applicationSettings: { properties: IStringDictionary<IConfigurationPropertySchema>; patternProperties: IStringDictionary<IConfigurationPropertySchema> } = { properties: {}, patternProperties: {} };
export const applicationMachineSettings: { properties: IStringDictionary<IConfigurationPropertySchema>; patternProperties: IStringDictionary<IConfigurationPropertySchema> } = { properties: {}, patternProperties: {} };
export const machineSettings: { properties: IStringDictionary<IConfigurationPropertySchema>; patternProperties: IStringDictionary<IConfigurationPropertySchema> } = { properties: {}, patternProperties: {} };
export const machineOverridableSettings: { properties: IStringDictionary<IConfigurationPropertySchema>; patternProperties: IStringDictionary<IConfigurationPropertySchema> } = { properties: {}, patternProperties: {} };
export const windowSettings: { properties: IStringDictionary<IConfigurationPropertySchema>; patternProperties: IStringDictionary<IConfigurationPropertySchema> } = { properties: {}, patternProperties: {} };
export const resourceSettings: { properties: IStringDictionary<IConfigurationPropertySchema>; patternProperties: IStringDictionary<IConfigurationPropertySchema> } = { properties: {}, patternProperties: {} };
export const resourceLanguageSettingsSchemaId = 'vscode://schemas/settings/resourceLanguage';
export const configurationDefaultsSchemaId = 'vscode://schemas/settings/configurationDefaults';
const contributionRegistry = Registry.as<IJSONContributionRegistry>(JSONExtensions.JSONContribution);
class ConfigurationRegistry implements IConfigurationRegistry {
private readonly registeredConfigurationDefaults: IConfigurationDefaults[] = [];
private readonly configurationDefaultsOverrides: Map<string, { configurationDefaultOverrides: IConfigurationDefaultOverride[]; configurationDefaultOverrideValue?: IConfigurationDefaultOverrideValue }>;
private readonly defaultLanguageConfigurationOverridesNode: IConfigurationNode;
private readonly configurationContributors: IConfigurationNode[];
private readonly configurationProperties: IStringDictionary<IRegisteredConfigurationPropertySchema>;
private readonly policyConfigurations: Map<PolicyName, string>;
private readonly excludedConfigurationProperties: IStringDictionary<IRegisteredConfigurationPropertySchema>;
private readonly resourceLanguageSettingsSchema: IJSONSchema;
private readonly overrideIdentifiers = new Set<string>();
private readonly _onDidSchemaChange = new Emitter<void>();
readonly onDidSchemaChange: Event<void> = this._onDidSchemaChange.event;
private readonly _onDidUpdateConfiguration = new Emitter<{ properties: ReadonlySet<string>; defaultsOverrides?: boolean }>();
readonly onDidUpdateConfiguration = this._onDidUpdateConfiguration.event;
constructor() {
this.configurationDefaultsOverrides = new Map();
this.defaultLanguageConfigurationOverridesNode = {
id: 'defaultOverrides',
title: nls.localize('defaultLanguageConfigurationOverrides.title', "Default Language Configuration Overrides"),
properties: {}
};
this.configurationContributors = [this.defaultLanguageConfigurationOverridesNode];
this.resourceLanguageSettingsSchema = {
properties: {},
patternProperties: {},
additionalProperties: true,
allowTrailingCommas: true,
allowComments: true
};
this.configurationProperties = {};
this.policyConfigurations = new Map<PolicyName, string>();
this.excludedConfigurationProperties = {};
contributionRegistry.registerSchema(resourceLanguageSettingsSchemaId, this.resourceLanguageSettingsSchema);
this.registerOverridePropertyPatternKey();
}
public registerConfiguration(configuration: IConfigurationNode, validate: boolean = true): void {
this.registerConfigurations([configuration], validate);
}
public registerConfigurations(configurations: IConfigurationNode[], validate: boolean = true): void {
const properties = new Set<string>();
this.doRegisterConfigurations(configurations, validate, properties);
contributionRegistry.registerSchema(resourceLanguageSettingsSchemaId, this.resourceLanguageSettingsSchema);
this._onDidSchemaChange.fire();
this._onDidUpdateConfiguration.fire({ properties });
}
public deregisterConfigurations(configurations: IConfigurationNode[]): void {
const properties = new Set<string>();
this.doDeregisterConfigurations(configurations, properties);
contributionRegistry.registerSchema(resourceLanguageSettingsSchemaId, this.resourceLanguageSettingsSchema);
this._onDidSchemaChange.fire();
this._onDidUpdateConfiguration.fire({ properties });
}
public updateConfigurations({ add, remove }: { add: IConfigurationNode[]; remove: IConfigurationNode[] }): void {
const properties = new Set<string>();
this.doDeregisterConfigurations(remove, properties);
this.doRegisterConfigurations(add, false, properties);
contributionRegistry.registerSchema(resourceLanguageSettingsSchemaId, this.resourceLanguageSettingsSchema);
this._onDidSchemaChange.fire();
this._onDidUpdateConfiguration.fire({ properties });
}
public registerDefaultConfigurations(configurationDefaults: IConfigurationDefaults[]): void {
const properties = new Set<string>();
this.doRegisterDefaultConfigurations(configurationDefaults, properties);
this._onDidSchemaChange.fire();
this._onDidUpdateConfiguration.fire({ properties, defaultsOverrides: true });
}
private doRegisterDefaultConfigurations(configurationDefaults: IConfigurationDefaults[], bucket: Set<string>) {
this.registeredConfigurationDefaults.push(...configurationDefaults);
const overrideIdentifiers: string[] = [];
for (const { overrides, source } of configurationDefaults) {
for (const key in overrides) {
bucket.add(key);
const configurationDefaultOverridesForKey = this.configurationDefaultsOverrides.get(key)
?? this.configurationDefaultsOverrides.set(key, { configurationDefaultOverrides: [] }).get(key)!;
const value = overrides[key];
configurationDefaultOverridesForKey.configurationDefaultOverrides.push({ value, source });
// Configuration defaults for Override Identifiers
if (OVERRIDE_PROPERTY_REGEX.test(key)) {
const newDefaultOverride = this.mergeDefaultConfigurationsForOverrideIdentifier(key, value, source, configurationDefaultOverridesForKey.configurationDefaultOverrideValue);
if (!newDefaultOverride) {
continue;
}
configurationDefaultOverridesForKey.configurationDefaultOverrideValue = newDefaultOverride;
this.updateDefaultOverrideProperty(key, newDefaultOverride, source);
overrideIdentifiers.push(...overrideIdentifiersFromKey(key));
}
// Configuration defaults for Configuration Properties
else {
const newDefaultOverride = this.mergeDefaultConfigurationsForConfigurationProperty(key, value, source, configurationDefaultOverridesForKey.configurationDefaultOverrideValue);
if (!newDefaultOverride) {
continue;
}
configurationDefaultOverridesForKey.configurationDefaultOverrideValue = newDefaultOverride;
const property = this.configurationProperties[key];
if (property) {
this.updatePropertyDefaultValue(key, property);
this.updateSchema(key, property);
}
}
}
}
this.doRegisterOverrideIdentifiers(overrideIdentifiers);
}
public deregisterDefaultConfigurations(defaultConfigurations: IConfigurationDefaults[]): void {
const properties = new Set<string>();
this.doDeregisterDefaultConfigurations(defaultConfigurations, properties);
this._onDidSchemaChange.fire();
this._onDidUpdateConfiguration.fire({ properties, defaultsOverrides: true });
}
private doDeregisterDefaultConfigurations(defaultConfigurations: IConfigurationDefaults[], bucket: Set<string>): void {
for (const defaultConfiguration of defaultConfigurations) {
const index = this.registeredConfigurationDefaults.indexOf(defaultConfiguration);
if (index !== -1) {
this.registeredConfigurationDefaults.splice(index, 1);
}
}
for (const { overrides, source } of defaultConfigurations) {
for (const key in overrides) {
const configurationDefaultOverridesForKey = this.configurationDefaultsOverrides.get(key);
if (!configurationDefaultOverridesForKey) {
continue;
}
const index = configurationDefaultOverridesForKey.configurationDefaultOverrides
.findIndex(configurationDefaultOverride => source ? configurationDefaultOverride.source?.id === source.id : configurationDefaultOverride.value === overrides[key]);
if (index === -1) {
continue;
}
configurationDefaultOverridesForKey.configurationDefaultOverrides.splice(index, 1);
if (configurationDefaultOverridesForKey.configurationDefaultOverrides.length === 0) {
this.configurationDefaultsOverrides.delete(key);
}
if (OVERRIDE_PROPERTY_REGEX.test(key)) {
let configurationDefaultOverrideValue: IConfigurationDefaultOverrideValue | undefined;
for (const configurationDefaultOverride of configurationDefaultOverridesForKey.configurationDefaultOverrides) {
configurationDefaultOverrideValue = this.mergeDefaultConfigurationsForOverrideIdentifier(key, configurationDefaultOverride.value, configurationDefaultOverride.source, configurationDefaultOverrideValue);
}
if (configurationDefaultOverrideValue && !types.isEmptyObject(configurationDefaultOverrideValue.value)) {
configurationDefaultOverridesForKey.configurationDefaultOverrideValue = configurationDefaultOverrideValue;
this.updateDefaultOverrideProperty(key, configurationDefaultOverrideValue, source);
} else {
this.configurationDefaultsOverrides.delete(key);
delete this.configurationProperties[key];
delete this.defaultLanguageConfigurationOverridesNode.properties![key];
}
} else {
let configurationDefaultOverrideValue: IConfigurationDefaultOverrideValue | undefined;
for (const configurationDefaultOverride of configurationDefaultOverridesForKey.configurationDefaultOverrides) {
configurationDefaultOverrideValue = this.mergeDefaultConfigurationsForConfigurationProperty(key, configurationDefaultOverride.value, configurationDefaultOverride.source, configurationDefaultOverrideValue);
}
configurationDefaultOverridesForKey.configurationDefaultOverrideValue = configurationDefaultOverrideValue;
const property = this.configurationProperties[key];
if (property) {
this.updatePropertyDefaultValue(key, property);
this.updateSchema(key, property);
}
}
bucket.add(key);
}
}
this.updateOverridePropertyPatternKey();
}
private updateDefaultOverrideProperty(key: string, newDefaultOverride: IConfigurationDefaultOverrideValue, source: IExtensionInfo | undefined): void {
const property: IRegisteredConfigurationPropertySchema = {
type: 'object',
default: newDefaultOverride.value,
description: nls.localize('defaultLanguageConfiguration.description', "Configure settings to be overridden for the {0} language.", getLanguageTagSettingPlainKey(key)),
$ref: resourceLanguageSettingsSchemaId,
defaultDefaultValue: newDefaultOverride.value,
source,
defaultValueSource: source
};
this.configurationProperties[key] = property;
this.defaultLanguageConfigurationOverridesNode.properties![key] = property;
}
private mergeDefaultConfigurationsForOverrideIdentifier(overrideIdentifier: string, configurationValueObject: IStringDictionary<any>, valueSource: IExtensionInfo | undefined, existingDefaultOverride: IConfigurationDefaultOverrideValue | undefined): IConfigurationDefaultOverrideValue | undefined {
const defaultValue = existingDefaultOverride?.value || {};
const source = existingDefaultOverride?.source ?? new Map<string, IExtensionInfo>();
// This should not happen
if (!(source instanceof Map)) {
console.error('objectConfigurationSources is not a Map');
return undefined;
}
for (const propertyKey of Object.keys(configurationValueObject)) {
const propertyDefaultValue = configurationValueObject[propertyKey];
const isObjectSetting = types.isObject(propertyDefaultValue) &&
(types.isUndefined(defaultValue[propertyKey]) || types.isObject(defaultValue[propertyKey]));
// If the default value is an object, merge the objects and store the source of each keys
if (isObjectSetting) {
defaultValue[propertyKey] = { ...(defaultValue[propertyKey] ?? {}), ...propertyDefaultValue };
// Track the source of each value in the object
if (valueSource) {
for (const objectKey in propertyDefaultValue) {
source.set(`${propertyKey}.${objectKey}`, valueSource);
}
}
}
// Primitive values are overridden
else {
defaultValue[propertyKey] = propertyDefaultValue;
if (valueSource) {
source.set(propertyKey, valueSource);
} else {
source.delete(propertyKey);
}
}
}
return { value: defaultValue, source };
}
private mergeDefaultConfigurationsForConfigurationProperty(propertyKey: string, value: any, valuesSource: IExtensionInfo | undefined, existingDefaultOverride: IConfigurationDefaultOverrideValue | undefined): IConfigurationDefaultOverrideValue | undefined {
const property = this.configurationProperties[propertyKey];
const existingDefaultValue = existingDefaultOverride?.value ?? property?.defaultDefaultValue;
let source: ConfigurationDefaultValueSource | undefined = valuesSource;
const isObjectSetting = types.isObject(value) &&
(
property !== undefined && property.type === 'object' ||
property === undefined && (types.isUndefined(existingDefaultValue) || types.isObject(existingDefaultValue))
);
// If the default value is an object, merge the objects and store the source of each keys
if (isObjectSetting) {
source = existingDefaultOverride?.source ?? new Map<string, IExtensionInfo>();
// This should not happen
if (!(source instanceof Map)) {
console.error('defaultValueSource is not a Map');
return undefined;
}
for (const objectKey in value) {
if (valuesSource) {
source.set(`${propertyKey}.${objectKey}`, valuesSource);
}
}
value = { ...(types.isObject(existingDefaultValue) ? existingDefaultValue : {}), ...value };
}
return { value, source };
}
public deltaConfiguration(delta: IConfigurationDelta): void {
// defaults: remove
let defaultsOverrides = false;
const properties = new Set<string>();
if (delta.removedDefaults) {
this.doDeregisterDefaultConfigurations(delta.removedDefaults, properties);
defaultsOverrides = true;
}
// defaults: add
if (delta.addedDefaults) {
this.doRegisterDefaultConfigurations(delta.addedDefaults, properties);
defaultsOverrides = true;
}
// configurations: remove
if (delta.removedConfigurations) {
this.doDeregisterConfigurations(delta.removedConfigurations, properties);
}
// configurations: add
if (delta.addedConfigurations) {
this.doRegisterConfigurations(delta.addedConfigurations, false, properties);
}
this._onDidSchemaChange.fire();
this._onDidUpdateConfiguration.fire({ properties, defaultsOverrides });
}
public notifyConfigurationSchemaUpdated(...configurations: IConfigurationNode[]) {
this._onDidSchemaChange.fire();
}
public registerOverrideIdentifiers(overrideIdentifiers: string[]): void {
this.doRegisterOverrideIdentifiers(overrideIdentifiers);
this._onDidSchemaChange.fire();
}
private doRegisterOverrideIdentifiers(overrideIdentifiers: string[]) {
for (const overrideIdentifier of overrideIdentifiers) {
this.overrideIdentifiers.add(overrideIdentifier);
}
this.updateOverridePropertyPatternKey();
}
private doRegisterConfigurations(configurations: IConfigurationNode[], validate: boolean, bucket: Set<string>): void {
configurations.forEach(configuration => {
this.validateAndRegisterProperties(configuration, validate, configuration.extensionInfo, configuration.restrictedProperties, undefined, bucket);
this.configurationContributors.push(configuration);
this.registerJSONConfiguration(configuration);
});
}
private doDeregisterConfigurations(configurations: IConfigurationNode[], bucket: Set<string>): void {
const deregisterConfiguration = (configuration: IConfigurationNode) => {
if (configuration.properties) {
for (const key in configuration.properties) {
bucket.add(key);
const property = this.configurationProperties[key];
if (property?.policy?.name) {
this.policyConfigurations.delete(property.policy.name);
}
delete this.configurationProperties[key];
this.removeFromSchema(key, configuration.properties[key]);
}
}
configuration.allOf?.forEach(node => deregisterConfiguration(node));
};
for (const configuration of configurations) {
deregisterConfiguration(configuration);
const index = this.configurationContributors.indexOf(configuration);
if (index !== -1) {
this.configurationContributors.splice(index, 1);
}
}
}
private validateAndRegisterProperties(configuration: IConfigurationNode, validate: boolean = true, extensionInfo: IExtensionInfo | undefined, restrictedProperties: string[] | undefined, scope: ConfigurationScope = ConfigurationScope.WINDOW, bucket: Set<string>): void {
scope = types.isUndefinedOrNull(configuration.scope) ? scope : configuration.scope;
const properties = configuration.properties;
if (properties) {
for (const key in properties) {
const property: IRegisteredConfigurationPropertySchema = properties[key];
if (validate && validateProperty(key, property)) {
delete properties[key];
continue;
}
property.source = extensionInfo;
// update default value
property.defaultDefaultValue = properties[key].default;
this.updatePropertyDefaultValue(key, property);
// update scope
if (OVERRIDE_PROPERTY_REGEX.test(key)) {
property.scope = undefined; // No scope for overridable properties `[${identifier}]`
} else {
property.scope = types.isUndefinedOrNull(property.scope) ? scope : property.scope;
property.restricted = types.isUndefinedOrNull(property.restricted) ? !!restrictedProperties?.includes(key) : property.restricted;
}
// Add to properties maps
// Property is included by default if 'included' is unspecified
if (properties[key].hasOwnProperty('included') && !properties[key].included) {
this.excludedConfigurationProperties[key] = properties[key];
delete properties[key];
continue;
} else {
this.configurationProperties[key] = properties[key];
if (properties[key].policy?.name) {
this.policyConfigurations.set(properties[key].policy!.name, key);
}
}
if (!properties[key].deprecationMessage && properties[key].markdownDeprecationMessage) {
// If not set, default deprecationMessage to the markdown source
properties[key].deprecationMessage = properties[key].markdownDeprecationMessage;
}
bucket.add(key);
}
}
const subNodes = configuration.allOf;
if (subNodes) {
for (const node of subNodes) {
this.validateAndRegisterProperties(node, validate, extensionInfo, restrictedProperties, scope, bucket);
}
}
}
// TODO: @sandy081 - Remove this method and include required info in getConfigurationProperties
getConfigurations(): IConfigurationNode[] {
return this.configurationContributors;
}
getConfigurationProperties(): IStringDictionary<IRegisteredConfigurationPropertySchema> {
return this.configurationProperties;
}
getPolicyConfigurations(): Map<PolicyName, string> {
return this.policyConfigurations;
}
getExcludedConfigurationProperties(): IStringDictionary<IRegisteredConfigurationPropertySchema> {
return this.excludedConfigurationProperties;
}
getRegisteredDefaultConfigurations(): IConfigurationDefaults[] {
return [...this.registeredConfigurationDefaults];
}
getConfigurationDefaultsOverrides(): Map<string, IConfigurationDefaultOverrideValue> {
const configurationDefaultsOverrides = new Map<string, IConfigurationDefaultOverrideValue>();
for (const [key, value] of this.configurationDefaultsOverrides) {
if (value.configurationDefaultOverrideValue) {
configurationDefaultsOverrides.set(key, value.configurationDefaultOverrideValue);
}
}
return configurationDefaultsOverrides;
}
private registerJSONConfiguration(configuration: IConfigurationNode) {
const register = (configuration: IConfigurationNode) => {
const properties = configuration.properties;
if (properties) {
for (const key in properties) {
this.updateSchema(key, properties[key]);
}
}
const subNodes = configuration.allOf;
subNodes?.forEach(register);
};
register(configuration);
}
private updateSchema(key: string, property: IConfigurationPropertySchema): void {
allSettings.properties[key] = property;
switch (property.scope) {
case ConfigurationScope.APPLICATION:
applicationSettings.properties[key] = property;
break;
case ConfigurationScope.MACHINE:
machineSettings.properties[key] = property;
break;
case ConfigurationScope.APPLICATION_MACHINE:
applicationMachineSettings.properties[key] = property;
break;
case ConfigurationScope.MACHINE_OVERRIDABLE:
machineOverridableSettings.properties[key] = property;
break;
case ConfigurationScope.WINDOW:
windowSettings.properties[key] = property;
break;
case ConfigurationScope.RESOURCE:
resourceSettings.properties[key] = property;
break;
case ConfigurationScope.LANGUAGE_OVERRIDABLE:
resourceSettings.properties[key] = property;
this.resourceLanguageSettingsSchema.properties![key] = property;
break;
}
}
private removeFromSchema(key: string, property: IConfigurationPropertySchema): void {
delete allSettings.properties[key];
switch (property.scope) {
case ConfigurationScope.APPLICATION:
delete applicationSettings.properties[key];
break;
case ConfigurationScope.MACHINE:
delete machineSettings.properties[key];
break;
case ConfigurationScope.APPLICATION_MACHINE:
delete applicationMachineSettings.properties[key];
break;
case ConfigurationScope.MACHINE_OVERRIDABLE:
delete machineOverridableSettings.properties[key];
break;
case ConfigurationScope.WINDOW:
delete windowSettings.properties[key];
break;
case ConfigurationScope.RESOURCE:
case ConfigurationScope.LANGUAGE_OVERRIDABLE:
delete resourceSettings.properties[key];
delete this.resourceLanguageSettingsSchema.properties![key];
break;
}
}
private updateOverridePropertyPatternKey(): void {
for (const overrideIdentifier of this.overrideIdentifiers.values()) {
const overrideIdentifierProperty = `[${overrideIdentifier}]`;
const resourceLanguagePropertiesSchema: IJSONSchema = {
type: 'object',
description: nls.localize('overrideSettings.defaultDescription', "Configure editor settings to be overridden for a language."),
errorMessage: nls.localize('overrideSettings.errorMessage', "This setting does not support per-language configuration."),
$ref: resourceLanguageSettingsSchemaId,
};
this.updatePropertyDefaultValue(overrideIdentifierProperty, resourceLanguagePropertiesSchema);
allSettings.properties[overrideIdentifierProperty] = resourceLanguagePropertiesSchema;
applicationSettings.properties[overrideIdentifierProperty] = resourceLanguagePropertiesSchema;
applicationMachineSettings.properties[overrideIdentifierProperty] = resourceLanguagePropertiesSchema;
machineSettings.properties[overrideIdentifierProperty] = resourceLanguagePropertiesSchema;
machineOverridableSettings.properties[overrideIdentifierProperty] = resourceLanguagePropertiesSchema;
windowSettings.properties[overrideIdentifierProperty] = resourceLanguagePropertiesSchema;
resourceSettings.properties[overrideIdentifierProperty] = resourceLanguagePropertiesSchema;
}
}
private registerOverridePropertyPatternKey(): void {
const resourceLanguagePropertiesSchema: IJSONSchema = {
type: 'object',
description: nls.localize('overrideSettings.defaultDescription', "Configure editor settings to be overridden for a language."),
errorMessage: nls.localize('overrideSettings.errorMessage', "This setting does not support per-language configuration."),
$ref: resourceLanguageSettingsSchemaId,
};
allSettings.patternProperties[OVERRIDE_PROPERTY_PATTERN] = resourceLanguagePropertiesSchema;
applicationSettings.patternProperties[OVERRIDE_PROPERTY_PATTERN] = resourceLanguagePropertiesSchema;
applicationMachineSettings.patternProperties[OVERRIDE_PROPERTY_PATTERN] = resourceLanguagePropertiesSchema;
machineSettings.patternProperties[OVERRIDE_PROPERTY_PATTERN] = resourceLanguagePropertiesSchema;
machineOverridableSettings.patternProperties[OVERRIDE_PROPERTY_PATTERN] = resourceLanguagePropertiesSchema;
windowSettings.patternProperties[OVERRIDE_PROPERTY_PATTERN] = resourceLanguagePropertiesSchema;
resourceSettings.patternProperties[OVERRIDE_PROPERTY_PATTERN] = resourceLanguagePropertiesSchema;
this._onDidSchemaChange.fire();
}
private updatePropertyDefaultValue(key: string, property: IRegisteredConfigurationPropertySchema): void {
const configurationdefaultOverride = this.configurationDefaultsOverrides.get(key)?.configurationDefaultOverrideValue;
let defaultValue = undefined;
let defaultSource = undefined;
if (configurationdefaultOverride
&& (!property.disallowConfigurationDefault || !configurationdefaultOverride.source) // Prevent overriding the default value if the property is disallowed to be overridden by configuration defaults from extensions
) {
defaultValue = configurationdefaultOverride.value;
defaultSource = configurationdefaultOverride.source;
}
if (types.isUndefined(defaultValue)) {
defaultValue = property.defaultDefaultValue;
defaultSource = undefined;
}
if (types.isUndefined(defaultValue)) {
defaultValue = getDefaultValue(property.type);
}
property.default = defaultValue;
property.defaultValueSource = defaultSource;
}
}
const OVERRIDE_IDENTIFIER_PATTERN = `\\[([^\\]]+)\\]`;
const OVERRIDE_IDENTIFIER_REGEX = new RegExp(OVERRIDE_IDENTIFIER_PATTERN, 'g');
export const OVERRIDE_PROPERTY_PATTERN = `^(${OVERRIDE_IDENTIFIER_PATTERN})+$`;
export const OVERRIDE_PROPERTY_REGEX = new RegExp(OVERRIDE_PROPERTY_PATTERN);
export function overrideIdentifiersFromKey(key: string): string[] {
const identifiers: string[] = [];
if (OVERRIDE_PROPERTY_REGEX.test(key)) {
let matches = OVERRIDE_IDENTIFIER_REGEX.exec(key);
while (matches?.length) {
const identifier = matches[1].trim();
if (identifier) {
identifiers.push(identifier);
}
matches = OVERRIDE_IDENTIFIER_REGEX.exec(key);
}
}
return distinct(identifiers);
}
export function keyFromOverrideIdentifiers(overrideIdentifiers: string[]): string {
return overrideIdentifiers.reduce((result, overrideIdentifier) => `${result}[${overrideIdentifier}]`, '');
}
export function getDefaultValue(type: string | string[] | undefined) {
const t = Array.isArray(type) ? (<string[]>type)[0] : <string>type;
switch (t) {
case 'boolean':
return false;
case 'integer':
case 'number':
return 0;
case 'string':
return '';
case 'array':
return [];
case 'object':
return {};
default:
return null;
}
}
const configurationRegistry = new ConfigurationRegistry();
Registry.add(Extensions.Configuration, configurationRegistry);
export function validateProperty(property: string, schema: IRegisteredConfigurationPropertySchema): string | null {
if (!property.trim()) {
return nls.localize('config.property.empty', "Cannot register an empty property");
}
if (OVERRIDE_PROPERTY_REGEX.test(property)) {
return nls.localize('config.property.languageDefault', "Cannot register '{0}'. This matches property pattern '\\\\[.*\\\\]$' for describing language specific editor settings. Use 'configurationDefaults' contribution.", property);
}
if (configurationRegistry.getConfigurationProperties()[property] !== undefined) {
return nls.localize('config.property.duplicate', "Cannot register '{0}'. This property is already registered.", property);
}
if (schema.policy?.name && configurationRegistry.getPolicyConfigurations().get(schema.policy?.name) !== undefined) {
return nls.localize('config.policy.duplicate', "Cannot register '{0}'. The associated policy {1} is already registered with {2}.", property, schema.policy?.name, configurationRegistry.getPolicyConfigurations().get(schema.policy?.name));
}
return null;
}
export function getScopes(): [string, ConfigurationScope | undefined][] {
const scopes: [string, ConfigurationScope | undefined][] = [];
const configurationProperties = configurationRegistry.getConfigurationProperties();
for (const key of Object.keys(configurationProperties)) {
scopes.push([key, configurationProperties[key].scope]);
}
scopes.push(['launch', ConfigurationScope.RESOURCE]);
scopes.push(['task', ConfigurationScope.RESOURCE]);
return scopes;
}
export function getAllConfigurationProperties(configurationNode: IConfigurationNode[]): IStringDictionary<IRegisteredConfigurationPropertySchema> {
const result: IStringDictionary<IRegisteredConfigurationPropertySchema> = {};
for (const configuration of configurationNode) {
const properties = configuration.properties;
if (types.isObject(properties)) {
for (const key in properties) {
result[key] = properties[key];
}
}
if (configuration.allOf) {
Object.assign(result, getAllConfigurationProperties(configuration.allOf));
}
}
return result;
}
export function parseScope(scope: string): ConfigurationScope {
switch (scope) {
case 'application':
return ConfigurationScope.APPLICATION;
case 'machine':
return ConfigurationScope.MACHINE;
case 'resource':
return ConfigurationScope.RESOURCE;
case 'machine-overridable':
return ConfigurationScope.MACHINE_OVERRIDABLE;
case 'language-overridable':
return ConfigurationScope.LANGUAGE_OVERRIDABLE;
default:
return ConfigurationScope.WINDOW;
}
}

109
crates/goose-bench/src/bench_config.rs
View file

@ -1,109 +0,0 @@
use crate::bench_work_dir::BenchmarkWorkDir;
use serde::{Deserialize, Serialize};
use std::fs;
use std::fs::read_to_string;
use std::path::PathBuf;
#[derive(Clone, Serialize, Deserialize, Debug)]
pub struct BenchToolShimOpt {
pub use_tool_shim: bool,
pub tool_shim_model: Option<String>,
}
#[derive(Clone, Serialize, Deserialize, Debug)]
pub struct BenchModel {
pub provider: String,
pub name: String,
pub parallel_safe: bool,
pub tool_shim: Option<BenchToolShimOpt>,
}
#[derive(Clone, Serialize, Deserialize, Debug)]
pub struct BenchEval {
pub selector: String,
pub post_process_cmd: Option<PathBuf>,
pub parallel_safe: bool,
}
#[derive(Clone, Serialize, Deserialize, Debug)]
pub struct BenchRunConfig {
pub models: Vec<BenchModel>,
pub evals: Vec<BenchEval>,
pub include_dirs: Vec<PathBuf>,
pub repeat: Option<usize>,
pub run_id: Option<String>,
pub output_dir: Option<PathBuf>,
pub eval_result_filename: String,
pub run_summary_filename: String,
pub env_file: Option<PathBuf>,
}
impl Default for BenchRunConfig {
fn default() -> Self {
BenchRunConfig {
models: vec![
BenchModel {
provider: "databricks".to_string(),
name: "goose".to_string(),
parallel_safe: true,
tool_shim: Some(BenchToolShimOpt {
use_tool_shim: false,
tool_shim_model: None,
}),
},
BenchModel {
provider: "databricks".to_string(),
name: "goose-claude-4-sonnet".to_string(),
parallel_safe: true,
tool_shim: None,
},
],
evals: vec![BenchEval {
selector: "core".into(),
post_process_cmd: None,
parallel_safe: true, // Default to true
}],
include_dirs: vec![],
repeat: Some(2),
run_id: None,
output_dir: None,
eval_result_filename: "eval-results.json".to_string(),
run_summary_filename: "run-results-summary.json".to_string(),
env_file: None,
}
}
}
impl BenchRunConfig {
pub fn from_string(cfg: String) -> anyhow::Result<Self> {
let mut config: Self = serde_json::from_str(cfg.as_str())?;
// update include_dirs to contain full-paths only
config.include_dirs = BenchmarkWorkDir::canonical_dirs(config.include_dirs);
Self::canonicalize_eval_post_proc_cmd(&mut config);
Ok(config)
}
fn canonicalize_eval_post_proc_cmd(config: &mut BenchRunConfig) {
// update eval post-process script paths to all be full-paths
config.evals.iter_mut().for_each(|eval| {
if let Some(post_process_cmd) = &eval.post_process_cmd {
let canon = BenchmarkWorkDir::canonical_dirs(vec![post_process_cmd.clone()]);
let full_path_cmd = canon[0].clone();
if !full_path_cmd.exists() {
panic!("BenchConfigError: Eval post-process command not found. File {:?} does not exist", full_path_cmd);
}
eval.post_process_cmd = Some(full_path_cmd);
}
});
}
pub fn from(cfg: PathBuf) -> anyhow::Result<Self> {
let config = Self::from_string(read_to_string(cfg)?)?;
Ok(config)
}
pub fn to_string(&self) -> anyhow::Result<String> {
Ok(serde_json::to_string_pretty(self)?)
}
pub fn save(&self, name: String) {
let config = self.to_string().unwrap();
fs::write(name, config).expect("Unable to write bench config file");
}
}

59
crates/goose-bench/src/bench_session.rs
View file

@ -1,59 +0,0 @@
use async_trait::async_trait;
use chrono::{DateTime, Utc};
use goose::conversation::Conversation;
use goose::session::session_manager::Session;
use serde::{Deserialize, Serialize};
use std::sync::Arc;
use tokio::sync::Mutex;
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct BenchAgentError {
pub message: String,
pub level: String, // ERROR, WARN, etc.
pub timestamp: DateTime<Utc>,
}
// avoid tying benchmarking to current session-impl.
#[async_trait]
pub trait BenchBaseSession: Send + Sync {
async fn headless(&mut self, message: String) -> anyhow::Result<()>;
fn message_history(&self) -> Conversation;
fn get_total_token_usage(&self) -> anyhow::Result<Option<i32>>;
async fn get_session(&self) -> anyhow::Result<Session>;
}
// struct for managing agent-session-access. to be passed to evals for benchmarking
pub struct BenchAgent {
session: Box<dyn BenchBaseSession>,
errors: Arc<Mutex<Vec<BenchAgentError>>>,
}
impl BenchAgent {
pub fn new(session: Box<dyn BenchBaseSession>) -> Self {
let errors = Arc::new(Mutex::new(Vec::new()));
Self { session, errors }
}
pub(crate) async fn prompt(&mut self, p: String) -> anyhow::Result<Conversation> {
// Clear previous errors
{
let mut errors = self.errors.lock().await;
errors.clear();
}
self.session.headless(p).await?;
Ok(self.session.message_history())
}
pub async fn get_errors(&self) -> Vec<BenchAgentError> {
let errors = self.errors.lock().await;
errors.clone()
}
pub(crate) async fn get_token_usage(&self) -> Option<i32> {
self.session.get_total_token_usage().ok().flatten()
}
pub(crate) async fn get_session(&self) -> anyhow::Result<Session> {
self.session.get_session().await
}
}

223
crates/goose-bench/src/bench_work_dir.rs
View file

@ -1,223 +0,0 @@
use anyhow::Context;
use chrono::Local;
use include_dir::{include_dir, Dir};
use serde::{Deserialize, Serialize};
use std::fs;
use std::io;
use std::path::Path;
use std::path::PathBuf;
use std::process::Command;
pub static BUILTIN_EVAL_ASSETS: Dir = include_dir!("$CARGO_MANIFEST_DIR/src/assets");
#[derive(Clone, Serialize, Deserialize, Debug)]
pub struct BenchmarkWorkDir {
pub base_path: PathBuf,
pub run_dir: PathBuf,
pub cwd: PathBuf,
pub run_id: Option<String>,
}
impl Default for BenchmarkWorkDir {
fn default() -> Self {
Self::new("work_dir".to_string(), Vec::new())
}
}
impl BenchmarkWorkDir {
pub fn new(work_dir_name: String, include_dirs: Vec<PathBuf>) -> Self {
let run_dir = std::env::current_dir().unwrap().canonicalize().unwrap();
let base_path = PathBuf::from(format!("./{}", work_dir_name));
fs::create_dir_all(&base_path).unwrap();
let base_path = PathBuf::from(&base_path).canonicalize().unwrap();
// abs paths from dir-strings
let dirs = Self::canonical_dirs(include_dirs);
// deep copy each dir
let _: Vec<_> = dirs
.iter()
.map(|d| Self::deep_copy(d.as_path(), base_path.as_path(), true))
.collect();
Self::copy_auto_included_dirs(&base_path);
std::env::set_current_dir(&base_path).unwrap();
BenchmarkWorkDir {
base_path: base_path.clone(),
run_dir,
cwd: base_path.clone(),
run_id: None,
}
}
pub fn init_experiment(output_dir: PathBuf) -> anyhow::Result<()> {
if !output_dir.is_absolute() {
anyhow::bail!(
"Internal Error: init_experiment received a non-absolute path: {}",
output_dir.display()
);
}
// create experiment folder
let current_time = Local::now().format("%H:%M:%S").to_string();
let current_date = Local::now().format("%Y-%m-%d").to_string();
let exp_folder_name = format!("benchmark-{}-{}", &current_date, &current_time);
let base_path = output_dir.join(exp_folder_name);
fs::create_dir_all(&base_path).with_context(|| {
format!(
"Failed to create benchmark directory: {}",
base_path.display()
)
})?;
std::env::set_current_dir(&base_path).with_context(|| {
format!(
"Failed to change working directory to: {}",
base_path.display()
)
})?;
Ok(())
}
pub fn canonical_dirs(include_dirs: Vec<PathBuf>) -> Vec<PathBuf> {
include_dirs
.iter()
.map(|d| {
let canon = d.canonicalize();
if canon.is_err() {
eprintln!("{:?} can't be canonicalized", d);
panic!();
}
canon.unwrap()
})
.collect::<Vec<_>>()
}
fn copy_auto_included_dirs(dest: &Path) {
let mut assets_dest = dest.to_path_buf();
assets_dest.push("assets");
if !assets_dest.exists() {
fs::create_dir_all(&assets_dest).unwrap();
}
BUILTIN_EVAL_ASSETS.extract(assets_dest).unwrap();
}
pub fn cd(&mut self, path: PathBuf) -> anyhow::Result<&mut Self> {
fs::create_dir_all(&path)?;
std::env::set_current_dir(&path)?;
self.cwd = path;
Ok(self)
}
pub(crate) fn _run_dir(&mut self) -> Option<PathBuf> {
if let Some(run_id) = &self.run_id {
let mut eval_dir = self.base_path.clone();
eval_dir.push(run_id);
return Some(eval_dir);
}
None
}
pub fn set_eval(&mut self, eval: &str, run_id: String) {
self.run_id = Some(run_id.clone());
let eval = eval.replace(":", std::path::MAIN_SEPARATOR_STR);
let mut eval_dir = self.base_path.clone();
eval_dir.push(run_id);
eval_dir.push(eval);
self.cd(eval_dir.clone())
.unwrap_or_else(|_| panic!("Failed to execute cd into {}", eval_dir.clone().display()));
}
fn chop_relative_base<P: AsRef<Path>>(path: P) -> anyhow::Result<PathBuf> {
let path = path.as_ref();
// Get the path components as an iterator
let mut components = path.components();
// Check the first component
if let Some(first) = components.next() {
use std::path::Component;
match first {
Component::ParentDir => Err(anyhow::anyhow!("RelativePathBaseError: Only paths relative to the current working directory are supported.")),
// If first component is "."
Component::CurDir => Ok(components.collect()),
// Otherwise, keep the full path
_ => {
// Create a new PathBuf
let mut result = PathBuf::new();
// Add back the first component
result.push(first);
// Add all remaining components
result.extend(components);
Ok(result)
}
}
} else {
// Empty path
Ok(PathBuf::new())
}
}
pub fn fs_get(&mut self, path: String) -> anyhow::Result<PathBuf> {
let p = PathBuf::from(&path);
if p.exists() {
return Ok(PathBuf::from(path));
}
if p.is_absolute() {
return Err(anyhow::anyhow!("AbsolutePathError: Only paths relative to the current working directory are supported."));
}
let asset_rel_path = Self::chop_relative_base(p.clone())
.unwrap_or_else(|_| panic!("AbsolutePathError: Only paths relative to the current working directory are supported."));
let here = PathBuf::from(".").canonicalize()?;
let artifact_at_root = self.base_path.clone().join(asset_rel_path);
Self::deep_copy(artifact_at_root.as_path(), here.as_path(), true)?;
Ok(PathBuf::from(path))
}
pub(crate) fn deep_copy<P, Q>(src: P, dst: Q, recursive: bool) -> io::Result<()>
where
P: AsRef<Path>,
Q: AsRef<Path>,
{
let src = src.as_ref();
let dst = dst.as_ref();
let mut cmd = Command::new("cp");
// Add -r flag if recursive is true
if recursive {
cmd.arg("-r");
}
// Add source and destination paths
cmd.arg(src).arg(dst);
// Execute the command
let output = cmd.output()?;
if output.status.success() {
Ok(())
} else {
let error_message = String::from_utf8_lossy(&output.stderr).to_string();
Err(io::Error::other(error_message))
}
}
pub fn save(&self) {
let work_dir = serde_json::to_string_pretty(&self).unwrap();
fs::write("work_dir.json", work_dir).expect("Unable to write work-dir as file");
}
}
impl Drop for BenchmarkWorkDir {
fn drop(&mut self) {
std::env::set_current_dir(&self.run_dir).unwrap();
}
}

94
crates/goose-bench/src/error_capture.rs
View file

@ -1,94 +0,0 @@
use crate::bench_session::BenchAgentError;
use chrono::Utc;
use once_cell::sync::Lazy;
use std::sync::Arc;
use std::sync::RwLock;
use tokio::sync::Mutex;
use tracing::{Event, Subscriber};
use tracing_subscriber::layer::Context;
use tracing_subscriber::Layer;
// Type alias to reduce complexity
type ErrorRegistry = RwLock<Option<Arc<Mutex<Vec<BenchAgentError>>>>>;
// Global registry for error vectors
static ERROR_REGISTRY: Lazy<ErrorRegistry> = Lazy::new(|| RwLock::new(None));
pub struct ErrorCaptureLayer;
impl Default for ErrorCaptureLayer {
fn default() -> Self {
Self
}
}
impl ErrorCaptureLayer {
pub fn new() -> Self {
Self
}
pub fn register_error_vector(errors: Arc<Mutex<Vec<BenchAgentError>>>) {
if let Ok(mut registry) = ERROR_REGISTRY.write() {
*registry = Some(errors);
}
}
}
impl<S> Layer<S> for ErrorCaptureLayer
where
S: Subscriber,
{
fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) {
// Only capture error and warning level events
if *event.metadata().level() <= tracing::Level::WARN {
let mut visitor = JsonVisitor::new();
event.record(&mut visitor);
if let Some(message) = visitor.recorded_fields.get("message") {
let error = BenchAgentError {
message: message.to_string(),
level: event.metadata().level().to_string(),
timestamp: Utc::now(),
};
// Get the current error vector from the registry
if let Ok(registry) = ERROR_REGISTRY.read() {
if let Some(errors) = registry.clone() {
tokio::spawn(async move {
let mut errors = errors.lock().await;
errors.push(error);
});
}
}
}
}
}
}
struct JsonVisitor {
recorded_fields: serde_json::Map<String, serde_json::Value>,
}
impl JsonVisitor {
fn new() -> Self {
Self {
recorded_fields: serde_json::Map::new(),
}
}
}
impl tracing::field::Visit for JsonVisitor {
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
self.recorded_fields.insert(
field.name().to_string(),
serde_json::Value::String(value.to_string()),
);
}
fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
self.recorded_fields.insert(
field.name().to_string(),
serde_json::Value::String(format!("{:?}", value)),
);
}
}

3
crates/goose-bench/src/eval_suites/core/computercontroller/mod.rs
View file

@ -1,3 +0,0 @@
// computer controller extension evals
mod script;
mod web_scrape;

86
crates/goose-bench/src/eval_suites/core/computercontroller/script.rs
View file

@ -1,86 +0,0 @@
// Create a new file called test.txt with the content 'Hello, World!
use crate::bench_session::BenchAgent;
use crate::bench_work_dir::BenchmarkWorkDir;
use crate::eval_suites::{
collect_baseline_metrics, metrics_hashmap_to_vec, EvalMetricValue, Evaluation,
ExtensionRequirements,
};
use crate::register_evaluation;
use async_trait::async_trait;
use goose::conversation::message::MessageContent;
use rmcp::model::Role;
use serde_json::{self, Value};
#[derive(Debug)]
pub struct ComputerControllerScript {}
impl ComputerControllerScript {
pub fn new() -> Self {
ComputerControllerScript {}
}
}
#[async_trait]
impl Evaluation for ComputerControllerScript {
async fn run(
&self,
agent: &mut BenchAgent,
_run_loc: &mut BenchmarkWorkDir,
) -> anyhow::Result<Vec<(String, EvalMetricValue)>> {
// Send the prompt to list files
let (messages, perf_metrics) =
collect_baseline_metrics(agent, "Make a beep sound".to_string()).await;
// Convert HashMap to Vec for our metrics
let mut metrics = metrics_hashmap_to_vec(perf_metrics);
let valid_tool_call = messages.iter().any(|msg| {
// Check if it's an assistant message
msg.role == Role::Assistant &&
// Check if any content item is a tool request for creating a file
msg.content.iter().any(|content| {
if let MessageContent::ToolRequest(tool_req) = content {
if let Ok(tool_call) = tool_req.tool_call.as_ref() {
// Check tool name is correct
if tool_call.name != "computercontroller__computer_control" {
return false;
}
// Parse the arguments as JSON
if let Ok(args) = serde_json::from_value::<Value>(serde_json::Value::Object(tool_call.arguments.clone().unwrap_or_default())) {
// Check all required parameters match exactly
args.get("script").and_then(Value::as_str).is_some_and(|s| s.contains("beep"))
} else {
false
}
} else {
false
}
} else {
false
}
})
});
metrics.push((
"Running os scripts".to_string(),
EvalMetricValue::Boolean(valid_tool_call),
));
Ok(metrics)
}
fn name(&self) -> &str {
"computercontroller_script"
}
fn required_extensions(&self) -> ExtensionRequirements {
ExtensionRequirements {
builtin: vec!["computercontroller".to_string()],
external: Vec::new(),
streamable_http: Vec::new(),
}
}
}
register_evaluation!(ComputerControllerScript);

89
crates/goose-bench/src/eval_suites/core/computercontroller/web_scrape.rs
View file

@ -1,89 +0,0 @@
// Create a new file called test.txt with the content 'Hello, World!
use crate::bench_session::BenchAgent;
use crate::bench_work_dir::BenchmarkWorkDir;
use crate::eval_suites::{
collect_baseline_metrics, metrics_hashmap_to_vec, EvalMetricValue, Evaluation,
ExtensionRequirements,
};
use crate::register_evaluation;
use async_trait::async_trait;
use goose::conversation::message::MessageContent;
use rmcp::model::Role;
use serde_json::{self, Value};
#[derive(Debug)]
pub struct ComputerControllerWebScrape {}
impl ComputerControllerWebScrape {
pub fn new() -> Self {
ComputerControllerWebScrape {}
}
}
#[async_trait]
impl Evaluation for ComputerControllerWebScrape {
async fn run(
&self,
agent: &mut BenchAgent,
_run_loc: &mut BenchmarkWorkDir,
) -> anyhow::Result<Vec<(String, EvalMetricValue)>> {
// Send the prompt to list files
let (messages, perf_metrics) = collect_baseline_metrics(
agent,
"What are the headlines on hackernews? Organize the list into categories.".to_string(),
)
.await;
// Convert HashMap to Vec for our metrics
let mut metrics = metrics_hashmap_to_vec(perf_metrics);
let valid_tool_call = messages.iter().any(|msg| {
// Check if it's an assistant message
msg.role == Role::Assistant &&
// Check if any content item is a tool request for creating a file
msg.content.iter().any(|content| {
if let MessageContent::ToolRequest(tool_req) = content {
if let Ok(tool_call) = tool_req.tool_call.as_ref() {
// Check tool name is correct
if tool_call.name != "computercontroller__web_scrape" {
return false;
}
// Parse the arguments as JSON
if let Ok(args) = serde_json::from_value::<Value>(serde_json::Value::Object(tool_call.arguments.clone().unwrap_or_default())) {
// Check all required parameters match exactly
args.get("url").and_then(Value::as_str).map(|s| s.trim_end_matches('/')) == Some("https://news.ycombinator.com")
} else {
false
}
} else {
false
}
} else {
false
}
})
});
metrics.push((
"Retrieve and scrape web pages".to_string(),
EvalMetricValue::Boolean(valid_tool_call),
));
Ok(metrics)
}
fn name(&self) -> &str {
"computercontroller_web_scrape"
}
fn required_extensions(&self) -> ExtensionRequirements {
ExtensionRequirements {
builtin: vec!["computercontroller".to_string()],
external: Vec::new(),
streamable_http: Vec::new(),
}
}
}
register_evaluation!(ComputerControllerWebScrape);

135
crates/goose-bench/src/eval_suites/core/developer/create_file.rs
View file

@ -1,135 +0,0 @@
// Create a new file called test.txt with the content 'Hello, World!
use crate::bench_session::BenchAgent;
use crate::bench_work_dir::BenchmarkWorkDir;
use crate::eval_suites::{
collect_baseline_metrics, metrics_hashmap_to_vec, EvalMetricValue, Evaluation,
ExtensionRequirements,
};
use crate::register_evaluation;
use async_trait::async_trait;
use goose::conversation::message::MessageContent;
use rmcp::model::Role;
use serde_json::{self, Value};
#[derive(Debug)]
pub struct DeveloperCreateFile {}
impl DeveloperCreateFile {
pub fn new() -> Self {
DeveloperCreateFile {}
}
}
#[async_trait]
impl Evaluation for DeveloperCreateFile {
async fn run(
&self,
agent: &mut BenchAgent,
_run_loc: &mut BenchmarkWorkDir,
) -> anyhow::Result<Vec<(String, EvalMetricValue)>> {
// Send the prompt to create and read
let (messages, perf_metrics) = collect_baseline_metrics(
agent,
"Create a new file called test.txt in the current directory with the content 'Hello, World!'. Then read the contents of the new file to confirm.".to_string()
).await;
// Convert HashMap to Vec for our metrics
let mut metrics = metrics_hashmap_to_vec(perf_metrics);
// Check for write operation
let write_tool_call = messages.iter().any(|msg| {
// Check if it's an assistant message
msg.role == Role::Assistant &&
// Check if any content item is a tool request for creating a file
msg.content.iter().any(|content| {
if let MessageContent::ToolRequest(tool_req) = content {
if let Ok(tool_call) = tool_req.tool_call.as_ref() {
// Check tool name is correct
if tool_call.name != "developer__text_editor" {
return false;
}
// Parse the arguments as JSON
if let Ok(args) = serde_json::from_value::<Value>(serde_json::Value::Object(tool_call.arguments.clone().unwrap_or_default())) {
// Check all required parameters match exactly
args.get("command").and_then(Value::as_str) == Some("write") &&
args.get("path").and_then(Value::as_str).is_some_and(|s| s.contains("test.txt")) &&
args.get("file_text").and_then(Value::as_str) == Some("Hello, World!")
} else {
false
}
} else {
false
}
} else {
false
}
})
});
// Check for read operation
let read_tool_call = messages.iter().any(|msg| {
// Check if it's an assistant message
msg.role == Role::Assistant &&
// Check if any content item is a tool request for reading a file
msg.content.iter().any(|content| {
if let MessageContent::ToolRequest(tool_req) = content {
if let Ok(tool_call) = tool_req.tool_call.as_ref() {
// Check tool name is correct
if tool_call.name != "developer__text_editor" {
return false;
}
// Parse the arguments as JSON
if let Ok(args) = serde_json::from_value::<Value>(serde_json::Value::Object(tool_call.arguments.clone().unwrap_or_default())) {
// Check all required parameters match exactly
args.get("command").and_then(Value::as_str) == Some("view") &&
args.get("path").and_then(Value::as_str).is_some_and(|s| s.contains("test.txt"))
} else {
false
}
} else {
false
}
} else {
false
}
})
});
metrics.push((
"Create file".to_string(),
EvalMetricValue::Boolean(write_tool_call),
));
metrics.push((
"Read file".to_string(),
EvalMetricValue::Boolean(read_tool_call),
));
metrics.push((
"Complete create and read".to_string(),
EvalMetricValue::Boolean(write_tool_call && read_tool_call),
));
metrics.push((
"score".to_string(),
EvalMetricValue::Float(((write_tool_call as u8) + (read_tool_call as u8)) as f64 / 2.0),
));
Ok(metrics)
}
fn name(&self) -> &str {
"developer_create_read_file"
}
fn required_extensions(&self) -> ExtensionRequirements {
ExtensionRequirements {
builtin: vec!["developer".to_string()],
external: Vec::new(),
streamable_http: Vec::new(),
}
}
}
register_evaluation!(DeveloperCreateFile);

94
crates/goose-bench/src/eval_suites/core/developer/list_files.rs
View file

@ -1,94 +0,0 @@
use crate::bench_session::BenchAgent;
use crate::bench_work_dir::BenchmarkWorkDir;
use crate::eval_suites::{
collect_baseline_metrics, metrics_hashmap_to_vec, EvalMetricValue, Evaluation,
ExtensionRequirements,
};
use crate::register_evaluation;
use async_trait::async_trait;
use goose::conversation::message::MessageContent;
use rmcp::model::Role;
use serde_json::{self, Value};
#[derive(Debug)]
pub struct DeveloperListFiles {}
impl DeveloperListFiles {
pub fn new() -> Self {
DeveloperListFiles {}
}
}
#[async_trait]
impl Evaluation for DeveloperListFiles {
async fn run(
&self,
agent: &mut BenchAgent,
_run_loc: &mut BenchmarkWorkDir,
) -> anyhow::Result<Vec<(String, EvalMetricValue)>> {
// Send the prompt to list files
let (messages, perf_metrics) =
collect_baseline_metrics(agent, "list the files in the current directory".to_string())
.await;
// Convert HashMap to Vec for our metrics
let mut metrics = metrics_hashmap_to_vec(perf_metrics);
// Check if the assistant makes appropriate tool calls
let valid_tool_call = messages.iter().any(|msg| {
// Check if it's an assistant message
msg.role == Role::Assistant &&
// Check if any content item is a tool request for listing files
msg.content.iter().any(|content| {
if let MessageContent::ToolRequest(tool_req) = content {
// Check if the tool call is for shell with ls or rg --files
if let Ok(tool_call) = tool_req.tool_call.as_ref() {
// Parse arguments as JSON Value first
if let Ok(args) = serde_json::from_value::<Value>(serde_json::Value::Object(tool_call.arguments.clone().unwrap_or_default())) {
tool_call.name == "developer__shell" &&
args.get("command")
.and_then(Value::as_str).is_some_and(|cmd| {
cmd.contains("ls ") ||
cmd.contains("ls\n") ||
cmd.contains("ls$") ||
cmd.contains("rg --files")
})
} else {
false
}
} else {
false
}
} else {
false
}
})
});
metrics.push((
"Using the shell command tool".to_string(),
EvalMetricValue::Boolean(valid_tool_call),
));
metrics.push((
"score".to_string(),
EvalMetricValue::Float((valid_tool_call as u8) as f64 / 1.0),
));
Ok(metrics)
}
fn name(&self) -> &str {
"developer_list_files"
}
fn required_extensions(&self) -> ExtensionRequirements {
ExtensionRequirements {
builtin: vec!["developer".to_string()],
external: Vec::new(),
streamable_http: Vec::new(),
}
}
}
register_evaluation!(DeveloperListFiles);

4
crates/goose-bench/src/eval_suites/core/developer/mod.rs
View file

@ -1,4 +0,0 @@
// developer extension evals
mod create_file;
mod list_files;
mod simple_repo_clone_test;

232
crates/goose-bench/src/eval_suites/core/developer/simple_repo_clone_test.rs
View file

@ -1,232 +0,0 @@
use crate::bench_session::BenchAgent;
use crate::bench_work_dir::BenchmarkWorkDir;
use crate::eval_suites::{
collect_baseline_metrics, metrics_hashmap_to_vec, EvalMetricValue, Evaluation,
ExtensionRequirements,
};
use crate::register_evaluation;
use async_trait::async_trait;
use goose::conversation::message::MessageContent;
use rmcp::model::Role;
use serde_json::{self, Value};
#[derive(Debug)]
pub struct SimpleRepoCloneTest {}
impl SimpleRepoCloneTest {
pub fn new() -> Self {
SimpleRepoCloneTest {}
}
}
#[async_trait]
impl Evaluation for SimpleRepoCloneTest {
async fn run(
&self,
agent: &mut BenchAgent,
_work_dir: &mut BenchmarkWorkDir,
) -> anyhow::Result<Vec<(String, EvalMetricValue)>> {
// Send the prompt to clone the repo and add a test
let (messages, perf_metrics) = collect_baseline_metrics(
agent,
"Clone the Git repository https://github.com/michaelneale/mcp-read-pdf to a temporary location. \
Then add a new test file that verifies the PDF reading functionality. The test should \
check if the PDF content can be read and processed correctly.".to_string(),
).await;
// Convert HashMap to Vec for our metrics
let mut metrics = metrics_hashmap_to_vec(perf_metrics);
// Check for git clone operation
let git_clone_executed = messages.iter().any(|msg| {
msg.role == Role::Assistant
&& msg.content.iter().any(|content| {
if let MessageContent::ToolRequest(tool_req) = content {
if let Ok(tool_call) = tool_req.tool_call.as_ref() {
if tool_call.name != "developer__shell" {
return false;
}
if let Ok(args) =
serde_json::from_value::<Value>(serde_json::Value::Object(
tool_call.arguments.clone().unwrap_or_default(),
))
{
let command = args.get("command").and_then(Value::as_str);
command.is_some_and(|cmd| {
cmd.contains("git clone")
&& cmd.contains("michaelneale/mcp-read-pdf")
})
} else {
false
}
} else {
false
}
} else {
false
}
})
});
// Check for exploring the repository structure
let repo_explored = messages.iter().any(|msg| {
msg.role == Role::Assistant
&& msg.content.iter().any(|content| {
if let MessageContent::ToolRequest(tool_req) = content {
if let Ok(tool_call) = tool_req.tool_call.as_ref() {
if tool_call.name != "developer__shell" {
return false;
}
if let Ok(args) =
serde_json::from_value::<Value>(serde_json::Value::Object(
tool_call.arguments.clone().unwrap_or_default(),
))
{
let command = args.get("command").and_then(Value::as_str);
command.is_some_and(|cmd| {
(cmd.contains("ls")
|| cmd.contains("find")
|| cmd.contains("rg"))
&& cmd.contains("mcp-read-pdf")
})
} else {
false
}
} else {
false
}
} else {
false
}
})
});
// Check for file creation to add a test
let test_added = messages.iter().any(|msg| {
msg.role == Role::Assistant
&& msg.content.iter().any(|content| {
if let MessageContent::ToolRequest(tool_req) = content {
if let Ok(tool_call) = tool_req.tool_call.as_ref() {
if tool_call.name != "developer__text_editor" {
return false;
}
if let Ok(args) =
serde_json::from_value::<Value>(serde_json::Value::Object(
tool_call.arguments.clone().unwrap_or_default(),
))
{
let command = args.get("command").and_then(Value::as_str);
let file_text = args.get("file_text").and_then(Value::as_str);
let path = args.get("path").and_then(Value::as_str);
command == Some("write")
&& path.is_some_and(|p| {
p.contains("test")
|| p.ends_with(".py")
|| p.ends_with(".js")
|| p.ends_with(".ts")
})
&& file_text.is_some_and(|text| {
text.contains("test")
|| text.contains("assert")
|| text.contains("expect")
|| text.contains("should")
})
} else {
false
}
} else {
false
}
} else {
false
}
})
});
// Check if the agent ran the test
let test_executed = messages.iter().any(|msg| {
msg.role == Role::Assistant
&& msg.content.iter().any(|content| {
if let MessageContent::ToolRequest(tool_req) = content {
if let Ok(tool_call) = tool_req.tool_call.as_ref() {
if tool_call.name != "developer__shell" {
return false;
}
if let Ok(args) =
serde_json::from_value::<Value>(serde_json::Value::Object(
tool_call.arguments.clone().unwrap_or_default(),
))
{
let command = args.get("command").and_then(Value::as_str);
command.is_some_and(|cmd| {
cmd.contains("test")
|| cmd.contains("pytest")
|| cmd.contains("jest")
|| cmd.contains("mocha")
|| (cmd.contains("node") && cmd.contains("test"))
|| (cmd.contains("python") && cmd.contains("test"))
})
} else {
false
}
} else {
false
}
} else {
false
}
})
});
// Add metrics
metrics.push((
"Git repo cloned".to_string(),
EvalMetricValue::Boolean(git_clone_executed),
));
metrics.push((
"Repository explored".to_string(),
EvalMetricValue::Boolean(repo_explored),
));
metrics.push((
"Test file added".to_string(),
EvalMetricValue::Boolean(test_added),
));
metrics.push((
"Test executed".to_string(),
EvalMetricValue::Boolean(test_executed),
));
metrics.push((
"Complete task".to_string(),
EvalMetricValue::Boolean(git_clone_executed && test_added),
));
metrics.push((
"score".to_string(),
EvalMetricValue::Float(
((git_clone_executed as u8) + (test_added as u8) + (test_executed as u8)) as f64
/ 3.0,
),
));
Ok(metrics)
}
fn name(&self) -> &str {
"simple_repo_clone_test"
}
fn required_extensions(&self) -> ExtensionRequirements {
ExtensionRequirements {
builtin: vec!["developer".to_string()],
external: Vec::new(),
streamable_http: Vec::new(),
}
}
}
register_evaluation!(SimpleRepoCloneTest);

108
crates/goose-bench/src/eval_suites/core/developer_image/image.rs
View file

@ -1,108 +0,0 @@
use crate::bench_session::BenchAgent;
use crate::bench_work_dir::BenchmarkWorkDir;
use crate::eval_suites::{
collect_baseline_metrics, metrics_hashmap_to_vec, EvalMetricValue, Evaluation,
ExtensionRequirements,
};
use crate::register_evaluation;
use async_trait::async_trait;
use goose::conversation::message::MessageContent;
use rmcp::model::Role;
use serde_json::{self, Value};
#[derive(Debug)]
pub struct DeveloperImage {}
impl DeveloperImage {
pub fn new() -> Self {
DeveloperImage {}
}
}
#[async_trait]
impl Evaluation for DeveloperImage {
async fn run(
&self,
agent: &mut BenchAgent,
_run_loc: &mut BenchmarkWorkDir,
) -> anyhow::Result<Vec<(String, EvalMetricValue)>> {
// Send the prompt to list files
let (messages, perf_metrics) = collect_baseline_metrics(
agent,
"Take a screenshot of the display 0 and describe what you see.".to_string(),
)
.await;
// Convert HashMap to Vec for our metrics
let mut metrics = metrics_hashmap_to_vec(perf_metrics);
// Check if the assistant makes appropriate tool calls and gets valid responses
let mut valid_tool_call = false;
let mut valid_response = false;
for msg in messages.iter() {
// Check for valid tool request
if msg.role == Role::Assistant {
for content in msg.content.iter() {
if let MessageContent::ToolRequest(tool_req) = content {
if let Ok(tool_call) = tool_req.tool_call.as_ref() {
if let Ok(args) =
serde_json::from_value::<Value>(serde_json::Value::Object(
tool_call.arguments.clone().unwrap_or_default(),
))
{
if tool_call.name == "developer__screen_capture"
&& (args.get("display").and_then(Value::as_i64) == Some(0))
{
valid_tool_call = true;
}
}
}
}
}
}
// Check for valid tool response
if msg.role == Role::User && valid_tool_call {
for content in msg.content.iter() {
if let MessageContent::ToolResponse(tool_resp) = content {
if let Ok(result) = &tool_resp.tool_result {
// Check each item in the result list
for item in &result.content {
if let Some(image) = item.as_image() {
// Image content already contains mime_type and data
if image.mime_type.starts_with("image/")
&& !image.data.is_empty()
{
valid_response = true;
break; // Found a valid image, no need to check further
}
}
}
}
}
}
}
}
// Both the tool call and response must be valid
metrics.push((
"Take a screenshot and upload images".to_string(),
EvalMetricValue::Boolean(valid_tool_call && valid_response),
));
Ok(metrics)
}
fn name(&self) -> &str {
"developer_image"
}
fn required_extensions(&self) -> ExtensionRequirements {
ExtensionRequirements {
builtin: vec!["developer".to_string()],
external: Vec::new(),
streamable_http: Vec::new(),
}
}
}
register_evaluation!(DeveloperImage);

1
crates/goose-bench/src/eval_suites/core/developer_image/mod.rs
View file

@ -1 +0,0 @@
mod image;

1
crates/goose-bench/src/eval_suites/core/developer_search_replace/mod.rs
View file

@ -1 +0,0 @@
mod search_replace;

116
crates/goose-bench/src/eval_suites/core/developer_search_replace/search_replace.rs
View file

@ -1,116 +0,0 @@
use crate::bench_session::BenchAgent;
use crate::bench_work_dir::BenchmarkWorkDir;
use crate::eval_suites::{
collect_baseline_metrics, metrics_hashmap_to_vec, EvalMetricValue, Evaluation,
ExtensionRequirements,
};
use crate::register_evaluation;
use async_trait::async_trait;
use std::fs;
#[derive(Debug)]
pub struct DeveloperSearchReplace {}
impl DeveloperSearchReplace {
pub fn new() -> Self {
DeveloperSearchReplace {}
}
}
#[async_trait]
impl Evaluation for DeveloperSearchReplace {
async fn run(
&self,
agent: &mut BenchAgent,
run_loc: &mut BenchmarkWorkDir,
) -> anyhow::Result<Vec<(String, EvalMetricValue)>> {
let _target_file = match run_loc.fs_get("./assets/kubernetes_swagger.json".to_string()) {
Ok(file) => file,
Err(_) => {
return Err(anyhow::anyhow!(
"Could not find kubernetes_swagger.json file"
))
}
};
let mut source_file = run_loc.base_path.clone();
source_file.push("assets/kubernetes_swagger.json");
// Send the prompt to modify the file
let (_messages, perf_metrics) = collect_baseline_metrics(
agent,
"Remove the io.k8s.api.admissionregistration.v1.ServiceReference definition block and replace with a new definition for io.k8s.api.admissionregistration.v1.FakeServiceReference. Update the fields in the definition as well to be consistent. Don't change the property names. Don't update any references to the old definition. Only modify the definition and it's description to 'FakeServiceReference simulates a reference to a fake service for testing purposes.'.The file to modify is kubernetes_swagger.json.".to_string()
).await;
// Convert HashMap to Vec for our metrics
let mut metrics = metrics_hashmap_to_vec(perf_metrics);
// Get the path to the modified file
let modified_file_path = std::env::current_dir()
.unwrap_or_default()
.join("kubernetes_swagger.json");
// Read the expected patch file from the assets directory
let patch_file_path = run_loc.base_path.join("assets").join("kubernetes.patch");
if !patch_file_path.exists() {
return Err(anyhow::anyhow!("Could not find patch file"));
}
let patch_content = fs::read_to_string(&patch_file_path)?
.lines()
.skip(4)
.collect::<Vec<&str>>()
.join("\n");
// Run git diff between modified and source files
let diff_output = std::process::Command::new("git")
.args([
"diff",
"--no-index",
source_file.to_str().unwrap(),
modified_file_path.to_str().unwrap(),
])
.output()?;
let actual_diff = String::from_utf8_lossy(&diff_output.stdout)
.to_string()
.lines()
.skip(4)
.collect::<Vec<&str>>()
.join("\n");
let mut changes_match = true;
// Compare the remaining lines
if actual_diff != patch_content {
println!("Diffs don't match!");
println!("Expected patch:\n{}", patch_content);
println!("Actual diff:\n{}", actual_diff);
changes_match = false;
}
metrics.push((
"Changes match expected patch".to_string(),
EvalMetricValue::Boolean(changes_match),
));
metrics.push((
"score".to_string(),
EvalMetricValue::Float((changes_match as u8) as f64 / 1.0),
));
Ok(metrics)
}
fn name(&self) -> &str {
"developer_search_replace"
}
fn required_extensions(&self) -> ExtensionRequirements {
ExtensionRequirements {
builtin: vec!["developer".to_string()],
external: Vec::new(),
streamable_http: Vec::new(),
}
}
}
register_evaluation!(DeveloperSearchReplace);

44
crates/goose-bench/src/eval_suites/core/example.rs
View file

@ -1,44 +0,0 @@
use crate::bench_session::BenchAgent;
use crate::bench_work_dir::BenchmarkWorkDir;
use crate::eval_suites::{EvalMetricValue, Evaluation, ExtensionRequirements};
use crate::register_evaluation;
use async_trait::async_trait;
// use std::fs;
pub struct ExampleEval {}
impl ExampleEval {
pub fn new() -> Self {
ExampleEval {}
}
}
#[async_trait]
impl Evaluation for ExampleEval {
async fn run(
&self,
agent: &mut BenchAgent,
_run_loc: &mut BenchmarkWorkDir,
) -> anyhow::Result<Vec<(String, EvalMetricValue)>> {
println!("ExampleEval - run");
let mut metrics = Vec::new();
let _ = agent.prompt("What can you do?".to_string()).await;
metrics.push(("example_metric".to_string(), EvalMetricValue::Boolean(true)));
metrics.push(("example_count".to_string(), EvalMetricValue::Integer(42)));
Ok(metrics)
}
fn name(&self) -> &str {
"example_eval"
}
fn required_extensions(&self) -> ExtensionRequirements {
ExtensionRequirements::default() // Example eval doesn't require any extensions
}
}
register_evaluation!(ExampleEval);

2
crates/goose-bench/src/eval_suites/core/memory/mod.rs
View file

@ -1,2 +0,0 @@
// memory extension evals
mod save_fact;

91
crates/goose-bench/src/eval_suites/core/memory/save_fact.rs
View file

@ -1,91 +0,0 @@
// Create a new file called test.txt with the content 'Hello, World!
use crate::bench_session::BenchAgent;
use crate::bench_work_dir::BenchmarkWorkDir;
use crate::eval_suites::{
collect_baseline_metrics, metrics_hashmap_to_vec, EvalMetricValue, Evaluation,
ExtensionRequirements,
};
use crate::register_evaluation;
use async_trait::async_trait;
use goose::conversation::message::MessageContent;
use rmcp::model::Role;
use serde_json::{self, Value};
#[derive(Debug)]
pub struct MemoryRememberMemory {}
impl MemoryRememberMemory {
pub fn new() -> Self {
MemoryRememberMemory {}
}
}
#[async_trait]
impl Evaluation for MemoryRememberMemory {
async fn run(
&self,
agent: &mut BenchAgent,
_run_loc: &mut BenchmarkWorkDir,
) -> anyhow::Result<Vec<(String, EvalMetricValue)>> {
// Send the prompt to list files
let (messages, perf_metrics) = collect_baseline_metrics(
agent,
"Save this fact: The capital of France is Paris.".to_string(),
)
.await;
// Convert HashMap to Vec for our metrics
let mut metrics = metrics_hashmap_to_vec(perf_metrics);
let valid_tool_call = messages.iter().any(|msg| {
// Check if it's an assistant message
msg.role == Role::Assistant &&
// Check if any content item is a tool request for creating a file
msg.content.iter().any(|content| {
if let MessageContent::ToolRequest(tool_req) = content {
if let Ok(tool_call) = tool_req.tool_call.as_ref() {
// Check tool name is correct
if tool_call.name != "memory__remember_memory" {
return false;
}
// Parse the arguments as JSON
if let Ok(args) = serde_json::from_value::<Value>(serde_json::Value::Object(tool_call.arguments.clone().unwrap_or_default())) {
// Check all required parameters match exactly
args.get("category").and_then(Value::as_str).is_some_and(|s| s.contains("fact")) &&
args.get("data").and_then(Value::as_str) == Some("The capital of France is Paris.") &&
args.get("is_global").and_then(Value::as_bool) == Some(true)
} else {
false
}
} else {
false
}
} else {
false
}
})
});
metrics.push((
"Saving facts".to_string(),
EvalMetricValue::Boolean(valid_tool_call),
));
Ok(metrics)
}
fn name(&self) -> &str {
"memory_remember_memory"
}
fn required_extensions(&self) -> ExtensionRequirements {
ExtensionRequirements {
builtin: vec!["memory".to_string()],
external: Vec::new(),
streamable_http: Vec::new(),
}
}
}
register_evaluation!(MemoryRememberMemory);

6
crates/goose-bench/src/eval_suites/core/mod.rs
View file

@ -1,6 +0,0 @@
mod computercontroller;
mod developer;
mod developer_image;
mod developer_search_replace;
mod example;
mod memory;

59
crates/goose-bench/src/eval_suites/evaluation.rs
View file

@ -1,59 +0,0 @@
use crate::bench_session::BenchAgent;
use crate::bench_work_dir::BenchmarkWorkDir;
use anyhow::Result;
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use std::fmt;
pub type Model = (String, String);
pub type Extension = String;
#[derive(Debug, Deserialize, Serialize, Clone)]
pub enum EvalMetricValue {
Integer(i64),
Float(f64),
String(String),
Boolean(bool),
}
impl fmt::Display for EvalMetricValue {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
EvalMetricValue::Integer(i) => write!(f, "{}", i),
EvalMetricValue::Float(fl) => write!(f, "{:.2}", fl),
EvalMetricValue::String(s) => write!(f, "{}", s),
EvalMetricValue::Boolean(b) => write!(f, "{}", b),
}
}
}
#[derive(Debug, Serialize)]
pub struct EvalMetric {
pub name: String,
pub value: EvalMetricValue,
}
#[derive(Debug, Default)]
pub struct ExtensionRequirements {
pub builtin: Vec<String>,
pub external: Vec<String>,
pub streamable_http: Vec<String>,
}
#[async_trait]
pub trait Evaluation: Send + Sync {
async fn run(
&self,
agent: &mut BenchAgent,
run_loc: &mut BenchmarkWorkDir,
) -> Result<Vec<(String, EvalMetricValue)>>;
fn name(&self) -> &str;
fn required_extensions(&self) -> ExtensionRequirements {
ExtensionRequirements {
builtin: Vec::new(),
external: Vec::new(),
streamable_http: Vec::new(),
}
}
}

130
crates/goose-bench/src/eval_suites/factory.rs
View file

@ -1,130 +0,0 @@
pub use super::Evaluation;
use regex::Regex;
use std::borrow::Cow;
use std::collections::HashMap;
use std::sync::{OnceLock, RwLock};
type EvaluationConstructor = fn() -> Box<dyn Evaluation>;
type Registry = &'static RwLock<HashMap<&'static str, EvaluationConstructor>>;
// Use std::sync::RwLock for interior mutability
static EVAL_REGISTRY: OnceLock<RwLock<HashMap<&'static str, EvaluationConstructor>>> =
OnceLock::new();
/// Initialize the registry if it hasn't been initialized
fn eval_registry() -> Registry {
EVAL_REGISTRY.get_or_init(|| RwLock::new(HashMap::new()))
}
/// Register a new evaluation version
pub fn register_eval(selector: &'static str, constructor: fn() -> Box<dyn Evaluation>) {
let registry = eval_registry();
if let Ok(mut map) = registry.write() {
map.insert(selector, constructor);
}
}
pub struct EvaluationSuite;
impl EvaluationSuite {
pub fn from(selector: &str) -> Option<Box<dyn Evaluation>> {
let registry = eval_registry();
let map = registry
.read()
.expect("Failed to read the benchmark evaluation registry.");
let constructor = map.get(selector)?;
let instance = constructor();
Some(instance)
}
pub fn registered_evals() -> Vec<&'static str> {
let registry = eval_registry();
let map = registry
.read()
.expect("Failed to read the benchmark evaluation registry.");
let evals: Vec<_> = map.keys().copied().collect();
evals
}
pub fn select(selectors: Vec<String>) -> HashMap<String, Vec<&'static str>> {
let eval_name_pattern = Regex::new(r":\w+$").unwrap();
let grouped_by_suite: HashMap<String, Vec<&'static str>> =
EvaluationSuite::registered_evals()
.into_iter()
.filter(|&eval| selectors.is_empty() || matches_any_selectors(eval, &selectors))
.fold(HashMap::new(), |mut suites, eval| {
let suite = match eval_name_pattern.replace(eval, "") {
Cow::Borrowed(s) => s.to_string(),
Cow::Owned(s) => s,
};
suites.entry(suite).or_default().push(eval);
suites
});
grouped_by_suite
}
pub fn available_selectors() -> HashMap<String, usize> {
let mut counts: HashMap<String, usize> = HashMap::new();
for selector in EvaluationSuite::registered_evals() {
let parts = selector.split(":").collect::<Vec<_>>();
for i in 0..parts.len() {
let sel = parts[..i + 1].join(":");
*counts.entry(sel).or_insert(0) += 1;
}
}
counts
}
}
fn matches_any_selectors(eval: &str, selectors: &Vec<String>) -> bool {
// selectors must prefix match exactly, no matching half-way in a word
// remove one level of nesting at a time and check exact match
let nesting_pattern = Regex::new(r":\w+$").unwrap();
for selector in selectors {
let mut level_up = eval.to_string();
while !level_up.is_empty() {
if level_up == *selector {
return true;
}
if !level_up.contains(":") {
break;
};
level_up = match nesting_pattern.replace(&level_up, "") {
Cow::Borrowed(s) => s.to_string(),
Cow::Owned(s) => s,
};
}
}
false
}
#[macro_export]
macro_rules! register_evaluation {
($evaluation_type:ty) => {
paste::paste! {
#[ctor::ctor]
#[allow(non_snake_case)]
fn [<__register_evaluation_ $evaluation_type>]() {
let mut path = std::path::PathBuf::from(file!());
path.set_extension("");
let eval_suites_dir = "eval_suites";
let eval_selector = {
let s = path.components()
.skip_while(|comp| comp.as_os_str() != eval_suites_dir)
.skip(1)
.map(|comp| comp.as_os_str().to_string_lossy().to_string())
.collect::<Vec<_>>()
.join(":");
Box::leak(s.into_boxed_str())
};
$crate::eval_suites::factory::register_eval(eval_selector, || {
Box::new(<$evaluation_type>::new())
});
}
}
};
}

109
crates/goose-bench/src/eval_suites/metrics.rs
View file

@ -1,109 +0,0 @@
use crate::bench_session::BenchAgent;
use crate::eval_suites::EvalMetricValue;
use goose::conversation::message::{Message, MessageContent};
use goose::conversation::Conversation;
use std::collections::HashMap;
use std::time::Instant;
/// Collect baseline metrics including execution time, tool usage, and token count
pub async fn collect_baseline_metrics(
agent: &mut BenchAgent,
prompt: String,
) -> (Conversation, HashMap<String, EvalMetricValue>) {
// Initialize metrics map
let mut metrics = HashMap::new();
// Start timer
let start_time = Instant::now();
// Execute prompt
let messages = match agent.prompt(prompt).await {
Ok(msgs) => msgs,
Err(e) => {
metrics.insert(
"prompt_error".to_string(),
EvalMetricValue::String(format!("Error: {}", e)),
);
Conversation::new_unvalidated(Vec::new())
}
};
// Calculate execution time
let execution_time = start_time.elapsed();
metrics.insert(
"prompt_execution_time_seconds".to_string(),
EvalMetricValue::Float(execution_time.as_secs_f64()),
);
// Count tool calls
let (total_tool_calls, tool_calls_by_name) = count_tool_calls(messages.messages());
metrics.insert(
"total_tool_calls".to_string(),
EvalMetricValue::Integer(total_tool_calls),
);
// Add tool calls by name metrics
for (tool_name, count) in tool_calls_by_name {
metrics.insert(
format!("tool_calls_{}", tool_name),
EvalMetricValue::Integer(count),
);
}
// Get token usage information if available
if let Some(token_count) = agent.get_token_usage().await {
metrics.insert(
"total_tokens".to_string(),
EvalMetricValue::Integer(token_count as i64),
);
}
(messages, metrics)
}
/// Count all tool calls in messages and categorize by tool name
fn count_tool_calls(messages: &[Message]) -> (i64, HashMap<String, i64>) {
let mut total_count = 0;
let mut counts_by_name = HashMap::new();
for message in messages {
for content in &message.content {
if let MessageContent::ToolRequest(tool_req) = content {
if let Ok(tool_call) = tool_req.tool_call.as_ref() {
total_count += 1;
// Count by name
*counts_by_name
.entry(tool_call.name.to_string())
.or_insert(0) += 1;
}
}
}
}
(total_count, counts_by_name)
}
/// Convert HashMap of metrics to Vec
pub fn metrics_hashmap_to_vec(
metrics: HashMap<String, EvalMetricValue>,
) -> Vec<(String, EvalMetricValue)> {
metrics.into_iter().collect()
}
/// Check if a specific tool was used in any of the messages
pub fn used_tool(messages: &[Message], tool_name: &str) -> bool {
messages.iter().any(|msg| {
msg.content.iter().any(|content| {
if let MessageContent::ToolRequest(tool_req) = content {
if let Ok(tool_call) = tool_req.tool_call.as_ref() {
tool_call.name.contains(tool_name)
} else {
false
}
} else {
false
}
})
})
}

11
crates/goose-bench/src/eval_suites/mod.rs
View file

@ -1,11 +0,0 @@
mod core;
mod evaluation;
mod factory;
mod metrics;
mod utils;
mod vibes;
pub use evaluation::*;
pub use factory::{register_eval, EvaluationSuite};
pub use metrics::*;
pub use utils::*;

32
crates/goose-bench/src/eval_suites/utils.rs
View file

@ -1,32 +0,0 @@
use crate::bench_work_dir::BenchmarkWorkDir;
use anyhow::{Context, Result};
use goose::conversation::message::Message;
use std::fs::File;
use std::io::Write;
use std::path::PathBuf;
/// Write the last agent message to a file
/// Returns the content of the message and an error if writing failed
pub fn write_response_to_file(
messages: &[Message],
_work_dir: &mut BenchmarkWorkDir, // Kept for API compatibility
filename: &str,
) -> Result<String> {
let last_msg = messages
.last()
.ok_or_else(|| anyhow::anyhow!("No messages to write to file"))?;
let text_content = last_msg.as_concat_text();
// Create a file in the current directory
let output_path = PathBuf::from(filename);
// Create and write to the file
let mut file = File::create(&output_path)
.with_context(|| format!("Failed to create file at {}", output_path.display()))?;
file.write_all(text_content.as_bytes())
.with_context(|| format!("Failed to write content to {}", output_path.display()))?;
Ok(text_content)
}

84
crates/goose-bench/src/eval_suites/vibes/blog_summary.rs
View file

@ -1,84 +0,0 @@
use crate::bench_session::BenchAgent;
use crate::bench_work_dir::BenchmarkWorkDir;
use crate::eval_suites::{
collect_baseline_metrics, metrics_hashmap_to_vec, write_response_to_file, EvalMetricValue,
Evaluation, ExtensionRequirements,
};
use crate::register_evaluation;
use async_trait::async_trait;
pub struct BlogSummary {}
impl BlogSummary {
pub fn new() -> Self {
BlogSummary {}
}
fn check_markdown_numbered_list(&self, text: &str) -> bool {
// Check if all numbers 1-5 exist in markdown numbered list format
(1..=5).all(|n| text.contains(&format!("{}.", n)))
}
}
#[async_trait]
impl Evaluation for BlogSummary {
async fn run(
&self,
agent: &mut BenchAgent,
run_loc: &mut BenchmarkWorkDir,
) -> anyhow::Result<Vec<(String, EvalMetricValue)>> {
println!("BlogSummary - run");
// Collect baseline metrics (execution time, token usage, tool calls)
let (response, perf_metrics) = collect_baseline_metrics(
agent,
"What are the top 5 most counterintuitive insights from this blog post? Format your response in Markdown with 5 numbered points (1. 2. 3. 4. 5.) https://huyenchip.com/2025/01/07/agents.html".to_string()
).await;
// Write response to file and get the text content
let response_text =
match write_response_to_file(response.messages(), run_loc, "blog_summary_output.txt") {
Ok(text) => text,
Err(e) => {
println!("Warning: Failed to write blog summary output: {}", e);
// If file write fails, still continue with the evaluation
response
.last()
.map_or_else(String::new, |msg| msg.as_concat_text())
}
};
// Convert HashMap to Vec for our metrics
let mut metrics = metrics_hashmap_to_vec(perf_metrics);
// Check if the content follows the markdown numbered list format
let has_markdown_list = self.check_markdown_numbered_list(&response_text);
metrics.push((
"valid_markdown_format".to_string(),
EvalMetricValue::Boolean(has_markdown_list),
));
// Check if the fetch tool was used
let used_fetch_tool = crate::eval_suites::used_tool(response.messages(), "fetch");
metrics.push((
"used_fetch_tool".to_string(),
EvalMetricValue::Boolean(used_fetch_tool),
));
Ok(metrics)
}
fn name(&self) -> &str {
"blog_summary"
}
fn required_extensions(&self) -> ExtensionRequirements {
ExtensionRequirements {
builtin: vec!["developer".to_string()],
external: vec!["uvx mcp-server-fetch".to_string()],
streamable_http: Vec::new(),
}
}
}
register_evaluation!(BlogSummary);

126
crates/goose-bench/src/eval_suites/vibes/flappy_bird.rs
View file

@ -1,126 +0,0 @@
use crate::bench_session::BenchAgent;
use crate::bench_work_dir::BenchmarkWorkDir;
use crate::eval_suites::{
collect_baseline_metrics, metrics_hashmap_to_vec, EvalMetricValue, Evaluation,
ExtensionRequirements,
};
use crate::register_evaluation;
use async_trait::async_trait;
use goose::conversation::message::MessageContent;
use rmcp::model::Role;
use serde_json::{self, Value};
use std::fs;
pub struct FlappyBird {}
impl FlappyBird {
pub fn new() -> Self {
FlappyBird {}
}
fn check_python_implementation(&self, content: &str) -> bool {
content.contains("import pygame") &&
content.contains("pygame.init()") &&
content.contains("while") && // Game loop
content.contains("pygame.event.get()") && // Event handling
content.contains("def main") && // Main function
content.contains("if __name__ == '__main__'") // Main guard
}
}
#[async_trait]
impl Evaluation for FlappyBird {
async fn run(
&self,
agent: &mut BenchAgent,
run_loc: &mut BenchmarkWorkDir,
) -> anyhow::Result<Vec<(String, EvalMetricValue)>> {
println!("FlappyBird - run");
// Collect baseline metrics (execution time, token usage, tool calls)
let (messages, perf_metrics) = collect_baseline_metrics(
agent,
"Create a Flappy Bird game in Python. Structure the code with a main function and use the if __name__ == '__main__': idiom. You must use pygame. The background color should be a light blue color. Pressing SPACE multiple times will accelerate the bird. The bird's shape should be a red circle. Place on the bottom some land colored as dark yellow chosen. Make a score shown on the top right side. Increment if you pass pipes and don't hit them. Make randomly spaced dark green pipes with enough space. When you lose, show the best score. Make the text inside the screen. Pressing q or Esc will quit the game. Restarting is pressing SPACE again. When trying to run the game, make sure to use pyenv and create the environment in the current working directory. The final game should be written to a file named flappy_bird.py. Remember to use your tools if applicable.".to_string()
).await;
// Convert HashMap to Vec for our metrics
let mut metrics = metrics_hashmap_to_vec(perf_metrics);
// Check if the agent used the text editor tool correctly
let valid_tool_call = messages.iter().any(|msg| {
msg.role == Role::Assistant
&& msg.content.iter().any(|content| {
if let MessageContent::ToolRequest(tool_req) = content {
if let Ok(tool_call) = tool_req.tool_call.as_ref() {
// Check tool name and basic parameters
if tool_call.name != "developer__text_editor" {
return false;
}
// Parse the arguments as JSON
if let Ok(args) =
serde_json::from_value::<Value>(serde_json::Value::Object(
tool_call.arguments.clone().unwrap_or_default(),
))
{
// Only check command is write and correct filename
args.get("command").and_then(Value::as_str) == Some("write")
&& args
.get("path")
.and_then(Value::as_str)
.is_some_and(|s| s.contains("flappy_bird.py"))
} else {
false
}
} else {
false
}
} else {
false
}
})
});
metrics.push((
"used_write_tool".to_string(),
EvalMetricValue::Boolean(valid_tool_call),
));
// If tool was used correctly, check the actual file content
let mut valid_implementation = false;
if valid_tool_call {
if let Ok(file_path) = run_loc.fs_get("flappy_bird.py".to_string()) {
if let Ok(content) = fs::read_to_string(file_path) {
valid_implementation = self.check_python_implementation(&content);
metrics.push((
"valid_implementation".to_string(),
EvalMetricValue::Boolean(valid_implementation),
));
}
}
}
metrics.push((
"score".to_string(),
EvalMetricValue::Float(
((valid_implementation as u8) + (valid_tool_call as u8)) as f64 / 2.0,
),
));
Ok(metrics)
}
fn name(&self) -> &str {
"flappy_bird"
}
fn required_extensions(&self) -> ExtensionRequirements {
ExtensionRequirements {
builtin: vec!["developer".to_string()],
external: Vec::new(),
streamable_http: Vec::new(),
}
}
}
register_evaluation!(FlappyBird);

136
crates/goose-bench/src/eval_suites/vibes/goose_wiki.rs
View file

@ -1,136 +0,0 @@
use crate::bench_session::BenchAgent;
use crate::bench_work_dir::BenchmarkWorkDir;
use crate::eval_suites::{
collect_baseline_metrics, metrics_hashmap_to_vec, EvalMetricValue, Evaluation,
ExtensionRequirements,
};
use crate::register_evaluation;
use async_trait::async_trait;
use goose::conversation::message::MessageContent;
use rmcp::model::Role;
use serde_json::{self, Value};
use std::fs;
pub struct GooseWiki {}
impl GooseWiki {
pub fn new() -> Self {
GooseWiki {}
}
fn check_html_implementation(&self, content: &str) -> bool {
// Check for basic structure
let has_basic_structure = content.contains("<html")
&& content.contains("</html>")
&& content.contains("<head")
&& content.contains("</head>")
&& content.contains("<body")
&& content.contains("</body>");
// Check for Wikipedia-style content
let has_wiki_elements = content.contains("<h1") && // Has headings
(content.contains("<h2") || content.contains("<h3")) && // Has subheadings
content.contains("Goose") && // Mentions Goose
content.contains("AI") && // Mentions AI
(content.contains("<p>") || content.contains("<div")); // Has paragraphs
has_basic_structure && has_wiki_elements
}
}
#[async_trait]
impl Evaluation for GooseWiki {
async fn run(
&self,
agent: &mut BenchAgent,
_run_loc: &mut BenchmarkWorkDir,
) -> anyhow::Result<Vec<(String, EvalMetricValue)>> {
println!("GooseWiki - run");
// Collect baseline metrics (execution time, token usage, tool calls)
let (messages, perf_metrics) = collect_baseline_metrics(
agent,
"Create a Wikipedia-style web page about Goose (Block's AI agent) in a new index.html file. The page should be a complete, well-structured HTML document with proper head and body sections. Use heading tags (h1, h2, h3) to organize the content into clear sections. Include comprehensive information about Goose organized in a way similar to how Wikipedia presents technical topics. Remember to use your tools if applicable.".to_string()
).await;
// Convert HashMap to Vec for our metrics
let mut metrics = metrics_hashmap_to_vec(perf_metrics);
// Check if the agent used the text editor tool to create index.html
let valid_tool_call = messages.iter().any(|msg| {
msg.role == Role::Assistant
&& msg.content.iter().any(|content| {
if let MessageContent::ToolRequest(tool_req) = content {
if let Ok(tool_call) = tool_req.tool_call.as_ref() {
// Check tool name is correct
if tool_call.name != "developer__text_editor" {
return false;
}
// Parse the arguments as JSON
if let Ok(args) =
serde_json::from_value::<Value>(serde_json::Value::Object(
tool_call.arguments.clone().unwrap_or_default(),
))
{
// Only check command is write and correct filename
args.get("command").and_then(Value::as_str) == Some("write")
&& args
.get("path")
.and_then(Value::as_str)
.is_some_and(|s| s.contains("index.html"))
} else {
false
}
} else {
false
}
} else {
false
}
})
});
metrics.push((
"used_write_tool".to_string(),
EvalMetricValue::Boolean(valid_tool_call),
));
let mut valid_implementation = false;
// If tool was used correctly, check the actual file content
if valid_tool_call {
if let Ok(file_path) = _run_loc.fs_get("index.html".to_string()) {
if let Ok(content) = fs::read_to_string(file_path) {
valid_implementation = self.check_html_implementation(&content);
metrics.push((
"valid_implementation".to_string(),
EvalMetricValue::Boolean(valid_implementation),
));
}
}
}
metrics.push((
"score".to_string(),
EvalMetricValue::Float(
((valid_implementation as u8) + (valid_tool_call as u8)) as f64 / 2.0,
),
));
Ok(metrics)
}
fn name(&self) -> &str {
"goose_wiki"
}
fn required_extensions(&self) -> ExtensionRequirements {
ExtensionRequirements {
builtin: vec!["developer".to_string()],
external: Vec::new(),
streamable_http: Vec::new(),
}
}
}
register_evaluation!(GooseWiki);

5
crates/goose-bench/src/eval_suites/vibes/mod.rs
View file

@ -1,5 +0,0 @@
mod blog_summary;
mod flappy_bird;
mod goose_wiki;
mod restaurant_research;
mod squirrel_census;

107
crates/goose-bench/src/eval_suites/vibes/restaurant_research.rs
View file

@ -1,107 +0,0 @@
use crate::bench_session::BenchAgent;
use crate::bench_work_dir::BenchmarkWorkDir;
use crate::eval_suites::{
collect_baseline_metrics, metrics_hashmap_to_vec, write_response_to_file, EvalMetricValue,
Evaluation, ExtensionRequirements,
};
use crate::register_evaluation;
use async_trait::async_trait;
pub struct RestaurantResearch {}
impl RestaurantResearch {
pub fn new() -> Self {
RestaurantResearch {}
}
fn check_markdown_bullets(&self, text: &str) -> bool {
// Check if there's at least one bullet point and proper markdown formatting
text.contains("- ") || text.contains("* ")
}
fn count_bullet_points(&self, text: &str) -> i64 {
// Count total bullet points (either - or * style)
let dash_bullets = text.matches("- ").count();
let star_bullets = text.matches("* ").count();
(dash_bullets + star_bullets) as i64
}
}
#[async_trait]
impl Evaluation for RestaurantResearch {
async fn run(
&self,
agent: &mut BenchAgent,
run_loc: &mut BenchmarkWorkDir,
) -> anyhow::Result<Vec<(String, EvalMetricValue)>> {
println!("RestaurantResearch - run");
// Collect baseline metrics (execution time, token usage, tool calls)
let (response, perf_metrics) = collect_baseline_metrics(
agent,
"Search the internet for and provide a current, detailed list of the best Sichuanese restaurants specifically in the East Village neighborhood of NYC. Format your response in Markdown using bullet points (either - or *) for each restaurant. For each restaurant include:
- Restaurant name and what they're known for
- Signature dishes
- Atmosphere/setting
- Any relevant details about reservations or dining experience
- What distinguishes them from others
Present the information in order of significance or quality. Focus specifically on Sichuanese establishments, not general Chinese restaurants. If you encounter a page you cannot access, try another one. Do not ask me for confirmation just conduct the searches yourself until you find the needed information. Remember to use your tools if applicable.".to_string()
).await;
// Write response to file and get the text content
let response_text = match write_response_to_file(
response.messages(),
run_loc,
"restaurant_research_output.txt",
) {
Ok(text) => text,
Err(e) => {
println!("Warning: Failed to write restaurant research output: {}", e);
// If file write fails, still continue with the evaluation
response
.last()
.map_or_else(String::new, |msg| msg.as_concat_text())
}
};
// Convert HashMap to Vec for our metrics
let mut metrics = metrics_hashmap_to_vec(perf_metrics);
// Check markdown formatting
let has_markdown_bullets = self.check_markdown_bullets(&response_text);
let bullet_count = self.count_bullet_points(&response_text);
metrics.push((
"valid_markdown_format".to_string(),
EvalMetricValue::Boolean(has_markdown_bullets),
));
metrics.push((
"bullet_point_count".to_string(),
EvalMetricValue::Integer(bullet_count),
));
// Check if the fetch tool was used
let used_fetch_tool = crate::eval_suites::used_tool(response.messages(), "fetch");
metrics.push((
"used_fetch_tool".to_string(),
EvalMetricValue::Boolean(used_fetch_tool),
));
Ok(metrics)
}
fn name(&self) -> &str {
"restaurant_research"
}
fn required_extensions(&self) -> ExtensionRequirements {
ExtensionRequirements {
builtin: vec!["developer".to_string()],
external: vec!["uvx mcp-server-fetch".to_string()],
streamable_http: Vec::new(),
}
}
}
register_evaluation!(RestaurantResearch);

178
crates/goose-bench/src/eval_suites/vibes/squirrel_census.rs
View file

@ -1,178 +0,0 @@
use crate::bench_session::BenchAgent;
use crate::bench_work_dir::BenchmarkWorkDir;
use crate::eval_suites::{
collect_baseline_metrics, metrics_hashmap_to_vec, EvalMetricValue, Evaluation,
ExtensionRequirements,
};
use crate::register_evaluation;
use async_trait::async_trait;
use goose::conversation::message::MessageContent;
use rmcp::model::Role;
use serde_json::{self, Value};
pub struct SquirrelCensus {}
impl SquirrelCensus {
pub fn new() -> Self {
SquirrelCensus {}
}
fn check_analysis_results(&self, text: &str) -> (bool, bool, bool) {
let text_lower = text.to_lowercase();
let has_central_manhattan =
text_lower.contains("central manhattan") && text.contains("174");
let has_tompkins = text_lower.contains("tompkins square park") && text.contains("59");
let has_gray = text_lower.contains("gray") || text_lower.contains("grey");
(has_central_manhattan, has_tompkins, has_gray)
}
}
#[async_trait]
impl Evaluation for SquirrelCensus {
#[allow(clippy::too_many_lines)]
async fn run(
&self,
agent: &mut BenchAgent,
run_loc: &mut BenchmarkWorkDir,
) -> anyhow::Result<Vec<(String, EvalMetricValue)>> {
println!("SquirrelCensus - run");
// Get the path to the squirrel data file
let squirrel_data_path = match run_loc.fs_get("./assets/squirrel-data.csv".to_string()) {
Ok(file) => file,
Err(_) => return Err(anyhow::anyhow!("Could not find squirrel-data.csv file")),
};
println!("squirrel_data_path: {:?}", squirrel_data_path);
// Collect baseline metrics (execution time, token usage, tool calls)
let (messages, perf_metrics) = collect_baseline_metrics(
agent,
format!(
"Create a Python script called analyze_squirrels.py that analyzes the CSV file at {}. Do not ask for any clarification or further instructions - proceed with the implementation as specified below.
The script should use pandas to answer these specific questions:
1. Which area (Area column) has the most squirrels spotted? For this area, what is the most common Primary Fur Color of squirrels?
2. Which specific park location (Park Name column) has the most squirrels spotted? For this location, what is the most common Primary Fur Color of squirrels?
The script should:
- Use pandas to read and analyze the data
- Print results in EXACTLY this format (including the markers):
[AREA_RESULT] <area_name> - <count> squirrels spotted
[AREA_COLOR] Most common fur color: <color> (<color_count> squirrels)
[PARK_RESULT] <park_name> - <count> squirrels spotted
[PARK_COLOR] Most common fur color: <color> (<color_count> squirrels)
After writing the script, run it using python3 and show the results. Do not ask for confirmation or further instructions. Remember to use your tools if applicable.",
squirrel_data_path.display()
)
).await;
// Convert HashMap to Vec for our metrics
let mut metrics = metrics_hashmap_to_vec(perf_metrics);
// Check if agent wrote the Python script
let wrote_script = messages.iter().any(|msg| {
msg.role == Role::Assistant
&& msg.content.iter().any(|content| {
if let MessageContent::ToolRequest(tool_req) = content {
if let Ok(tool_call) = tool_req.tool_call.as_ref() {
if tool_call.name != "developer__text_editor" {
return false;
}
if let Ok(args) =
serde_json::from_value::<Value>(serde_json::Value::Object(
tool_call.arguments.clone().unwrap_or_default(),
))
{
args.get("command").and_then(Value::as_str) == Some("write")
&& args
.get("path")
.and_then(Value::as_str)
.is_some_and(|s| s.contains("analyze_squirrels.py"))
} else {
false
}
} else {
false
}
} else {
false
}
})
});
// Check if agent ran the script
let ran_script = messages.iter().any(|msg| {
msg.role == Role::Assistant
&& msg.content.iter().any(|content| {
if let MessageContent::ToolRequest(tool_req) = content {
if let Ok(tool_call) = tool_req.tool_call.as_ref() {
if tool_call.name != "developer__shell" {
return false;
}
if let Ok(args) =
serde_json::from_value::<Value>(serde_json::Value::Object(
tool_call.arguments.clone().unwrap_or_default(),
))
{
args.get("command")
.and_then(Value::as_str)
.is_some_and(|s| {
s.contains("python") && s.contains("analyze_squirrels.py")
})
} else {
false
}
} else {
false
}
} else {
false
}
})
});
// Check the last message for correct results
let correct_results = if let Some(last_msg) = messages.last() {
let text_content = last_msg.as_concat_text();
let (has_central_manhattan, has_tompkins, has_gray) =
self.check_analysis_results(&text_content);
has_central_manhattan && has_tompkins && has_gray
} else {
false
};
metrics.push((
"wrote_script".to_string(),
EvalMetricValue::Boolean(wrote_script),
));
metrics.push((
"ran_script".to_string(),
EvalMetricValue::Boolean(ran_script),
));
metrics.push((
"score".to_string(),
EvalMetricValue::Float((correct_results as u8) as f64 / 1.0),
));
Ok(metrics)
}
fn name(&self) -> &str {
"squirrel_census"
}
fn required_extensions(&self) -> ExtensionRequirements {
ExtensionRequirements {
builtin: vec!["developer".to_string()],
external: Vec::new(),
streamable_http: Vec::new(),
}
}
}
register_evaluation!(SquirrelCensus);

8
crates/goose-bench/src/lib.rs
View file

@ -1,8 +0,0 @@
pub mod bench_config;
pub mod bench_session;
pub mod bench_work_dir;
pub mod error_capture;
pub mod eval_suites;
pub mod reporting;
pub mod runners;
pub mod utilities;

133
crates/goose-bench/src/reporting.rs
View file

@ -1,133 +0,0 @@
use crate::bench_session::BenchAgentError;
use crate::eval_suites::EvalMetricValue;
use chrono::Local;
use serde::{Deserialize, Serialize};
use std::fmt;
/// Represents a single evaluation result
#[derive(Default, Deserialize, Serialize)]
pub struct EvaluationResult {
pub name: String,
pub metrics: Vec<(String, EvalMetricValue)>,
pub errors: Vec<BenchAgentError>,
}
/// Represents results for an entire suite
#[derive(Default, Deserialize, Serialize)]
pub struct SuiteResult {
pub name: String,
pub evaluations: Vec<EvaluationResult>,
}
/// Contains all benchmark results and metadata
#[derive(Default, Deserialize, Serialize)]
pub struct BenchmarkResults {
pub provider: String,
pub start_time: String,
pub suites: Vec<SuiteResult>,
}
impl EvaluationResult {
pub fn new(name: String) -> Self {
Self {
name,
metrics: Vec::new(),
errors: Vec::new(),
}
}
pub fn add_metric(&mut self, name: String, metric: EvalMetricValue) {
self.metrics.push((name, metric));
}
pub fn add_error(&mut self, error: BenchAgentError) {
self.errors.push(error);
}
}
impl SuiteResult {
pub fn new(name: String) -> Self {
Self {
name,
evaluations: Vec::new(),
}
}
pub fn add_evaluation(&mut self, eval: EvaluationResult) {
self.evaluations.push(eval);
}
}
impl BenchmarkResults {
pub fn new(provider: String) -> Self {
Self {
provider,
start_time: Local::now().format("%Y-%m-%d %H:%M:%S").to_string(),
suites: Vec::new(),
}
}
pub fn add_suite(&mut self, suite: SuiteResult) {
self.suites.push(suite);
}
/// Generate a summary of the benchmark results
pub fn summary(&self) -> String {
let mut summary = String::new();
summary.push_str(&format!("Benchmark Summary - {}\n", self.provider));
summary.push_str(&format!("Run at: {}\n\n", self.start_time));
for suite in &self.suites {
summary.push_str(&format!(
"Suite: {} ({} evaluations)\n",
suite.name,
suite.evaluations.len()
));
// Count total metrics and errors
let total_metrics: usize = suite.evaluations.iter().map(|e| e.metrics.len()).sum();
let total_errors: usize = suite.evaluations.iter().map(|e| e.errors.len()).sum();
summary.push_str(&format!(" Total metrics: {}\n", total_metrics));
if total_errors > 0 {
summary.push_str(&format!(" Total errors: {}\n", total_errors));
}
}
summary
}
}
impl fmt::Display for BenchmarkResults {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
writeln!(f, "Benchmark Results")?;
writeln!(f, "Provider: {}", self.provider)?;
writeln!(f, "Start Time: {}", self.start_time)?;
writeln!(f)?;
for suite in &self.suites {
writeln!(f, "Suite: {}", suite.name)?;
for eval in &suite.evaluations {
writeln!(f, " Evaluation: {}", eval.name)?;
for (metric_name, metric_value) in &eval.metrics {
writeln!(f, " {}: {}", metric_name, metric_value)?;
}
if !eval.errors.is_empty() {
writeln!(f, " Errors:")?;
for error in &eval.errors {
writeln!(
f,
" [{}] {}: {}",
error.timestamp.format("%H:%M:%S"),
error.level,
error.message
)?;
}
}
writeln!(f)?;
}
}
Ok(())
}
}

94
crates/goose-bench/src/runners/bench_runner.rs
View file

@ -1,94 +0,0 @@
use crate::bench_config::{BenchModel, BenchRunConfig};
use crate::bench_work_dir::BenchmarkWorkDir;
use crate::eval_suites::EvaluationSuite;
use crate::runners::model_runner::ModelRunner;
use crate::utilities::{await_process_exits, parallel_bench_cmd};
use anyhow::Context;
use std::path::PathBuf;
#[derive(Clone)]
pub struct BenchRunner {
config: BenchRunConfig,
}
impl BenchRunner {
pub fn new(config_path: PathBuf) -> anyhow::Result<BenchRunner> {
let config = BenchRunConfig::from(config_path.clone())?;
let resolved_output_dir = match &config.output_dir {
Some(path) => {
if !path.is_absolute() {
anyhow::bail!(
"Config Error in '{}': 'output_dir' must be an absolute path, but found relative path: {}",
config_path.display(),
path.display()
);
}
path.clone()
}
None => std::env::current_dir().context(
"Failed to get current working directory to use as default output directory",
)?,
};
BenchmarkWorkDir::init_experiment(resolved_output_dir)?;
config.save("config.cfg".to_string());
Ok(BenchRunner { config })
}
pub fn from(config: String) -> anyhow::Result<BenchRunner> {
let config = BenchRunConfig::from_string(config)?;
Ok(BenchRunner { config })
}
pub fn run(&mut self) -> anyhow::Result<()> {
// split models that must run serial from those that can be run in parallel
let (parallel_models, serial_models): &(Vec<BenchModel>, Vec<BenchModel>) = &self
.config
.models
.clone()
.into_iter()
.partition(|model| model.parallel_safe);
// exec parallel models
let mut parallel_models_handle = Vec::new();
for model in parallel_models {
self.config.models = vec![model.clone()];
let cfg = self.config.to_string()?;
let model_handle = parallel_bench_cmd("eval-model".to_string(), cfg, Vec::new());
parallel_models_handle.push(model_handle);
}
// exec serial models
for model in serial_models {
self.config.models = vec![model.clone()];
ModelRunner::from(self.config.to_string()?)?.run()?;
}
await_process_exits(&mut parallel_models_handle, Vec::new());
Ok(())
}
pub fn list_selectors(_config: Option<PathBuf>) -> anyhow::Result<()> {
let selector_eval_counts = EvaluationSuite::available_selectors();
let mut keys: Vec<_> = selector_eval_counts.keys().collect();
keys.sort();
let max_key_len = keys.iter().map(|k| k.len()).max().unwrap_or(0);
println!(
"selector {} => Eval Count",
" ".repeat(max_key_len - "selector".len())
);
println!("{}", "-".repeat(max_key_len + 6));
for selector in keys {
println!(
"{} {} => {}",
selector,
" ".repeat(max_key_len - selector.len()),
selector_eval_counts.get(selector).unwrap()
);
}
Ok(())
}
}

189
crates/goose-bench/src/runners/eval_runner.rs
View file

@ -1,189 +0,0 @@
use crate::bench_config::{BenchEval, BenchModel, BenchRunConfig};
use crate::bench_session::BenchAgent;
use crate::bench_work_dir::BenchmarkWorkDir;
use crate::eval_suites::{EvaluationSuite, ExtensionRequirements};
use crate::reporting::EvaluationResult;
use crate::utilities::await_process_exits;
use anyhow::{bail, Context, Result};
use std::env;
use std::fs;
use std::future::Future;
use std::path::PathBuf;
use std::process::Command;
use std::time::{SystemTime, UNIX_EPOCH};
use tracing;
#[derive(Clone)]
pub struct EvalRunner {
config: BenchRunConfig,
}
impl EvalRunner {
pub fn from(config: String) -> Result<EvalRunner> {
let config = BenchRunConfig::from_string(config)
.context("Failed to parse evaluation configuration")?;
Ok(EvalRunner { config })
}
fn create_work_dir(&self, config: &BenchRunConfig) -> Result<BenchmarkWorkDir> {
let goose_model = config
.models
.first()
.context("No model specified in configuration")?;
let model_name = goose_model.name.clone();
let provider_name = goose_model.provider.clone();
// construct work-dir name to have a shim component only if shim configured to be used
let work_dir_name_shim = {
let mut shim_name = "".to_string();
if let Some(shim_opt) = &goose_model.tool_shim {
if shim_opt.use_tool_shim {
let shim_model = if let Some(shim_model) = &shim_opt.tool_shim_model {
shim_model.clone()
} else {
"default".to_string()
};
shim_name = format!("-{}-shim-model", shim_model);
}
}
shim_name
};
let include_dir = config.include_dirs.clone();
let work_dir_name = format!("{}-{}{}", provider_name, model_name, work_dir_name_shim);
let work_dir = BenchmarkWorkDir::new(work_dir_name, include_dir);
Ok(work_dir)
}
pub async fn run<F, Fut>(&mut self, agent_generator: F) -> Result<()>
where
F: Fn(ExtensionRequirements, String) -> Fut,
Fut: Future<Output = BenchAgent> + Send,
{
let mut work_dir = self
.create_work_dir(&self.config)
.context("Failed to create evaluation work directory")?;
let bench_eval = self
.config
.evals
.first()
.context("No evaluations specified in configuration")?;
let run_id = &self
.config
.run_id
.clone()
.unwrap_or_else(|| "run-0".to_string());
let run_id = format!("run-{}", run_id.clone());
// create entire dir subtree for eval and cd into dir for running eval
work_dir.set_eval(&bench_eval.selector, run_id);
tracing::info!("Set evaluation directory for {}", bench_eval.selector);
if let Some(eval) = EvaluationSuite::from(&bench_eval.selector) {
let now_stamp = SystemTime::now()
.duration_since(UNIX_EPOCH)
.context("Failed to get current timestamp")?
.as_nanos();
let session_id = format!("{}-{}", bench_eval.selector.clone(), now_stamp);
let mut agent = agent_generator(eval.required_extensions(), session_id).await;
tracing::info!("Agent created for {}", eval.name());
let mut result = EvaluationResult::new(eval.name().to_string());
match eval.run(&mut agent, &mut work_dir).await {
Ok(metrics) => {
tracing::info!("Evaluation run successful with {} metrics", metrics.len());
for (name, metric) in metrics {
result.add_metric(name, metric);
}
}
Err(e) => {
tracing::error!("Evaluation run failed: {}", e);
}
}
// Add any errors that occurred
let errors = agent.get_errors().await;
tracing::info!("Agent reported {} errors", errors.len());
for error in errors {
result.add_error(error);
}
// Write results to file
let eval_results = serde_json::to_string_pretty(&result)
.context("Failed to serialize evaluation results to JSON")?;
let eval_results_file = env::current_dir()
.context("Failed to get current directory")?
.join(&self.config.eval_result_filename);
fs::write(&eval_results_file, &eval_results).with_context(|| {
format!(
"Failed to write evaluation results to {}",
eval_results_file.display()
)
})?;
tracing::info!(
"Wrote evaluation results to {}",
eval_results_file.display()
);
self.config.save("config.cfg".to_string());
work_dir.save();
// handle running post-process cmd if configured
if let Some(cmd) = &bench_eval.post_process_cmd {
tracing::info!("Running post-process command: {:?}", cmd);
let handle = Command::new(cmd)
.arg(&eval_results_file)
.spawn()
.with_context(|| {
format!("Failed to execute post-process command: {:?}", cmd)
})?;
await_process_exits(&mut [handle], Vec::new());
}
// copy session file into eval-dir
let here = env::current_dir()
.context("Failed to get current directory")?
.canonicalize()
.context("Failed to canonicalize current directory path")?;
let session = agent.get_session().await?;
let session_json = serde_json::to_string_pretty(&session)
.context("Failed to serialize session to JSON")?;
fs::write(here.join("session.json"), session_json)
.context("Failed to write session JSON to evaluation directory")?;
tracing::info!("Evaluation completed successfully");
} else {
tracing::error!("No evaluation found for selector: {}", bench_eval.selector);
bail!("No evaluation found for selector: {}", bench_eval.selector);
}
Ok(())
}
pub fn path_for_eval(model: &BenchModel, eval: &BenchEval, run_id: String) -> PathBuf {
let provider = model.provider.clone();
let model = model.name.clone();
let eval_path = &eval.selector.replace(":", std::path::MAIN_SEPARATOR_STR);
let eval_results_location = format!(
"{}-{}/run-{}{}{}",
&provider,
model,
run_id,
std::path::MAIN_SEPARATOR_STR,
eval_path
);
PathBuf::from(eval_results_location.clone())
}
}

81
crates/goose-bench/src/runners/metric_aggregator.rs
View file

@ -1,81 +0,0 @@
use anyhow::{bail, ensure, Context, Result};
use std::path::PathBuf;
use tracing;
pub struct MetricAggregator;
impl MetricAggregator {
/// Generate leaderboard and aggregated metrics CSV files from benchmark directory
pub fn generate_csv_from_benchmark_dir(benchmark_dir: &PathBuf) -> Result<()> {
use std::process::Command;
// Step 1: Run prepare_aggregate_metrics.py to create aggregate_metrics.csv files
let prepare_script_path = std::env::current_dir()
.context("Failed to get current working directory")?
.join("scripts")
.join("bench-postprocess-scripts")
.join("prepare_aggregate_metrics.py");
ensure!(
prepare_script_path.exists(),
"Prepare script not found: {}",
prepare_script_path.display()
);
tracing::info!(
"Preparing aggregate metrics from benchmark directory: {}",
benchmark_dir.display()
);
let output = Command::new(&prepare_script_path)
.arg("--benchmark-dir")
.arg(benchmark_dir)
.output()
.context("Failed to execute prepare_aggregate_metrics.py script")?;
if !output.status.success() {
let error_message = String::from_utf8_lossy(&output.stderr);
bail!("Failed to prepare aggregate metrics: {}", error_message);
}
let success_message = String::from_utf8_lossy(&output.stdout);
tracing::info!("{}", success_message);
// Step 2: Run generate_leaderboard.py to create the final leaderboard
let leaderboard_script_path = std::env::current_dir()
.context("Failed to get current working directory")?
.join("scripts")
.join("bench-postprocess-scripts")
.join("generate_leaderboard.py");
ensure!(
leaderboard_script_path.exists(),
"Leaderboard script not found: {}",
leaderboard_script_path.display()
);
tracing::info!(
"Generating leaderboard from benchmark directory: {}",
benchmark_dir.display()
);
let output = Command::new(&leaderboard_script_path)
.arg("--benchmark-dir")
.arg(benchmark_dir)
.arg("--leaderboard-output")
.arg("leaderboard.csv")
.arg("--union-output")
.arg("all_metrics.csv")
.output()
.context("Failed to execute generate_leaderboard.py script")?;
if !output.status.success() {
let error_message = String::from_utf8_lossy(&output.stderr);
bail!("Failed to generate leaderboard: {}", error_message);
}
let success_message = String::from_utf8_lossy(&output.stdout);
tracing::info!("{}", success_message);
Ok(())
}
}

4
crates/goose-bench/src/runners/mod.rs
View file

@ -1,4 +0,0 @@
pub mod bench_runner;
pub mod eval_runner;
pub mod metric_aggregator;
pub mod model_runner;

248
crates/goose-bench/src/runners/model_runner.rs
View file

@ -1,248 +0,0 @@
use crate::bench_config::{BenchEval, BenchModel, BenchRunConfig};
use crate::eval_suites::EvaluationSuite;
use crate::reporting::{BenchmarkResults, SuiteResult};
use crate::runners::eval_runner::EvalRunner;
use crate::utilities::{await_process_exits, parallel_bench_cmd};
use anyhow::{Context, Result};
use dotenvy::from_path_iter;
use std::collections::HashMap;
use std::fs::read_to_string;
use std::path::PathBuf;
use std::process::Child;
use std::thread;
use tracing;
#[derive(Clone)]
pub struct ModelRunner {
config: BenchRunConfig,
}
impl ModelRunner {
pub fn from(config: String) -> Result<ModelRunner> {
let config =
BenchRunConfig::from_string(config).context("Failed to parse configuration")?;
Ok(ModelRunner { config })
}
pub fn run(&self) -> Result<()> {
let model = self
.config
.models
.first()
.context("No model specified in config")?;
let suites = self.collect_evals_for_run();
let mut handles = vec![];
for i in 0..self.config.repeat.unwrap_or(1) {
let self_copy = self.clone();
let model_clone = model.clone();
let suites_clone = suites.clone();
let handle = thread::spawn(move || -> Result<()> {
self_copy.run_benchmark(&model_clone, suites_clone, i.to_string())
});
handles.push(handle);
}
await_process_exits(&mut Vec::new(), handles);
let mut all_runs_results: Vec<BenchmarkResults> = Vec::new();
for i in 0..self.config.repeat.unwrap_or(1) {
match self.collect_run_results(model.clone(), suites.clone(), i.to_string()) {
Ok(run_results) => all_runs_results.push(run_results),
Err(e) => {
tracing::error!("Failed to collect results for run {}: {}", i, e)
}
}
}
Ok(())
}
fn run_benchmark(
&self,
model: &BenchModel,
suites: HashMap<String, Vec<BenchEval>>,
run_id: String,
) -> Result<()> {
let mut results_handles = HashMap::<String, Vec<Child>>::new();
// Load environment variables from file if specified
let mut envs = self.toolshim_envs();
if let Some(env_file) = &self.config.env_file {
let env_vars = ModelRunner::load_env_file(env_file).context(format!(
"Failed to load environment file: {}",
env_file.display()
))?;
envs.extend(env_vars);
}
envs.push(("GOOSE_MODEL".to_string(), model.clone().name));
envs.push(("GOOSE_PROVIDER".to_string(), model.clone().provider));
// Only run in parallel if the model is parallel_safe
let run_parallel = model.parallel_safe;
for (suite, evals) in suites.iter() {
results_handles.insert((*suite).clone(), Vec::new());
// Group evaluations by parallel_safe
let mut parallel_evals = Vec::new();
let mut sequential_evals = Vec::new();
for eval in evals {
if eval.parallel_safe && run_parallel {
parallel_evals.push(eval);
} else {
sequential_evals.push(eval);
}
}
// Run parallel-safe evaluations in parallel
if !parallel_evals.is_empty() {
for eval_selector in &parallel_evals {
let mut config_copy = self.config.clone();
config_copy.run_id = Some(run_id.clone());
config_copy.evals = vec![(*eval_selector).clone()];
let cfg = config_copy
.to_string()
.context("Failed to serialize configuration")?;
let handle = parallel_bench_cmd("exec-eval".to_string(), cfg, envs.clone());
results_handles.get_mut(suite).unwrap().push(handle);
}
}
// Run non-parallel-safe evaluations sequentially
for eval_selector in &sequential_evals {
let mut config_copy = self.config.clone();
config_copy.run_id = Some(run_id.clone());
config_copy.evals = vec![(*eval_selector).clone()];
let cfg = config_copy
.to_string()
.context("Failed to serialize configuration")?;
let handle = parallel_bench_cmd("exec-eval".to_string(), cfg, envs.clone());
// Wait for this process to complete before starting the next one
let mut child_procs = vec![handle];
await_process_exits(&mut child_procs, Vec::new());
}
}
// Wait for any remaining parallel processes to complete
for (_, child_procs) in results_handles.iter_mut() {
await_process_exits(child_procs, Vec::new());
}
Ok(())
}
fn collect_run_results(
&self,
model: BenchModel,
suites: HashMap<String, Vec<BenchEval>>,
run_id: String,
) -> Result<BenchmarkResults> {
let mut results = BenchmarkResults::new(model.provider.clone());
let mut summary_path: Option<PathBuf> = None;
for (suite, evals) in suites.iter() {
let mut suite_result = SuiteResult::new(suite.clone());
for eval_selector in evals {
let mut eval_path =
EvalRunner::path_for_eval(&model, eval_selector, run_id.clone());
eval_path.push(self.config.eval_result_filename.clone());
let content = read_to_string(&eval_path).with_context(|| {
format!(
"Failed to read evaluation results from {}",
eval_path.display()
)
})?;
let eval_result = serde_json::from_str(&content)
.context("Failed to parse evaluation results JSON")?;
suite_result.add_evaluation(eval_result);
// use current eval to determine where the summary should be written
if summary_path.is_none() {
let mut result = PathBuf::new();
let mut iter = eval_path.components();
if let Some(first) = iter.next() {
result.push(first);
if let Some(second) = iter.next() {
result.push(second);
}
}
summary_path = Some(result);
}
}
results.add_suite(suite_result);
}
if let Some(path) = summary_path {
let mut run_summary = PathBuf::new();
run_summary.push(path);
run_summary.push(&self.config.run_summary_filename);
let output_str = serde_json::to_string_pretty(&results)
.context("Failed to serialize benchmark results to JSON")?;
std::fs::write(&run_summary, &output_str).with_context(|| {
format!(
"Failed to write results summary to {}",
run_summary.display()
)
})?;
}
Ok(results)
}
fn collect_evals_for_run(&self) -> HashMap<String, Vec<BenchEval>> {
// convert suites map {suite_name => [eval_selector_str] to map suite_name => [BenchEval]
let mut result: HashMap<String, Vec<BenchEval>> = HashMap::new();
for eval in self.config.evals.iter() {
let selected_suites = EvaluationSuite::select(vec![eval.selector.clone()]);
for (suite, evals) in selected_suites {
let entry: &mut Vec<BenchEval> = result.entry(suite).or_default();
entry.reserve(evals.len());
for suite_eval in evals {
let mut updated_eval = eval.clone();
updated_eval.selector = suite_eval.to_string();
entry.push(updated_eval);
}
}
}
result
}
fn toolshim_envs(&self) -> Vec<(String, String)> {
// read tool-shim preference from config, set respective env vars accordingly
let mut shim_envs: Vec<(String, String)> = Vec::new();
if let Some(model) = self.config.models.first() {
if let Some(shim_opt) = &model.tool_shim {
if shim_opt.use_tool_shim {
shim_envs.push(("GOOSE_TOOLSHIM".to_string(), "true".to_string()));
if let Some(shim_model) = &shim_opt.tool_shim_model {
shim_envs.push((
"GOOSE_TOOLSHIM_OLLAMA_MODEL".to_string(),
shim_model.clone(),
));
}
}
}
}
shim_envs
}
fn load_env_file(path: &PathBuf) -> Result<Vec<(String, String)>> {
let iter =
from_path_iter(path).context("Failed to read environment variables from file")?;
let env_vars = iter
.map(|item| item.context("Failed to parse environment variable"))
.collect::<Result<_, _>>()?;
Ok(env_vars)
}
}

37
crates/goose-bench/src/utilities.rs
View file

@ -1,37 +0,0 @@
use anyhow::Result;
use std::env;
use std::process::{Child, Command};
use std::thread::JoinHandle;
use tracing;
pub fn await_process_exits(child_processes: &mut [Child], handles: Vec<JoinHandle<Result<()>>>) {
for child in child_processes.iter_mut() {
match child.wait() {
Ok(status) => tracing::info!("Child exited with status: {}", status),
Err(e) => tracing::error!("Error waiting for child: {}", e),
}
}
for handle in handles {
match handle.join() {
Ok(_res) => (),
Err(e) => {
// Handle thread panic
tracing::error!("Thread panicked: {:?}", e);
}
}
}
}
pub fn parallel_bench_cmd(bench_cmd: String, config: String, envs: Vec<(String, String)>) -> Child {
let current_exe = env::current_exe().expect("Failed to get current executable path");
let mut cmd = Command::new(current_exe);
cmd.arg("bench").arg(bench_cmd).arg("--config").arg(config);
for (key, value) in envs.into_iter() {
cmd.env(key, value);
}
cmd.spawn().expect("Failed to spawn child process")
}

1
crates/goose-cli/Cargo.toml
View file

@ -17,7 +17,6 @@ path = "src/main.rs"
[dependencies]
goose = { path = "../goose" }
goose-acp = { path = "../goose-acp" }
goose-bench = { path = "../goose-bench" }
goose-mcp = { path = "../goose-mcp" }
rmcp = { workspace = true }
clap = { version = "4.4", features = ["derive"] }

90
crates/goose-cli/src/cli.rs
View file

@ -10,7 +10,6 @@ use goose_mcp::{
AutoVisualiserRouter, ComputerControllerServer, DeveloperServer, MemoryServer, TutorialServer,
};
use crate::commands::bench::agent_generator;
use crate::commands::configure::{configure_telemetry_consent_dialog, handle_configure};
use crate::commands::info::handle_info;
use crate::commands::project::{handle_project_default, handle_projects_interactive};
@ -31,11 +30,6 @@ use crate::session::{build_session, SessionBuilderConfig};
use goose::agents::Container;
use goose::session::session_manager::SessionType;
use goose::session::SessionManager;
use goose_bench::bench_config::BenchRunConfig;
use goose_bench::runners::bench_runner::BenchRunner;
use goose_bench::runners::eval_runner::EvalRunner;
use goose_bench::runners::metric_aggregator::MetricAggregator;
use goose_bench::runners::model_runner::ModelRunner;
use std::io::Read;
use std::path::PathBuf;
use tracing::warn;
@ -598,60 +592,6 @@ enum SchedulerCommand {
CronHelp {},
}
#[derive(Subcommand)]
pub enum BenchCommand {
#[command(name = "init-config", about = "Create a new starter-config")]
InitConfig {
#[arg(short, long, help = "filename with extension for generated config")]
name: String,
},
#[command(about = "Run all benchmarks from a config")]
Run {
#[arg(
short,
long,
help = "A config file generated by the config-init command"
)]
config: PathBuf,
},
#[command(about = "List all available selectors")]
Selectors {
#[arg(
short,
long,
help = "A config file generated by the config-init command"
)]
config: Option<PathBuf>,
},
#[command(name = "eval-model", about = "Run an eval of model")]
EvalModel {
#[arg(short, long, help = "A serialized config file for the model only.")]
config: String,
},
#[command(name = "exec-eval", about = "run a single eval")]
ExecEval {
#[arg(short, long, help = "A serialized config file for the eval only.")]
config: String,
},
#[command(
name = "generate-leaderboard",
about = "Generate a leaderboard CSV from benchmark results"
)]
GenerateLeaderboard {
#[arg(
short,
long,
help = "Path to the benchmark directory containing model evaluation results"
)]
benchmark_dir: PathBuf,
},
}
#[derive(Subcommand)]
enum RecipeCommand {
/// Validate a recipe file
@ -862,13 +802,6 @@ enum Command {
reconfigure: bool,
},
/// Evaluate system configuration across a range of practical tasks
#[command(about = "Evaluate system configuration across a range of practical tasks")]
Bench {
#[command(subcommand)]
cmd: BenchCommand,
},
/// Start a web server with a chat interface
#[command(about = "Experimental: Start a web server with a chat interface")]
Web {
@ -1018,7 +951,6 @@ fn get_command_name(command: &Option<Command>) -> &'static str {
Some(Command::Run { .. }) => "run",
Some(Command::Schedule { .. }) => "schedule",
Some(Command::Update { .. }) => "update",
Some(Command::Bench { .. }) => "bench",
Some(Command::Recipe { .. }) => "recipe",
Some(Command::Web { .. }) => "web",
Some(Command::Term { .. }) => "term",
@ -1029,7 +961,7 @@ fn get_command_name(command: &Option<Command>) -> &'static str {
async fn handle_mcp_command(server: McpCommand) -> Result<()> {
let name = server.name();
crate::logging::setup_logging(Some(&format!("mcp-{name}")), None)?;
let _ = crate::logging::setup_logging(Some(&format!("mcp-{name}")));
match server {
McpCommand::AutoVisualiser => serve(AutoVisualiserRouter::new()).await?,
McpCommand::ComputerController => serve(ComputerControllerServer::new()).await?,
@ -1426,25 +1358,6 @@ async fn handle_schedule_command(command: SchedulerCommand) -> Result<()> {
}
}
async fn handle_bench_command(cmd: BenchCommand) -> Result<()> {
match cmd {
BenchCommand::Selectors { config } => BenchRunner::list_selectors(config)?,
BenchCommand::InitConfig { name } => {
let mut config = BenchRunConfig::default();
let cwd = std::env::current_dir()?;
config.output_dir = Some(cwd);
config.save(name);
}
BenchCommand::Run { config } => BenchRunner::new(config)?.run()?,
BenchCommand::EvalModel { config } => ModelRunner::from(config)?.run()?,
BenchCommand::ExecEval { config } => EvalRunner::from(config)?.run(agent_generator).await?,
BenchCommand::GenerateLeaderboard { benchmark_dir } => {
MetricAggregator::generate_csv_from_benchmark_dir(&benchmark_dir)?
}
}
Ok(())
}
fn handle_recipe_subcommand(command: RecipeCommand) -> Result<()> {
match command {
RecipeCommand::Validate { recipe_name } => handle_validate(&recipe_name),
@ -1597,7 +1510,6 @@ pub async fn cli() -> anyhow::Result<()> {
crate::commands::update::update(canary, reconfigure)?;
Ok(())
}
Some(Command::Bench { cmd }) => handle_bench_command(cmd).await,
Some(Command::Recipe { command }) => handle_recipe_subcommand(command),
Some(Command::Web {
port,

76
crates/goose-cli/src/commands/bench.rs
View file

@ -1,76 +0,0 @@
use crate::cli::StreamableHttpOptions;
use crate::session::build_session;
use crate::session::SessionBuilderConfig;
use crate::{logging, CliSession};
use async_trait::async_trait;
use goose::conversation::Conversation;
use goose::session::session_manager::Session;
use goose_bench::bench_session::{BenchAgent, BenchBaseSession};
use goose_bench::eval_suites::ExtensionRequirements;
use std::sync::Arc;
use tokio::sync::Mutex;
// allow session obj to be used in benchmarking
#[async_trait]
impl BenchBaseSession for CliSession {
async fn headless(&mut self, message: String) -> anyhow::Result<()> {
self.headless(message).await
}
fn message_history(&self) -> Conversation {
self.message_history()
}
fn get_total_token_usage(&self) -> anyhow::Result<Option<i32>> {
// Since the trait requires sync but the session method is async,
// we need to block on the async call
tokio::task::block_in_place(|| {
tokio::runtime::Handle::current().block_on(self.get_total_token_usage())
})
}
async fn get_session(&self) -> anyhow::Result<Session> {
self.get_session().await
}
}
pub async fn agent_generator(
requirements: ExtensionRequirements,
session_id: String,
) -> BenchAgent {
let streamable_http_extensions: Vec<StreamableHttpOptions> = requirements
.streamable_http
.iter()
.map(|s| StreamableHttpOptions {
url: s.clone(),
timeout: goose::config::DEFAULT_EXTENSION_TIMEOUT,
})
.collect();
let base_session = build_session(SessionBuilderConfig {
session_id: Some(session_id),
resume: false,
fork: false,
no_session: false,
extensions: requirements.external,
streamable_http_extensions,
builtins: requirements.builtin,
no_profile: true,
recipe: None,
additional_system_prompt: None,
provider: None,
model: None,
debug: false,
max_tool_repetitions: None,
interactive: false, // Benchmarking is non-interactive
scheduled_job_id: None,
max_turns: None,
quiet: false,
output_format: "text".to_string(),
container: None,
})
.await;
let bench_agent = BenchAgent::new(Box::new(base_session));
let errors = Some(Arc::new(Mutex::new(bench_agent.get_errors().await)));
logging::setup_logging(Some("bench"), errors).expect("Failed to initialize logging");
bench_agent
}

1
crates/goose-cli/src/commands/mod.rs
View file

@ -1,4 +1,3 @@
pub mod bench;
pub mod configure;
pub mod info;
pub mod project;

2
crates/goose-cli/src/commands/web.rs
View file

@ -238,7 +238,7 @@ pub async fn handle_web(
no_auth: bool,
) -> Result<()> {
validate_network_auth(&host, &auth_token, no_auth);
crate::logging::setup_logging(Some("goose-web"), None)?;
crate::logging::setup_logging(Some("goose-web"))?;
let (provider_name, model) = get_provider_and_model();
let agent = create_agent(&provider_name, &model).await?;

28
crates/goose-cli/src/logging.rs
View file

@ -1,7 +1,5 @@
use anyhow::{Context, Result};
use std::sync::Arc;
use std::sync::Once;
use tokio::sync::Mutex;
use tracing_appender::rolling::Rotation;
use tracing_subscriber::{
filter::LevelFilter, fmt, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Layer,
@ -9,8 +7,6 @@ use tracing_subscriber::{
};
use goose::tracing::{langfuse_layer, otlp_layer};
use goose_bench::bench_session::BenchAgentError;
use goose_bench::error_capture::ErrorCaptureLayer;
// Used to ensure we only set up tracing once
static INIT: Once = Once::new();
@ -20,27 +16,14 @@ static INIT: Once = Once::new();
/// - File-based logging with JSON formatting (DEBUG level)
/// - No console output (all logs go to files only)
/// - Optional Langfuse integration (DEBUG level)
/// - Optional error capture layer for benchmarking
pub fn setup_logging(
name: Option<&str>,
error_capture: Option<Arc<Mutex<Vec<BenchAgentError>>>>,
) -> Result<()> {
setup_logging_internal(name, error_capture, false)
pub fn setup_logging(name: Option<&str>) -> Result<()> {
setup_logging_internal(name, false)
}
/// Internal function that allows bypassing the Once check for testing
fn setup_logging_internal(
name: Option<&str>,
error_capture: Option<Arc<Mutex<Vec<BenchAgentError>>>>,
force: bool,
) -> Result<()> {
fn setup_logging_internal(name: Option<&str>, force: bool) -> Result<()> {
let mut result = Ok(());
// Register the error vector if provided
if let Some(errors) = error_capture {
ErrorCaptureLayer::register_error_vector(errors);
}
let mut setup = || {
result = (|| {
let log_dir = goose::logging::prepare_log_directory("cli", true)?;
@ -84,11 +67,6 @@ fn setup_logging_internal(
// Console logging disabled for CLI - all logs go to files only
];
// Only add ErrorCaptureLayer if not in test mode
if !force {
layers.push(ErrorCaptureLayer::new().boxed());
}
if !force {
if let Ok((otlp_tracing_layer, otlp_metrics_layer, otlp_logs_layer)) =
otlp_layer::init_otlp()

2
crates/goose-cli/src/main.rs
View file

@ -3,7 +3,7 @@ use goose_cli::cli::cli;
#[tokio::main]
async fn main() -> Result<()> {
if let Err(e) = goose_cli::logging::setup_logging(None, None) {
if let Err(e) = goose_cli::logging::setup_logging(None) {
eprintln!("Warning: Failed to initialize logging: {}", e);
}

72
evals/open-model-gym/.gitignore vendored Normal file
View file

@ -0,0 +1,72 @@
# Dependencies
node_modules/
.pnpm-store/
.workdir/
report.html
# Build outputs
dist/
build/
out/
.next/
.nuxt/
.output/
.opencode-root
suite/.pi-root
# TypeScript
*.tsbuildinfo
*.d.ts.map
.g3/
# Logs
logs/
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
pnpm-debug.log*
# Runtime data
pids/
*.pid
*.seed
*.pid.lock
# Coverage & testing
coverage/
.nyc_output/
.jest/
# Caches
.cache/
.parcel-cache/
.turbo/
.eslintcache
.stylelintcache
*.swp
*.swo
# IDE & editors
.idea/
.vscode/
*.sublime-project
*.sublime-workspace
# OS files
.DS_Store
Thumbs.db
# Environment variables
.env
.env.local
.env.*.local
# Lock files (optional - uncomment if you don't want to track)
# package-lock.json
# yarn.lock
# pnpm-lock.yaml
# Temporary files
tmp/
temp/
*.tmp

60
evals/open-model-gym/Justfile Normal file
View file

@ -0,0 +1,60 @@
# Agent Runner - Test Suite (supports goose and opencode)
# Default recipe
default: run
# Full test run - all scenarios, all agents, 3 repetitions (worst kept)
run: _install
cd suite && npm run test
# Quick test - file-editing + everyday-app-automation, single run each (no repetition)
test: _install
cd suite && npx tsx src/runner.ts --scenario=file-editing,everyday-app-automation --run-count=1
# Run a specific scenario (all agents, 3 reps)
scenario name: _install
cd suite && npx tsx src/runner.ts --scenario={{name}}
# Run against a specific agent (all scenarios, 3 reps)
agent name: _install
cd suite && npx tsx src/runner.ts --agent={{name}}
# Open report in browser
report:
open report.html
# Install all dependencies
install:
cd suite && npm install
cd mcp-harness && npm install && npm run build
@# Install pi-mcp-adapter for Pi runner MCP support
@pi list 2>/dev/null | grep -q "pi-mcp-adapter" || pi install npm:pi-mcp-adapter
# Build TypeScript
build: _install
cd suite && npm run build
# Clear the test cache
clear-cache:
cd suite && npx tsx src/runner.ts --clear-cache
# Run tests ignoring cache (force fresh runs)
run-fresh: _install
cd suite && npx tsx src/runner.ts --no-cache
# Show cache stats
cache-stats:
@if [ -f suite/.cache/index.json ]; then \
echo "Cache entries: $$(cat suite/.cache/index.json | grep -o '"[a-f0-9]\{16\}":' | wc -l | tr -d ' ')"; \
echo "Cache size: $$(du -sh suite/.cache 2>/dev/null | cut -f1 || echo '0')"; \
else \
echo "No cache found"; \
fi
# Internal: install if node_modules missing, always rebuild mcp-harness
_install:
@[ -d suite/node_modules ] || (cd suite && npm install)
@[ -d mcp-harness/node_modules ] || (cd mcp-harness && npm install)
@cd mcp-harness && npm run build
@# Ensure pi-mcp-adapter is installed for Pi runner
@pi list 2>/dev/null | grep -q "pi-mcp-adapter" || pi install npm:pi-mcp-adapter

291
evals/open-model-gym/README.md Normal file
View file

@ -0,0 +1,291 @@
# Open Model Gym
Run agent tests across a matrix of **models × runners × scenarios**.
It isn't hard for any agent to do ok with opus, but lets scale things in the other direction. What do we have to break things down to.
<img width="1768" height="1133" alt="image" src="https://github.com/user-attachments/assets/29915659-ee6b-4a8b-ba5e-58420b168b43" />
## Quick Start
```bash
just install # one-time setup
just run # run full matrix (3 reps each)
just report # view results
```
## How It Works
The test harness runs every combination of models, runners, and scenarios defined in your matrix. Each test runs multiple times (default 3) and keeps the **worst result** — if a test fails even once, it's marked failed. This catches flaky passes.
## Configuration
Edit `config.yaml` to define your test matrix:
### Models
LLMs to test against. Supports any provider (Anthropic, OpenAI, Ollama, etc.):
```yaml
models:
- name: opus
provider: anthropic
model: claude-opus-4-5-20251101
- name: qwen3-coder
provider: ollama
model: qwen3-coder:64k
- name: gpt4
provider: openai
model: gpt-4-turbo
```
### Runners
Agent frameworks that execute the tests. Each runner has its own binary, type, and configuration:
```yaml
runners:
# Goose agent with extensions
- name: goose-full
type: goose
bin: goose # path to binary (can be absolute)
extensions: [developer, todo, skills]
stdio:
- node mcp-harness/dist/index.js
# OpenCode agent
- name: opencode
type: opencode
bin: opencode # path to binary
stdio:
- node mcp-harness/dist/index.js
# Custom goose binary path
- name: goose-dev
type: goose
bin: /path/to/my/goose-dev
extensions: [developer]
```
**Supported runner types:**
- `goose` — [Goose](https://github.com/block/goose) agent framework
- `opencode` — [OpenCode](https://opencode.ai) agent framework
- `pi` — [Pi](https://github.com/badlogic/pi-mono) coding agent
## Runner Details
Each runner has different setup requirements, MCP integration methods, and session handling.
### Goose
[Goose](https://github.com/block/goose) is Block's open-source coding agent with built-in MCP support.
**Setup:** Install via `brew install goose` or from source.
**MCP Integration:** Native support. The harness writes a `config.yaml` to an isolated `.goose-root/` directory with extensions and MCP servers:
```yaml
extensions:
developer:
enabled: true
mcp_harness:
type: stdio
enabled: true
cmd: node
args: [mcp-harness/dist/index.js]
```
**Session Handling:** Uses `--name <session>` for named sessions, `--resume` to continue:
- Turn 1: `goose run -i <prompt> --name <session>`
- Turn 2+: `goose run -i <prompt> --name <session> --resume`
- Single-turn: `goose run -i <prompt> --no-session`
### OpenCode
[OpenCode](https://opencode.ai) is a terminal-based coding agent.
**Setup:** Install via their website or package manager.
**MCP Integration:** Native support. The harness writes an `opencode.json` config to the workdir:
```json
{
"mcp": {
"harness": {
"type": "local",
"command": ["node", "mcp-harness/dist/index.js"],
"enabled": true
}
},
"model": "anthropic/claude-opus-4-5-20251101"
}
```
**Session Handling:** Uses `--continue` to resume the last session in the working directory:
- Turn 1: `opencode run "<prompt>"`
- Turn 2+: `opencode run --continue "<prompt>"`
⚠️ OpenCode doesn't support named sessions, so multi-turn scenarios exclude it.
### Pi
[Pi](https://github.com/badlogic/pi-mono) is a lightweight coding agent that requires an adapter for MCP support.
**Setup:**
```bash
# Install Pi
npm install -g @anthropic/pi # or from source
# Install the MCP adapter (required for MCP tools)
pi install npm:pi-mcp-adapter
```
The `just install` recipe auto-installs pi-mcp-adapter if missing.
**MCP Integration:** Via [pi-mcp-adapter](https://github.com/nicobailon/pi-mcp-adapter). The harness dynamically writes a `.pi-mcp.json` config to the workdir:
```json
{
"mcpServers": {
"harness": {
"command": "node",
"args": ["mcp-harness/dist/index.js"],
"lifecycle": "eager",
"env": { "MCP_HARNESS_LOG": "<workdir>/tool-calls.log" }
}
},
"settings": { "directTools": true }
}
```
Key settings:
- `directTools: true` — Registers MCP tools directly in Pi's tool list (no wrapper)
- `lifecycle: "eager"` — Connects to MCP servers at startup
**Model Configuration:** Pi requires custom models (like Ollama) to be defined in `models.json`. The harness automatically generates this config in an isolated `.pi-root/` directory and sets `PI_CODING_AGENT_DIR` to use it:
```json
{
"providers": {
"ollama": {
"baseUrl": "http://localhost:11434/v1",
"api": "openai-completions",
"apiKey": "ollama",
"models": [{ "id": "model-name", "name": "Model Name", ... }]
}
}
}
```
The harness copies `auth.json` from your real Pi config (`~/.pi/agent/`) so API keys work.
**Session Handling:** Uses `--session <path>` for file-based sessions, `--continue` to resume:
- Turn 1: `pi -p --session <path> "<prompt>"`
- Turn 2+: `pi -p --continue --session <path> "<prompt>"`
- Single-turn: `pi -p --no-session "<prompt>"`
The `-p` flag runs Pi in non-interactive "print" mode for automation
### Matrix
Define which scenarios run against which models/runners:
```yaml
matrix:
- scenario: file-editing
models: [opus, qwen3-coder] # omit to run all models
runners: [goose-full, opencode] # omit to run all runners
- scenario: everyday-app-automation
# runs against ALL models and ALL runners
```
## Scenarios
Scenarios live in `suite/scenarios/` as YAML files:
```yaml
name: file-editing
description: Create and edit files
prompt: |
1. Create joke.md containing a short joke
2. Edit hello.rs to add a debug function
setup:
hello.rs: |
fn main() { println!("Hello!"); }
validate:
- type: file_exists
path: joke.md
- type: file_matches
path: hello.rs
regex: "fn\\s+debug"
```
### Validation Rules
| Rule | Description |
|------|-------------|
| `file_exists` | File exists at path |
| `file_not_empty` | File exists and has content |
| `file_contains` | File contains literal string |
| `file_matches` | File matches regex pattern |
| `command_succeeds` | Shell command exits 0 |
| `tool_called` | MCP tool was called with matching args (regex supported) |
**Tool call validation example:**
```yaml
validate:
- type: tool_called
tool: slack_search_messages
args:
query: /quarterly.?review/ # regex pattern
- type: tool_called
tool: jira_create_issue
args:
summary: /Q1.*Review/
description: /David Brown/
```
## MCP Harness
Mock MCP server providing simulated tools for testing agent tool-use without hitting real APIs.
```bash
cd mcp-harness && npm install && npm run build
```
**Available tools:** gdrive, sheets, salesforce, slack, calendar, gmail, jira, github
Each tool returns realistic mock data. Tool calls are logged to `tool-calls.log` in the workdir for validation.
## Commands
| Command | Description |
|---------|-------------|
| `just run` | Full test run (3 reps each, worst kept) |
| `just test` | Quick run (1 rep each) |
| `just scenario <name>` | Run specific scenario |
| `just agent <name>` | Run specific agent |
| `just report` | Open HTML results |
### CLI Flags
```bash
# Filter by scenario, model, or runner
npx tsx src/runner.ts --scenario=file-editing --model=opus --runner=goose
# Control repetition count
npx tsx src/runner.ts --run-count=5
# Don't auto-open browser
npx tsx src/runner.ts --no-open
```
## Output
- `report.html` — Live-updating HTML matrix showing pass/fail status, duration, and validation details
- `logs/` — Full agent output logs for each run

498
evals/open-model-gym/agent-gym-report-2026-02-03.html Normal file
View file

File diff suppressed because one or more lines are too long

85
evals/open-model-gym/config.yaml Normal file
View file

@ -0,0 +1,85 @@
# =============================================================================
# Models - the LLMs to test
# =============================================================================
models:
- name: opus
provider: anthropic
model: claude-opus-4-5-20251101
- name: glm-4.7-flash
provider: ollama
model: glm-4.7-flash:latest
# too slow on 64g:
#- name: frob/qwen3-coder-next:latest
# provider: ollama
# model: frob/qwen3-coder-next:latest
- name: kimi-k2.5
provider: ollama
model: kimi-k2.5:cloud
- name: gpt-oss-120b
provider: ollama
model: gpt-oss:120b-cloud
- name: gpt-oss-20b
provider: ollama
model: gpt-oss:20b
- name: qwen3-coder:latest
provider: ollama
model: qwen3-coder:latest
# good but too slow on 64G
#- name: nemotron-3-nano
# provider: ollama
# model: nemotron-3-nano:latest
# =============================================================================
# Runners - agent frameworks with their specific configurations
# =============================================================================
# Each runner has its own binary, extensions/config, and isolated config directory
runners:
# - name: goose
# type: goose
# bin: goose
# extensions: [developer]
# stdio:
# - node mcp-harness/dist/index.js
- name: goose-full
type: goose
bin: goose
extensions: [developer, todo, skills, code_execution, extensionmanager]
stdio:
- node mcp-harness/dist/index.js
- name: opencode
type: opencode
bin: opencode
stdio:
- node mcp-harness/dist/index.js
- name: pi
type: pi
bin: pi
# Pi takes provider/model from the test matrix, not config
# MCP support via pi-mcp-adapter: `pi install npm:pi-mcp-adapter`
stdio:
- node mcp-harness/dist/index.js
# =============================================================================
# Test Matrix
# =============================================================================
# scenarios × models × runners
# - Omit 'models' to run against ALL models
# - Omit 'runners' to run against ALL runners
matrix:
# Single-turn scenarios: all models × all runners
- scenario: everyday-app-automation
- scenario: file-editing
# Multi-turn: goose and pi only (opencode doesn't support session continuation)
- scenario: multi-turn-edit
runners: [pi, goose-full]

BIN
evals/open-model-gym/gym.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 2.3 MiB

87
evals/open-model-gym/mcp-harness/README.md Normal file
View file

@ -0,0 +1,87 @@
# MCP Harness
A simulated MCP server with realistic fake tools for testing. Provides mock implementations of common business integrations without requiring actual API credentials.
## Tools Included (35 tools)
### Google Drive
- `gdrive_search` - Search files by name, content, or type
- `gdrive_read_file` - Read file contents
- `gdrive_create_file` - Create new files
- `gdrive_share_file` - Share files with users
### Google Sheets
- `sheets_read` - Read spreadsheet data
- `sheets_write` - Write/update cells
- `sheets_append` - Append rows
- `sheets_create` - Create new spreadsheets
### Salesforce
- `salesforce_query` - Execute SOQL queries
- `salesforce_get_record` - Get record by ID
- `salesforce_create_record` - Create records
- `salesforce_update_record` - Update records
- `salesforce_search` - SOSL search
### Slack
- `slack_send_message` - Send messages
- `slack_get_messages` - Get channel messages
- `slack_search_messages` - Search messages
- `slack_list_channels` - List channels
- `slack_get_user_info` - Get user info
- `slack_set_status` - Set user status
### Google Calendar
- `calendar_list_events` - List events
- `calendar_create_event` - Create events
- `calendar_update_event` - Update events
- `calendar_delete_event` - Delete events
### Gmail
- `gmail_search` - Search emails
- `gmail_read_message` - Read email content
- `gmail_send` - Send emails
- `gmail_create_draft` - Create drafts
### Jira
- `jira_search_issues` - Search with JQL
- `jira_get_issue` - Get issue details
- `jira_create_issue` - Create issues
- `jira_update_issue` - Update issues
- `jira_add_comment` - Add comments
### GitHub
- `github_search_repos` - Search repositories
- `github_list_issues` - List issues
- `github_create_issue` - Create issues
- `github_list_prs` - List pull requests
## Setup
```bash
npm install
npm run build
```
## Run
```bash
npm run start
# or
./run.sh
```
## MCP Config
Add to your MCP client config:
```json
{
"mcpServers": {
"harness": {
"command": "node",
"args": ["/path/to/mcp-harness/dist/index.js"]
}
}
}
```

1161
evals/open-model-gym/mcp-harness/package-lock.json generated Normal file
View file

File diff suppressed because it is too large Load diff

18
evals/open-model-gym/mcp-harness/package.json Normal file
View file

@ -0,0 +1,18 @@
{
"name": "mcp-harness",
"version": "1.0.0",
"description": "Simulated real-world MCP tools for testing - Google Drive, Sheets, Salesforce, Slack, and more",
"private": true,
"type": "module",
"scripts": {
"build": "tsc -p tsconfig.json",
"start": "node dist/index.js"
},
"dependencies": {
"@modelcontextprotocol/sdk": "^1.0.0"
},
"devDependencies": {
"@types/node": "^25.2.0",
"typescript": "^5.6.3"
}
}

3
evals/open-model-gym/mcp-harness/run.sh Executable file
View file

@ -0,0 +1,3 @@
#!/bin/bash
cd "$(dirname "$0")"
npm run build && npm run start

1040
evals/open-model-gym/mcp-harness/src/index.ts Normal file
View file

File diff suppressed because it is too large Load diff

14
evals/open-model-gym/mcp-harness/tsconfig.json Normal file
View file

@ -0,0 +1,14 @@
{
"compilerOptions": {
"target": "ES2022",
"module": "NodeNext",
"moduleResolution": "NodeNext",
"outDir": "dist",
"rootDir": "src",
"strict": true,
"esModuleInterop": true,
"skipLibCheck": true,
"declaration": true
},
"include": ["src/**/*"]
}

4
evals/open-model-gym/suite/.gitignore vendored Normal file
View file

@ -0,0 +1,4 @@
node_modules/
.workdir/
.goose-root/
.cache/

1083
evals/open-model-gym/suite/package-lock.json generated Normal file
View file

File diff suppressed because it is too large Load diff

19
evals/open-model-gym/suite/package.json Normal file
View file

@ -0,0 +1,19 @@
{
"name": "agent-runner",
"version": "0.1.0",
"type": "module",
"scripts": {
"build": "tsc",
"test": "tsx src/runner.ts",
"test:scenario": "tsx src/runner.ts --scenario"
},
"dependencies": {
"glob": "^11.0.0",
"yaml": "^2.4.0"
},
"devDependencies": {
"@types/node": "^22.0.0",
"tsx": "^4.19.0",
"typescript": "^5.5.0"
}
}

47
evals/open-model-gym/suite/scenarios/everyday-app-automation.yaml Normal file
View file

@ -0,0 +1,47 @@
name: everyday-app-automation
description: Multi-step workflow using everyday app tools (Slack, Jira, Calendar) with data dependencies
prompt: |
Using the available tools, complete these tasks:
1. Search Slack for messages mentioning "quarterly review"
2. Look up the user who posted the message about quarterly review to get their full name
3. Create a Jira issue titled "Q1 Review Follow-ups" with a description that includes the name of the person who posted the Slack message
4. Create a calendar event for next Monday at 2pm called "Review Discussion"
5. Write a summary of what you did to a file called workflow-log.md
tags:
- complex
- multi-step
- mcp-harness
- data-flow
validate:
# Check workflow summary was written
- type: file_exists
path: workflow-log.md
- type: file_not_empty
path: workflow-log.md
# Check Slack search was called with the right query
- type: tool_called
tool: slack_search_messages
args:
query: /quarterly.?review/
# Check that the agent looked up the user info (data dependency: requires reading user ID from search results)
- type: tool_called
tool: slack_get_user_info
args:
userId: /U004/
# Check Jira issue was created with expected title and includes the user's name (data dependency: requires reading name from user info)
- type: tool_called
tool: jira_create_issue
args:
summary: /q1.?review|follow.?up/
description: /David.?Brown/
# Check calendar event was created with expected title
- type: tool_called
tool: calendar_create_event
args:
summary: /review.?discussion/

108
evals/open-model-gym/suite/scenarios/file-editing.yaml Normal file
View file

@ -0,0 +1,108 @@
name: file-editing
description: Navigate a small codebase and make a targeted edit
prompt: |
The User struct in user.rs is missing a display_name() method.
Add a method that returns the full name formatted as "first_name last_name".
tags:
- file-editing
- code-navigation
setup:
Cargo.toml: |
[package]
name = "user-service"
version = "0.1.0"
edition = "2021"
src/main.rs: |
mod models;
mod utils;
use models::user::User;
fn main() {
let user = User::new("Alice", "Smith", "alice@example.com");
println!("Created user: {}", user.email());
}
src/models/mod.rs: |
pub mod user;
src/models/user.rs: |
pub struct User {
first_name: String,
last_name: String,
email: String,
}
impl User {
pub fn new(first_name: &str, last_name: &str, email: &str) -> Self {
Self {
first_name: first_name.to_string(),
last_name: last_name.to_string(),
email: email.to_string(),
}
}
pub fn email(&self) -> &str {
&self.email
}
pub fn first_name(&self) -> &str {
&self.first_name
}
pub fn last_name(&self) -> &str {
&self.last_name
}
}
src/utils/mod.rs: |
pub mod formatting;
src/utils/formatting.rs: |
pub fn capitalize(s: &str) -> String {
let mut chars = s.chars();
match chars.next() {
None => String::new(),
Some(c) => c.to_uppercase().collect::<String>() + chars.as_str(),
}
}
validate:
# The edit was made to the correct file
- type: file_exists
path: src/models/user.rs
name: user.rs exists
# Method was added
- type: file_matches
path: src/models/user.rs
regex: "fn\\s+display_name"
name: display_name() added
# Method returns a String or &str
- type: file_matches
path: src/models/user.rs
regex: "display_name.*->.*String|display_name.*->.*str"
name: has return type
# Original code preserved
- type: file_contains
path: src/models/user.rs
pattern: "pub fn email"
name: email() preserved
- type: file_contains
path: src/models/user.rs
pattern: "pub fn first_name"
name: first_name() preserved
# Other files untouched
- type: file_exists
path: src/main.rs
name: main.rs exists
- type: file_contains
path: src/main.rs
pattern: "mod models"
name: main.rs unchanged
# Code compiles
- type: command_succeeds
command: "cargo build"
name: cargo build

78
evals/open-model-gym/suite/scenarios/multi-turn-edit.yaml Normal file
View file

@ -0,0 +1,78 @@
name: multi-turn-edit
description: Multi-turn conversation - add a method, then rename it
tags:
- file-editing
- multi-turn
setup:
Cargo.toml: |
[package]
name = "user-service"
version = "0.1.0"
edition = "2021"
src/main.rs: |
mod models;
use models::user::User;
fn main() {
let user = User::new("Alice", "Smith");
println!("User: {} {}", user.first_name(), user.last_name());
}
src/models/mod.rs: |
pub mod user;
src/models/user.rs: |
pub struct User {
first_name: String,
last_name: String,
}
impl User {
pub fn new(first_name: &str, last_name: &str) -> Self {
Self {
first_name: first_name.to_string(),
last_name: last_name.to_string(),
}
}
pub fn first_name(&self) -> &str {
&self.first_name
}
pub fn last_name(&self) -> &str {
&self.last_name
}
}
turns:
- prompt: |
Add an email() method to the User struct in src/models/user.rs.
It should return a generated email in the format "first_name.last_name@example.com" (lowercase).
validate:
- type: file_matches
path: src/models/user.rs
regex: "fn\\s+email"
name: email() added
- type: command_succeeds
command: "cargo build"
name: compiles after turn 1
- prompt: |
Actually, can you rename the email() method to generated_email() instead?
Make sure to update any references.
validate:
- type: file_matches
path: src/models/user.rs
regex: "fn\\s+generated_email"
name: renamed to generated_email()
- type: file_not_matches
path: src/models/user.rs
regex: "fn\\s+email\\s*\\("
name: old email() removed
- type: command_succeeds
command: "cargo build"
name: compiles after turn 2

BIN
evals/open-model-gym/suite/src/gym.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 2.3 MiB

1504
evals/open-model-gym/suite/src/runner.ts Normal file
View file

File diff suppressed because it is too large Load diff

74
evals/open-model-gym/suite/src/types.ts Normal file
View file

@ -0,0 +1,74 @@
export interface AgentConfig {
model: string;
provider: string;
/** Extensions (runner knows which are platform vs builtin) */
extensions?: string[];
/** Stdio extension commands (for custom MCP servers) */
stdio?: string[];
/** Path to goose binary (default: "goose") */
"goose-bin"?: string;
temperature?: number;
maxTokens?: number;
}
export interface Scenario {
name: string;
description: string;
prompt?: string;
/** Files to create before running (relative paths) */
setup?: Record<string, string>;
/** Validation rules to check after agent completes (single-turn) */
validate?: ValidationRule[];
/** Multi-turn conversation (alternative to single prompt+validate) */
turns?: Turn[];
/** Tags for filtering scenarios */
tags?: string[];
}
/** A single turn in a multi-turn conversation */
export interface Turn {
/** The prompt for this turn */
prompt: string;
/** Validation rules to check after this turn completes */
validate: ValidationRule[];
}
export type ValidationRule =
| { type: "file_exists"; path: string; name?: string }
| { type: "file_contains"; path: string; pattern: string; name?: string }
| { type: "file_matches"; path: string; regex: string; name?: string }
| { type: "file_not_matches"; path: string; regex: string; name?: string }
| { type: "file_not_empty"; path: string; name?: string }
| { type: "command_succeeds"; command: string; name?: string }
| { type: "tool_called"; tool: string; args?: Record<string, string | RegExp>; name?: string }
| { type: "custom"; fn: string; name?: string };
export interface TestRun {
scenario: Scenario;
config: AgentConfig;
workdir: string;
startTime: Date;
endTime?: Date;
status: "pending" | "running" | "passed" | "failed";
errors?: string[];
}
export interface TestResult {
run: TestRun;
validations: Array<{
rule: ValidationRule;
passed: boolean;
message?: string;
}>;
}
export interface SuiteConfig {
/** Agent configurations to permute */
agents: AgentConfig[];
/** Scenarios to run */
scenarios: string[];
/** Base directory for test workspaces */
workdir: string;
/** Parallel execution count */
parallel?: number;
}

184
evals/open-model-gym/suite/src/validator.ts Normal file
View file

@ -0,0 +1,184 @@
import { existsSync, readFileSync, statSync } from "node:fs";
import { execSync } from "node:child_process";
import { join } from "node:path";
import type { ValidationRule } from "./types.js";
export interface ValidationResult {
passed: boolean;
message?: string;
}
export function validateRule(
rule: ValidationRule,
workdir: string
): ValidationResult {
switch (rule.type) {
case "file_exists": {
const fullPath = join(workdir, rule.path);
const exists = existsSync(fullPath);
return {
passed: exists,
message: exists ? undefined : `File not found: ${rule.path}`,
};
}
case "file_not_empty": {
const fullPath = join(workdir, rule.path);
if (!existsSync(fullPath)) {
return { passed: false, message: `File not found: ${rule.path}` };
}
const stat = statSync(fullPath);
return {
passed: stat.size > 0,
message: stat.size > 0 ? undefined : `File is empty: ${rule.path}`,
};
}
case "file_contains": {
const fullPath = join(workdir, rule.path);
if (!existsSync(fullPath)) {
return { passed: false, message: `File not found: ${rule.path}` };
}
const content = readFileSync(fullPath, "utf-8");
const contains = content.includes(rule.pattern);
return {
passed: contains,
message: contains
? undefined
: `File ${rule.path} does not contain: ${rule.pattern}`,
};
}
case "file_matches": {
const fullPath = join(workdir, rule.path);
if (!existsSync(fullPath)) {
return { passed: false, message: `File not found: ${rule.path}` };
}
const content = readFileSync(fullPath, "utf-8");
const regex = new RegExp(rule.regex);
const matches = regex.test(content);
return {
passed: matches,
message: matches
? undefined
: `File ${rule.path} does not match regex: ${rule.regex}`,
};
}
case "file_not_matches": {
const fullPath = join(workdir, rule.path);
if (!existsSync(fullPath)) {
return { passed: false, message: `File not found: ${rule.path}` };
}
const content = readFileSync(fullPath, "utf-8");
const regex = new RegExp(rule.regex);
const matches = regex.test(content);
return {
passed: !matches,
message: !matches
? undefined
: `File ${rule.path} should not match regex: ${rule.regex}`,
};
}
case "command_succeeds": {
try {
execSync(rule.command, { cwd: workdir, stdio: "pipe" });
return { passed: true };
} catch (err) {
return {
passed: false,
message: `Command failed: ${rule.command}`,
};
}
}
case "tool_called": {
const logPath = join(workdir, "tool-calls.log");
if (!existsSync(logPath)) {
return { passed: false, message: "tool-calls.log not found" };
}
const content = readFileSync(logPath, "utf-8");
const lines = content.trim().split("\n").filter(Boolean);
// Find all calls to the specified tool
const matchingCalls = lines
.map((line) => {
try {
return JSON.parse(line);
} catch {
return null;
}
})
.filter((entry) => entry?.tool === rule.tool);
if (matchingCalls.length === 0) {
return { passed: false, message: `Tool not called: ${rule.tool}` };
}
// If no arg requirements, just check tool was called
if (!rule.args) {
return { passed: true };
}
// Check if any call matches the arg requirements
for (const call of matchingCalls) {
const args = call.arguments || {};
let allMatch = true;
for (const [key, expected] of Object.entries(rule.args)) {
const actual = args[key];
if (actual === undefined) {
allMatch = false;
break;
}
// If expected starts/ends with /, treat as regex pattern
if (typeof expected === "string" && expected.startsWith("/") && expected.endsWith("/")) {
const pattern = new RegExp(expected.slice(1, -1), "i");
if (!pattern.test(String(actual))) {
allMatch = false;
break;
}
} else {
// Exact match (case-insensitive for strings)
const actualStr = String(actual).toLowerCase();
const expectedStr = String(expected).toLowerCase();
if (!actualStr.includes(expectedStr)) {
allMatch = false;
break;
}
}
}
if (allMatch) {
return { passed: true };
}
}
return {
passed: false,
message: `Tool ${rule.tool} called but args didn't match: expected ${JSON.stringify(rule.args)}`,
};
}
case "custom": {
// Custom validators loaded dynamically
return { passed: false, message: "Custom validators not yet implemented" };
}
default:
return { passed: false, message: `Unknown rule type` };
}
}
export function validateAll(
rules: ValidationRule[],
workdir: string
): Array<{ rule: ValidationRule; result: ValidationResult }> {
return rules.map((rule) => ({
rule,
result: validateRule(rule, workdir),
}));
}

12
evals/open-model-gym/suite/tsconfig.json Normal file
View file

@ -0,0 +1,12 @@
{
"compilerOptions": {
"target": "ES2022",
"module": "NodeNext",
"moduleResolution": "NodeNext",
"strict": true,
"esModuleInterop": true,
"skipLibCheck": true,
"outDir": "dist"
},
"include": ["src"]
}