From a437ffd034b756d450e17c3d4f7e359ce625cfe1 Mon Sep 17 00:00:00 2001 From: rUv Date: Sat, 27 Jun 2026 10:48:36 -0400 Subject: [PATCH] feat(timesfm): real-model tests + GPU/batch optimization + ruvector-timesfm crate + metaharness (#608) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(timesfm): GPU/device optimization + ruvector-timesfm integration crate timesfm: - cuda/metal features now imply candle (so `--features cuda` alone compiles the numeric path); add timesfm::select_device() (TIMESFM_DEVICE=cpu|cuda|metal) and use it in the bench instead of hardcoding Device::Cpu. - Validated real-weight decode on RTX 5080: 45.2 ms (CPU) -> 3.97 ms (cuda) = ~11.4x, parity preserved (max-abs 8.58e-6). Note: decode at h<=128 is a single forward pass (horizon_len=128), so KV-cache is a no-op there; GPU/f16 are the real levers. Derive serde on PruneDecision for the MCP boundary. ruvector-timesfm (new crate): RuVector-facing integration. - Forecaster: load-once, forecast(series, horizon) -> point + calibrated p10..p90 quantile bands. - anomaly: forecast-band detection (flag observed points outside their p10/p90). - sweep::EarlyStopper: ADR-191 TimesFM-driven early-stopping for ruflo/Darwin sweeps (wraps prune::decide_prune with min_history + confidence gate). - ruvector-timesfm-forecast: JSON-in/out CLI = the time_series_forecast MCP tool entry point. - telemetry_anomaly example (flags injected spikes on real weights), integration tests (5 candle + 3 pure-logic, all green; gated/skip without 814MB weights). clippy --all-targets -D warnings clean (both feature states); fmt clean. Co-Authored-By: claude-flow * feat(harness): add generated timesfm metaharness bundle (ADR-041) Authentic output of the agent-harness-generator (create-agent-harness v0.2.7, kernel 0.1.2) synthesizing an engineering-pod harness for the TimesFM forecasting crates. Template vertical:coding (the generator's recommended rust-crate-harness archetype); host claude-code. - score: scaffoldReady, 6/6 hard constraints, toolSafety 100, compileConfidence 90 - genome: repo_type rust, topology maintainer/tester/security, risk 0.37, mcp_surface local_default_deny - witness: .harness/manifest.sha256 over .harness/manifest.json, verified valid (7c45ab91…). PROVENANCE.md records the repro command, score, genome, witness, and the link to the time_series_forecast MCP tool (ruvector-timesfm-forecast). Co-Authored-By: claude-flow * feat(ruvector-timesfm): batched forecasting (throughput path) Forecaster::forecast_batch forecasts B equal-length series in one model call. Measured on real weights (B=32, ctx=256, h=64): - CPU: 27 -> 166 forecasts/s (6.16x), bit-exact vs per-series - cuda: 244 -> 2078 forecasts/s (8.45x), rel diff 1.7e-4 (GPU reduction order) Adds the throughput example (sequential vs batched + correctness check with a relative tolerance for GPU) and a real-model batch-parity integration test. Co-Authored-By: claude-flow * feat(harness): Darwin evolve via OpenRouter, key sourced from GCP Secret Manager Adds scripts/evolve-openrouter.{sh,mjs} to optimize the timesfm-harness with Darwin Mode's OpenRouter LLM mutator (library-only; not CLI-exposed). The OpenRouter API key is fetched from GCP Secret Manager at runtime (gcloud secrets versions access OPENROUTER_API_KEY, project cognitum-20260110) and exported only into the run's process — never stored in the repo/dotfile/logs. Driver resolves @metaharness/darwin (devDependency) or DARWIN_DIST for local monorepo runs. Validated: real-sandbox evolve (1 gen x 2 children, google/gemini-2.5-flash) scored baseline 0.985 with safety 1.0 and zero secret-exposure flags; ~$0.003. Mutations pass the validateGeneratedCode gate and only promote on measured improvement. PROVENANCE.md documents usage. Co-Authored-By: claude-flow * feat(timesfm): int8/int4 weight quantization (QLinear + load_quantized) Adds QLinear (full-precision or ggml-quantized weight via QMatMul) threaded through the decoder; PatchedTimeSeriesDecoder::load_quantized(cfg, vb, dtype) quantizes the 2 ResidualBlocks + 20 transformer layers (embeddings/norms/scaling stay f32). Exposed as Forecaster::load_quantized(.., Quant::Q8_0|Q4_0). Measured on real weights (CPU, ctx=512/h=128) — quant is a MEMORY win, not a CPU-speed win (dequant overhead dominates the small 16-patch matmuls): f32 : 46 ms 814 MB Q8_0 : 242 ms ~212 MB (4x smaller) rel err 3.5e-3 (recommended) Q4_0 : 246 ms ~112 MB (7x smaller) rel err 3.1e-2 All outputs finite. f32 path unchanged (QLinear::Full == prior Linear; parity still 8.58e-6). quant_bench example + Q8_0 integration test added. Co-Authored-By: claude-flow * feat(ruvector-timesfm): forecast-driven HNSW rebuild scheduler (vector-db hook) rebuild module: forecast an index's recall-drift curve with TimesFM and advise WHEN to rebuild — schedule the rebuild to land just before the conservative (p10) recall forecast crosses a floor, instead of fixed-schedule or after-the-fact. Forecaster::advise_rebuild(recall_history, floor, horizon, lead_steps) -> RebuildAdvice{rebuild_now, steps_until_floor, ...}. Ties into the ruvector-diskann recall-trigger work. Pure-logic + real-model tests. Co-Authored-By: claude-flow * feat(timesfm): f16-on-load path (Forecaster::load_f16) + GPU bench Run the forward in f16 (f16 weights/activations). Three localized dtype fixes make the path f16-clean (attention mask coerce, decode padding dtype, RevIN scalar-extraction slices); the f32 path is untouched (parity still 8.583e-6). Forecaster gains a dtype field + load_f16; forecast/forecast_batch build inputs in the load dtype and surface f32 to callers. Measured RTX 5080 (B=32, ctx=256, h=64): batched f32 2082 -> f16 3261 forecasts/s (1.57x), sequential 238 -> 303/s. f16 forecasts within rel 2e-2 of f32. (CPU f16 is slower, like quant — GPU is where f16 pays off.) f16 + Q8 remain the two precision knobs: f16 for GPU latency, Q8_0 for edge memory. Co-Authored-By: claude-flow --------- Co-authored-by: ruvnet --- Cargo.lock | 13 + Cargo.toml | 2 + crates/ruvector-timesfm/Cargo.toml | 63 ++++ .../examples/telemetry_anomaly.rs | 74 +++++ .../ruvector-timesfm/examples/throughput.rs | 99 ++++++ crates/ruvector-timesfm/src/anomaly.rs | 114 +++++++ crates/ruvector-timesfm/src/bin/forecast.rs | 70 +++++ crates/ruvector-timesfm/src/forecast_types.rs | 52 ++++ crates/ruvector-timesfm/src/forecaster.rs | 284 ++++++++++++++++++ crates/ruvector-timesfm/src/lib.rs | 76 +++++ crates/ruvector-timesfm/src/rebuild.rs | 82 +++++ crates/ruvector-timesfm/src/sweep.rs | 133 ++++++++ crates/ruvector-timesfm/tests/integration.rs | 235 +++++++++++++++ crates/timesfm/Cargo.toml | 11 +- crates/timesfm/examples/bench.rs | 7 +- crates/timesfm/examples/quant_bench.rs | 101 +++++++ crates/timesfm/src/lib.rs | 26 ++ crates/timesfm/src/model.rs | 194 ++++++++++-- crates/timesfm/src/prune.rs | 2 +- .../.claude-plugin/plugin.json | 23 ++ .../.claude/commands/doctor.md | 12 + .../.claude/commands/review-diff.md | 10 + .../timesfm-harness/.claude/settings.json | 40 +++ .../.claude/skills/evolve/SKILL.md | 53 ++++ .../.claude/skills/plan-change/SKILL.md | 15 + .../timesfm-harness/.harness/manifest.json | 40 +++ .../timesfm-harness/.harness/manifest.sha256 | 1 + harnesses/timesfm-harness/CLAUDE.md | 32 ++ harnesses/timesfm-harness/LICENSE | 21 ++ harnesses/timesfm-harness/PROVENANCE.md | 110 +++++++ harnesses/timesfm-harness/README.md | 30 ++ .../timesfm-harness/__tests__/smoke.test.ts | 37 +++ harnesses/timesfm-harness/bin/cli.js | 100 ++++++ harnesses/timesfm-harness/package.json | 44 +++ .../scripts/evolve-openrouter.mjs | 84 ++++++ .../scripts/evolve-openrouter.sh | 33 ++ .../timesfm-harness/src/agents/architect.ts | 7 + .../timesfm-harness/src/agents/implementer.ts | 7 + .../timesfm-harness/src/agents/reviewer.ts | 7 + .../timesfm-harness/src/agents/test-writer.ts | 7 + harnesses/timesfm-harness/src/init.ts | 21 ++ harnesses/timesfm-harness/tsconfig.json | 19 ++ harnesses/timesfm-harness/vitest.config.ts | 23 ++ 43 files changed, 2381 insertions(+), 33 deletions(-) create mode 100644 crates/ruvector-timesfm/Cargo.toml create mode 100644 crates/ruvector-timesfm/examples/telemetry_anomaly.rs create mode 100644 crates/ruvector-timesfm/examples/throughput.rs create mode 100644 crates/ruvector-timesfm/src/anomaly.rs create mode 100644 crates/ruvector-timesfm/src/bin/forecast.rs create mode 100644 crates/ruvector-timesfm/src/forecast_types.rs create mode 100644 crates/ruvector-timesfm/src/forecaster.rs create mode 100644 crates/ruvector-timesfm/src/lib.rs create mode 100644 crates/ruvector-timesfm/src/rebuild.rs create mode 100644 crates/ruvector-timesfm/src/sweep.rs create mode 100644 crates/ruvector-timesfm/tests/integration.rs create mode 100644 crates/timesfm/examples/quant_bench.rs create mode 100644 harnesses/timesfm-harness/.claude-plugin/plugin.json create mode 100644 harnesses/timesfm-harness/.claude/commands/doctor.md create mode 100644 harnesses/timesfm-harness/.claude/commands/review-diff.md create mode 100644 harnesses/timesfm-harness/.claude/settings.json create mode 100644 harnesses/timesfm-harness/.claude/skills/evolve/SKILL.md create mode 100644 harnesses/timesfm-harness/.claude/skills/plan-change/SKILL.md create mode 100644 harnesses/timesfm-harness/.harness/manifest.json create mode 100644 harnesses/timesfm-harness/.harness/manifest.sha256 create mode 100644 harnesses/timesfm-harness/CLAUDE.md create mode 100644 harnesses/timesfm-harness/LICENSE create mode 100644 harnesses/timesfm-harness/PROVENANCE.md create mode 100644 harnesses/timesfm-harness/README.md create mode 100644 harnesses/timesfm-harness/__tests__/smoke.test.ts create mode 100644 harnesses/timesfm-harness/bin/cli.js create mode 100644 harnesses/timesfm-harness/package.json create mode 100644 harnesses/timesfm-harness/scripts/evolve-openrouter.mjs create mode 100755 harnesses/timesfm-harness/scripts/evolve-openrouter.sh create mode 100644 harnesses/timesfm-harness/src/agents/architect.ts create mode 100644 harnesses/timesfm-harness/src/agents/implementer.ts create mode 100644 harnesses/timesfm-harness/src/agents/reviewer.ts create mode 100644 harnesses/timesfm-harness/src/agents/test-writer.ts create mode 100644 harnesses/timesfm-harness/src/init.ts create mode 100644 harnesses/timesfm-harness/tsconfig.json create mode 100644 harnesses/timesfm-harness/vitest.config.ts diff --git a/Cargo.lock b/Cargo.lock index e1aae09e5..2d1a57854 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10574,6 +10574,19 @@ dependencies = [ name = "ruvector-temporal-tensor" version = "2.2.3" +[[package]] +name = "ruvector-timesfm" +version = "2.2.3" +dependencies = [ + "anyhow", + "candle-core 0.9.2", + "candle-nn 0.9.2", + "serde", + "serde_json", + "thiserror 2.0.18", + "timesfm", +] + [[package]] name = "ruvector-tiny-dancer-core" version = "2.2.3" diff --git a/Cargo.toml b/Cargo.toml index c82190e3d..c2f4d345c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -270,6 +270,8 @@ members = [ "crates/ruvector-maxsim", # TimesFM 1.0 200M decoder-only patched time-series Transformer (candle, ADR-189/191) "crates/timesfm", + # RuVector integration for TimesFM: Forecaster + anomaly bands + sweep early-stopping + "crates/ruvector-timesfm", ] resolver = "2" diff --git a/crates/ruvector-timesfm/Cargo.toml b/crates/ruvector-timesfm/Cargo.toml new file mode 100644 index 000000000..3f1f3799f --- /dev/null +++ b/crates/ruvector-timesfm/Cargo.toml @@ -0,0 +1,63 @@ +[package] +name = "ruvector-timesfm" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true +description = "RuVector integration for the TimesFM time-series foundation model: a high-level quantile Forecaster, forecast-band anomaly detection, and TimesFM-driven early-stopping for optimization sweeps (ADR-191)" +keywords = ["timesfm", "forecasting", "time-series", "anomaly", "ruvector"] +categories = ["science", "algorithms"] +publish = true + +[lib] +name = "ruvector_timesfm" +path = "src/lib.rs" + +# The forecast CLI (MCP `time_series_forecast` tool entry point) needs the +# numeric path, so it only builds with the candle feature. +[[bin]] +name = "ruvector-timesfm-forecast" +path = "src/bin/forecast.rs" +required-features = ["candle"] + +[[example]] +name = "telemetry_anomaly" +required-features = ["candle"] + +[[example]] +name = "throughput" +required-features = ["candle"] + +[features] +default = [] +# Activates the numeric path (TimesFM candle inference). Off by default so a +# stock `cargo build --workspace` stays light, mirroring the `timesfm` crate. +candle = ["timesfm/candle", "dep:candle-core", "dep:candle-nn"] +cuda = ["candle", "timesfm/cuda", "candle-core/cuda", "candle-nn/cuda"] +metal = ["candle", "timesfm/metal", "candle-core/metal", "candle-nn/metal"] + +[dependencies] +timesfm = { path = "../timesfm", version = "2.2.3" } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +thiserror = { workspace = true } + +# Only needed for the numeric path (Device/Tensor + VarBuilder at the boundary). +candle-core = { version = "0.9", optional = true } +candle-nn = { version = "0.9", optional = true } + +[dev-dependencies] +anyhow = { workspace = true } + +# Research-tier crate, doc/style churn deferred — mirror timesfm's lint posture. +[lints.rust] +unexpected_cfgs = { level = "allow", priority = -1 } + +[lints.clippy] +pedantic = { level = "allow", priority = -2 } +correctness = { level = "deny", priority = -1 } +suspicious = { level = "deny", priority = -1 } +too_many_arguments = "allow" +needless_range_loop = "allow" diff --git a/crates/ruvector-timesfm/examples/telemetry_anomaly.rs b/crates/ruvector-timesfm/examples/telemetry_anomaly.rs new file mode 100644 index 000000000..e59f9ca95 --- /dev/null +++ b/crates/ruvector-timesfm/examples/telemetry_anomaly.rs @@ -0,0 +1,74 @@ +//! Forecast-band anomaly detection on a synthetic telemetry series. +//! +//! Models the RuVector/host-telemetry use case (disk-fill, GPU memory, query +//! load): learn the normal daily rhythm, forecast the next window, and flag +//! observed points that leave their p10/p90 band. Run with real weights: +//! +//! ```ignore +//! cargo run -p ruvector-timesfm --features candle --release --example telemetry_anomaly \ +//! -- /tmp/timesfm-parity/timesfm.safetensors +//! ``` +//! +//! Skips cleanly (exit 0) when weights are absent — never fabricates a result. + +use ruvector_timesfm::Forecaster; + +/// A daily-seasonal telemetry signal: sinusoidal load + slow upward drift. +fn synth_series(n: usize) -> Vec { + (0..n) + .map(|t| { + let day = (t as f32 / 24.0) * std::f32::consts::TAU; + 50.0 + 20.0 * day.sin() + 0.05 * t as f32 + }) + .collect() +} + +fn main() -> anyhow::Result<()> { + let weights = std::env::args() + .nth(1) + .or_else(|| std::env::var("TIMESFM_WEIGHTS").ok()) + .unwrap_or_else(|| "/tmp/timesfm-parity/timesfm.safetensors".into()); + if !std::path::Path::new(&weights).exists() { + eprintln!("SKIP telemetry_anomaly: weights missing ({weights}). Not fabricating a result."); + return Ok(()); + } + + let device = timesfm::select_device()?; + let forecaster = Forecaster::load(&weights, device)?; + + // 256 points of history; observe the next 32, with two injected spikes. + let history = synth_series(256); + let mut observed = synth_series_window(256, 32); + observed[8] += 60.0; // sudden spike (e.g. disk write storm) + observed[20] -= 45.0; // sudden drop (e.g. sensor dropout) + + let report = forecaster.detect_anomalies(&history, &observed, 0)?; + println!( + "telemetry_anomaly: scored {} steps, {} anomalies", + report.points.len(), + report.n_anomalies + ); + for a in report.anomalies() { + println!( + " step {:2}: observed={:7.2} band=[{:7.2}, {:7.2}] expected={:7.2} deviation={:+.2}×band", + a.index, a.observed, a.lower, a.upper, a.expected, a.deviation + ); + } + // The two injected spikes should be the dominant anomalies. + assert!( + report.n_anomalies >= 2, + "expected the two injected spikes to be flagged, got {}", + report.n_anomalies + ); + Ok(()) +} + +/// Continuation of `synth_series` starting at `offset`, `len` points long. +fn synth_series_window(offset: usize, len: usize) -> Vec { + (offset..offset + len) + .map(|t| { + let day = (t as f32 / 24.0) * std::f32::consts::TAU; + 50.0 + 20.0 * day.sin() + 0.05 * t as f32 + }) + .collect() +} diff --git a/crates/ruvector-timesfm/examples/throughput.rs b/crates/ruvector-timesfm/examples/throughput.rs new file mode 100644 index 000000000..bfd32a0df --- /dev/null +++ b/crates/ruvector-timesfm/examples/throughput.rs @@ -0,0 +1,99 @@ +//! Batched throughput bench: forecast B series sequentially vs. in one batched +//! model call, and verify the batched path matches the per-series path. +//! +//! ```ignore +//! cargo run -p ruvector-timesfm --features candle --release --example throughput \ +//! -- /tmp/timesfm-parity/timesfm.safetensors +//! # GPU: +//! TIMESFM_DEVICE=cuda cargo run -p ruvector-timesfm --features cuda --release \ +//! --example throughput -- /tmp/timesfm-parity/timesfm.safetensors +//! ``` +//! Skips cleanly (exit 0) when weights are absent. + +use std::time::Instant; + +use ruvector_timesfm::Forecaster; + +fn synth(seed: usize, n: usize) -> Vec { + (0..n) + .map(|t| { + let phase = seed as f32 * 0.7; + 50.0 + 12.0 * ((t as f32 / 16.0) + phase).sin() + 0.03 * t as f32 + }) + .collect() +} + +fn main() -> anyhow::Result<()> { + let weights = std::env::args() + .nth(1) + .or_else(|| std::env::var("TIMESFM_WEIGHTS").ok()) + .unwrap_or_else(|| "/tmp/timesfm-parity/timesfm.safetensors".into()); + if !std::path::Path::new(&weights).exists() { + eprintln!("SKIP throughput: weights missing ({weights})."); + return Ok(()); + } + + let device = timesfm::select_device()?; + let prec = std::env::var("TIMESFM_PRECISION").unwrap_or_else(|_| "f32".into()); + let dev_label = format!( + "{}/{prec}", + std::env::var("TIMESFM_DEVICE").unwrap_or_else(|_| "cpu".into()) + ); + let f = if prec == "f16" { + Forecaster::load_f16(&weights, device)? + } else { + Forecaster::load(&weights, device)? + }; + + let batch_size = 32usize; + let ctx = 256usize; + let horizon = 64usize; + let series: Vec> = (0..batch_size).map(|s| synth(s, ctx)).collect(); + + // Warm up. + let _ = f.forecast_batch(&series, horizon, 0)?; + + // Sequential. + let t = Instant::now(); + let seq: Vec<_> = series + .iter() + .map(|s| f.forecast(s, horizon)) + .collect::>()?; + let seq_ms = t.elapsed().as_secs_f64() * 1000.0; + + // Batched. + let t = Instant::now(); + let batched = f.forecast_batch(&series, horizon, 0)?; + let batch_ms = t.elapsed().as_secs_f64() * 1000.0; + + // Correctness: batched must match sequential. On CPU this is bit-exact; on + // GPU, batched vs per-row matmuls reduce in a different order, so compare + // with a *relative* tolerance (scaled by the series magnitude) rather than a + // tight absolute one. + let mut max_abs = 0f32; + let mut scale = 1e-6f32; + for (a, b) in seq.iter().zip(batched.iter()) { + for (x, y) in a.point.iter().zip(b.point.iter()) { + max_abs = max_abs.max((x - y).abs()); + scale = scale.max(x.abs()); + } + } + let rel = max_abs / scale; + + println!( + "throughput [{dev_label}] B={batch_size} ctx={ctx} h={horizon}:\n \ + sequential: {seq_ms:8.2} ms total = {:7.0} forecasts/s\n \ + batched: {batch_ms:8.2} ms total = {:7.0} forecasts/s ({:.2}x)\n \ + batched-vs-sequential max-abs-diff = {max_abs:.3e} (rel {rel:.3e})", + batch_size as f64 / (seq_ms / 1000.0), + batch_size as f64 / (batch_ms / 1000.0), + seq_ms / batch_ms, + ); + + // Relative tolerance: bit-exact on CPU (~0), GPU reduction-order ~1e-4 rel. + assert!( + rel < 1e-3, + "batched path diverged from sequential: rel {rel:.3e} (abs {max_abs:.3e})" + ); + Ok(()) +} diff --git a/crates/ruvector-timesfm/src/anomaly.rs b/crates/ruvector-timesfm/src/anomaly.rs new file mode 100644 index 000000000..20a274d2f --- /dev/null +++ b/crates/ruvector-timesfm/src/anomaly.rs @@ -0,0 +1,114 @@ +//! Forecast-band anomaly detection. +//! +//! Forecast the expected window with TimesFM, then flag observed points that +//! fall outside their `[p10, p90]` quantile band. The band width is the model's +//! own calibrated uncertainty, so a point counts as anomalous only when it +//! leaves the range the model itself considered plausible — no hand-tuned +//! Z-score threshold per series. + +use serde::{Deserialize, Serialize}; + +use crate::forecast_types::Forecast; + +/// One observed point scored against its forecast band. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct AnomalyPoint { + /// Step index within the observed window. + pub index: usize, + /// The observed value. + pub observed: f32, + /// Model's expected value (p50) for this step. + pub expected: f32, + /// Lower band (p10). + pub lower: f32, + /// Upper band (p90). + pub upper: f32, + /// Signed distance *outside* the band: positive above `upper`, negative + /// below `lower`, `0.0` when inside the band. Normalized by band width. + pub deviation: f32, + /// `true` when the observed value fell outside `[lower, upper]`. + pub is_anomaly: bool, +} + +/// Result of scoring an observed window against a forecast. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct AnomalyReport { + /// Per-step scoring, one entry per observed value. + pub points: Vec, + /// Number of points flagged as anomalous. + pub n_anomalies: usize, +} + +impl AnomalyReport { + /// The flagged points only. + #[must_use] + pub fn anomalies(&self) -> Vec<&AnomalyPoint> { + self.points.iter().filter(|p| p.is_anomaly).collect() + } +} + +/// Score `observed` against the bands in `forecast`. +/// +/// Compares each observed value to the `[p10, p90]` band of the matching +/// forecast step. `deviation` is normalized by band width so it is comparable +/// across series of different scales. The number of points scored is +/// `min(observed.len(), forecast.horizon())`. +#[must_use] +pub fn score_window(forecast: &Forecast, observed: &[f32]) -> AnomalyReport { + let p10 = forecast.p10(); + let p50 = forecast.p50(); + let p90 = forecast.p90(); + let n = observed.len().min(forecast.horizon()); + + let mut points = Vec::with_capacity(n); + let mut n_anomalies = 0; + for i in 0..n { + let (lower, expected, upper, obs) = (p10[i], p50[i], p90[i], observed[i]); + // Band width; guard against a degenerate zero-width band. + let width = (upper - lower).abs().max(f32::EPSILON); + let deviation = if obs > upper { + (obs - upper) / width + } else if obs < lower { + (obs - lower) / width // negative + } else { + 0.0 + }; + let is_anomaly = deviation != 0.0; + if is_anomaly { + n_anomalies += 1; + } + points.push(AnomalyPoint { + index: i, + observed: obs, + expected, + lower, + upper, + deviation, + is_anomaly, + }); + } + + AnomalyReport { + points, + n_anomalies, + } +} + +#[cfg(feature = "candle")] +impl crate::Forecaster { + /// Forecast `observed.len()` steps from `history`, then score each observed + /// value against its `[p10, p90]` band. The forecast never sees `observed`, + /// so this is a genuine out-of-sample anomaly check. + pub fn detect_anomalies( + &self, + history: &[f32], + observed: &[f32], + freq_id: u32, + ) -> crate::Result { + if observed.is_empty() { + return Err(crate::Error::Invalid("observed window is empty".into())); + } + let forecast = self.forecast_with_freq(history, observed.len(), freq_id)?; + Ok(score_window(&forecast, observed)) + } +} diff --git a/crates/ruvector-timesfm/src/bin/forecast.rs b/crates/ruvector-timesfm/src/bin/forecast.rs new file mode 100644 index 000000000..07341c2b5 --- /dev/null +++ b/crates/ruvector-timesfm/src/bin/forecast.rs @@ -0,0 +1,70 @@ +//! `ruvector-timesfm-forecast` — JSON-in/JSON-out forecasting CLI. +//! +//! This is the shell-out entry point for the RuVector `time_series_forecast` +//! MCP tool: an agent (or the MCP server) writes a JSON request on stdin and +//! reads a JSON forecast on stdout. The device is chosen via `TIMESFM_DEVICE` +//! (`cpu` | `cuda` | `metal`, default cpu). +//! +//! Request: `{"weights":"/path/timesfm.safetensors","series":[...],"horizon":64,"freq_id":0}` +//! Response: `{"horizon":64,"point":[...],"p10":[...],"p50":[...],"p90":[...]}` +//! +//! Run: `echo '{"weights":"...","series":[...],"horizon":32}' | ruvector-timesfm-forecast` + +use std::io::Read; + +use ruvector_timesfm::{Error, Forecaster, Result}; +use serde::{Deserialize, Serialize}; + +#[derive(Deserialize)] +struct Request { + weights: String, + series: Vec, + horizon: usize, + #[serde(default)] + freq_id: u32, +} + +#[derive(Serialize)] +struct Response { + horizon: usize, + device: String, + point: Vec, + p10: Vec, + p50: Vec, + p90: Vec, +} + +fn run() -> Result<()> { + let mut buf = String::new(); + std::io::stdin().read_to_string(&mut buf)?; + if buf.trim().is_empty() { + return Err(Error::Invalid( + "no JSON request on stdin (expected {weights, series, horizon})".into(), + )); + } + let req: Request = serde_json::from_str(&buf)?; + + let device = timesfm::select_device()?; + let device_label = std::env::var("TIMESFM_DEVICE").unwrap_or_else(|_| "cpu".into()); + + let forecaster = Forecaster::load(&req.weights, device)?; + let forecast = forecaster.forecast_with_freq(&req.series, req.horizon, req.freq_id)?; + + let resp = Response { + horizon: forecast.horizon(), + device: device_label, + p10: forecast.p10(), + p50: forecast.p50(), + p90: forecast.p90(), + point: forecast.point, + }; + println!("{}", serde_json::to_string(&resp)?); + Ok(()) +} + +fn main() { + if let Err(e) = run() { + eprintln!("ruvector-timesfm-forecast: {e}"); + std::process::exit(1); + } +} diff --git a/crates/ruvector-timesfm/src/forecast_types.rs b/crates/ruvector-timesfm/src/forecast_types.rs new file mode 100644 index 000000000..7b2a184ab --- /dev/null +++ b/crates/ruvector-timesfm/src/forecast_types.rs @@ -0,0 +1,52 @@ +//! Plain forecast data types — available without the `candle` feature so +//! callers can hold/serialize forecasts in code that doesn't pull in the model. + +use serde::{Deserialize, Serialize}; + +/// Number of quantile channels TimesFM emits (p10, p20, …, p90). +pub const NUM_QUANTILES: usize = 9; + +/// A horizon forecast: a point (mean) estimate per step plus the nine +/// calibrated quantiles (p10..p90) per step. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Forecast { + /// Point (mean-channel) forecast, one value per horizon step. + pub point: Vec, + /// Per-step quantiles, `[p10, p20, p30, p40, p50, p60, p70, p80, p90]`. + /// `quantiles.len() == point.len()`. + pub quantiles: Vec<[f32; NUM_QUANTILES]>, +} + +impl Forecast { + /// Horizon length (number of forecast steps). + #[must_use] + pub fn horizon(&self) -> usize { + self.point.len() + } + + /// Extract quantile channel `q` (`0 => p10 … 8 => p90`) across all steps. + /// Out-of-range indices are clamped into `0..NUM_QUANTILES`. + #[must_use] + pub fn quantile(&self, q: usize) -> Vec { + let q = q.min(NUM_QUANTILES - 1); + self.quantiles.iter().map(|row| row[q]).collect() + } + + /// The p10 (lower) band, one value per step. + #[must_use] + pub fn p10(&self) -> Vec { + self.quantile(0) + } + + /// The p50 (median) band, one value per step. + #[must_use] + pub fn p50(&self) -> Vec { + self.quantile(4) + } + + /// The p90 (upper) band, one value per step. + #[must_use] + pub fn p90(&self) -> Vec { + self.quantile(8) + } +} diff --git a/crates/ruvector-timesfm/src/forecaster.rs b/crates/ruvector-timesfm/src/forecaster.rs new file mode 100644 index 000000000..0c09cf1ce --- /dev/null +++ b/crates/ruvector-timesfm/src/forecaster.rs @@ -0,0 +1,284 @@ +//! The high-level [`Forecaster`] — load TimesFM weights once, forecast many. + +use std::path::Path; + +use candle_core::quantized::GgmlDType; +use candle_core::{DType, Device, IndexOp, Tensor}; +use timesfm::config::TimesfmConfig; +use timesfm::model::PatchedTimeSeriesDecoder; +use timesfm::prune::{decide_prune, PruneDecision}; + +use crate::forecast_types::{Forecast, NUM_QUANTILES}; +use crate::{Error, Result}; + +/// Weight quantization for loading. `Q8_0` (int8) shrinks the model ~4× +/// (~212 MB) with good accuracy (rel error ~3e-3); `Q4_0` (int4) ~7× (~112 MB) +/// with more error (~3e-2). These are **memory** wins for edge deployment — on a +/// small-context model like TimesFM-200M, quantized CPU matmuls are *slower* +/// than f32 (dequant overhead dominates), so prefer f32/GPU for latency. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Quant { + /// int8 weights — ~4× smaller, ~3e-3 relative error. + Q8_0, + /// int4 weights — ~7× smaller, ~3e-2 relative error. + Q4_0, +} + +impl Quant { + fn dtype(self) -> GgmlDType { + match self { + Quant::Q8_0 => GgmlDType::Q8_0, + Quant::Q4_0 => GgmlDType::Q4_0, + } + } +} + +/// A loaded TimesFM 1.0 200M forecaster bound to a compute device. +/// +/// Construct once (weight load + mmap is the expensive part), then call +/// [`Forecaster::forecast`] repeatedly. +pub struct Forecaster { + model: PatchedTimeSeriesDecoder, + device: Device, + /// Activation dtype the forward runs in (F32 by default, F16 for `load_f16`). + dtype: DType, +} + +impl Forecaster { + /// Load TimesFM 1.0 200M weights (a converted `safetensors` file) onto + /// `device`. Use [`timesfm::select_device`] to pick CPU/cuda/metal from the + /// `TIMESFM_DEVICE` env var. + pub fn load(weights: impl AsRef, device: Device) -> Result { + let cfg = TimesfmConfig::timesfm_1p0_200m(); + Self::load_with_config(weights, cfg, device) + } + + /// Load with an explicit [`TimesfmConfig`] (e.g. a test/tiny variant). + pub fn load_with_config( + weights: impl AsRef, + cfg: TimesfmConfig, + device: Device, + ) -> Result { + let path = weights.as_ref(); + if !path.exists() { + return Err(Error::Invalid(format!( + "weights file not found: {}", + path.display() + ))); + } + // SAFETY: from_mmaped_safetensors is unsafe because it mmaps a file the + // caller asserts is a valid safetensors blob; load() validates existence + // and candle validates the header/dtypes on read. + let vb = unsafe { + candle_nn::VarBuilder::from_mmaped_safetensors( + &[path.to_path_buf()], + DType::F32, + &device, + )? + }; + let model = PatchedTimeSeriesDecoder::load(cfg, vb)?; + Ok(Self { + model, + device, + dtype: DType::F32, + }) + } + + /// Load weights as **f16** and run the forward in f16. On GPU this can cut + /// latency; on CPU it is typically slower (f16 emulation). Forecasts match + /// f32 only to ~f16 precision (rel error ~1e-3). + pub fn load_f16(weights: impl AsRef, device: Device) -> Result { + let path = weights.as_ref(); + if !path.exists() { + return Err(Error::Invalid(format!( + "weights file not found: {}", + path.display() + ))); + } + let cfg = TimesfmConfig::timesfm_1p0_200m(); + // SAFETY: see load_with_config. Weights are cast to f16 on load. + let vb = unsafe { + candle_nn::VarBuilder::from_mmaped_safetensors( + &[path.to_path_buf()], + DType::F16, + &device, + )? + }; + let model = PatchedTimeSeriesDecoder::load(cfg, vb)?; + Ok(Self { + model, + device, + dtype: DType::F16, + }) + } + + /// Load with weights quantized to int8/int4 ([`Quant`]) — a ~4–7× smaller + /// resident model for memory-constrained (edge/Pi) deployment. See [`Quant`] + /// for the accuracy/latency tradeoff. + pub fn load_quantized(weights: impl AsRef, device: Device, quant: Quant) -> Result { + let path = weights.as_ref(); + if !path.exists() { + return Err(Error::Invalid(format!( + "weights file not found: {}", + path.display() + ))); + } + let cfg = TimesfmConfig::timesfm_1p0_200m(); + // SAFETY: see load_with_config. + let vb = unsafe { + candle_nn::VarBuilder::from_mmaped_safetensors( + &[path.to_path_buf()], + DType::F32, + &device, + )? + }; + let model = PatchedTimeSeriesDecoder::load_quantized(cfg, vb, quant.dtype())?; + Ok(Self { + model, + device, + dtype: DType::F32, + }) + } + + /// The device this forecaster runs on. + #[must_use] + pub fn device(&self) -> &Device { + &self.device + } + + /// The underlying model (for callers that need the raw decode API). + #[must_use] + pub fn model(&self) -> &PatchedTimeSeriesDecoder { + &self.model + } + + /// Forecast `horizon` steps ahead of `series`, using the default (finest) + /// frequency bucket. Returns point + quantile bands. + pub fn forecast(&self, series: &[f32], horizon: usize) -> Result { + self.forecast_with_freq(series, horizon, 0) + } + + /// Forecast `horizon` steps with an explicit frequency id (0 = high/fine, + /// 1 = medium, 2 = low — TimesFM's frequency buckets). + pub fn forecast_with_freq( + &self, + series: &[f32], + horizon: usize, + freq_id: u32, + ) -> Result { + if series.is_empty() { + return Err(Error::Invalid("series must be non-empty".into())); + } + if horizon == 0 { + return Err(Error::Invalid("horizon must be > 0".into())); + } + + let k = series.len(); + let input_ts = + Tensor::from_vec(series.to_vec(), (1, k), &self.device)?.to_dtype(self.dtype)?; + let input_padding = Tensor::zeros((1, k), self.dtype, &self.device)?; + let freq = Tensor::from_vec(vec![freq_id], (1, 1), &self.device)?; + + // (point [1, h], full [1, h, num_outputs]); channel 0 = mean, 1..=9 = p10..p90. + let (point_t, full_t) = self + .model + .decode(&input_ts, &input_padding, &freq, horizon)?; + + // Outputs come back in the forward dtype; surface f32 to callers. + let point: Vec = point_t.i(0)?.to_dtype(DType::F32)?.to_vec1()?; + let full: Vec> = full_t.i(0)?.to_dtype(DType::F32)?.to_vec2()?; // [h][num_outputs] + + let quantiles: Vec<[f32; NUM_QUANTILES]> = full + .iter() + .map(|row| { + let mut q = [0f32; NUM_QUANTILES]; + // row[0] is the mean; quantiles live at indices 1..=9. + for (j, slot) in q.iter_mut().enumerate() { + *slot = row.get(j + 1).copied().unwrap_or(row[0]); + } + q + }) + .collect(); + + Ok(Forecast { point, quantiles }) + } + + /// Forecast a **batch** of equal-length series in a single model call — + /// the throughput path. All series must share one length (the common case + /// for windowed telemetry / multi-series dashboards); returns one + /// [`Forecast`] per input row. For ragged lengths, call [`Self::forecast`] + /// per series. + pub fn forecast_batch( + &self, + series_batch: &[Vec], + horizon: usize, + freq_id: u32, + ) -> Result> { + if series_batch.is_empty() { + return Err(Error::Invalid("series batch is empty".into())); + } + if horizon == 0 { + return Err(Error::Invalid("horizon must be > 0".into())); + } + let k = series_batch[0].len(); + if k == 0 { + return Err(Error::Invalid("series must be non-empty".into())); + } + if series_batch.iter().any(|s| s.len() != k) { + return Err(Error::Invalid( + "forecast_batch requires equal-length series; use forecast() per series for ragged input".into(), + )); + } + let b = series_batch.len(); + let flat: Vec = series_batch.iter().flatten().copied().collect(); + let input_ts = Tensor::from_vec(flat, (b, k), &self.device)?.to_dtype(self.dtype)?; + let input_padding = Tensor::zeros((b, k), self.dtype, &self.device)?; + let freq = Tensor::from_vec(vec![freq_id; b], (b, 1), &self.device)?; + + let (point_t, full_t) = self + .model + .decode(&input_ts, &input_padding, &freq, horizon)?; + let point_t = point_t.to_dtype(DType::F32)?; + let full_t = full_t.to_dtype(DType::F32)?; + + let mut out = Vec::with_capacity(b); + for row in 0..b { + let point: Vec = point_t.i(row)?.to_vec1()?; + let full: Vec> = full_t.i(row)?.to_vec2()?; + let quantiles: Vec<[f32; NUM_QUANTILES]> = full + .iter() + .map(|r| { + let mut q = [0f32; NUM_QUANTILES]; + for (j, slot) in q.iter_mut().enumerate() { + *slot = r.get(j + 1).copied().unwrap_or(r[0]); + } + q + }) + .collect(); + out.push(Forecast { point, quantiles }); + } + Ok(out) + } + + /// PRUNE/CONTINUE decision for a partial optimization curve (lower = better), + /// forecasting toward `target_iters` total against a viability `threshold`. + /// Thin pass-through to [`timesfm::prune::decide_prune`] using this + /// forecaster's model + device; see [`crate::sweep`] for the gated wrapper. + pub fn prune_decision( + &self, + curve: &[f32], + target_iters: usize, + threshold: f32, + ) -> Result { + if curve.is_empty() { + return Err(Error::Invalid("curve must be non-empty".into())); + } + Ok(decide_prune( + &self.model, + curve, + target_iters, + threshold, + &self.device, + )?) + } +} diff --git a/crates/ruvector-timesfm/src/lib.rs b/crates/ruvector-timesfm/src/lib.rs new file mode 100644 index 000000000..751aff009 --- /dev/null +++ b/crates/ruvector-timesfm/src/lib.rs @@ -0,0 +1,76 @@ +//! # ruvector-timesfm +//! +//! RuVector-facing integration for the [`timesfm`] TimesFM 1.0 200M time-series +//! foundation model. The base `timesfm` crate is a faithful, parity-validated +//! candle port of the model; this crate wraps it in the three things RuVector +//! and ruflo actually call: +//! +//! 1. [`Forecaster`] — a one-call quantile forecaster: load weights once, +//! `forecast(series, horizon)` → point forecast **plus calibrated p10..p90 +//! bands** ([`Forecast`]). +//! 2. [`anomaly`] — forecast-band anomaly detection: forecast the expected +//! window, then flag observed points that fall outside their p10/p90 band +//! (host/vector-db telemetry: disk-fill, GPU memory, query load). +//! 3. [`sweep`] — TimesFM-driven early stopping for optimization sweeps +//! (ADR-191 §2): an [`sweep::EarlyStopper`] that wraps +//! [`timesfm::prune::decide_prune`] with `min_history` + a confidence gate so +//! ruflo/Darwin runs can kill doomed genomes early. +//! +//! ## Feature gating +//! +//! The numeric path lives behind the **`candle`** feature (and `cuda`/`metal` +//! which imply it), mirroring `timesfm`. Without it, only the plain data types +//! ([`Forecast`], [`anomaly::AnomalyReport`], [`sweep::EarlyStopper`]) compile — +//! the inference methods are gated. This keeps a stock +//! `cargo build --workspace` light. +//! +//! ```ignore +//! use ruvector_timesfm::Forecaster; +//! let f = Forecaster::load("/path/timesfm.safetensors", timesfm::select_device()?)?; +//! let forecast = f.forecast(&history, 64)?; +//! let (lo, hi) = (forecast.p10(), forecast.p90()); +//! ``` + +pub mod anomaly; +mod forecast_types; +pub mod rebuild; +pub mod sweep; + +pub use forecast_types::Forecast; + +#[cfg(feature = "candle")] +mod forecaster; +#[cfg(feature = "candle")] +pub use forecaster::{Forecaster, Quant}; + +// Re-export the underlying model crate so callers can reach config/prune types +// (and `select_device`) without a second dependency. +pub use timesfm; + +/// Crate error type. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// Input validation failure (empty series, zero horizon, …). + #[error("invalid input: {0}")] + Invalid(String), + + /// Error bubbling up from the underlying `timesfm` crate. + #[error("timesfm error: {0}")] + Timesfm(#[from] timesfm::Error), + + /// candle tensor error (numeric path only). + #[cfg(feature = "candle")] + #[error("candle error: {0}")] + Candle(#[from] candle_core::Error), + + /// I/O error (weight loading, CLI). + #[error("io error: {0}")] + Io(#[from] std::io::Error), + + /// JSON (de)serialization error (CLI / serde boundary). + #[error("json error: {0}")] + Json(#[from] serde_json::Error), +} + +/// Crate result alias. +pub type Result = std::result::Result; diff --git a/crates/ruvector-timesfm/src/rebuild.rs b/crates/ruvector-timesfm/src/rebuild.rs new file mode 100644 index 000000000..8047e584d --- /dev/null +++ b/crates/ruvector-timesfm/src/rebuild.rs @@ -0,0 +1,82 @@ +//! Forecast-driven vector-index maintenance: decide *when* to rebuild an HNSW +//! index from its recall-drift history. +//! +//! RuVector ANN indexes lose recall as they accrue deletes/updates (see the +//! `ruvector-diskann` recall-trigger work). Instead of rebuilding on a fixed +//! schedule or only after recall has already dropped, forecast the recall curve +//! with TimesFM and schedule the rebuild to land *just before* recall crosses a +//! floor — fewer rebuilds at equal-or-better served recall. + +use serde::{Deserialize, Serialize}; + +use crate::forecast_types::Forecast; + +/// Advice on whether/when to rebuild a vector index. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RebuildAdvice { + /// Rebuild now: recall is already at/under the floor, or the conservative + /// (p10) forecast crosses the floor within `lead_steps`. + pub rebuild_now: bool, + /// Forecast steps until the **median (p50)** recall is predicted to drop + /// below the floor (`None` if it never does within the horizon). + pub steps_until_floor: Option, + /// Same, but for the conservative **lower band (p10)** — the early-warning + /// signal `rebuild_now` acts on. + pub steps_until_floor_p10: Option, + /// Recall floor the advice was taken against (echoed for auditability). + pub floor: f32, + /// The recall forecast (point + bands) the advice was derived from. + pub forecast: Forecast, +} + +/// Decide rebuild timing from a recall forecast. +/// +/// `recall` curves are *higher = better*, so "crossing the floor" means the +/// forecast falling **below** `floor`. `lead_steps` is how many steps of +/// look-ahead trigger an immediate rebuild (e.g. enough time for a rebuild to +/// finish before recall actually degrades). Acts on the **p10** lower band so +/// the trigger is conservative. +#[must_use] +pub fn advise_from_forecast( + forecast: Forecast, + last_observed_recall: f32, + floor: f32, + lead_steps: usize, +) -> RebuildAdvice { + let first_below = |band: &[f32]| band.iter().position(|&r| r < floor); + let p50 = forecast.p50(); + let p10 = forecast.p10(); + let steps_until_floor = first_below(&p50); + let steps_until_floor_p10 = first_below(&p10); + + let rebuild_now = last_observed_recall <= floor + || matches!(steps_until_floor_p10, Some(s) if s <= lead_steps); + + RebuildAdvice { + rebuild_now, + steps_until_floor, + steps_until_floor_p10, + floor, + forecast, + } +} + +#[cfg(feature = "candle")] +impl crate::Forecaster { + /// Forecast `horizon` steps of recall from `recall_history` and advise on + /// rebuild timing against `floor` (see [`advise_from_forecast`]). + pub fn advise_rebuild( + &self, + recall_history: &[f32], + floor: f32, + horizon: usize, + lead_steps: usize, + ) -> crate::Result { + if recall_history.is_empty() { + return Err(crate::Error::Invalid("recall_history is empty".into())); + } + let forecast = self.forecast(recall_history, horizon)?; + let last = *recall_history.last().unwrap(); + Ok(advise_from_forecast(forecast, last, floor, lead_steps)) + } +} diff --git a/crates/ruvector-timesfm/src/sweep.rs b/crates/ruvector-timesfm/src/sweep.rs new file mode 100644 index 000000000..287a005b4 --- /dev/null +++ b/crates/ruvector-timesfm/src/sweep.rs @@ -0,0 +1,133 @@ +//! TimesFM-driven early stopping for optimization sweeps (ADR-191 §2). +//! +//! Generalizes the `timesfm::prune` example into a reusable, configurable +//! [`EarlyStopper`] for ruflo / Darwin sweeps: feed the champion metric curve +//! (lower = better) and get a [`StopDecision`]. The stopper adds the two gates +//! the raw [`timesfm::prune::decide_prune`] leaves to the caller — a +//! `min_history` warm-up and a `confidence` floor — so a sweep can wire it in +//! with a single `evaluate` call. + +use serde::{Deserialize, Serialize}; +use timesfm::prune::PruneDecision; + +/// Configurable early-stopping policy. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct EarlyStopper { + /// Viability threshold (same units as the curve; lower = better). A run is a + /// PRUNE candidate when its forecast plateau is above this. + pub threshold: f32, + /// Total iterations the run is budgeted for (the forecast horizon is + /// `target_iters - len(curve)`). + pub target_iters: usize, + /// Don't decide until at least this many points are observed (warm-up). + pub min_history: usize, + /// Only `stop` when `decide_prune`'s confidence is at least this. Lets + /// borderline runs continue. + pub confidence_gate: f32, + /// TimesFM frequency bucket (0 = fine/per-iteration, the right one for + /// optimization curves). + pub freq_id: u32, +} + +impl Default for EarlyStopper { + fn default() -> Self { + Self { + threshold: 0.05, + target_iters: 1000, + min_history: 16, + confidence_gate: 0.6, + freq_id: 0, + } + } +} + +impl EarlyStopper { + /// New stopper with the given viability threshold and iteration budget; + /// other fields take their [`Default`] values. + #[must_use] + pub fn new(threshold: f32, target_iters: usize) -> Self { + Self { + threshold, + target_iters, + ..Self::default() + } + } + + /// Set the warm-up (minimum observed points before a decision is made). + #[must_use] + pub fn with_min_history(mut self, min_history: usize) -> Self { + self.min_history = min_history; + self + } + + /// Set the confidence floor for acting on a PRUNE. + #[must_use] + pub fn with_confidence_gate(mut self, gate: f32) -> Self { + self.confidence_gate = gate; + self + } +} + +/// Outcome of [`EarlyStopper::evaluate`]. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct StopDecision { + /// `true` ⇒ kill the run now (forecast plateaus above threshold AND the + /// confidence gate is cleared AND warm-up satisfied). + pub stop: bool, + /// Human/audit-readable reason for the decision. + pub reason: String, + /// The underlying forecast decision, when one was computed (`None` during + /// warm-up). + pub decision: Option, +} + +#[cfg(feature = "candle")] +impl EarlyStopper { + /// Evaluate a partial champion curve and decide whether to stop the run. + /// + /// Returns `stop = false` (with `decision = None`) while the curve is + /// shorter than `min_history`. Otherwise forecasts with `forecaster` and + /// applies the threshold + confidence gate. + pub fn evaluate( + &self, + forecaster: &crate::Forecaster, + curve: &[f32], + ) -> crate::Result { + if curve.len() < self.min_history { + return Ok(StopDecision { + stop: false, + reason: format!( + "warm-up: {}/{} observations before deciding", + curve.len(), + self.min_history + ), + decision: None, + }); + } + + let decision = forecaster.prune_decision(curve, self.target_iters, self.threshold)?; + let stop = decision.prune && decision.confidence >= self.confidence_gate; + let reason = + if stop { + format!( + "PRUNE: forecast plateau {:.4} > threshold {:.4} (confidence {:.3} ≥ gate {:.3})", + decision.forecast_plateau, self.threshold, decision.confidence, self.confidence_gate + ) + } else if decision.prune { + format!( + "CONTINUE: plateau {:.4} > threshold but confidence {:.3} < gate {:.3}", + decision.forecast_plateau, decision.confidence, self.confidence_gate + ) + } else { + format!( + "CONTINUE: forecast plateau {:.4} ≤ threshold {:.4} (or already viable)", + decision.forecast_plateau, self.threshold + ) + }; + Ok(StopDecision { + stop, + reason, + decision: Some(decision), + }) + } +} diff --git a/crates/ruvector-timesfm/tests/integration.rs b/crates/ruvector-timesfm/tests/integration.rs new file mode 100644 index 000000000..4a309aaed --- /dev/null +++ b/crates/ruvector-timesfm/tests/integration.rs @@ -0,0 +1,235 @@ +//! Integration tests. The pure-logic tests always run; the real-model tests are +//! gated on the `candle` feature AND the local weights, skipping cleanly (never +//! fabricating a pass) when the 814 MB safetensors are absent. + +use ruvector_timesfm::anomaly::score_window; +use ruvector_timesfm::sweep::EarlyStopper; +use ruvector_timesfm::Forecast; + +fn forecast_from_bands(p10: &[f32], p50: &[f32], p90: &[f32]) -> Forecast { + let quantiles = (0..p50.len()) + .map(|i| { + // p10..p90; fill the in-between channels by interpolation (only + // p10/p50/p90 are asserted by the anomaly logic). + [ + p10[i], p10[i], p10[i], p50[i], p50[i], p50[i], p90[i], p90[i], p90[i], + ] + }) + .collect(); + Forecast { + point: p50.to_vec(), + quantiles, + } +} + +#[test] +fn forecast_quantile_accessors() { + let f = forecast_from_bands(&[1.0, 2.0], &[5.0, 6.0], &[9.0, 10.0]); + assert_eq!(f.horizon(), 2); + assert_eq!(f.p10(), vec![1.0, 2.0]); + assert_eq!(f.p50(), vec![5.0, 6.0]); + assert_eq!(f.p90(), vec![9.0, 10.0]); +} + +#[test] +fn anomaly_flags_out_of_band_points() { + let f = forecast_from_bands(&[0.0, 0.0, 0.0], &[5.0, 5.0, 5.0], &[10.0, 10.0, 10.0]); + // inside band, above band, below band. + let observed = [5.0, 25.0, -15.0]; + let report = score_window(&f, &observed); + assert_eq!(report.points.len(), 3); + assert_eq!(report.n_anomalies, 2); + assert!(!report.points[0].is_anomaly); + assert!(report.points[1].is_anomaly && report.points[1].deviation > 0.0); + assert!(report.points[2].is_anomaly && report.points[2].deviation < 0.0); +} + +#[test] +fn early_stopper_warms_up_before_deciding() { + let stopper = EarlyStopper::new(0.05, 1000).with_min_history(16); + // Build a StopDecision via the non-candle path is not possible (evaluate is + // gated), but the config + Default surface is exercised here. + assert_eq!(stopper.min_history, 16); + assert_eq!(stopper.threshold, 0.05); + assert_eq!(EarlyStopper::default().confidence_gate, 0.6); +} + +#[test] +fn rebuild_advice_triggers_before_floor() { + use ruvector_timesfm::rebuild::advise_from_forecast; + // Recall (higher=better) declining: p50 crosses 0.90 floor at step 3, p10 at step 1. + let p10 = [0.92, 0.88, 0.85, 0.82, 0.80]; + let p50 = [0.95, 0.93, 0.91, 0.89, 0.87]; + let p90 = [0.98, 0.97, 0.96, 0.95, 0.94]; + let f = forecast_from_bands(&p10, &p50, &p90); + // lead_steps=2: p10 dips below floor at step 1 (<=2) ⇒ rebuild now. + let a = advise_from_forecast(f, 0.95, 0.90, 2); + assert!(a.rebuild_now); + assert_eq!(a.steps_until_floor, Some(3)); + assert_eq!(a.steps_until_floor_p10, Some(1)); + + // Healthy: recall holds above floor ⇒ no rebuild. + let f2 = forecast_from_bands(&[0.95; 5], &[0.97; 5], &[0.99; 5]); + let b = advise_from_forecast(f2, 0.97, 0.90, 2); + assert!(!b.rebuild_now && b.steps_until_floor.is_none()); +} + +#[cfg(feature = "candle")] +mod real_model { + use ruvector_timesfm::Forecaster; + + const WEIGHTS: &str = "/tmp/timesfm-parity/timesfm.safetensors"; + + fn skip() -> bool { + if !std::path::Path::new(WEIGHTS).exists() { + eprintln!("SKIP real-model test: weights missing ({WEIGHTS})."); + true + } else { + false + } + } + + #[test] + fn forecast_shapes_and_band_ordering() -> anyhow::Result<()> { + if skip() { + return Ok(()); + } + let device = timesfm::select_device()?; + let f = Forecaster::load(WEIGHTS, device)?; + let series: Vec = (0..256) + .map(|t| (t as f32 / 12.0).sin() * 10.0 + 50.0) + .collect(); + let forecast = f.forecast(&series, 64)?; + assert_eq!(forecast.horizon(), 64); + assert_eq!(forecast.point.len(), 64); + // All forecast values finite; quantiles monotone p10 <= p50 <= p90. + for i in 0..64 { + assert!(forecast.point[i].is_finite()); + let (lo, mid, hi) = (forecast.p10()[i], forecast.p50()[i], forecast.p90()[i]); + assert!(lo.is_finite() && mid.is_finite() && hi.is_finite()); + assert!(lo <= hi, "p10 {lo} > p90 {hi} at step {i}"); + } + Ok(()) + } + + #[test] + fn early_stopper_prunes_doomed_run() -> anyhow::Result<()> { + if skip() { + return Ok(()); + } + use ruvector_timesfm::sweep::EarlyStopper; + let device = timesfm::select_device()?; + let f = Forecaster::load(WEIGHTS, device)?; + // doomed: decays toward 0.20, never reaches the 0.05 threshold. + let doomed: Vec = (0..128) + .map(|t| 0.20 + 0.75 * (-(t as f32) / 16.0).exp()) + .collect(); + let stopper = EarlyStopper::new(0.05, 1000) + .with_min_history(16) + .with_confidence_gate(0.5); + let d = stopper.evaluate(&f, &doomed)?; + assert!(d.stop, "doomed run should stop: {}", d.reason); + + // warm-up: too few points → never stop. + let short = &doomed[..8]; + let d2 = stopper.evaluate(&f, short)?; + assert!(!d2.stop && d2.decision.is_none()); + Ok(()) + } + + #[test] + fn f16_load_forecasts_close_to_f32() -> anyhow::Result<()> { + if skip() { + return Ok(()); + } + let device = timesfm::select_device()?; + let series: Vec = (0..256) + .map(|t| (t as f32 / 13.0).sin() * 7.0 + 48.0) + .collect(); + let f32m = Forecaster::load(WEIGHTS, device.clone())?; + let ref_fc = f32m.forecast(&series, 32)?; + let f16m = Forecaster::load_f16(WEIGHTS, device)?; + let f16_fc = f16m.forecast(&series, 32)?; + assert!(f16_fc.point.iter().all(|x| x.is_finite()), "f16 non-finite"); + let scale = ref_fc.point.iter().fold(1e-6f32, |m, v| m.max(v.abs())); + let max_abs = ref_fc + .point + .iter() + .zip(f16_fc.point.iter()) + .fold(0f32, |m, (a, b)| m.max((a - b).abs())); + // f16 has ~3 decimal digits; allow a loose relative bound. + assert!( + max_abs / scale < 2e-2, + "f16 diverged from f32: rel {:.3e}", + max_abs / scale + ); + Ok(()) + } + + #[test] + fn quantized_load_forecasts_close_to_f32() -> anyhow::Result<()> { + if skip() { + return Ok(()); + } + use ruvector_timesfm::Quant; + let device = timesfm::select_device()?; + let series: Vec = (0..256) + .map(|t| (t as f32 / 11.0).sin() * 9.0 + 45.0) + .collect(); + + let f32m = Forecaster::load(WEIGHTS, device.clone())?; + let ref_fc = f32m.forecast(&series, 32)?; + + // Q8_0 stays close to f32 (relative error ~3e-3 measured); assert a + // generous bound and that every value is finite. + let q8 = Forecaster::load_quantized(WEIGHTS, device, Quant::Q8_0)?; + let q8_fc = q8.forecast(&series, 32)?; + let scale = ref_fc.point.iter().fold(1e-6f32, |m, v| m.max(v.abs())); + let max_abs = ref_fc + .point + .iter() + .zip(q8_fc.point.iter()) + .fold(0f32, |m, (a, b)| m.max((a - b).abs())); + assert!(q8_fc.point.iter().all(|x| x.is_finite()), "Q8_0 non-finite"); + assert!( + max_abs / scale < 5e-2, + "Q8_0 diverged from f32: rel {:.3e}", + max_abs / scale + ); + Ok(()) + } + + #[test] + fn batched_matches_per_series() -> anyhow::Result<()> { + if skip() { + return Ok(()); + } + let device = timesfm::select_device()?; + let f = Forecaster::load(WEIGHTS, device)?; + let batch: Vec> = (0..4) + .map(|s| { + (0..128) + .map(|t| ((t as f32 + s as f32) / 9.0).sin() * 8.0 + 40.0) + .collect() + }) + .collect(); + let batched = f.forecast_batch(&batch, 32, 0)?; + assert_eq!(batched.len(), 4); + for (i, series) in batch.iter().enumerate() { + let single = f.forecast(series, 32)?; + // CPU bit-exact; GPU within reduction-order noise (relative). + let scale = single.point.iter().fold(1e-6f32, |m, v| m.max(v.abs())); + let max_abs = single + .point + .iter() + .zip(batched[i].point.iter()) + .fold(0f32, |m, (a, b)| m.max((a - b).abs())); + assert!( + max_abs / scale < 1e-3, + "row {i} batched vs single rel {:.3e}", + max_abs / scale + ); + } + Ok(()) + } +} diff --git a/crates/timesfm/Cargo.toml b/crates/timesfm/Cargo.toml index 8fe862790..316ae1c3d 100644 --- a/crates/timesfm/Cargo.toml +++ b/crates/timesfm/Cargo.toml @@ -28,15 +28,20 @@ required-features = ["candle"] name = "bench24" required-features = ["candle"] +[[example]] +name = "quant_bench" +required-features = ["candle"] + [features] default = [] # Activates the candle inference path (hand-rolled decoder + VarBuilder # safetensors loading). Off by default so stock `cargo build --workspace` # stays light, mirroring ruvector-hailo's `cpu-fallback`/`candle` pattern. candle = ["candle-core", "candle-nn", "candle-transformers", "tokenizers"] -# Acceleration backends (opt-in alongside `candle`). -metal = ["candle-core/metal", "candle-nn/metal"] -cuda = ["candle-core/cuda", "candle-nn/cuda"] +# Acceleration backends. Each implies `candle` so `--features cuda` alone +# compiles the numeric path on the GPU (no need to also pass `candle`). +metal = ["candle", "candle-core/metal", "candle-nn/metal"] +cuda = ["candle", "candle-core/cuda", "candle-nn/cuda"] # HuggingFace download path (google/timesfm-1.0-200m safetensors). hub = ["hf-hub"] diff --git a/crates/timesfm/examples/bench.rs b/crates/timesfm/examples/bench.rs index 5a2e3e554..2684e2ffa 100644 --- a/crates/timesfm/examples/bench.rs +++ b/crates/timesfm/examples/bench.rs @@ -5,7 +5,7 @@ use std::time::Instant; use timesfm::config::TimesfmConfig; use timesfm::model::PatchedTimeSeriesDecoder; fn main() -> anyhow::Result<()> { - let device = Device::Cpu; + let device = timesfm::select_device()?; let weights = std::env::var("TIMESFM_WEIGHTS") .unwrap_or("/tmp/timesfm-parity/timesfm.safetensors".into()); let cfg = TimesfmConfig::timesfm_1p0_200m(); @@ -28,9 +28,10 @@ fn main() -> anyhow::Result<()> { let thr = std::thread::available_parallelism() .map(|x| x.get()) .unwrap_or(0); + let dev_label = std::env::var("TIMESFM_DEVICE").unwrap_or_else(|_| "cpu".into()); println!( - "TimesFM-200M decode(ctx=512,h=128) CPU: {:.2} ms/forecast (mean of {} iters, {} threads)", - per, n, thr + "TimesFM-200M decode(ctx=512,h=128) [{}]: {:.2} ms/forecast (mean of {} iters, {} threads)", + dev_label, per, n, thr ); Ok(()) } diff --git a/crates/timesfm/examples/quant_bench.rs b/crates/timesfm/examples/quant_bench.rs new file mode 100644 index 000000000..754ae5ef3 --- /dev/null +++ b/crates/timesfm/examples/quant_bench.rs @@ -0,0 +1,101 @@ +//! Quantized-inference bench: load TimesFM-200M at f32 / Q8_0 (int8) / Q4_0 +//! (int4), and report each variant's decode latency and forecast error vs the +//! f32 model on the same input. Real weights only. +//! +//! ```ignore +//! cargo run -p timesfm --features candle --release --example quant_bench -- \ +//! /tmp/timesfm-parity/timesfm.safetensors +//! ``` +//! Skips cleanly (exit 0) when weights are absent. + +use std::time::Instant; + +use candle_core::quantized::GgmlDType; +use candle_core::{DType, IndexOp, Tensor}; +use candle_nn::VarBuilder; +use timesfm::config::TimesfmConfig; +use timesfm::model::PatchedTimeSeriesDecoder; + +fn load(weights: &str, quant: Option) -> anyhow::Result { + let device = timesfm::select_device()?; + let cfg = TimesfmConfig::timesfm_1p0_200m(); + let vb = unsafe { VarBuilder::from_mmaped_safetensors(&[weights], DType::F32, &device)? }; + Ok(match quant { + None => PatchedTimeSeriesDecoder::load(cfg, vb)?, + Some(dt) => PatchedTimeSeriesDecoder::load_quantized(cfg, vb, dt)?, + }) +} + +fn decode_point( + model: &PatchedTimeSeriesDecoder, + ctx: &Tensor, + pad: &Tensor, + freq: &Tensor, + h: usize, +) -> anyhow::Result> { + let (point, _full) = model.decode(ctx, pad, freq, h)?; + Ok(point.i(0)?.to_vec1()?) +} + +fn main() -> anyhow::Result<()> { + let weights = std::env::args() + .nth(1) + .or_else(|| std::env::var("TIMESFM_WEIGHTS").ok()) + .unwrap_or_else(|| "/tmp/timesfm-parity/timesfm.safetensors".into()); + if !std::path::Path::new(&weights).exists() { + eprintln!("SKIP quant_bench: weights missing ({weights})."); + return Ok(()); + } + + let device = timesfm::select_device()?; + let ctx_len = 512usize; + let horizon = 128usize; + // Deterministic input series. + let series: Vec = (0..ctx_len) + .map(|t| ((t as f32) / 24.0).sin() * 10.0 + 50.0 + 0.01 * t as f32) + .collect(); + let ctx = Tensor::from_vec(series, (1, ctx_len), &device)?; + let pad = Tensor::zeros((1, ctx_len), DType::F32, &device)?; + let freq = Tensor::from_vec(vec![0u32], (1, 1), &device)?; + + // f32 reference. + let m32 = load(&weights, None)?; + let _ = decode_point(&m32, &ctx, &pad, &freq, horizon)?; // warm + let t = Instant::now(); + let ref32 = decode_point(&m32, &ctx, &pad, &freq, horizon)?; + let ms32 = t.elapsed().as_secs_f64() * 1000.0; + let approx_f32_mb = 200.0; // 200M params × 4 bytes ≈ 800MB on disk; resident ≈ mmap. + println!("f32 : {ms32:7.2} ms (reference) weights≈814 MB on disk"); + + for (name, dt, bytes_per_w) in [ + ("Q8_0", GgmlDType::Q8_0, 1.06_f64), + ("Q4_0", GgmlDType::Q4_0, 0.56_f64), + ] { + let m = load(&weights, Some(dt))?; + let _ = decode_point(&m, &ctx, &pad, &freq, horizon)?; // warm + let t = Instant::now(); + let q = decode_point(&m, &ctx, &pad, &freq, horizon)?; + let ms = t.elapsed().as_secs_f64() * 1000.0; + + let n_bad = q.iter().filter(|x| !x.is_finite()).count(); + let mut max_abs = 0f32; + let mut sum_abs = 0f64; + let mut scale = 1e-6f32; + for (a, b) in q.iter().zip(ref32.iter()) { + max_abs = max_abs.max((a - b).abs()); + sum_abs += (a - b).abs() as f64; + scale = scale.max(b.abs()); + } + let mae = sum_abs / horizon as f64; + let approx_mb = approx_f32_mb * 4.0 * (bytes_per_w / 4.0); // vs f32 4 bytes/w + println!( + "{name:7}: {ms:7.2} ms ({:.2}× vs f32) MAE={mae:.3e} max-abs={max_abs:.3e} rel={:.3e} weights≈{:.0} MB {}", + ms32 / ms, + max_abs / scale, + approx_mb, + if n_bad == 0 { "finite-ok" } else { "NON-FINITE!" }, + ); + anyhow::ensure!(n_bad == 0, "{name} produced non-finite forecasts"); + } + Ok(()) +} diff --git a/crates/timesfm/src/lib.rs b/crates/timesfm/src/lib.rs index 3db195c55..29b1d8850 100644 --- a/crates/timesfm/src/lib.rs +++ b/crates/timesfm/src/lib.rs @@ -59,6 +59,32 @@ pub use model::{ TimesFMAttention, TimesFMDecoderLayer, TransformerMLP, }; +/// Select the compute device from the `TIMESFM_DEVICE` env var +/// (`cpu` | `cuda` | `metal`), defaulting to CPU. +/// +/// `cuda`/`metal` only resolve to a real accelerator when the corresponding +/// crate feature is enabled (`--features cuda` / `--features metal`); otherwise +/// the request logs a notice and falls back to CPU so examples/benches still +/// run. This keeps every example, bench, test, and downstream crate selecting +/// the device the same way instead of hardcoding `Device::Cpu`. +#[cfg(feature = "candle")] +pub fn select_device() -> candle_core::Result { + use candle_core::Device; + match std::env::var("TIMESFM_DEVICE").ok().as_deref() { + #[cfg(feature = "cuda")] + Some("cuda") => Device::new_cuda(0), + #[cfg(feature = "metal")] + Some("metal") => Device::new_metal(0), + Some(other @ ("cuda" | "metal")) => { + eprintln!( + "TIMESFM_DEVICE={other} requested but the `{other}` feature is not enabled; using CPU" + ); + Ok(Device::Cpu) + } + _ => Ok(Device::Cpu), + } +} + /// Crate-level error type. Wraps candle errors when the `candle` feature is on. #[derive(Debug, thiserror::Error)] pub enum Error { diff --git a/crates/timesfm/src/model.rs b/crates/timesfm/src/model.rs index 395db5dc5..24ce50d1d 100644 --- a/crates/timesfm/src/model.rs +++ b/crates/timesfm/src/model.rs @@ -16,11 +16,76 @@ //! embedding (NOT RoPE); //! * RevIN-style per-series instance normalization around the whole stack. +use candle_core::quantized::{GgmlDType, QMatMul, QTensor}; use candle_core::{DType, Device, IndexOp, Result, Tensor, D}; use candle_nn::{ops, Embedding, LayerNorm, Linear, Module, RmsNorm, VarBuilder}; use crate::config::TimesfmConfig; +// --------------------------------------------------------------------------- +// QLinear: a Linear that is either full-precision (f32) or quantized (the +// weight matrix is stored as a ggml QTensor and matmul'd via QMatMul). Used to +// thread optional int8/int4 weight quantization through the whole decoder +// without changing any forward() math. Bias stays f32. +// --------------------------------------------------------------------------- + +/// A linear layer that is either full-precision or weight-quantized. +pub enum QLinear { + /// Full-precision `candle_nn::Linear` (`x·Wᵀ + b`). + Full(Linear), + /// Quantized weight (`QMatMul`) plus an optional f32 bias. + Quant { + /// Quantized weight matmul. + mm: QMatMul, + /// Optional f32 bias added after the matmul. + bias: Option, + }, +} + +impl QLinear { + /// Load a `[out_dim, in_dim]` weight (+ bias) from `vb`. When `quant` is + /// `Some`, the weight is quantized to that ggml dtype (e.g. `Q8_0`, `Q4_0`) + /// at load time; otherwise a full-precision `Linear` is built. + pub fn load( + in_dim: usize, + out_dim: usize, + vb: VarBuilder, + quant: Option, + ) -> Result { + match quant { + None => Ok(QLinear::Full(candle_nn::linear(in_dim, out_dim, vb)?)), + Some(dtype) => { + let w = vb.get((out_dim, in_dim), "weight")?; + let bias = vb.get(out_dim, "bias").ok(); + // ggml block size is 32; every TimesFM-200M inner dim is a + // multiple of 32, but fall back to full precision rather than + // erroring if a future config violates that. + let mm = if in_dim % 32 == 0 { + let qt = QTensor::quantize(&w, dtype)?; + QMatMul::from_qtensor(qt)? + } else { + QMatMul::Tensor(w) + }; + Ok(QLinear::Quant { mm, bias }) + } + } + } + + /// `x·Wᵀ (+ b)`. + pub fn forward(&self, xs: &Tensor) -> Result { + match self { + QLinear::Full(l) => l.forward(xs), + QLinear::Quant { mm, bias } => { + let y = mm.forward(xs)?; + match bias { + Some(b) => y.broadcast_add(b), + None => Ok(y), + } + } + } + } +} + /// `1.442695041 = 1 / ln(2)`. Folded into the query scaling so the softmax runs /// in base-2 just like the reference implementation. const LOG2_E: f64 = 1.442_695_041; @@ -31,17 +96,28 @@ const LOG2_E: f64 = 1.442_695_041; /// `hidden = SiLU(hidden_layer(x)); out = output_layer(hidden) + residual_layer(x)`. pub struct ResidualBlock { - hidden_layer: Linear, - output_layer: Linear, - residual_layer: Linear, + hidden_layer: QLinear, + output_layer: QLinear, + residual_layer: QLinear, } impl ResidualBlock { pub fn load(in_dim: usize, hid_dim: usize, out_dim: usize, vb: VarBuilder) -> Result { + Self::load_quant(in_dim, hid_dim, out_dim, vb, None) + } + + /// As [`Self::load`], optionally quantizing the three weight matrices. + pub fn load_quant( + in_dim: usize, + hid_dim: usize, + out_dim: usize, + vb: VarBuilder, + quant: Option, + ) -> Result { Ok(Self { - hidden_layer: candle_nn::linear(in_dim, hid_dim, vb.pp("hidden_layer"))?, - output_layer: candle_nn::linear(hid_dim, out_dim, vb.pp("output_layer"))?, - residual_layer: candle_nn::linear(in_dim, out_dim, vb.pp("residual_layer"))?, + hidden_layer: QLinear::load(in_dim, hid_dim, vb.pp("hidden_layer"), quant)?, + output_layer: QLinear::load(hid_dim, out_dim, vb.pp("output_layer"), quant)?, + residual_layer: QLinear::load(in_dim, out_dim, vb.pp("residual_layer"), quant)?, }) } @@ -108,8 +184,8 @@ impl PositionalEmbedding { // --------------------------------------------------------------------------- pub struct TimesFMAttention { - qkv_proj: Linear, - o_proj: Linear, + qkv_proj: QLinear, + o_proj: QLinear, /// Learnable per-head-dim scaling parameter, shape `[head_dim]`. scaling: Tensor, num_heads: usize, @@ -120,11 +196,21 @@ pub struct TimesFMAttention { impl TimesFMAttention { pub fn load(cfg: &TimesfmConfig, vb: VarBuilder) -> Result { - let qkv_proj = candle_nn::linear(cfg.hidden_size, cfg.qkv_dim(), vb.pp("qkv_proj"))?; - let o_proj = candle_nn::linear( + Self::load_quant(cfg, vb, None) + } + + /// As [`Self::load`], optionally quantizing the qkv/o projection weights. + pub fn load_quant( + cfg: &TimesfmConfig, + vb: VarBuilder, + quant: Option, + ) -> Result { + let qkv_proj = QLinear::load(cfg.hidden_size, cfg.qkv_dim(), vb.pp("qkv_proj"), quant)?; + let o_proj = QLinear::load( cfg.num_heads * cfg.head_dim, cfg.hidden_size, vb.pp("o_proj"), + quant, )?; let scaling = vb.get(cfg.head_dim, "scaling")?; Ok(Self { @@ -178,7 +264,9 @@ impl TimesFMAttention { // scores [B, heads, N, N]. q already carries the scaling, so no extra // 1/sqrt(d) factor here. let scores = q.matmul(&k.transpose(2, 3)?.contiguous()?)?; - let scores = scores.broadcast_add(mask)?; + // The additive mask is built in f32; coerce it to the score dtype so an + // f16 forward (f16 weights/activations) doesn't dtype-mismatch here. + let scores = scores.broadcast_add(&mask.to_dtype(scores.dtype())?)?; let probs = ops::softmax_last_dim(&scores)?; let ctx = probs.matmul(&v)?; // [B, heads, N, hd] @@ -214,18 +302,35 @@ fn softplus(x: &Tensor) -> Result { pub struct TransformerMLP { layer_norm: LayerNorm, - gate_proj: Linear, - down_proj: Linear, + gate_proj: QLinear, + down_proj: QLinear, } impl TransformerMLP { pub fn load(cfg: &TimesfmConfig, vb: VarBuilder) -> Result { + Self::load_quant(cfg, vb, None) + } + + /// As [`Self::load`], optionally quantizing the gate/down projection weights. + pub fn load_quant( + cfg: &TimesfmConfig, + vb: VarBuilder, + quant: Option, + ) -> Result { let layer_norm = candle_nn::layer_norm(cfg.hidden_size, cfg.rms_norm_eps, vb.pp("layer_norm"))?; - let gate_proj = - candle_nn::linear(cfg.hidden_size, cfg.intermediate_size, vb.pp("gate_proj"))?; - let down_proj = - candle_nn::linear(cfg.intermediate_size, cfg.hidden_size, vb.pp("down_proj"))?; + let gate_proj = QLinear::load( + cfg.hidden_size, + cfg.intermediate_size, + vb.pp("gate_proj"), + quant, + )?; + let down_proj = QLinear::load( + cfg.intermediate_size, + cfg.hidden_size, + vb.pp("down_proj"), + quant, + )?; Ok(Self { layer_norm, gate_proj, @@ -254,14 +359,23 @@ pub struct TimesFMDecoderLayer { impl TimesFMDecoderLayer { pub fn load(cfg: &TimesfmConfig, vb: VarBuilder) -> Result { + Self::load_quant(cfg, vb, None) + } + + /// As [`Self::load`], optionally quantizing the attention/MLP weights. + pub fn load_quant( + cfg: &TimesfmConfig, + vb: VarBuilder, + quant: Option, + ) -> Result { Ok(Self { input_layernorm: candle_nn::rms_norm( cfg.hidden_size, cfg.rms_norm_eps, vb.pp("input_layernorm"), )?, - self_attn: TimesFMAttention::load(cfg, vb.pp("self_attn"))?, - mlp: TransformerMLP::load(cfg, vb.pp("mlp"))?, + self_attn: TimesFMAttention::load_quant(cfg, vb.pp("self_attn"), quant)?, + mlp: TransformerMLP::load_quant(cfg, vb.pp("mlp"), quant)?, }) } @@ -287,9 +401,18 @@ pub struct StackedDecoder { impl StackedDecoder { pub fn load(cfg: &TimesfmConfig, vb: VarBuilder) -> Result { + Self::load_quant(cfg, vb, None) + } + + /// As [`Self::load`], optionally quantizing every layer's weights. + pub fn load_quant( + cfg: &TimesfmConfig, + vb: VarBuilder, + quant: Option, + ) -> Result { let mut layers = Vec::with_capacity(cfg.num_layers); for i in 0..cfg.num_layers { - layers.push(TimesFMDecoderLayer::load(cfg, vb.pp(i))?); + layers.push(TimesFMDecoderLayer::load_quant(cfg, vb.pp(i), quant)?); } Ok(Self { layers }) } @@ -325,20 +448,36 @@ pub struct PatchedTimeSeriesDecoder { impl PatchedTimeSeriesDecoder { pub fn load(cfg: TimesfmConfig, vb: VarBuilder) -> Result { - let input_ff_layer = ResidualBlock::load( + Self::load_quant(cfg, vb, None) + } + + /// Load with all weight matrices quantized to `dtype` (e.g. + /// [`GgmlDType::Q8_0`] for ~2× smaller / int8, [`GgmlDType::Q4_0`] for ~4× + /// smaller / int4). The patch-embed/output `ResidualBlock`s and the 20 + /// transformer layers are quantized; the frequency embedding, layernorms and + /// the learnable attention scaling stay f32 (tiny + precision-sensitive). + pub fn load_quantized(cfg: TimesfmConfig, vb: VarBuilder, dtype: GgmlDType) -> Result { + Self::load_quant(cfg, vb, Some(dtype)) + } + + fn load_quant(cfg: TimesfmConfig, vb: VarBuilder, quant: Option) -> Result { + let input_ff_layer = ResidualBlock::load_quant( cfg.input_ff_in_dim(), cfg.hidden_size, cfg.hidden_size, vb.pp("input_ff_layer"), + quant, )?; let freq_emb = candle_nn::embedding(cfg.num_freq, cfg.hidden_size, vb.pp("freq_emb"))?; let position_emb = PositionalEmbedding::new(cfg.hidden_size); - let stacked_transformer = StackedDecoder::load(&cfg, vb.pp("stacked_transformer"))?; - let horizon_ff_layer = ResidualBlock::load( + let stacked_transformer = + StackedDecoder::load_quant(&cfg, vb.pp("stacked_transformer"), quant)?; + let horizon_ff_layer = ResidualBlock::load_quant( cfg.hidden_size, cfg.hidden_size, cfg.horizon_ff_out_dim(), vb.pp("horizon_ff_layer"), + quant, )?; Ok(Self { input_ff_layer, @@ -519,8 +658,10 @@ impl PatchedTimeSeriesDecoder { first_idx[row] }; // [P] for this row's chosen patch. - let arr = x.i((row, patch as usize, ..))?; - let msk = keep.i((row, patch as usize, ..))?; + // RevIN stats are computed in f32 via scalar extraction; coerce the + // slices so an f16 forward (f16 `x`/`keep`) doesn't trip to_scalar. + let arr = x.i((row, patch as usize, ..))?.to_dtype(DType::F32)?; + let msk = keep.i((row, patch as usize, ..))?.to_dtype(DType::F32)?; let cnt = msk.sum_all()?.to_scalar::()?.max(1.0); let sum = (arr.mul(&msk)?).sum_all()?.to_scalar::()?; let mu = sum / cnt; @@ -578,7 +719,8 @@ impl PatchedTimeSeriesDecoder { // append the mean chunk to the context for the next step. context = Tensor::cat(&[&context, &mean], 1)?; - let new_pad = Tensor::zeros((b, output_patch_len), DType::F32, device)?; + // Match the running padding dtype (f32, or f16 for an f16 forward). + let new_pad = Tensor::zeros((b, output_patch_len), padding.dtype(), device)?; padding = Tensor::cat(&[&padding, &new_pad], 1)?; } diff --git a/crates/timesfm/src/prune.rs b/crates/timesfm/src/prune.rs index 0334c59a1..38feffffe 100644 --- a/crates/timesfm/src/prune.rs +++ b/crates/timesfm/src/prune.rs @@ -51,7 +51,7 @@ use crate::model::PatchedTimeSeriesDecoder; use crate::Result; /// A prune/continue decision plus the evidence behind it. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] pub struct PruneDecision { /// `true` ⇒ the run is forecast to plateau *worse* than the viability /// threshold and should be killed; `false` ⇒ let it keep running. diff --git a/harnesses/timesfm-harness/.claude-plugin/plugin.json b/harnesses/timesfm-harness/.claude-plugin/plugin.json new file mode 100644 index 000000000..475fa45d2 --- /dev/null +++ b/harnesses/timesfm-harness/.claude-plugin/plugin.json @@ -0,0 +1,23 @@ +{ + "name": "timesfm-harness", + "version": "0.1.0", + "description": "TimesFM 1.0 200M decoder-only time-series forecasting inference crate (Rust/candle) — engineering pod harness", + "author": { + "displayName": "Generated by metaharness", + "url": "https://www.npmjs.com/package/metaharness" + }, + "license": "MIT", + "categories": [ + "agent-harness", + "metaharness-scaffold", + "Engineering", + "software-engineering" + ], + "tags": [ + "metaharness", + "agent-harness", + "vertical:coding", + "software-engineering" + ], + "homepage": "https://github.com/ruvnet/agent-harness-generator" +} diff --git a/harnesses/timesfm-harness/.claude/commands/doctor.md b/harnesses/timesfm-harness/.claude/commands/doctor.md new file mode 100644 index 000000000..7fa0c94f9 --- /dev/null +++ b/harnesses/timesfm-harness/.claude/commands/doctor.md @@ -0,0 +1,12 @@ +--- +description: "Health-check the harness: kernel load, MCP wiring, memory backend, host adapter." +--- + +Run a full health check and print a PASS/FAIL table. + +1. Kernel loads and `kernelInfo().version` matches package.json. +2. The MCP server starts and lists its tools. +3. The memory backend is reachable. +4. The configured host adapter is present. + +Exit non-zero if any check fails. diff --git a/harnesses/timesfm-harness/.claude/commands/review-diff.md b/harnesses/timesfm-harness/.claude/commands/review-diff.md new file mode 100644 index 000000000..5ed67e902 --- /dev/null +++ b/harnesses/timesfm-harness/.claude/commands/review-diff.md @@ -0,0 +1,10 @@ +--- +description: "Review the current working diff for correctness, security, and reuse." +--- + +Review the current git diff. + +1. `git diff` to read the change. +2. Report only high-confidence findings as `file:line — issue — fix`. +3. Separate bugs from nits. +4. End with APPROVE or REQUEST-CHANGES and a one-line reason. diff --git a/harnesses/timesfm-harness/.claude/settings.json b/harnesses/timesfm-harness/.claude/settings.json new file mode 100644 index 000000000..5cea77620 --- /dev/null +++ b/harnesses/timesfm-harness/.claude/settings.json @@ -0,0 +1,40 @@ +{ + "permissions": { + "allow": [ + "Bash(npx timesfm-harness*)", + "mcp__timesfm-harness__*", + "mcp__code_index__*", + "Bash(npm test*)", + "Bash(npm run*)", + "Bash(git diff*)", + "Bash(git status*)", + "Bash(git log*)" + ], + "deny": [ + "Read(./.env)", + "Read(./.env.*)", + "Bash(git push*)", + "Bash(rm -rf*)" + ] + }, + "mcpServers": { + "timesfm-harness": { + "command": "npx", + "args": [ + "-y", + "timesfm-harness@latest", + "mcp", + "start" + ] + }, + "code_index": { + "command": "npx", + "args": [ + "-y", + "timesfm-harness@latest", + "mcp", + "index" + ] + } + } +} diff --git a/harnesses/timesfm-harness/.claude/skills/evolve/SKILL.md b/harnesses/timesfm-harness/.claude/skills/evolve/SKILL.md new file mode 100644 index 000000000..37c2f5a83 --- /dev/null +++ b/harnesses/timesfm-harness/.claude/skills/evolve/SKILL.md @@ -0,0 +1,53 @@ +--- +name: evolve +description: "Evolve this harness with Darwin Mode — frozen model, evolving harness (real, sandboxed, safety-gated)." +--- + +# evolve — Darwin Mode self-improvement + +`timesfm-harness` ships with **Darwin Mode** (`@metaharness/darwin`, ADR-070…146): the model +is frozen; the *harness* evolves. Each generation mutates ONE of the 7 surface files +(planner, contextBuilder, reviewer, retry/tool/memory/score policy), sandboxes each +child, scores it, and keeps only variants that *measurably* improve — building an +archive of successful descendants. + +## Run it + +```bash +npm run evolve # real substrate: runs your test command per variant (deterministic mutator — no API key, no network) +npm run evolve:dry # mock substrate: fast, fully offline, no test execution +``` + +Or directly: + +```bash +npx metaharness-darwin evolve . --sandbox real --generations 3 --children 4 +``` + +## Safety (secure by default) + +- **Deterministic mutator** is the default — **no network, no API key, air-gapped**. +- Every mutation passes the `validateGeneratedCode` gate: no new imports, network, + filesystem, shell, env access, or dependencies — pure refactor/tuning only. +- Mutations run in a **sandbox**; only variants that pass your tests are archived. +- Nothing is promoted without measured improvement (guard against Goodharting). + +See `@metaharness/darwin` for selection strategies (`--selection`, `--crossover`, +`--curriculum`), statistical gates (`--fdr`, `--bench`), and the real-LLM mutator (library API). + +## What the benchmarks taught us (measured, full SWE-bench Lite 300) + +Defaults worth carrying into how you evolve and run this harness (full evidence + CIs in +`@metaharness/darwin`'s `LEARNINGS.md` / `bench/results/RESULTS.md`): + +1. **Closed-loop repair is the #1 lever (~2×).** Feeding test/compiler failure back and retrying took + resolve-rate 7.7% → 15.3% on the *same cheap model*. Iterate against ground truth, don't single-shot. +2. **Cheap-first + cost-aware routing.** Track **$/resolve**, not just resolve-rate; a cheap model + resolved 31× cheaper per fix than a frontier one. Reserve frontier for *measured* capability gaps. +3. **Tier the models (Barbarian & Scholar).** Cheap sweep + frontier on *only the residual* = 33.3% + at ~6× lower cost than running frontier everywhere. +4. **Put the output-format contract in a system message + example**, and size prompts to the model's + real context window — this alone took a weak local model from 0% to ~50% valid output. +5. **Only trust batch evaluation of the final artifact** — in-loop counters drift 1.5–5×. +6. **The harness multiplies the model; it can't rescue one below the task's reasoning floor.** Pick + the smallest model *above* the floor, then let evolution do the rest. diff --git a/harnesses/timesfm-harness/.claude/skills/plan-change/SKILL.md b/harnesses/timesfm-harness/.claude/skills/plan-change/SKILL.md new file mode 100644 index 000000000..22fb03f56 --- /dev/null +++ b/harnesses/timesfm-harness/.claude/skills/plan-change/SKILL.md @@ -0,0 +1,15 @@ +--- +name: plan-change +description: "Turn a feature request into a minimal, file-level implementation plan before any code." +--- + +# plan-change + +Produce an implementation plan for a requested change. + +1. Restate the goal in one sentence. +2. List the files to touch and why. +3. Name the smallest interface that satisfies it. +4. Flag anything that ripples beyond three files or widens a permission. + +Hand the plan to the implementer; do not write code in this step. diff --git a/harnesses/timesfm-harness/.harness/manifest.json b/harnesses/timesfm-harness/.harness/manifest.json new file mode 100644 index 000000000..57a1adcc3 --- /dev/null +++ b/harnesses/timesfm-harness/.harness/manifest.json @@ -0,0 +1,40 @@ +{ + "schema": 1, + "generator": "0.1.0", + "template": "vertical:coding", + "template_version": "0.0.0", + "vars": { + "name": "timesfm-harness", + "description": "TimesFM 1.0 200M decoder-only time-series forecasting inference crate (Rust/candle) — engineering pod harness", + "host": "claude-code" + }, + "hosts": [ + "claude-code" + ], + "files": { + ".claude/commands/doctor.md": "2f1475fa0ed34729cb2ed9d6cecf29e99781da0a56afef133361e3ff0a17bb52", + ".claude/commands/review-diff.md": "2ab52f01487bfe67335f4de5193d7a6fb0f72f646411114613d8fa5da071ef2b", + ".claude/settings.json": "c6b680b7c2ce4c0c4125dbb36f34c99b826e7d75c355e780d988660880013a77", + ".claude/skills/plan-change/SKILL.md": "84e1c44ca264b999ebf80cd8fa0e5ebf554275bbd8023b1530ad053a44482e30", + ".claude-plugin/plugin.json": "4fb680b03453b73b7fff4050ec175cf30169d6b0d58941ce12dbf3292b2a26de", + "CLAUDE.md": "739055b4a227cffdd525fae343da2feb98f9fb8d06cd7f7d6139bac85caf7c20", + "README.md": "697541069108fde6038232cc4853d7ae0d22e3bc6283864444020bc48e309124", + "__tests__/smoke.test.ts": "99b3f65522fc146893f127af5795b4e165817edfaa70f313926979c9239bb76a", + "bin/cli.js": "0e8fba4c852cdb9544f7ddcb3e7936864f30ca53d5a75891ba933fede7902840", + "package.json": "11859320c6a86ac9637316325bfb65fdfaeca3ffa9d0839fb1c831b5e7601eb5", + "src/agents/architect.ts": "b07d7dd64b5ecf406ee6ccce545f44b367f1eb699f7c10c5884ca9f4c59b27b9", + "src/agents/implementer.ts": "705f008d7d83af785b306895f7abaa6cbba9d6f9e95cfe475648d499684c093d", + "src/agents/reviewer.ts": "476725f65d80d0e603b615e4d432fe6d0c058b7dc49ce484a7b62ee5edba5bc4", + "src/agents/test-writer.ts": "53f3ac9d344694c1a058586d90bb66af1478b31ef69d8bb44c20a09cf9215581", + "src/init.ts": "e32e046b54f72088962b4cb840811bb931752da512a9386818f6ba4b1354a76c", + "tsconfig.json": "8b4e730a1aa39162ac574455d7a98e1881f5313ca80ffe503b9652dcf0c76b9d", + "vitest.config.ts": "e9e94875611ab1cd602c1f6923902c2dab7953a75ac022962973639d1937587a", + "LICENSE": "a471cd13d1c9d05bd3104774af74facd230076758ee25320e549fe0de08a376a", + ".claude/skills/evolve/SKILL.md": "298f66d6e5d2cf2c5c6c6f1bce193b75849e092e455aa5086a03428ffe766902" + }, + "generated_at": "2026-06-25T20:42:21.001Z", + "meta": { + "surface": "cli", + "kernel_version": "0.1.2" + } +} \ No newline at end of file diff --git a/harnesses/timesfm-harness/.harness/manifest.sha256 b/harnesses/timesfm-harness/.harness/manifest.sha256 new file mode 100644 index 000000000..dfa8d39b0 --- /dev/null +++ b/harnesses/timesfm-harness/.harness/manifest.sha256 @@ -0,0 +1 @@ +7c45ab915393da6e43141935ce884d1718b3dcabaf2454f3fe32519999e32a7c diff --git a/harnesses/timesfm-harness/CLAUDE.md b/harnesses/timesfm-harness/CLAUDE.md new file mode 100644 index 000000000..984485fcb --- /dev/null +++ b/harnesses/timesfm-harness/CLAUDE.md @@ -0,0 +1,32 @@ +# timesfm-harness + +TimesFM 1.0 200M decoder-only time-series forecasting inference crate (Rust/candle) — engineering pod harness + +> Advanced Coding harness · domain: `software-engineering`. Generated with [create-agent-harness](https://github.com/ruvnet/agent-harness-generator). + +## Behavioral rules + +- Use the harness's MCP tools (`mcp__timesfm-harness__*`) for orchestration +- Memory and routing are handled by the kernel — you don't need to learn them +- Defer destructive operations to the user + +## Agents + +| Agent | Tier | Role | +|---|---|---| +| `architect` | opus | Designs the change before code is written. | +| `implementer` | sonnet | Writes code that matches the surrounding style. | +| `reviewer` | opus | Hunts correctness bugs in the diff. | +| `test-writer` | sonnet | Adds the missing tests for the change. | +## Skills + +- `/plan-change` — Turn a feature request into a minimal, file-level implementation plan before any code. + +## Commands + +- `doctor` — Health-check the harness: kernel load, MCP wiring, memory backend, host adapter. +- `review-diff` — Review the current working diff for correctness, security, and reuse. + +## Architecture + +This harness uses [@metaharness/kernel](https://www.npmjs.com/package/@metaharness/kernel) — a Rust-compiled WASM module with a NAPI-RS native fallback — so the same code runs identically on every platform. diff --git a/harnesses/timesfm-harness/LICENSE b/harnesses/timesfm-harness/LICENSE new file mode 100644 index 000000000..a432ee129 --- /dev/null +++ b/harnesses/timesfm-harness/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 timesfm-harness authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/harnesses/timesfm-harness/PROVENANCE.md b/harnesses/timesfm-harness/PROVENANCE.md new file mode 100644 index 000000000..b6e50f0e6 --- /dev/null +++ b/harnesses/timesfm-harness/PROVENANCE.md @@ -0,0 +1,110 @@ +# timesfm-harness — provenance + +This directory is a **metaharness bundle** (ADR-041 "metaharness as program +synthesis") generated for the `crates/timesfm` + `crates/ruvector-timesfm` +TimesFM forecasting work, using the `agent-harness-generator` at +`ruvnet/agent-harness-generator`. It is the authentic generator output, not +hand-authored. + +## How it was generated (reproducible) + +Generator: `create-agent-harness` CLI **v0.2.7**, kernel `0.1.2`, +binary `packages/create-agent-harness/dist/bin.js`. + +```bash +cd /path/to/agent-harness-generator +BIN=packages/create-agent-harness/dist/bin.js + +# Feasibility scorecard + repo genome (read-only analysis of the crate) +node $BIN score /path/to/ruvector/crates/timesfm --json +node $BIN genome /path/to/ruvector/crates/timesfm --json + +# Synthesize the bundle (engineering-pod vertical, Claude Code host) +node $BIN timesfm-harness \ + --template vertical:coding --host claude-code \ + --description "TimesFM 1.0 200M decoder-only time-series forecasting inference crate (Rust/candle) — engineering pod harness" \ + --target +``` + +`vertical:coding` (engineering pod: architect / implementer / reviewer / +test-writer over code memory) was the generator's own recommended template for +this `rust-crate-harness` archetype — the right fit for developing a Rust +inference library. + +## Score (feasibility scorecard) + +```json +{ "schema":1, "repo":"timesfm", "harnessFit":52, "compileConfidence":90, + "taskCoverage":79, "toolSafety":100, "memoryUsefulness":34, + "estCostPerRunUsd":0.048, "recommendedMode":"CLI + MCP", + "archetype":"rust-crate-harness", "template":"vertical:coding", + "scaffoldReady":true, "hardConstraints":"6/6" } +``` + +## Genome (repo synthesis verdict) + +```json +{ "repo_type":"rust", "agent_topology":["maintainer","tester","security"], + "risk_score":0.37, "mcp_surface":"local_default_deny", + "test_confidence":0.5, "publish_readiness":0.55 } +``` + +## Witness / provenance (ADR-011) + +`.harness/manifest.json` records `schema:1`, the template/vars/hosts, and a +SHA-256 for every emitted file. `.harness/manifest.sha256` is the witness over +the manifest: + +``` +manifest witness = 7c45ab915393da6e43141935ce884d1718b3dcabaf2454f3fe32519999e32a7c +``` + +Verify integrity: + +```bash +sha256sum .harness/manifest.json # must equal the contents of .harness/manifest.sha256 +``` + +Verified valid at commit time. + +## Connection to the RuVector TimesFM work + +The harness governs an engineering pod for the forecasting crates. The +runtime forecasting capability it would orchestrate is the +`time_series_forecast` MCP tool, implemented by the +`ruvector-timesfm-forecast` CLI in `crates/ruvector-timesfm` (JSON in → +point + p10/p50/p90 out). + +## Optimizing the harness — Darwin evolve via OpenRouter (key from GCP) + +The harness ships a Darwin-Mode self-improvement loop (the `evolve` skill). Its +default mutator is deterministic (air-gapped, no key). The **LLM mutator** +(`OpenRouterMutator`, ADR-071) is library-only — not exposed by the +`metaharness-darwin` CLI — so `scripts/evolve-openrouter.{sh,mjs}` wire it into +the `evolve()` engine. + +The OpenRouter API key is **sourced from GCP Secret Manager at runtime** and +exported only into the run's process — never stored in the repo, a dotfile, or +the logs: + +```bash +# real sandbox, key fetched from GCP secret OPENROUTER_API_KEY (project cognitum-20260110) +./scripts/evolve-openrouter.sh +# tune cost/scope: +GENERATIONS=1 CHILDREN=2 SANDBOX=mock ./scripts/evolve-openrouter.sh +# overrides: OPENROUTER_SECRET, GCP_PROJECT, DARWIN_MUTATOR_MODEL, DARWIN_DIST +``` + +Validated run (real sandbox, 1 gen × 2 children, `google/gemini-2.5-flash`): +baseline scored **0.985** (taskSuccess 1.0, testPassRate 1.0, safety 1.0, zero +secret-exposure/destructive/hallucination flags); 2 real OpenRouter mutations, +~$0.003. Every mutation passes the `validateGeneratedCode` safety gate (no new +imports/network/shell/env) and only promotes on measured improvement. + +## Notes on this generator version (v0.2.7) + +- `mint` emits `.harness/manifest.json` + `.harness/manifest.sha256` (the + witness). The MCP policy is embedded in `.claude/settings.json` + (default-deny). It does **not** emit separate `.harness/genome.json` / + `.harness/mcp-policy.json` files — `genome` is a read-only command (captured + above) and the policy lives in settings.json. diff --git a/harnesses/timesfm-harness/README.md b/harnesses/timesfm-harness/README.md new file mode 100644 index 000000000..6abe16eaa --- /dev/null +++ b/harnesses/timesfm-harness/README.md @@ -0,0 +1,30 @@ +# timesfm-harness + +TimesFM 1.0 200M decoder-only time-series forecasting inference crate (Rust/candle) — engineering pod harness + +> **Advanced Coding** — Architect → implement → review → test, with a code-index MCP and push-guarded git perms. +> +> Generated with [`create-agent-harness`](https://github.com/ruvnet/agent-harness-generator). Multi-host scaffolding with a kernel that resolves native → wasm → js (js backend in the published beta; see `harness doctor`). + +## Install + +```bash +npm install -g timesfm-harness +timesfm-harness init +timesfm-harness doctor +``` + +## Agents + +| Agent | Role | +|---|---| +| `architect` | Designs the change before code is written. | +| `implementer` | Writes code that matches the surrounding style. | +| `reviewer` | Hunts correctness bugs in the diff. | +| `test-writer` | Adds the missing tests for the change. | + +This harness ships with the **claude-code** adapter. + +## License + +MIT diff --git a/harnesses/timesfm-harness/__tests__/smoke.test.ts b/harnesses/timesfm-harness/__tests__/smoke.test.ts new file mode 100644 index 000000000..37146318e --- /dev/null +++ b/harnesses/timesfm-harness/__tests__/smoke.test.ts @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: MIT +// Generated by metaharness — a real smoke test for timesfm-harness. +// +// This is NOT a placeholder: it boots the actual kernel + host adapter the +// harness depends on, so `npm test` fails loudly if @metaharness/kernel or +// @metaharness/host-claude-code is missing, broken, or version-skewed. It is the +// fastest signal that `npm install` produced a runnable harness. + +import { describe, it, expect } from 'vitest'; +import { loadKernel } from '@metaharness/kernel'; +import adapter from '@metaharness/host-claude-code'; +import { run } from '../bin/cli.js'; + +describe('timesfm-harness — install smoke test', () => { + it('loads the kernel and reports a version + a known backend', async () => { + const kernel = await loadKernel(); + const info = kernel.kernelInfo(); + expect(typeof info.version).toBe('string'); + expect(info.version.length).toBeGreaterThan(0); + expect(['native', 'wasm', 'js']).toContain(kernel.backend); + }); + + it('resolves the host adapter with a name', () => { + expect(typeof adapter.name).toBe('string'); + expect(adapter.name.length).toBeGreaterThan(0); + }); + + it('the CLI doctor command succeeds (exit 0)', async () => { + const code = await run(['doctor']); + expect(code).toBe(0); + }); + + it('an unknown CLI command exits non-zero', async () => { + const code = await run(['definitely-not-a-command']); + expect(code).not.toBe(0); + }); +}); diff --git a/harnesses/timesfm-harness/bin/cli.js b/harnesses/timesfm-harness/bin/cli.js new file mode 100644 index 000000000..92ce8369e --- /dev/null +++ b/harnesses/timesfm-harness/bin/cli.js @@ -0,0 +1,100 @@ +#!/usr/bin/env node +// SPDX-License-Identifier: MIT +// Generated by metaharness — the `timesfm-harness` CLI entry point. +// +// This is plain ESM JavaScript on purpose: it runs as-is via `npx timesfm-harness` +// with NO build step. `npm run build` (tsc) is only needed if you extend the +// TypeScript in src/. The published package ships this file directly (see the +// "bin" + "files" fields in package.json), so `npx timesfm-harness` works the moment +// `npm install` has resolved @metaharness/kernel + @metaharness/host-claude-code. + +import { loadKernel } from '@metaharness/kernel'; +import adapter from '@metaharness/host-claude-code'; + +const HARNESS_NAME = 'timesfm-harness'; + +/** `timesfm-harness init` — boot the kernel + host adapter and report status. */ +async function init() { + const kernel = await loadKernel(); + const info = kernel.kernelInfo(); + console.log(`${HARNESS_NAME} — kernel ${info.version} (${kernel.backend})`); + console.log(`Host adapter: ${adapter.name}`); + console.log(`Run \`${HARNESS_NAME} doctor\` to verify the install.`); + return 0; +} + +/** `timesfm-harness doctor` — verify the install end-to-end (kernel + host resolve). */ +async function doctor() { + const kernel = await loadKernel(); + const info = kernel.kernelInfo(); + const checks = [ + ['kernel loads', !!kernel], + ['kernel reports a version', typeof info.version === 'string' && info.version.length > 0], + ['kernel backend is native|wasm|js', ['native', 'wasm', 'js'].includes(kernel.backend)], + ['host adapter has a name', typeof adapter?.name === 'string' && adapter.name.length > 0], + ]; + let ok = true; + for (const [label, pass] of checks) { + console.log(`${pass ? 'PASS' : 'FAIL'} ${label}`); + if (!pass) ok = false; + } + console.log( + ok + ? `\n${HARNESS_NAME}: all checks passed (kernel ${info.version}, ${kernel.backend} backend, host ${adapter.name})` + : `\n${HARNESS_NAME}: doctor found problems`, + ); + return ok ? 0 : 1; +} + +/** + * Dispatch one CLI invocation. Exported (not just run on import) so a test can + * drive it without spawning a subprocess. Returns the intended exit code. + */ +export async function run(argv) { + const cmd = argv[0] ?? 'init'; + switch (cmd) { + case 'init': + return init(); + case 'doctor': + return doctor(); + case '--version': + case '-v': { + const kernel = await loadKernel(); + console.log(kernel.version()); + return 0; + } + case '--help': + case '-h': + console.log(`Usage: ${HARNESS_NAME} \n\n init boot the kernel + host adapter (default)\n doctor verify the install end-to-end\n --version print the kernel version`); + return 0; + default: + console.error(`Unknown command: ${cmd}. Try \`${HARNESS_NAME} --help\`.`); + return 2; + } +} + +// CLI guard: execute only when invoked directly (not when imported by a test). +// npm's bin shims pass a NON-normalized argv[1] (e.g. ".../.bin/..//bin/cli.js" +// on Windows) and may differ in case, so realpath BOTH sides before comparing — +// a naive string === misses the npx/shim path and the CLI silently no-ops. +import { fileURLToPath } from 'node:url'; +import { realpathSync } from 'node:fs'; +import { argv } from 'node:process'; +const invokedDirectly = (() => { + if (!argv[1]) return false; + try { + const a = realpathSync(argv[1]); + const b = realpathSync(fileURLToPath(import.meta.url)); + return process.platform === 'win32' ? a.toLowerCase() === b.toLowerCase() : a === b; + } catch { + return false; + } +})(); +if (invokedDirectly) { + run(argv.slice(2)) + .then((code) => process.exit(code)) + .catch((err) => { + console.error(err); + process.exit(1); + }); +} diff --git a/harnesses/timesfm-harness/package.json b/harnesses/timesfm-harness/package.json new file mode 100644 index 000000000..37d64a6db --- /dev/null +++ b/harnesses/timesfm-harness/package.json @@ -0,0 +1,44 @@ +{ + "name": "timesfm-harness", + "version": "0.1.0", + "description": "TimesFM 1.0 200M decoder-only time-series forecasting inference crate (Rust/candle) — engineering pod harness", + "license": "MIT", + "type": "module", + "bin": { + "timesfm-harness": "bin/cli.js" + }, + "files": [ + "bin/**", + "dist/**", + "src/**", + "tsconfig.json", + ".claude/**", + "CLAUDE.md", + "README.md", + "LICENSE" + ], + "scripts": { + "build": "tsc", + "test": "vitest run", + "init": "node ./bin/cli.js init", + "doctor": "node ./bin/cli.js doctor", + "evolve": "metaharness-darwin evolve . --sandbox real --generations 3 --children 4", + "evolve:dry": "metaharness-darwin evolve . --sandbox mock --generations 2 --children 3" + }, + "dependencies": { + "@metaharness/kernel": "^0.1.0", + "@metaharness/host-claude-code": "^0.1.0" + }, + "devDependencies": { + "@types/node": "^20.0.0", + "typescript": "^5.4.0", + "vitest": "^3.0.0", + "@metaharness/darwin": "^0.2.2" + }, + "engines": { + "node": ">=20.0.0" + }, + "publishConfig": { + "access": "public" + } +} diff --git a/harnesses/timesfm-harness/scripts/evolve-openrouter.mjs b/harnesses/timesfm-harness/scripts/evolve-openrouter.mjs new file mode 100644 index 000000000..b871634b0 --- /dev/null +++ b/harnesses/timesfm-harness/scripts/evolve-openrouter.mjs @@ -0,0 +1,84 @@ +// Drive Darwin-Mode `evolve` with the OpenRouter LLM mutator (ADR-071). +// +// The OpenRouter mutator is library-only (not exposed by the `metaharness-darwin` +// CLI), so this small driver wires it into the evolve engine. The API key is read +// by the mutator from OPENROUTER_API_KEY (env) — `evolve-openrouter.sh` populates +// that from GCP Secret Manager at runtime; the key is never stored in the repo. +// +// Resolution: prefer the installed `@metaharness/darwin` devDependency; fall back +// to DARWIN_DIST= for monorepo/local runs. +// +// node scripts/evolve-openrouter.mjs [harness-dir] +// +// Env: GENERATIONS, CHILDREN, SANDBOX(real|mock|agent), DARWIN_MUTATOR_MODEL, +// CONCURRENCY, SEED. + +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +let darwin; +try { + darwin = await import('@metaharness/darwin'); +} catch { + const dist = process.env.DARWIN_DIST; + if (!dist) { + console.error( + 'evolve-openrouter: install @metaharness/darwin (npm i) or set DARWIN_DIST to its dist/index.js', + ); + process.exit(1); + } + darwin = await import(dist); +} +const { evolve, OpenRouterMutator } = darwin; + +const here = dirname(fileURLToPath(import.meta.url)); +const repoRoot = resolve(process.argv[2] || resolve(here, '..')); + +const generations = Number(process.env.GENERATIONS || '2'); +const children = Number(process.env.CHILDREN || '3'); +const sandboxMode = process.env.SANDBOX || 'real'; +const model = process.env.DARWIN_MUTATOR_MODEL || 'google/gemini-2.5-flash'; + +if (!process.env.OPENROUTER_API_KEY) { + console.error('evolve-openrouter: OPENROUTER_API_KEY not set (use evolve-openrouter.sh to source it from GCP).'); + process.exit(1); +} + +const generator = new OpenRouterMutator({ model, maxTokens: 1800, temperature: 0.4 }); + +const t0 = process.hrtime.bigint(); +const result = await evolve({ + repoRoot, + workRoot: `${repoRoot}/.metaharness/work`, + generations, + childrenPerGeneration: children, + concurrency: Number(process.env.CONCURRENCY || '2'), + seed: Number(process.env.SEED || '7'), + promotionDelta: 0.05, + tasks: ['run repository test suite', 'verify generated harness safety', 'check trace quality'], + sandboxMode, + generator, + tieBreaker: 'insertion', + selection: 'score', +}); +const ms = Number(process.hrtime.bigint() - t0) / 1e6; + +console.log( + 'EVOLVE_RESULT ' + + JSON.stringify( + { + model, + sandboxMode, + generations, + children, + wallMs: Math.round(ms), + baselineScore: result?.baseline?.score, + winnerScore: result?.winner?.score, + improved: (result?.winner?.score ?? -Infinity) > (result?.baseline?.score ?? Infinity), + winnerLineage: result?.winnerLineage, + mutatorTelemetry: generator.telemetry, + }, + null, + 2, + ), +); diff --git a/harnesses/timesfm-harness/scripts/evolve-openrouter.sh b/harnesses/timesfm-harness/scripts/evolve-openrouter.sh new file mode 100755 index 000000000..e277df061 --- /dev/null +++ b/harnesses/timesfm-harness/scripts/evolve-openrouter.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Evolve the timesfm-harness with Darwin Mode using the OpenRouter LLM mutator, +# sourcing the OpenRouter API key from GCP Secret Manager at runtime. +# +# The key is fetched fresh on every run and exported only into this process's +# environment — it is NEVER written to the repo, a dotfile, or the logs. +# +# Usage: +# ./scripts/evolve-openrouter.sh # real sandbox, 2 gens x 3 children +# GENERATIONS=1 CHILDREN=2 SANDBOX=mock ./scripts/evolve-openrouter.sh +# +# Env overrides: +# OPENROUTER_SECRET GCP secret name (default: OPENROUTER_API_KEY) +# GCP_PROJECT GCP project (default: cognitum-20260110) +# DARWIN_MUTATOR_MODEL OpenRouter model (default: google/gemini-2.5-flash) +# DARWIN_DIST path to darwin dist (for monorepo/local runs without npm i) +set -euo pipefail + +SECRET="${OPENROUTER_SECRET:-OPENROUTER_API_KEY}" +PROJECT="${GCP_PROJECT:-cognitum-20260110}" + +if ! command -v gcloud >/dev/null 2>&1; then + echo "evolve-openrouter: gcloud not found; cannot source the OpenRouter key from GCP." >&2 + exit 1 +fi + +# Fetch the key from GCP Secret Manager into this process only. +OPENROUTER_API_KEY="$(gcloud secrets versions access latest --secret="$SECRET" --project="$PROJECT")" +export OPENROUTER_API_KEY +export DARWIN_MUTATOR_MODEL="${DARWIN_MUTATOR_MODEL:-google/gemini-2.5-flash}" + +HARNESS_DIR="$(cd "$(dirname "$0")/.." && pwd)" +exec node "$HARNESS_DIR/scripts/evolve-openrouter.mjs" "$HARNESS_DIR" diff --git a/harnesses/timesfm-harness/src/agents/architect.ts b/harnesses/timesfm-harness/src/agents/architect.ts new file mode 100644 index 000000000..8e791ac50 --- /dev/null +++ b/harnesses/timesfm-harness/src/agents/architect.ts @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: MIT +// Architect agent — Designs the change before code is written. + +export const SYSTEM_PROMPT = `You are the architect. Before any code is written you produce the smallest design that satisfies the request: the files to touch, the interfaces to add, and the trade-offs. You never write the implementation — you hand a crisp plan to the implementer. Prefer reuse over new abstractions; call out any change that ripples beyond three files. You operate inside the timesfm-harness harness; defer destructive actions to the user.`; + +export const NAME = 'architect'; +export const TIER = 'opus' as const; diff --git a/harnesses/timesfm-harness/src/agents/implementer.ts b/harnesses/timesfm-harness/src/agents/implementer.ts new file mode 100644 index 000000000..d3fce1013 --- /dev/null +++ b/harnesses/timesfm-harness/src/agents/implementer.ts @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: MIT +// Implementer agent — Writes code that matches the surrounding style. + +export const SYSTEM_PROMPT = `You implement the architect's plan. Match the existing code's naming, comment density, and idioms — your diff should read like the person who wrote the file kept writing. Make the minimal change; do not refactor unrelated code. Leave the tests to the test-writer unless asked. You operate inside the timesfm-harness harness; defer destructive actions to the user.`; + +export const NAME = 'implementer'; +export const TIER = 'sonnet' as const; diff --git a/harnesses/timesfm-harness/src/agents/reviewer.ts b/harnesses/timesfm-harness/src/agents/reviewer.ts new file mode 100644 index 000000000..98c8eb919 --- /dev/null +++ b/harnesses/timesfm-harness/src/agents/reviewer.ts @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: MIT +// Reviewer agent — Hunts correctness bugs in the diff. + +export const SYSTEM_PROMPT = `You review diffs for correctness, security, and reuse. Report only high-confidence findings, each with a file:line and a concrete fix. Distinguish a bug (will break) from a nit (style). Never approve a change that widens a permission, swallows an error, or ships a secret. You operate inside the timesfm-harness harness; defer destructive actions to the user.`; + +export const NAME = 'reviewer'; +export const TIER = 'opus' as const; diff --git a/harnesses/timesfm-harness/src/agents/test-writer.ts b/harnesses/timesfm-harness/src/agents/test-writer.ts new file mode 100644 index 000000000..70ba2290c --- /dev/null +++ b/harnesses/timesfm-harness/src/agents/test-writer.ts @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: MIT +// Test Writer agent — Adds the missing tests for the change. + +export const SYSTEM_PROMPT = `You write the tests the change needs: the happy path, the boundary, and the one failure mode most likely to regress. Mirror the project's existing test style and runner. A test that cannot fail is worse than no test — assert behaviour, not implementation. You operate inside the timesfm-harness harness; defer destructive actions to the user.`; + +export const NAME = 'test-writer'; +export const TIER = 'sonnet' as const; diff --git a/harnesses/timesfm-harness/src/init.ts b/harnesses/timesfm-harness/src/init.ts new file mode 100644 index 000000000..cd537d1c8 --- /dev/null +++ b/harnesses/timesfm-harness/src/init.ts @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: MIT +// Generated by create-agent-harness — your harness's `timesfm-harness init` entry. + +import { loadKernel } from '@metaharness/kernel'; +import adapter from '@metaharness/host-claude-code'; + +const HARNESS_NAME = 'timesfm-harness'; + +async function main(): Promise { + const kernel = await loadKernel(); + const info = kernel.kernelInfo(); + console.log(`${HARNESS_NAME} — kernel ${info.version} (${kernel.backend})`); + console.log(`Host adapter: ${adapter.name}`); + console.log(`Run \`${HARNESS_NAME} doctor\` to verify the install.`); + return 0; +} + +main().then(c => process.exit(c)).catch(err => { + console.error(err); + process.exit(1); +}); diff --git a/harnesses/timesfm-harness/tsconfig.json b/harnesses/timesfm-harness/tsconfig.json new file mode 100644 index 000000000..4f908fa45 --- /dev/null +++ b/harnesses/timesfm-harness/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022"], + "outDir": "./dist", + "rootDir": "./src", + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["src/**/*.ts"], + "exclude": ["node_modules", "dist", "__tests__"] +} diff --git a/harnesses/timesfm-harness/vitest.config.ts b/harnesses/timesfm-harness/vitest.config.ts new file mode 100644 index 000000000..ccfd43196 --- /dev/null +++ b/harnesses/timesfm-harness/vitest.config.ts @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: MIT +// Generated by metaharness. Strips the `#!/usr/bin/env node` shebang from +// importable entrypoints (e.g. bin/cli.js) before Vite parses them — Vite/esbuild +// (used internally by Vitest) does NOT strip shebangs, so importing a shebanged +// module throws `SyntaxError: Invalid or unexpected token`. See issue #44. +// Has no effect on direct CLI execution or `npm run doctor` (those bypass Vite). +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + plugins: [ + { + name: 'strip-shebang', + enforce: 'pre', + transform(code: string) { + if (code.startsWith('#!')) { + // Replace with a blank line so source line numbers stay aligned. + return { code: code.replace(/^#![^\n]*/, ''), map: null }; + } + return null; + }, + }, + ], +});