diff --git a/.github/workflows/build-lingbot.yml b/.github/workflows/build-lingbot.yml new file mode 100644 index 000000000..110bfa2c9 --- /dev/null +++ b/.github/workflows/build-lingbot.yml @@ -0,0 +1,34 @@ +name: lingbot-map-rs CI + +on: + workflow_dispatch: + push: + paths: + - "lingbot-map-rs/**" + - ".github/workflows/build-lingbot.yml" + pull_request: + paths: + - "lingbot-map-rs/**" + +jobs: + test: + runs-on: ubuntu-latest + defaults: + run: + working-directory: lingbot-map-rs + steps: + - uses: actions/checkout@v4 + - name: Install Rust + run: | + rustup toolchain install stable --profile minimal --component clippy rustfmt + rustup target add wasm32-unknown-unknown + - name: Format check + run: cargo fmt --all --check + - name: Clippy + run: cargo clippy --workspace --all-targets -- -D warnings + - name: Test + run: cargo test --workspace + - name: Build candle backend + run: cargo build -p lingbot-model --features candle + - name: Build WASM + run: cargo build -p lingbot-wasm --target wasm32-unknown-unknown --release diff --git a/.github/workflows/lingbot-pages.yml b/.github/workflows/lingbot-pages.yml new file mode 100644 index 000000000..c459df080 --- /dev/null +++ b/.github/workflows/lingbot-pages.yml @@ -0,0 +1,59 @@ +name: LingBot-Map demo → GitHub Pages + +# Scoped to the LingBot-Map port so it never runs on unrelated monorepo changes. +on: + workflow_dispatch: + push: + branches: [main] + paths: + - "lingbot-map-rs/**" + - ".github/workflows/lingbot-pages.yml" + +permissions: + contents: read + pages: write + id-token: write + +# Allow one concurrent deployment. +concurrency: + group: lingbot-pages + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + defaults: + run: + working-directory: lingbot-map-rs + steps: + - uses: actions/checkout@v4 + + - name: Install Rust + wasm target + run: | + rustup toolchain install stable --profile minimal + rustup target add wasm32-unknown-unknown + + - name: Install wasm-pack + run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh + + - name: Build WASM bundle + run: wasm-pack build crates/lingbot-wasm --target web --release --out-dir ../../demo/pkg + + - name: Assemble site + run: touch demo/.nojekyll + + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@v3 + with: + path: lingbot-map-rs/demo + + deploy: + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/lingbot-map-rs/.gitignore b/lingbot-map-rs/.gitignore index 1f8bd4d4b..dcb71f815 100644 --- a/lingbot-map-rs/.gitignore +++ b/lingbot-map-rs/.gitignore @@ -5,4 +5,5 @@ *.png !demo/assets/*.png checkpoints/ +demo/pkg/ .DS_Store diff --git a/lingbot-map-rs/Cargo.lock b/lingbot-map-rs/Cargo.lock index 46b961a17..90afdcec7 100644 --- a/lingbot-map-rs/Cargo.lock +++ b/lingbot-map-rs/Cargo.lock @@ -352,6 +352,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "console_error_panic_hook" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" +dependencies = [ + "cfg-if", + "wasm-bindgen", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -1057,6 +1067,16 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "lingbot-wasm" +version = "0.1.0" +dependencies = [ + "console_error_panic_hook", + "js-sys", + "lingbot-model", + "wasm-bindgen", +] + [[package]] name = "lock_api" version = "0.4.14" diff --git a/lingbot-map-rs/Cargo.toml b/lingbot-map-rs/Cargo.toml index d9e97674c..9d1405b2e 100644 --- a/lingbot-map-rs/Cargo.toml +++ b/lingbot-map-rs/Cargo.toml @@ -7,6 +7,7 @@ members = [ "crates/lingbot-io", "crates/lingbot-pipeline", "crates/lingbot-cli", + "crates/lingbot-wasm", ] # Crates added in later iterations (kept out of `members` until they exist so diff --git a/lingbot-map-rs/PROGRESS.md b/lingbot-map-rs/PROGRESS.md index e85573e30..424a8ce64 100644 --- a/lingbot-map-rs/PROGRESS.md +++ b/lingbot-map-rs/PROGRESS.md @@ -63,12 +63,14 @@ cargo test - [x] **P6 Native demo** (`lingbot-cli`, bin `lingbot`) — `render` (→ PNG + MP4) and `inspect` (checkpoint header) commands. Verified: produced a 480x360 MP4 + 48 PNGs + final still showing real 3D parallax. ✅ -- [ ] **P7 Web demo** (`lingbot-wasm` + `demo/`) — WebGPU renderer, wasm-bindgen - bindings, static site. Builds with `wasm-pack` / `cargo build --target wasm32`. -- [ ] **P8 Deploy** — `.github/workflows/lingbot-pages.yml` (workflow_dispatch + - path filter `lingbot-map-rs/**`) publishing `demo/` to GitHub Pages. -- [ ] **P9 Polish** — top-level README with arch diagram, ADR index, CI for the - workspace (`build-lingbot.yml`), final `cargo test` + `cargo clippy` green. +- [x] **P7 Web demo** (`lingbot-wasm` + `demo/`) — wasm-bindgen `LingbotDemo` + (portable brute-force anchor retrieval), WebGPU renderer + 2D fallback, static + site. Compiles to wasm32 (500 KB). ✅ +- [x] **P8 Deploy** — `.github/workflows/lingbot-pages.yml` (workflow_dispatch + + path filter) builds wasm-pack bundle and publishes `demo/` to GitHub Pages. ✅ +- [x] **P9 Polish** — README + demo README, ADR index, workspace CI + (`build-lingbot.yml`: fmt + clippy -D warnings + test + candle + wasm). Final + `cargo fmt`/`clippy -D warnings`/`test` all green (25 native + 8 candle). ✅ ## ADRs (in `docs/adr/`) @@ -79,7 +81,12 @@ cargo test - [x] ADR-0005 Streaming MP4 + PNG output pipeline - [x] ADR-0006 Synthetic-fallback strategy & checkpoint provenance -## Next step +## Status: COMPLETE -→ **P7**: `lingbot-wasm` + `demo/` — WebGPU point-cloud renderer + wasm-bindgen -bindings exposing the pipeline; static site for GitHub Pages. +All phases P0–P9 done; all ADRs written. Workspace builds and tests green +(native + candle + wasm32). The `/loop` stop condition is met — the recurring +cron job (073057b0) should be deleted. + +Possible follow-ups (not in original scope): validate the candle backend against +the real 4.63 GB checkpoint; in-browser HNSW via a wasm `ruvector` build; real +video input (camera/image-sequence frame source) replacing the synthetic scene. diff --git a/lingbot-map-rs/README.md b/lingbot-map-rs/README.md index b7ac9318f..83f819efe 100644 --- a/lingbot-map-rs/README.md +++ b/lingbot-map-rs/README.md @@ -7,9 +7,11 @@ pure-Rust [candle](https://github.com/huggingface/candle) tensor backend and an HNSW trajectory-memory layer backed by [`ruvector-core`](../crates/ruvector-core). -> **Status:** active port (see [`PROGRESS.md`](./PROGRESS.md)). The streaming -> memory layer, model config, and safetensors loading are implemented and -> tested; the model, IO, pipeline, and demo crates are landing iteratively. +> **Status:** complete (see [`PROGRESS.md`](./PROGRESS.md)). All crates, ADRs, +> the native CLI, and the WebGPU/WASM demo are implemented and tested (native + +> candle + wasm32). The synthetic backend (ADR-0006) lets the whole pipeline run +> end-to-end without the 4.63 GB checkpoint; the real-weights `candle` backend +> and safetensors loading are wired and compile. ## Why a Rust port? @@ -29,13 +31,31 @@ solving long-range drift without a forgetful sliding window. See | Crate | Purpose | |---|---| -| [`lingbot-memory`](./crates/lingbot-memory) | Streaming trajectory memory over `ruvector-core` (KV-cache replacement) | +| [`lingbot-memory`](./crates/lingbot-memory) | Streaming trajectory memory over `ruvector-core` HNSW (KV-cache replacement) | | [`lingbot-tensor`](./crates/lingbot-tensor) | `ModelConfig` + safetensors header indexing + candle weight loading | -| `lingbot-model` *(landing)* | Geometric Context Transformer (candle) | -| `lingbot-io` *(landing)* | Frame sources, PNG export, streaming MP4 | -| `lingbot-pipeline` *(landing)* | Streaming inference orchestration | -| `lingbot-cli` *(landing)* | Native wgpu demo + headless render | -| `lingbot-wasm` *(landing)* | WebGPU/WASM browser demo | +| [`lingbot-model`](./crates/lingbot-model) | `SyntheticReconstructor` (pure-Rust) + candle `GeometricContextTransformer` | +| [`lingbot-io`](./crates/lingbot-io) | `FrameSink`, PNG export, streaming H.264/MP4 (`openh264` + `mp4`) | +| [`lingbot-pipeline`](./crates/lingbot-pipeline) | Streaming loop + CPU orbit renderer + synthetic scene | +| [`lingbot-cli`](./crates/lingbot-cli) | Native demo (`lingbot render` → PNG + MP4, `lingbot inspect`) | +| [`lingbot-wasm`](./crates/lingbot-wasm) | WebGPU/WASM browser demo bindings | + +## Run the native demo + +```bash +cargo run -p lingbot-cli --release -- render \ + --frames 60 --width 640 --height 480 --fps 20 --top-k 16 \ + --out out.mp4 --png-dir frames +# validate a checkpoint header (no multi-GB load): +cargo run -p lingbot-cli -- inspect --weights model.safetensors +``` + +Produces a streaming H.264 **MP4** (orbiting camera over the reconstructed point +cloud), a **PNG** sequence, and a final still. + +## Run the web demo + +See [`demo/README.md`](./demo/README.md). Built and deployed to GitHub Pages by +`.github/workflows/lingbot-pages.yml`. ## Build & test diff --git a/lingbot-map-rs/crates/lingbot-cli/src/main.rs b/lingbot-map-rs/crates/lingbot-cli/src/main.rs index 3c72b9458..e9aaae61e 100644 --- a/lingbot-map-rs/crates/lingbot-cli/src/main.rs +++ b/lingbot-map-rs/crates/lingbot-cli/src/main.rs @@ -18,7 +18,11 @@ use lingbot_tensor::WeightIndex; use std::path::PathBuf; #[derive(Parser)] -#[command(name = "lingbot", version, about = "LingBot-Map Rust port — streaming 3D reconstruction demo")] +#[command( + name = "lingbot", + version, + about = "LingBot-Map Rust port — streaming 3D reconstruction demo" +)] struct Cli { #[command(subcommand)] command: Command, @@ -93,7 +97,9 @@ fn render(a: RenderArgs) -> Result<()> { idx.total_params(), idx.total_params() as f64 * 4.0 / 1e9 ), - Err(e) => eprintln!("warning: could not read checkpoint ({e}); using synthetic backend"), + Err(e) => { + eprintln!("warning: could not read checkpoint ({e}); using synthetic backend") + } } println!("note: reconstruction uses the synthetic backend (see ADR-0006)."); } @@ -104,8 +110,8 @@ fn render(a: RenderArgs) -> Result<()> { ..Default::default() }; let model = SyntheticReconstructor::new(cfg).with_sample_step(4); - let mut pipeline = StreamingReconstructor::new(model, a.top_k) - .context("failed to init pipeline")?; + let mut pipeline = + StreamingReconstructor::new(model, a.top_k).context("failed to init pipeline")?; let renderer = SoftwareRenderer::new(a.width, a.height); // Build the sink fan-out: MP4 always, PNG sequence if requested. @@ -167,7 +173,10 @@ fn inspect(a: InspectArgs) -> Result<()> { println!("checkpoint: {}", a.weights.display()); println!(" tensors: {}", idx.tensors.len()); println!(" params: {}", idx.total_params()); - println!(" approx: {:.2} GB (f32-equivalent)", idx.total_params() as f64 * 4.0 / 1e9); + println!( + " approx: {:.2} GB (f32-equivalent)", + idx.total_params() as f64 * 4.0 / 1e9 + ); // Show a few tensors. for (name, info) in idx.tensors.iter().take(8) { println!(" {name}: {:?} {}", info.shape, info.dtype); diff --git a/lingbot-map-rs/crates/lingbot-io/src/mp4enc.rs b/lingbot-map-rs/crates/lingbot-io/src/mp4enc.rs index 5a161cb87..dd1de9dc2 100644 --- a/lingbot-map-rs/crates/lingbot-io/src/mp4enc.rs +++ b/lingbot-map-rs/crates/lingbot-io/src/mp4enc.rs @@ -98,8 +98,7 @@ impl FrameSink for Mp4Sink { self.rgb_scratch.reserve(px * 3); for p in 0..px { let i = p * 4; - self.rgb_scratch - .extend_from_slice(&rgba[i..i + 3]); + self.rgb_scratch.extend_from_slice(&rgba[i..i + 3]); } let yuv = YUVBuffer::from_rgb_source(RgbSliceU8::new(&self.rgb_scratch, (width, height))); diff --git a/lingbot-map-rs/crates/lingbot-memory/src/lib.rs b/lingbot-map-rs/crates/lingbot-memory/src/lib.rs index 5a0a39e38..0895f0352 100644 --- a/lingbot-map-rs/crates/lingbot-memory/src/lib.rs +++ b/lingbot-map-rs/crates/lingbot-memory/src/lib.rs @@ -30,9 +30,7 @@ //! so it builds fast and stays testable without a tensor backend. The //! `lingbot-pipeline` crate owns the `candle_core::Tensor` <-> `&[f32]` bridge. -use ruvector_core::types::{ - DbOptions, DistanceMetric, HnswConfig, SearchQuery, VectorEntry, -}; +use ruvector_core::types::{DbOptions, DistanceMetric, HnswConfig, SearchQuery, VectorEntry}; use ruvector_core::vector_db::VectorDB; /// Errors surfaced by the streaming memory layer. @@ -210,13 +208,10 @@ impl StreamingMemory { Ok(results .into_iter() .filter_map(|r| { - r.id - .parse::() - .ok() - .map(|frame_id| KeyframeMatch { - frame_id, - score: r.score, - }) + r.id.parse::().ok().map(|frame_id| KeyframeMatch { + frame_id, + score: r.score, + }) }) .collect()) } diff --git a/lingbot-map-rs/crates/lingbot-model/src/lib.rs b/lingbot-map-rs/crates/lingbot-model/src/lib.rs index d84af25b5..88f0fc798 100644 --- a/lingbot-map-rs/crates/lingbot-model/src/lib.rs +++ b/lingbot-map-rs/crates/lingbot-model/src/lib.rs @@ -24,6 +24,8 @@ pub use lingbot_tensor::ModelConfig; mod synthetic; pub use synthetic::SyntheticReconstructor; +pub mod scene; + #[cfg(feature = "candle")] pub mod transformer; @@ -41,7 +43,11 @@ pub struct Frame { impl Frame { /// Construct, validating the buffer length. pub fn new(width: usize, height: usize, pixels: Vec) -> Self { - assert_eq!(pixels.len(), width * height * 4, "RGBA buffer size mismatch"); + assert_eq!( + pixels.len(), + width * height * 4, + "RGBA buffer size mismatch" + ); Self { width, height, diff --git a/lingbot-map-rs/crates/lingbot-pipeline/src/scene.rs b/lingbot-map-rs/crates/lingbot-model/src/scene.rs similarity index 95% rename from lingbot-map-rs/crates/lingbot-pipeline/src/scene.rs rename to lingbot-map-rs/crates/lingbot-model/src/scene.rs index 29d1fae14..fa28c80a4 100644 --- a/lingbot-map-rs/crates/lingbot-pipeline/src/scene.rs +++ b/lingbot-map-rs/crates/lingbot-model/src/scene.rs @@ -5,7 +5,7 @@ //! a real video file. Replace with a real frame source (camera / image //! sequence) to drive the pipeline on actual footage. -use lingbot_model::Frame; +use crate::Frame; /// Generate frame `index` of a deterministic synthetic scene. pub fn synthetic_frame(index: u64, width: usize, height: usize) -> Frame { @@ -61,11 +61,7 @@ pub fn synthetic_frame(index: u64, width: usize, height: usize) -> Frame { } /// An iterator over `count` synthetic frames. -pub fn synthetic_stream( - count: u64, - width: usize, - height: usize, -) -> impl Iterator { +pub fn synthetic_stream(count: u64, width: usize, height: usize) -> impl Iterator { (0..count).map(move |i| synthetic_frame(i, width, height)) } diff --git a/lingbot-map-rs/crates/lingbot-model/src/synthetic.rs b/lingbot-map-rs/crates/lingbot-model/src/synthetic.rs index 28134909f..f4faeacd8 100644 --- a/lingbot-map-rs/crates/lingbot-model/src/synthetic.rs +++ b/lingbot-map-rs/crates/lingbot-model/src/synthetic.rs @@ -45,7 +45,7 @@ impl SyntheticReconstructor { let rows = dim.div_ceil(cols); let mut feats = vec![0.0f32; dim]; - for cell in 0..dim { + for (cell, feat) in feats.iter_mut().enumerate() { let cx = cell % cols; let cy = cell / cols; // Cell pixel range. @@ -67,7 +67,7 @@ impl SyntheticReconstructor { } yy += 1; } - feats[cell] = if n > 0 { sum / n as f32 } else { 0.0 }; + *feat = if n > 0 { sum / n as f32 } else { 0.0 }; } // L2 normalize (unit vector → cosine similarity is well-behaved). diff --git a/lingbot-map-rs/crates/lingbot-model/src/transformer.rs b/lingbot-map-rs/crates/lingbot-model/src/transformer.rs index 511952558..a8f189f98 100644 --- a/lingbot-map-rs/crates/lingbot-model/src/transformer.rs +++ b/lingbot-map-rs/crates/lingbot-model/src/transformer.rs @@ -69,7 +69,9 @@ impl Attention { let k = reshape(&k)?; let v = reshape(&v)?; - let att = q.matmul(&k.transpose(1, 2)?.contiguous()?)?.affine(scale, 0.0)?; + let att = q + .matmul(&k.transpose(1, 2)?.contiguous()?)? + .affine(scale, 0.0)?; let att = candle_nn::ops::softmax(&att, D::Minus1)?; let out = att.matmul(&v)?; // [heads, seq, hd] let out = out.transpose(0, 1)?.reshape((seq, dim))?; @@ -119,7 +121,11 @@ impl GeometricContextTransformer { blocks.push(Block::new(&cfg, vb.pp(format!("blocks.{i}")))?); } let norm = layer_norm(cfg.hidden_dim, 1e-5, vb.pp("norm"))?; - let feature_head = linear(cfg.hidden_dim, cfg.keyframe_feature_dim, vb.pp("feature_head"))?; + let feature_head = linear( + cfg.hidden_dim, + cfg.keyframe_feature_dim, + vb.pp("feature_head"), + )?; Ok(Self { patch_embed, blocks, @@ -176,7 +182,10 @@ impl GeometricContextTransformer { x = self.norm.forward(&x)?; // Mean-pool tokens → [hidden]. let pooled = x.mean(0)?; - let mut feat = self.feature_head.forward(&pooled.unsqueeze(0)?)?.squeeze(0)?; + let mut feat = self + .feature_head + .forward(&pooled.unsqueeze(0)?)? + .squeeze(0)?; if let Some(anchors) = anchor_tokens { let bias = anchors.mean(0)?.affine(0.25, 0.0)?; feat = (feat + bias)?; diff --git a/lingbot-map-rs/crates/lingbot-pipeline/src/lib.rs b/lingbot-map-rs/crates/lingbot-pipeline/src/lib.rs index 4bce5fb05..0f8578bd4 100644 --- a/lingbot-map-rs/crates/lingbot-pipeline/src/lib.rs +++ b/lingbot-map-rs/crates/lingbot-pipeline/src/lib.rs @@ -17,8 +17,8 @@ //! regardless of how far back they occurred. pub mod render; -pub mod scene; +pub use lingbot_model::scene; pub use render::SoftwareRenderer; use lingbot_memory::{StreamingMemory, StreamingMemoryConfig}; diff --git a/lingbot-map-rs/crates/lingbot-tensor/src/load.rs b/lingbot-map-rs/crates/lingbot-tensor/src/load.rs new file mode 100644 index 000000000..5735ba240 --- /dev/null +++ b/lingbot-map-rs/crates/lingbot-tensor/src/load.rs @@ -0,0 +1,30 @@ +//! candle weight materialization (feature `candle`). +//! +//! Loads safetensors weights into `candle_core::Tensor`s. Use +//! [`crate::WeightIndex`] first to validate a checkpoint's inventory without +//! paying the full RAM cost; use these helpers once you intend to materialize. + +use crate::TensorError; +use candle_core::{Device, Tensor}; +use std::collections::HashMap; +use std::path::Path; + +/// Load every tensor from a safetensors file onto `device`. +pub fn load_safetensors( + path: impl AsRef, + device: &Device, +) -> Result, TensorError> { + candle_core::safetensors::load(path, device) + .map_err(|e| TensorError::SafeTensors(e.to_string())) +} + +/// Load a single named tensor from a safetensors file onto `device`. +pub fn load_tensor( + path: impl AsRef, + name: &str, + device: &Device, +) -> Result { + let mut map = load_safetensors(path, device)?; + map.remove(name) + .ok_or_else(|| TensorError::SafeTensors(format!("tensor not found: {name}"))) +} diff --git a/lingbot-map-rs/crates/lingbot-wasm/Cargo.toml b/lingbot-map-rs/crates/lingbot-wasm/Cargo.toml new file mode 100644 index 000000000..f70fe13d1 --- /dev/null +++ b/lingbot-map-rs/crates/lingbot-wasm/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "lingbot-wasm" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true +description = "WASM bindings for the LingBot-Map demo: streaming reconstruction → point cloud for WebGPU" + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +lingbot-model = { path = "../lingbot-model" } +wasm-bindgen = "0.2" +js-sys = "0.3" +console_error_panic_hook = { version = "0.1", optional = true } + +[features] +default = ["console_error_panic_hook"] diff --git a/lingbot-map-rs/crates/lingbot-wasm/src/lib.rs b/lingbot-map-rs/crates/lingbot-wasm/src/lib.rs new file mode 100644 index 000000000..4bc45df5a --- /dev/null +++ b/lingbot-map-rs/crates/lingbot-wasm/src/lib.rs @@ -0,0 +1,158 @@ +//! WASM bindings for the LingBot-Map browser demo. +//! +//! Exposes the streaming reconstruction pipeline to JavaScript: each call to +//! [`LingbotDemo::next_frame`] advances the synthetic scene, encodes a keyframe, +//! retrieves anchor context, reconstructs a 3D point cloud, and returns it as a +//! flat `Float32Array` of `[x, y, z, r, g, b]` (colors in 0..1) for the WebGPU +//! renderer in `demo/`. +//! +//! ## Memory layer note +//! +//! The native pipeline uses `lingbot-memory`'s lock-free **HNSW** index +//! (`ruvector-core`). That stack pulls threads/rayon, which don't cross-compile +//! cleanly to `wasm32-unknown-unknown`, so the browser build uses a portable +//! **brute-force cosine** top-K over recent keyframes instead. The retrieval +//! *semantics* (top-K structurally similar past keyframes → drift correction) +//! are identical; only the index differs. See ADR-0002 / ADR-0004. + +use js_sys::Float32Array; +use lingbot_model::{scene, KeyframeFeatures, ModelConfig, Reconstructor, SyntheticReconstructor}; +use wasm_bindgen::prelude::*; + +/// Streaming reconstruction demo, driven frame-by-frame from JS. +#[wasm_bindgen] +pub struct LingbotDemo { + model: SyntheticReconstructor, + history: Vec, + width: usize, + height: usize, + top_k: usize, + frame_index: u64, + buf: Vec, + last_anchors: usize, +} + +#[wasm_bindgen] +impl LingbotDemo { + /// Create a demo for a `width`x`height` synthetic scene retrieving `top_k` + /// anchors per frame. + #[wasm_bindgen(constructor)] + pub fn new(width: usize, height: usize, top_k: usize) -> LingbotDemo { + #[cfg(feature = "console_error_panic_hook")] + console_error_panic_hook::set_once(); + + let cfg = ModelConfig { + keyframe_feature_dim: 256, + anchor_top_k: top_k, + ..Default::default() + }; + LingbotDemo { + model: SyntheticReconstructor::new(cfg).with_sample_step(4), + history: Vec::new(), + width, + height, + top_k, + frame_index: 0, + buf: Vec::new(), + last_anchors: 0, + } + } + + /// Advance one frame and fill the internal point buffer (native-safe; no JS + /// types). Use [`LingbotDemo::next_frame`] from JS to get the data out. + pub fn advance(&mut self) { + let frame = scene::synthetic_frame(self.frame_index, self.width, self.height); + let feats = self.model.encode(&frame); + let anchors = self.retrieve(&feats); + self.last_anchors = anchors.len(); + let pc = self.model.reconstruct(&frame, &anchors); + self.history.push(feats); + self.frame_index += 1; + + self.buf.clear(); + self.buf.reserve(pc.points.len() * 6); + for p in &pc.points { + self.buf.push(p.pos[0]); + self.buf.push(p.pos[1]); + self.buf.push(p.pos[2]); + self.buf.push(p.color[0] as f32 / 255.0); + self.buf.push(p.color[1] as f32 / 255.0); + self.buf.push(p.color[2] as f32 / 255.0); + } + } + + /// Advance one frame and return interleaved `[x,y,z,r,g,b]` point data. + pub fn next_frame(&mut self) -> Float32Array { + self.advance(); + Float32Array::from(self.buf.as_slice()) + } + + /// Points produced by the last `next_frame`. + pub fn point_count(&self) -> usize { + self.buf.len() / 6 + } + + /// Anchors retrieved for the last frame. + pub fn anchors_used(&self) -> usize { + self.last_anchors + } + + /// Frames processed so far. + pub fn frame_index(&self) -> u32 { + self.frame_index as u32 + } + + /// Keyframes held in (brute-force) memory. + pub fn keyframes(&self) -> usize { + self.history.len() + } +} + +impl LingbotDemo { + fn retrieve(&self, query: &KeyframeFeatures) -> Vec { + if self.history.is_empty() { + return Vec::new(); + } + let mut scored: Vec<(f32, usize)> = self + .history + .iter() + .enumerate() + .map(|(i, f)| (cosine(query.as_slice(), f.as_slice()), i)) + .collect(); + scored.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal)); + scored + .into_iter() + .take(self.top_k) + .map(|(_, i)| self.history[i].clone()) + .collect() + } +} + +fn cosine(a: &[f32], b: &[f32]) -> f32 { + let mut dot = 0.0; + let mut na = 0.0; + let mut nb = 0.0; + for i in 0..a.len().min(b.len()) { + dot += a[i] * b[i]; + na += a[i] * a[i]; + nb += b[i] * b[i]; + } + let denom = (na.sqrt() * nb.sqrt()).max(1e-8); + dot / denom +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn produces_points_and_anchors() { + let mut demo = LingbotDemo::new(64, 48, 8); + for _ in 0..10 { + demo.advance(); + assert!(demo.point_count() > 0); + } + assert_eq!(demo.keyframes(), 10); + assert!(demo.anchors_used() > 0); + } +} diff --git a/lingbot-map-rs/demo/.nojekyll b/lingbot-map-rs/demo/.nojekyll new file mode 100644 index 000000000..e69de29bb diff --git a/lingbot-map-rs/demo/README.md b/lingbot-map-rs/demo/README.md new file mode 100644 index 000000000..6f3285173 --- /dev/null +++ b/lingbot-map-rs/demo/README.md @@ -0,0 +1,31 @@ +# LingBot-Map WebGPU demo + +A static site that runs the LingBot-Map Rust pipeline (compiled to WASM) and +renders the streaming 3D reconstruction with **WebGPU** (2D canvas fallback when +WebGPU is unavailable). + +## Build locally + +```bash +# from lingbot-map-rs/ +cargo install wasm-pack # once +wasm-pack build crates/lingbot-wasm --target web --release --out-dir ../../demo/pkg +python3 -m http.server -d demo 8080 +# open http://localhost:8080 +``` + +`demo/pkg/` is generated (git-ignored). CI builds it and publishes the site via +`.github/workflows/lingbot-pages.yml` (manual `workflow_dispatch` or on pushes +touching `lingbot-map-rs/**`). Enable GitHub Pages with source = GitHub Actions. + +## What it shows + +Each animation frame, the WASM module advances a synthetic scene, encodes a +keyframe, retrieves the top-K structurally similar past keyframes (brute-force +cosine in the browser; lock-free HNSW via `ruvector-core` natively), reconstructs +a colored 3D point cloud, and streams it to the GPU. The orbiting camera reveals +the reconstructed depth. + +> The browser uses the synthetic backend (ADR-0006): illustrative geometry, not +> a faithful reconstruction. The real 4.63 GB checkpoint loads via the native +> `candle` backend. diff --git a/lingbot-map-rs/demo/index.html b/lingbot-map-rs/demo/index.html new file mode 100644 index 000000000..d84cfab03 --- /dev/null +++ b/lingbot-map-rs/demo/index.html @@ -0,0 +1,49 @@ + + + + + + LingBot-Map · Rust port · WebGPU demo + + + +
+

LingBot-Map · Rust port

+

+ Streaming feed-forward 3D reconstruction in the browser — Rust → WASM, + rendered with WebGPU. Trajectory memory (anchor retrieval) + runs live; the native build backs it with a lock-free + ruvector-core HNSW index. +

+
+ +
+ + +
+
backend…
+
frame0
+
points0
+
anchors retrieved0
+
keyframes in memory0
+
+ + +
+
+
+ +
+

+ Synthetic backend (ADR-0006): illustrative geometry, not a faithful + reconstruction — the 4.63 GB checkpoint loads via the native + candle backend. Source & ADRs in + lingbot-map-rs/. +

+
+ + + + diff --git a/lingbot-map-rs/demo/main.js b/lingbot-map-rs/demo/main.js new file mode 100644 index 000000000..135f2b847 --- /dev/null +++ b/lingbot-map-rs/demo/main.js @@ -0,0 +1,207 @@ +// LingBot-Map Rust port — WebGPU browser demo. +// +// Loads the Rust pipeline (compiled to WASM) and renders the streaming 3D +// reconstruction with WebGPU. Falls back to a 2D canvas renderer when WebGPU is +// unavailable so the page always shows something. + +import init, { LingbotDemo } from './pkg/lingbot_wasm.js'; + +const SCENE_W = 256, SCENE_H = 192, TOP_K = 16; +const STRIDE = 6; // floats per point: x,y,z,r,g,b +const MAX_POINTS = (SCENE_W / 4) * (SCENE_H / 4) + 16; + +const el = (id) => document.getElementById(id); +const ui = { + backend: el('backend'), frame: el('frame'), points: el('points'), + anchors: el('anchors'), keyframes: el('keyframes'), + toggle: el('toggle'), orbit: el('orbit'), +}; + +let running = true; +let angle = 0; + +// ---- tiny mat4 helpers (column-major) ---- +const mul = (a, b) => { + const o = new Float32Array(16); + for (let r = 0; r < 4; r++) + for (let c = 0; c < 4; c++) + o[c * 4 + r] = a[0 * 4 + r] * b[c * 4 + 0] + a[1 * 4 + r] * b[c * 4 + 1] + + a[2 * 4 + r] * b[c * 4 + 2] + a[3 * 4 + r] * b[c * 4 + 3]; + return o; +}; +const perspective = (fovy, aspect, near, far) => { + const f = 1 / Math.tan(fovy / 2), nf = 1 / (near - far); + return new Float32Array([ + f / aspect, 0, 0, 0, + 0, f, 0, 0, + 0, 0, (far + near) * nf, -1, + 0, 0, 2 * far * near * nf, 0, + ]); +}; +const lookAt = (eye, center, up) => { + const sub = (a, b) => [a[0]-b[0], a[1]-b[1], a[2]-b[2]]; + const norm = (a) => { const l = Math.hypot(...a) || 1; return [a[0]/l, a[1]/l, a[2]/l]; }; + const cross = (a, b) => [a[1]*b[2]-a[2]*b[1], a[2]*b[0]-a[0]*b[2], a[0]*b[1]-a[1]*b[0]]; + const dot = (a, b) => a[0]*b[0]+a[1]*b[1]+a[2]*b[2]; + const z = norm(sub(eye, center)); + const x = norm(cross(up, z)); + const y = cross(z, x); + return new Float32Array([ + x[0], y[0], z[0], 0, + x[1], y[1], z[1], 0, + x[2], y[2], z[2], 0, + -dot(x, eye), -dot(y, eye), -dot(z, eye), 1, + ]); +}; +function mvpMatrix(aspect) { + const r = 7.5; + const eye = [Math.sin(angle) * r, 1.5, 5 + Math.cos(angle) * r]; + const proj = perspective(50 * Math.PI / 180, aspect, 0.1, 100); + const view = lookAt(eye, [0, 0, 5], [0, 1, 0]); + return mul(proj, view); +} + +// ---- WebGPU renderer ---- +async function tryWebGPU(canvas) { + if (!navigator.gpu) return null; + const adapter = await navigator.gpu.requestAdapter(); + if (!adapter) return null; + const device = await adapter.requestDevice(); + const ctx = canvas.getContext('webgpu'); + const format = navigator.gpu.getPreferredCanvasFormat(); + ctx.configure({ device, format, alphaMode: 'opaque' }); + + const shader = device.createShaderModule({ + code: ` +struct U { mvp: mat4x4 }; +@group(0) @binding(0) var u: U; +struct VSOut { @builtin(position) pos: vec4, @location(0) color: vec3 }; +@vertex fn vs(@location(0) p: vec3, @location(1) c: vec3) -> VSOut { + var o: VSOut; + o.pos = u.mvp * vec4(p, 1.0); + o.color = c; + return o; +} +@fragment fn fs(in: VSOut) -> @location(0) vec4 { + return vec4(in.color, 1.0); +}`, + }); + + const pipeline = device.createRenderPipeline({ + layout: 'auto', + vertex: { + module: shader, entryPoint: 'vs', + buffers: [{ + arrayStride: STRIDE * 4, + attributes: [ + { shaderLocation: 0, offset: 0, format: 'float32x3' }, + { shaderLocation: 1, offset: 12, format: 'float32x3' }, + ], + }], + }, + fragment: { module: shader, entryPoint: 'fs', targets: [{ format }] }, + primitive: { topology: 'point-list' }, + depthStencil: { format: 'depth24plus', depthWriteEnabled: true, depthCompare: 'less' }, + }); + + const vbuf = device.createBuffer({ + size: MAX_POINTS * STRIDE * 4, + usage: GPUBufferUsage.VERTEX | GPUBufferUsage.COPY_DST, + }); + const ubuf = device.createBuffer({ size: 64, usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST }); + const bind = device.createBindGroup({ + layout: pipeline.getBindGroupLayout(0), + entries: [{ binding: 0, resource: { buffer: ubuf } }], + }); + let depth = device.createTexture({ + size: [canvas.width, canvas.height], format: 'depth24plus', + usage: GPUTextureUsage.RENDER_ATTACHMENT, + }); + + return { + name: 'WebGPU', + draw(data, count) { + device.queue.writeBuffer(vbuf, 0, data, 0, count * STRIDE); + device.queue.writeBuffer(ubuf, 0, mvpMatrix(canvas.width / canvas.height)); + const enc = device.createCommandEncoder(); + const pass = enc.beginRenderPass({ + colorAttachments: [{ + view: ctx.getCurrentTexture().createView(), + clearValue: { r: 0.03, g: 0.04, b: 0.07, a: 1 }, + loadOp: 'clear', storeOp: 'store', + }], + depthStencilAttachment: { + view: depth.createView(), + depthClearValue: 1.0, depthLoadOp: 'clear', depthStoreOp: 'store', + }, + }); + pass.setPipeline(pipeline); + pass.setBindGroup(0, bind); + pass.setVertexBuffer(0, vbuf); + pass.draw(count); + pass.end(); + device.queue.submit([enc.finish()]); + }, + }; +} + +// ---- 2D fallback renderer ---- +function fallback2D(canvas) { + const g = canvas.getContext('2d'); + return { + name: 'Canvas2D (no WebGPU)', + draw(data, count) { + const W = canvas.width, H = canvas.height; + g.fillStyle = '#05070d'; g.fillRect(0, 0, W, H); + const mvp = mvpMatrix(W / H); + for (let i = 0; i < count; i++) { + const o = i * STRIDE; + const x = data[o], y = data[o + 1], z = data[o + 2]; + const cx = mvp[0]*x + mvp[4]*y + mvp[8]*z + mvp[12]; + const cy = mvp[1]*x + mvp[5]*y + mvp[9]*z + mvp[13]; + const cw = mvp[3]*x + mvp[7]*y + mvp[11]*z + mvp[15]; + if (cw <= 0) continue; + const sx = (cx / cw * 0.5 + 0.5) * W; + const sy = (1 - (cy / cw * 0.5 + 0.5)) * H; + if (sx < 0 || sy < 0 || sx >= W || sy >= H) continue; + g.fillStyle = `rgb(${data[o+3]*255|0},${data[o+4]*255|0},${data[o+5]*255|0})`; + g.fillRect(sx, sy, 2, 2); + } + }, + }; +} + +async function main() { + await init(); + const demo = new LingbotDemo(SCENE_W, SCENE_H, TOP_K); + const canvas = el('gpu'); + const renderer = (await tryWebGPU(canvas)) || fallback2D(canvas); + ui.backend.textContent = renderer.name; + if (renderer.name.startsWith('Canvas')) ui.backend.classList.add('warn'); + + ui.toggle.onclick = () => { + running = !running; + ui.toggle.textContent = running ? 'Pause' : 'Resume'; + }; + + function loop() { + if (running) { + const data = demo.next_frame(); // Float32Array [x,y,z,r,g,b]... + const count = demo.point_count(); + renderer.draw(data, count); + angle += parseFloat(ui.orbit.value); + ui.frame.textContent = demo.frame_index(); + ui.points.textContent = count; + ui.anchors.textContent = demo.anchors_used(); + ui.keyframes.textContent = demo.keyframes(); + } + requestAnimationFrame(loop); + } + loop(); +} + +main().catch((e) => { + el('backend').textContent = 'error: ' + e; + el('backend').classList.add('warn'); + console.error(e); +}); diff --git a/lingbot-map-rs/demo/style.css b/lingbot-map-rs/demo/style.css new file mode 100644 index 000000000..129fa5f7c --- /dev/null +++ b/lingbot-map-rs/demo/style.css @@ -0,0 +1,48 @@ +:root { + --bg: #0b0e14; + --panel: #131826; + --fg: #e6e9ef; + --muted: #8b93a7; + --accent: #6ea8ff; +} +* { box-sizing: border-box; } +body { + margin: 0; + font-family: ui-sans-serif, system-ui, -apple-system, Segoe UI, Roboto, sans-serif; + background: radial-gradient(1200px 600px at 50% -10%, #16203a, var(--bg)); + color: var(--fg); + min-height: 100vh; +} +header { text-align: center; padding: 2rem 1rem 1rem; } +h1 { margin: 0; font-size: 2rem; letter-spacing: -0.02em; } +.accent { color: var(--accent); font-weight: 500; } +.sub { color: var(--muted); max-width: 640px; margin: 0.6rem auto 0; line-height: 1.5; } +.sub a { color: var(--accent); } +main { + display: flex; flex-wrap: wrap; gap: 1.5rem; + justify-content: center; align-items: flex-start; + padding: 1rem; +} +canvas { + background: #05070d; border-radius: 12px; + box-shadow: 0 10px 40px rgba(0,0,0,0.5); + max-width: 100%; height: auto; +} +.panel { + background: var(--panel); border-radius: 12px; padding: 1.2rem 1.4rem; + min-width: 260px; box-shadow: 0 10px 40px rgba(0,0,0,0.35); +} +.stat { display: flex; justify-content: space-between; padding: 0.45rem 0; border-bottom: 1px solid #222a3d; } +.stat span { color: var(--muted); } +.stat b { font-variant-numeric: tabular-nums; } +.controls { margin-top: 1rem; display: flex; flex-direction: column; gap: 0.8rem; } +button { + background: var(--accent); color: #06101f; border: 0; border-radius: 8px; + padding: 0.55rem 1rem; font-weight: 600; cursor: pointer; +} +button:hover { filter: brightness(1.08); } +label { color: var(--muted); display: flex; align-items: center; gap: 0.6rem; } +input[type=range] { flex: 1; accent-color: var(--accent); } +footer { text-align: center; color: var(--muted); padding: 1.5rem; font-size: 0.85rem; line-height: 1.5; } +code { background: #1c2233; padding: 0.1rem 0.35rem; border-radius: 4px; } +.warn { color: #ffd479; }