diff --git a/docs/adr/ADR-040-causal-atlas-rvf-runtime-planet-detection.md b/docs/adr/ADR-040-causal-atlas-rvf-runtime-planet-detection.md index cb8a4041d..1a74e6ad8 100644 --- a/docs/adr/ADR-040-causal-atlas-rvf-runtime-planet-detection.md +++ b/docs/adr/ADR-040-causal-atlas-rvf-runtime-planet-detection.md @@ -25,19 +25,10 @@ retained archetypes. ### Existing Capabilities (ADR-003 through ADR-008) -| Component | Package | Relevant APIs | -|-----------|---------|---------------| -| **RVF segments** | `@ruvector/rvf`, `@ruvector/rvf-node` | `embedKernel`, `extractKernel`, `embedEbpf`, `segments`, `derive` | -| **HNSW indexing** | `@ruvector/rvf-node` | `ingestBatch`, `query`, `compact`, HNSW with metadata filters | -| **Witness chains** | `@ruvector/rvf-node`, `RvfSolver` | `verifyWitness`, SHAKE-256 witness chains, signed root hash | -| **Graph transactions** | `NativeAccelerator` | `graphTransaction`, `graphBatchInsert`, Cypher queries | -| **SIMD embeddings** | `@ruvector/ruvllm` | 768-dim SIMD embed, cosine/dot/L2, HNSW memory search | -| **SONA learning** | `SonaLearningBackend` | Micro-LoRA, trajectory recording, EWC++ | -| **Federated coordination** | `FederatedSessionManager` | Cross-agent trajectories, warm-start patterns | -| **Contrastive training** | `ContrastiveTrainer` | InfoNCE, hard negative mining, 3-stage curriculum | -| **Adaptive index** | `AdaptiveIndexTuner` | 5-tier compression, Matryoshka truncation, health monitoring | -| **Kernel embedding** | `KernelBuilder` (ADR-008) | Minimal Linux boot from KERNEL_SEG + INITRD_SEG | -| **Lazy model download** | `ChatInference` (ADR-008) | Deferred GGUF load on first inference call | +RVF segments, HNSW indexing, SHAKE-256 witness chains, graph transactions, +768-dim SIMD embeddings, SONA learning, federated coordination, contrastive +training, adaptive index tuning, kernel embedding (ADR-008), and lazy model +download (ADR-008). See ADR-003 through ADR-008 for full API details. ### What This ADR Adds @@ -119,26 +110,7 @@ Following the lazy-download pattern established in ADR-008 for GGUF models: INDEX_SEG, a new witness root is committed, and the RVF is sealed into a reproducible snapshot -``` -Download state machine: - - [boot] ──first-inference──> [downloading-tier-1] - │ │ - │ (offline demo works) │ (progress: 0-100%) - │ │ - ▼ ▼ - [tier-0-only] [tier-1-ready] - │ - rvf ingest --expand - │ - ▼ - [tier-2-ready] - │ - life pipeline activated - │ - ▼ - [tier-3-ready] ──seal──> [sealed-snapshot] -``` +**State machine**: `[boot]` -> `[tier-0-only]` (offline demo) -> `[tier-1-ready]` (first inference) -> `[tier-2-ready]` (`rvf ingest --expand`) -> `[tier-3-ready]` (life pipeline) -> `[sealed-snapshot]`. Each tier download: - Resumes from last byte on interruption (HTTP Range headers) @@ -168,197 +140,50 @@ Extends the ADR-003 segment model with domain-specific segments. ### Core Entities -```typescript -interface Event { - id: string; - t_start: number; // epoch seconds - t_end: number; - domain: 'kepler' | 'tess' | 'jwst' | 'sdss' | 'derived'; - payload_hash: string; // SHA-256 of raw data window - provenance: Provenance; -} - -interface Observation { - id: string; - instrument: string; // 'kepler-lc' | 'tess-ffi' | 'jwst-nirspec' | ... - target_id: string; // e.g., KIC or TIC identifier - data_pointer: string; // segment offset into VEC_SEG - calibration_version: string; - provenance: Provenance; -} - -interface InteractionEdge { - src_event_id: string; - dst_event_id: string; - type: 'causal' | 'periodicity' | 'shape_similarity' | 'co_occurrence' | 'spatial'; - weight: number; - lag: number; // temporal lag in seconds - confidence: number; - provenance: Provenance; -} - -interface Boundary { - boundary_id: string; - partition_left_set_hash: string; - partition_right_set_hash: string; - cut_weight: number; - cut_witness: string; // witness chain reference - stability_score: number; -} - -interface Candidate { - candidate_id: string; - category: 'planet' | 'life'; - evidence_pointers: string[]; // event and edge IDs - score: number; - uncertainty: number; - publishable: boolean; // based on POLICY_SEG rules - witness_trace: string; // WITNESS_SEG reference for replay -} - -interface Provenance { - source: string; // 'mast-kepler' | 'mast-tess' | 'mast-jwst' | ... - download_witness: string; // witness chain entry for the download - transform_chain: string[]; // ordered list of transform IDs applied - timestamp: string; // ISO-8601 -} -``` +| Entity | Key Fields | Description | +|--------|-----------|-------------| +| **Event** | `id`, `t_start`, `t_end`, `domain`, `payload_hash`, `provenance` | Time-windowed observation or derived result. Domain: kepler, tess, jwst, sdss, derived | +| **Observation** | `id`, `instrument`, `target_id`, `data_pointer`, `calibration_version` | Raw instrument measurement with VEC_SEG offset | +| **InteractionEdge** | `src_event_id`, `dst_event_id`, `type`, `weight`, `lag`, `confidence` | Typed relationship: causal, periodicity, shape_similarity, co_occurrence, spatial | +| **Boundary** | `boundary_id`, `partition_*_hash`, `cut_weight`, `cut_witness`, `stability_score` | Graph partition boundary with witness chain reference | +| **Candidate** | `candidate_id`, `category`, `evidence_pointers[]`, `score`, `uncertainty`, `publishable`, `witness_trace` | Planet or life candidate with POLICY_SEG-gated publishability | +| **Provenance** | `source`, `download_witness`, `transform_chain[]`, `timestamp` | Full lineage from download through every transform | ### Domain Adapters -#### Planet Transit Adapter - -``` -Input: flux time series + cadence metadata (Kepler/TESS FITS) -Output: Event nodes for windows - InteractionEdges for periodicity hints and shape similarity - Candidate nodes for dip detections -``` - -#### Spectrum Adapter - -``` -Input: wavelength, flux, error arrays (JWST NIRSpec, etc.) -Output: Event nodes for band windows - InteractionEdges for molecule feature co-occurrence - Disequilibrium score components -``` - -#### Cosmic Web Adapter (optional, Phase 2+) - -``` -Input: galaxy positions and redshifts (SDSS) -Output: Graph of spatial adjacency and filament membership -``` +| Adapter | Input | Output | +|---------|-------|--------| +| **Planet Transit** | Flux time series + cadence metadata (Kepler/TESS FITS) | Event nodes, periodicity/shape InteractionEdges, dip Candidates | +| **Spectrum** | Wavelength, flux, error arrays (JWST NIRSpec, etc.) | Band Event nodes, molecule co-occurrence edges, disequilibrium scores | +| **Cosmic Web** (Phase 2+) | Galaxy positions and redshifts (SDSS) | Spatial adjacency graph with filament membership | ## The Four System Constructs ### 1. Compressed Causal Atlas -**Definition**: A partial order of events plus minimal sufficient descriptors -to reproduce derived edges. +Partial order of events plus minimal sufficient descriptors to reproduce derived edges. -**Construction**: +**Construction**: (1) Window light curves at scales 2h/12h/3d/27d. (2) Extract features: flux derivatives, autocorrelation peaks, wavelet energy, matched filter response. (3) Embed via RuVector SIMD into VEC_SEG. (4) Add causal edges where window A precedes B and improves predictability (prediction gain, POLICY_SEG constrained). (5) Compress: top-k parents per node, retain boundary witnesses, delta-encode into DELTA_SEG. -1. **Windowing** — Light curves into overlapping windows at multiple scales - - Scales: 2 hours, 12 hours, 3 days, 27 days - -2. **Feature extraction** — Robust features per window - - Flux derivative statistics - - Autocorrelation peaks - - Wavelet energy bands - - Transit-shaped matched filter response - -3. **Embedding** — RuVector SIMD embed per window, stored in VEC_SEG - -4. **Causal edges** — Add edge when window A precedes window B and improves - predictability of B (conditional mutual information proxy or prediction gain, - subject to POLICY_SEG constraints) - - Edge weight: prediction gain magnitude - - Provenance: exact windows, transform IDs, threshold used - -5. **Atlas compression** - - Keep only top-k causal parents per node - - Retain stable boundary witnesses - - Delta-encode updates into DELTA_SEG - -**Output API**: - -| Endpoint | Returns | -|----------|---------| -| `atlas.query(event_id)` | Parents, children, plus provenance | -| `atlas.trace(candidate_id)` | Minimal causal chain for a candidate | +**API**: `atlas.query(event_id)` returns parents/children with provenance. `atlas.trace(candidate_id)` returns minimal causal chain. ### 2. Coherence Field Index -**Definition**: A field over the atlas graph that assigns coherence pressure -and cut stability over time. +Field over the atlas graph assigning coherence pressure and cut stability over time. Signals: cut pressure (min-cut values), partition entropy (cluster size distribution), disagreement (cross-detector rate), drift (embedding distribution shift). -**Signals**: +**Algorithm**: Maintain partition tree with dynamic min-cut on incremental changes. Each epoch: compute cut witnesses for top boundaries, emit to GRAPH_SEG, append to WITNESS_SEG. Index boundaries by descriptor: cut value, partition sizes, curvature proxy, churn. -| Signal | Description | -|--------|-------------| -| Cut pressure | Minimum cut values over selected subgraphs | -| Partition entropy | Distribution of cluster sizes and churn rate | -| Disagreement | Cross-detector disagreement rate | -| Drift | Embedding distribution shift in sliding window | - -**Algorithm**: - -1. Maintain a partition tree. Update with dynamic min-cut on incremental - graph changes -2. For each update epoch: - - Compute cut witnesses for top boundaries - - Emit boundary events into GRAPH_SEG - - Append witness record into WITNESS_SEG -3. Index boundaries via descriptor vector: - - Cut value, partition sizes, local graph curvature proxy, recent churn - -**Query API**: - -| Endpoint | Returns | -|----------|---------| -| `coherence.get(target_id, epoch)` | Field values for target at epoch | -| `boundary.nearest(descriptor)` | Similar historical boundary states via INDEX_SEG | +**API**: `coherence.get(target_id, epoch)` returns field values. `boundary.nearest(descriptor)` returns similar historical states via INDEX_SEG. ### 3. Multi-Scale Interaction Memory -**Definition**: A memory that retains interactions at multiple time resolutions -with strict budget control. - -**Three tiers**: - -| Tier | Resolution | Content | -|------|-----------|---------| -| **S** | Seconds to minutes | High-fidelity deltas | -| **M** | Hours to days | Aggregated deltas | -| **L** | Weeks to months | Boundary summaries and archetypes | - -**Retention rules**: -1. Preserve events that are boundary-critical -2. Preserve events that are candidate evidence -3. Compress everything else via archetype clustering in INDEX_SEG - -**Mechanism**: -- DELTA_SEG is append-only -- Periodic compaction produces a new RVF root with a witness proof of - preservation rules applied +Tiered retention with strict budget control: **S** (seconds-minutes, high-fidelity deltas), **M** (hours-days, aggregated), **L** (weeks-months, boundary summaries and archetypes). Retention preserves boundary-critical events and candidate evidence; compresses the rest via archetype clustering. DELTA_SEG is append-only; periodic compaction produces a new RVF root with witness proof. ### 4. Boundary Evolution Tracker -**Definition**: A tracker that treats boundaries as primary objects that evolve -over time. +Treats boundaries as primary objects evolving over time -- the holographic design principle. Boundaries are stored and indexed as first-class objects; interior state is reconstructable from boundary witnesses and retained archetypes. -**This is where the holographic flavor is implemented.** You primarily store -and index boundaries, and treat interior state as reconstructable from boundary -witnesses and retained archetypes. - -**Output API**: - -| Endpoint | Returns | -|----------|---------| -| `boundary.timeline(target_id)` | Boundary evolution over time | -| `boundary.alerts` | Alerts when: cut pressure spikes, boundary identity flips, disagreement exceeds threshold, drift persists beyond policy | +**API**: `boundary.timeline(target_id)` returns boundary evolution. `boundary.alerts` fires on cut pressure spikes, boundary identity flips, disagreement threshold breaches, or persistent drift. ## Planet Detection Pipeline @@ -443,139 +268,9 @@ thermodynamic relaxation. **Output**: Life candidate list with explicit uncertainty + required follow-up observations list generated by POLICY_SEG rules. -## Microlensing Detection Pipeline (M0-M3) +### Microlensing & Cross-Domain Extensions -Extends the transit pipeline to gravitational microlensing events for rogue -planet and exomoon candidate detection. - -### Additional Data Sources - -| Source | Access | Reference | -|--------|--------|-----------| -| OGLE-IV microlensing events | Public FTP | [ogle.astrouw.edu.pl](https://ogle.astrouw.edu.pl/ogle4/ews/ews.html) | -| MOA-II microlensing alerts | Public archive | [www.massey.ac.nz/~iabond/moa](https://www.massey.ac.nz/~iabond/moa/) | -| Gaia astrometric catalog | ESA public | [gea.esac.esa.int](https://gea.esac.esa.int/archive/) | -| Roman Space Telescope | Upcoming | High-cadence microlensing survey | - -### Stage M0: Ingest - -Microlensing light curve ingestion from OGLE/MOA archives. -- Normalize photometry across surveys (MOA-II: ~15 min cadence, 0.008σ floor; OGLE-IV: ~40 min cadence, 0.005σ floor) -- Segment into event windows around magnification peaks -- Store as RVF Event nodes with survey-specific metadata - -### Stage M1: Single-Lens Detection - -Standard Paczynski curve fitting for point-source point-lens (PSPL) events. -- Two-phase fit: coarse grid search + fine refinement -- Linear regression for source flux (F_s) and blending flux (F_b) -- Parameter estimation: Einstein crossing time (t_E), impact parameter (u_0), peak time (t_0) -- Create Candidate nodes for events exceeding SNR threshold - -### Stage M2: Anomaly Detection via MRF/Mincut - -Residual analysis after best-fit PSPL subtraction using Markov Random Field -optimization with graph cut inference. - -**Three-statistic lambda computation:** -1. Excess chi2: window fit quality relative to global reduced chi2 -2. Runs test: temporal coherence of residual sign changes -3. Gaussian bump fit: localized perturbation chi2 improvement - -**Differential normalization:** Compare each window's signal to tau-space -neighbors, producing z-scores that are ~0 for uniform fit quality and positive -only for localized anomalies. - -**Graph construction:** -- Temporal chain: consecutive time windows (alpha weight) -- RuVector kNN edges: embedding similarity from residual features (beta weight) -- Edmonds-Karp BFS max-flow solver for s-t mincut - -### Stage M3: Coherence Gating - -Dynamic mincut separates competing explanations (noise vs planet vs moon vs -binary lens) using support region analysis. - -| Component | Role | -|-----------|------| -| RuVector embeddings | Cross-survey memory; retrieve similar anomalies | -| HNSW index | Fast similarity search across microlensing events | -| Dynamic mincut | Separate competing explanations | -| Witness chain | Full provenance from raw photometry to candidate | - -**Important constraint:** Dynamic mincut provides coherent support extraction -but cannot replace lens modeling. The physics solver (PSPL/binary) provides -local evidence; mincut provides spatial coherence. - -## Cross-Domain Graph-Cut Applications - -The MRF/mincut + RuVector architecture generalizes beyond astronomy to any -domain requiring coherent anomaly detection in structured data. - -### Implemented Verticals - -| Vertical | Example | Graph Topology | Anomaly Types | -|----------|---------|----------------|---------------| -| **Medical Imaging** | `medical_graphcut.rs` | 4-connected spatial grid, gradient-weighted edges | Lesion segmentation (T1-MRI, T2-MRI, CT) | -| **Genomics** | `genomic_graphcut.rs` | Linear chain + kNN similarity | CNV gains/losses, LOH, cancer drivers (TP53, BRCA1, EGFR, MYC) | -| **Financial Fraud** | `financial_fraud_graphcut.rs` | Temporal chain + merchant edges + kNN | Card-not-present, account takeover, card clone, synthetic, refund | -| **Supply Chain** | `supply_chain_graphcut.rs` | Tier chain + geographic + kNN | Quality defect, shortage, price anomaly, delay, counterfeit, demand shock | -| **Cybersecurity** | `cyber_threat_graphcut.rs` | Source/destination chain + kNN | Port scan, brute force, exfiltration, C2 beacon, DDoS, lateral movement | -| **Climate** | `climate_graphcut.rs` | Spatial adjacency + gradient + kNN | Heat wave, pollution spike, drought, ocean warming, cold snap, sensor fault | - -### Common Architecture - -All verticals share: -1. **Domain-specific data generator** with realistic parameters from published datasets -2. **Feature extraction** producing 32-dim embeddings for RuVector storage -3. **Graph construction** combining domain topology (spatial/temporal/chain) with kNN similarity edges -4. **Edmonds-Karp BFS** s-t mincut solver (identical implementation) -5. **RVF integration**: witness chains, filtered metadata queries, lineage derivation -6. **Evaluation**: comparison against threshold baseline showing graph-cut improvement - -### Additional Real Data Sources - -| Domain | Source | Access | -|--------|--------|--------| -| Genomics | TCGA (The Cancer Genome Atlas) | [portal.gdc.cancer.gov](https://portal.gdc.cancer.gov/) | -| Genomics | ClinVar variants | [ncbi.nlm.nih.gov/clinvar](https://www.ncbi.nlm.nih.gov/clinvar/) | -| Genomics | COSMIC (somatic mutations) | [cancer.sanger.ac.uk/cosmic](https://cancer.sanger.ac.uk/cosmic) | -| Medical | LIDC-IDRI (lung CT) | [cancerimagingarchive.net](https://www.cancerimagingarchive.net/) | -| Medical | BraTS (brain tumors) | [synapse.org](https://www.synapse.org/) | -| Cybersecurity | CICIDS2017 | [unb.ca/cic](https://www.unb.ca/cic/datasets/ids-2017.html) | -| Climate | NOAA GHCN | [ncei.noaa.gov](https://www.ncei.noaa.gov/) | -| Climate | EPA AQI | [epa.gov/aqs](https://www.epa.gov/aqs) | - -## Measured Results - -Results from running examples with `cargo run --example --release`. - -### Astronomy - -| Pipeline | Metric | Value | Notes | -|----------|--------|-------|-------| -| `planet_detection` | Recall@10 | 8/10 confirmed | Synthetic Kepler-like targets | -| `exomoon_graphcut` | Precision | 0.25 | Chang-Refsdal perturbative limit | -| `exomoon_graphcut` | Recall | 0.25 | Limited by ~2-3σ per-window SNR | -| `exomoon_graphcut` | F1 | 0.25 | 30 events, 12 with moons | -| `real_microlensing` | Planets rediscovered | 2/4 | OGLE-2005-BLG-390, OGLE-2016-BLG-1195 | -| `microlensing_detection` | Events processed | 20 | Synthetic PSPL + anomalies | - -### Medical Imaging - -| Modality | Graph Cut Dice | Threshold Dice | Improvement | -|----------|---------------|----------------|-------------| -| T1-MRI | 0.44-0.59 | 0.32-0.46 | +27-39% | -| T2-MRI | 0.50-0.62 | 0.38-0.48 | +29-32% | -| CT | 0.48-0.58 | 0.35-0.44 | +32-37% | - -### Genomics - -| Platform | Sensitivity | Specificity | Drivers Found | -|----------|-------------|-------------|---------------| -| WGS (30x) | 0.91-0.95 | 0.66-0.97 | 4/4 (TP53, BRCA1, EGFR, MYC) | -| WES (100x) | 0.88-0.93 | 0.70-0.95 | 4/4 | -| Panel (500x) | 0.93-0.97 | 0.72-0.98 | 4/4 | +> See [ADR-040b: Microlensing & Graph-Cut Extensions](ADR-040b-microlensing-graphcut-extensions.md) for M0-M3 pipeline, cross-domain applications, measured results, and Rust crate structure. ## Runtime and Portability @@ -630,167 +325,9 @@ WS /ws/live # real-time boundary alerts, pipeline pr 3. No remote writes without explicit policy toggle in POLICY_SEG 4. Downloaded data verified against MANIFEST_SEG hashes before ingestion -## Three.js Visualization Dashboard +### Dashboard Architecture -The RVF embeds a Vite-bundled Three.js dashboard in `DASHBOARD_SEG`. The -runtime HTTP server serves it at `/` (root). All visualizations are driven -by the same API endpoints the CLI uses, so every rendered frame corresponds -to queryable, witness-backed data. - -### Architecture - -``` -DASHBOARD_SEG (inside RVF) - dist/ - index.html # Vite SPA entry - assets/ - main.[hash].js # Three.js + D3 + app logic (tree-shaken) - main.[hash].css # Tailwind/minimal styles - worker.js # Web Worker for graph layout - -Runtime serves: - GET / -> DASHBOARD_SEG/dist/index.html - GET /assets/* -> DASHBOARD_SEG/dist/assets/* - GET /api/* -> JSON API (atlas, coherence, candidates, etc.) - WS /ws/live -> Live streaming of boundary alerts and pipeline progress -``` - -**Build pipeline**: Vite builds the dashboard at package time into a single -tree-shaken bundle. The bundle is embedded into `DASHBOARD_SEG` during RVF -assembly. No Node.js required at runtime — the dashboard is pure static -assets served by the existing HTTP server. - -### Dashboard Views - -#### V1: Causal Atlas Explorer (Three.js 3D) - -Interactive 3D force-directed graph of the causal atlas. - -| Feature | Implementation | -|---------|---------------| -| **Node rendering** | `THREE.InstancedMesh` for events — color by domain (Kepler=blue, TESS=cyan, JWST=gold, derived=white) | -| **Edge rendering** | `THREE.LineSegments` with opacity mapped to edge weight | -| **Causal flow** | Animated particles along causal edges showing temporal direction | -| **Scale selector** | Toggle between window scales (2h, 12h, 3d, 27d) — re-layouts graph | -| **Candidate highlight** | Click candidate in sidebar to trace its causal chain in 3D, dimming unrelated nodes | -| **Witness replay** | Step through witness chain entries, animating graph state forward/backward | -| **LOD** | Level-of-detail: far=boundary nodes only, mid=top-k events, close=full subgraph | - -Data source: `GET /api/atlas/query`, `GET /api/atlas/trace` - -#### V2: Coherence Field Heatmap (Three.js + shader) - -Real-time coherence field rendered as a colored surface over the atlas graph. - -| Feature | Implementation | -|---------|---------------| -| **Field surface** | `THREE.PlaneGeometry` subdivided grid, vertex colors from coherence values | -| **Cut pressure** | Red hotspots where cut pressure is high, cool blue where stable | -| **Partition boundaries** | Glowing wireframe lines at partition cuts | -| **Time scrubber** | Scrub through epochs to see coherence evolution | -| **Drift overlay** | Toggle to show embedding drift as animated vector arrows | -| **Alert markers** | Pulsing icons at boundary alert locations | - -Data source: `GET /api/coherence`, `GET /api/boundary/timeline`, `WS /ws/live` - -#### V3: Planet Candidate Dashboard (2D panels + 3D orbit) - -Split view combining data panels with 3D orbital visualization. - -| Panel | Content | -|-------|---------| -| **Ranked list** | Sortable table: candidate ID, score, uncertainty, period, SNR, publishable status | -| **Light curve viewer** | Interactive D3 chart: raw flux, detrended flux, transit model overlay, per-window score | -| **Phase-folded plot** | All transits folded at detected period, with confidence band | -| **3D orbit preview** | `THREE.Line` showing inferred orbital path around host star, sized by uncertainty | -| **Evidence trace** | Expandable tree showing witness chain from raw data to final score | -| **Score breakdown** | Radar chart: SNR, shape consistency, period stability, coherence stability | - -Data source: `GET /api/candidates/planet`, `GET /api/candidates/:id/trace` - -#### V4: Life Candidate Dashboard (2D panels + 3D molecule) - -Split view for spectral disequilibrium analysis. - -| Panel | Content | -|-------|---------| -| **Ranked list** | Sortable table: candidate ID, disequilibrium score, uncertainty, molecule flags, publishable | -| **Spectrum viewer** | Interactive D3 chart: wavelength vs flux, molecule absorption bands highlighted | -| **Molecule presence matrix** | Heatmap of detected molecule families vs confidence | -| **3D molecule overlay** | `THREE.Sprite` labels at absorption wavelengths in a 3D wavelength space | -| **Reaction graph** | Force-directed graph of molecule co-occurrences vs equilibrium expectations | -| **Confound panel** | Bar chart: stellar activity penalty, contamination risk, repeatability score | - -Data source: `GET /api/candidates/life`, `GET /api/candidates/:id/trace` - -#### V5: System Status Dashboard - -Operational health and download progress. - -| Panel | Content | -|-------|---------| -| **Download progress** | Per-tier progress bars with byte counts and ETA | -| **Segment sizes** | Stacked bar chart of RVF segment utilization | -| **Memory tiers** | S/M/L tier fill levels and compaction history | -| **Witness chain** | Scrolling log of recent witness entries with hash preview | -| **Pipeline status** | P0/P1/P2 and L0/L1/L2 stage indicators with event counts | -| **Performance** | Query latency histogram, events/second throughput | - -Data source: `GET /api/status`, `GET /api/memory/tiers`, `WS /ws/live` - -### WebSocket Live Stream - -```typescript -// WS /ws/live — server pushes events as they happen -interface LiveEvent { - type: 'boundary_alert' | 'candidate_new' | 'candidate_update' | - 'download_progress' | 'witness_commit' | 'pipeline_stage' | - 'coherence_update'; - timestamp: string; - data: Record; -} -``` - -The dashboard subscribes on connect and updates all views in real-time as -pipelines process data and boundaries evolve. - -### Vite Build Configuration - -```typescript -// vite.config.ts for dashboard build -import { defineConfig } from 'vite'; - -export default defineConfig({ - build: { - outDir: 'dist/dashboard', - assetsDir: 'assets', - rollupOptions: { - output: { - manualChunks: { - three: ['three'], // ~150 KB gzipped - d3: ['d3-scale', 'd3-axis', 'd3-shape', 'd3-selection'], - }, - }, - }, - }, -}); -``` - -**Bundle budget**: < 500 KB gzipped total (Three.js ~150 KB, D3 subset ~30 KB, -app logic ~50 KB, styles ~10 KB). The dashboard adds minimal overhead to the -RVF artifact. - -### Design Decision: D5 — Dashboard Embedded in RVF - -The Three.js dashboard is bundled at build time and embedded in `DASHBOARD_SEG` -rather than served from an external CDN or requiring a separate install. This -ensures: - -1. **Fully offline**: Works without network after boot -2. **Version-locked**: Dashboard always matches the API version it queries -3. **Single artifact**: One RVF file = runtime + data + visualization -4. **Witness-aligned**: Dashboard renders exactly the data the witness chain - can verify +> See [ADR-040a: Planet Detection Dashboard](ADR-040a-planet-detection-dashboard.md) for Views V1-V5, WebSocket streaming, Vite build configuration, and design decision D5. ## Package Structure @@ -816,37 +353,7 @@ packages/agentdb-causal-atlas/ ProgressiveDownloader.ts # Tiered lazy download with resume DataManifest.ts # URL + hash + size manifests KernelBuilder.ts # Reuse/extend from ADR-008 - dashboard/ # Vite + Three.js visualization app - vite.config.ts # Build config — outputs to dist/dashboard/ - index.html # SPA entry point - src/ - main.ts # App bootstrap, router, WS connection - api.ts # Typed fetch wrappers for /api/* endpoints - ws.ts # WebSocket client for /ws/live - views/ - AtlasExplorer.ts # V1: 3D causal atlas (Three.js force graph) - CoherenceHeatmap.ts # V2: Coherence field surface + cut pressure - PlanetDashboard.ts # V3: Planet candidates + light curves + 3D orbit - LifeDashboard.ts # V4: Life candidates + spectra + molecule graph - StatusDashboard.ts # V5: System health, downloads, witness log - three/ - AtlasGraph.ts # InstancedMesh nodes, LineSegments edges, particles - CoherenceSurface.ts # PlaneGeometry with vertex-colored field - OrbitPreview.ts # Orbital path visualization - CausalFlow.ts # Animated particles along causal edges - LODController.ts # Level-of-detail: boundary → top-k → full - charts/ - LightCurveChart.ts # D3 flux time series with transit overlay - SpectrumChart.ts # D3 wavelength vs flux with molecule bands - RadarChart.ts # Score breakdown radar - MoleculeMatrix.ts # Heatmap of molecule presence vs confidence - components/ - Sidebar.ts # Candidate list, filters, search - TimeScrubber.ts # Epoch scrubber for coherence replay - WitnessLog.ts # Scrolling witness chain entries - DownloadProgress.ts # Tier progress bars - styles/ - main.css # Minimal Tailwind or hand-rolled styles + dashboard/ # See ADR-040a for full dashboard tree tests/ causal-atlas.test.ts planet-detection.test.ts @@ -854,28 +361,11 @@ packages/agentdb-causal-atlas/ progressive-download.test.ts coherence-field.test.ts boundary-tracker.test.ts - dashboard.test.ts # Dashboard build + API integration tests ``` ### Rust Implementation -The pipeline examples are implemented in Rust using the RVF runtime crates: - -| Crate | Role | -|-------|------| -| `rvf-types` | Core types, segment definitions, derivation types | -| `rvf-runtime` | RvfStore, HNSW indexing, metadata filters, queries | -| `rvf-crypto` | SHAKE-256 witness chains, verification | -| `rvf-wire` | Wire format serialization | -| `rvf-manifest` | Segment manifests and policies | -| `rvf-index` | Progressive HNSW index construction | -| `rvf-quant` | Vector quantization (PQ, SQ) | -| `rvf-kernel` | Kernel/initrd segment embedding | -| `rvf-launch` | Boot sequence and runtime launch | -| `rvf-ebpf` | eBPF socket/syscall policies | -| `rvf-server` | HTTP/WS server for dashboard and API | - -Examples location: `examples/rvf/examples/` +> See [ADR-040b](ADR-040b-microlensing-graphcut-extensions.md#rust-implementation) for the full Rust crate table. Examples location: `examples/rvf/examples/` ## Implementation Phases @@ -883,55 +373,45 @@ Examples location: `examples/rvf/examples/` **Scope**: Kepler and TESS only. No spectra. No life scoring. -1. Implement `ProgressiveDownloader` with tier-0 curated dataset (100 Kepler targets) -2. Implement `PlanetTransitAdapter` for FITS light curve ingestion -3. Implement `CausalAtlas` with windowing, feature extraction, SIMD embedding -4. Implement `PlanetDetection` pipeline (P0-P2) -5. Implement `WITNESS_SEG` with SHAKE-256 chain +1. `ProgressiveDownloader` with tier-0 curated dataset (100 Kepler targets) +2. `PlanetTransitAdapter` for FITS light curve ingestion +3. `CausalAtlas` with windowing, feature extraction, SIMD embedding +4. `PlanetDetection` pipeline (P0-P2) +5. `WITNESS_SEG` with SHAKE-256 chain 6. CLI: `rvf run`, `rvf query planet list`, `rvf trace` 7. HTTP: `/api/candidates/planet`, `/api/atlas/trace` -8. Dashboard: Vite scaffold, V1 Atlas Explorer (Three.js 3D graph), V3 Planet - Dashboard (ranked list + light curve chart), V5 Status Dashboard (download - progress + witness log). Embedded in `DASHBOARD_SEG`, served at `/` -9. WebSocket `/ws/live` for real-time pipeline progress +8. Dashboard shell: V1, V3, V5 views (see ADR-040a), WebSocket `/ws/live` **Acceptance**: 1,000 Kepler targets, top-100 ranked list includes >= 80 confirmed planets, every item replays to same score and witness root on two -machines. Dashboard renders atlas graph and candidate list in browser. +machines. -### Phase 2: Coherence Field + Boundary Tracker + Dashboard V2 (v0.2) +### Phase 2: Coherence Field + Boundary Tracker (v0.2) -1. Implement `CoherenceField` with dynamic min-cut, partition entropy -2. Implement `BoundaryTracker` with timeline and alerts -3. Implement `MultiScaleMemory` with S/M/L tiers and budget control -4. Add coherence gating to planet pipeline +1. `CoherenceField` with dynamic min-cut, partition entropy +2. `BoundaryTracker` with timeline and alerts +3. `MultiScaleMemory` with S/M/L tiers and budget control +4. Coherence gating added to planet pipeline 5. HTTP: `/api/coherence`, `/api/boundary/*`, `/api/memory/tiers` -6. Dashboard: V2 Coherence Heatmap (Three.js field surface + cut pressure - overlay + time scrubber), boundary alert markers via WebSocket +6. Dashboard V2 Coherence Heatmap (see ADR-040a) -### Phase 3: Life Candidate Pipeline + Dashboard V4 (v0.3) +### Phase 3: Life Candidate Pipeline (v0.3) -1. Implement `SpectrumAdapter` for JWST/archive spectral data -2. Implement `LifeCandidate` pipeline (L0-L2) -3. Implement disequilibrium scoring with reaction plausibility graph -4. Tier-3 progressive download for spectral data -5. CLI: `rvf query life list` -6. HTTP: `/api/candidates/life` -7. Dashboard: V4 Life Dashboard (spectrum viewer + molecule presence matrix - + reaction graph + confound panel) +1. `SpectrumAdapter` for JWST/archive spectral data +2. `LifeCandidate` pipeline (L0-L2) with disequilibrium scoring +3. Tier-3 progressive download for spectral data +4. CLI: `rvf query life list`; HTTP: `/api/candidates/life` +5. Dashboard V4 Life Dashboard (see ADR-040a) -**Acceptance**: Published spectra with known atmospheric detections vs nulls, -AUC > 0.8, every score includes confound penalties and provenance trace. -Dashboard renders spectrum analysis in browser. +**Acceptance**: AUC > 0.8 on published spectra with known atmospheric +detections vs nulls, every score includes confound penalties and provenance. ### Phase 4: Cosmic Web + Full Integration (v0.4) 1. `CosmicWebAdapter` for SDSS spatial graph 2. Cross-domain coherence (planet candidates enriched by large-scale context) -3. Dashboard: 3D cosmic web view, cross-domain candidate linking -4. Full offline demo with sealed RVF snapshot -5. `rvf ingest --expand` for tier-2 bulk download -6. Dashboard polish: LOD optimization, mobile-responsive layout, dark/light theme +3. Full offline demo with sealed RVF snapshot +4. `rvf ingest --expand` for tier-2 bulk download ## Evaluation Plan diff --git a/examples/rvf/examples/real_microlensing.rs b/examples/rvf/examples/real_microlensing.rs index 8625b982d..0ab90ec74 100644 --- a/examples/rvf/examples/real_microlensing.rs +++ b/examples/rvf/examples/real_microlensing.rs @@ -1,12 +1,6 @@ -//! Real Microlensing Data Analysis Pipeline -//! -//! Downloads and analyzes real microlensing light curves from public surveys: -//! - OGLE Early Warning System (EWS) -//! - MOA Alerts -//! -//! Features progressive download manifest with real OGLE/MOA event parameters, -//! cross-survey normalization, and anomaly detection via graph-cut residual analysis. -//! +//! Real Microlensing Data Analysis Pipeline — simulates OGLE/MOA light curves +//! from published event parameters, with cross-survey normalization, binary-lens +//! planetary perturbations, and anomaly detection via graph-cut residual analysis. //! Run: cargo run --example real_microlensing --release use rvf_runtime::{ @@ -22,317 +16,87 @@ const FIELD_SOURCE: u16 = 0; const FIELD_EVENT: u16 = 1; const FIELD_ANOMALY_TYPE: u16 = 2; -// --------------------------------------------------------------------------- -// LCG deterministic random -// --------------------------------------------------------------------------- -fn lcg_f64(state: &mut u64) -> f64 { - *state = state.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); - ((*state >> 33) as f64) / (u32::MAX as f64) -} - -fn lcg_normal(state: &mut u64) -> f64 { - let u1 = lcg_f64(state).max(1e-15); - let u2 = lcg_f64(state); +const LCG_MUL: u64 = 6364136223846793005; +const LCG_INC: u64 = 1442695040888963407; +fn lcg_f64(s: &mut u64) -> f64 { *s = s.wrapping_mul(LCG_MUL).wrapping_add(LCG_INC); ((*s >> 33) as f64) / (u32::MAX as f64) } +fn lcg_normal(s: &mut u64) -> f64 { + let (u1, u2) = (lcg_f64(s).max(1e-15), lcg_f64(s)); (-2.0 * u1.ln()).sqrt() * (2.0 * std::f64::consts::PI * u2).cos() } - fn random_vector(dim: usize, seed: u64) -> Vec { - let mut v = Vec::with_capacity(dim); let mut x = seed.wrapping_add(1); - for _ in 0..dim { - x = x.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); - v.push(((x >> 33) as f32) / (u32::MAX as f32) - 0.5); - } - v + (0..dim).map(|_| { x = x.wrapping_mul(LCG_MUL).wrapping_add(LCG_INC); ((x >> 33) as f32) / (u32::MAX as f32) - 0.5 }).collect() } -// --------------------------------------------------------------------------- -// Photometric data types -// --------------------------------------------------------------------------- - #[derive(Debug, Clone)] -struct Observation { - hjd: f64, mag: f64, mag_err: f64, flux: f64, flux_err: f64, -} - +struct Observation { hjd: f64, mag: f64, mag_err: f64, flux: f64, flux_err: f64 } #[derive(Debug, Clone)] struct MicrolensingEvent { - name: String, - source: String, - observations: Vec, - baseline_mag: f64, - peak_mag: f64, - t0_est: f64, - t_e_est: f64, - u0_est: f64, + name: String, source: String, observations: Vec, + baseline_mag: f64, peak_mag: f64, t0_est: f64, t_e_est: f64, u0_est: f64, } - -// --------------------------------------------------------------------------- -// Enhanced OGLE format parser -// --------------------------------------------------------------------------- - -/// Parse OGLE EWS photometry. Handles HJD and JD time formats, magnitude and -/// flux columns (detected by value range: mag > 10, flux < 100), comment lines, -/// and malformed/NaN/inf values gracefully. -fn parse_ogle_format(data: &str, event_name: &str) -> MicrolensingEvent { - let mut observations = Vec::new(); - - for line in data.lines() { - let line = line.trim(); - if line.is_empty() || line.starts_with('#') || line.starts_with("\\") { continue; } - - let parts: Vec<&str> = line.split_whitespace().collect(); - if parts.len() < 3 { continue; } - - let (hjd, val, err) = match (parts[0].parse::(), parts[1].parse::(), parts[2].parse::()) { - (Ok(t), Ok(v), Ok(e)) => (t, v, e), - _ => continue, - }; - - // Skip NaN/inf values - if hjd.is_nan() || hjd.is_infinite() || val.is_nan() || val.is_infinite() - || err.is_nan() || err.is_infinite() { continue; } - - // JD -> HJD conversion: subtract 2450000 if time looks like full JD - let hjd = if hjd > 2400000.0 { hjd - 2450000.0 } else { hjd }; - - // Detect magnitude vs flux by value range: magnitudes > 10, flux < 100 - let (mag, mag_err) = if val > 10.0 { - (val, err) // already magnitude - } else if val > 0.0 { - // Flux column: convert to magnitude using arbitrary zero-point 22.0 - let m = -2.5 * val.log10() + 22.0; - let me = 2.5 / (val * 2.302585) * err; - (m, me) - } else { continue; }; - - if mag_err <= 0.0 || mag_err > 1.0 || mag < 5.0 || mag > 25.0 { continue; } - - observations.push(Observation { hjd, mag, mag_err, flux: 0.0, flux_err: 0.0 }); - } - - finalize_event(observations, event_name, "OGLE") -} - -// --------------------------------------------------------------------------- -// MOA format parser -// --------------------------------------------------------------------------- - -/// Parse MOA-II photometry. MOA column order: JD, flux, flux_error (no magnitude). -/// Normalizes to relative flux using baseline estimated from first/last 20% of data. -fn parse_moa_format(data: &str, event_name: &str) -> MicrolensingEvent { - let mut observations = Vec::new(); - - for line in data.lines() { - let line = line.trim(); - if line.is_empty() || line.starts_with('#') { continue; } - - let parts: Vec<&str> = line.split_whitespace().collect(); - if parts.len() < 3 { continue; } - - let (jd, flux, flux_err) = match (parts[0].parse::(), parts[1].parse::(), parts[2].parse::()) { - (Ok(t), Ok(f), Ok(e)) => (t, f, e), - _ => continue, - }; - - if jd.is_nan() || flux.is_nan() || flux_err.is_nan() { continue; } - if flux.is_infinite() || flux_err.is_infinite() { continue; } - if flux <= 0.0 || flux_err <= 0.0 { continue; } - - let hjd = if jd > 2400000.0 { jd - 2450000.0 } else { jd }; - let mag = -2.5 * flux.log10() + 22.0; - let mag_err = 2.5 / (flux * 2.302585) * flux_err; - - if mag_err <= 0.0 || mag_err > 1.0 { continue; } - observations.push(Observation { hjd, mag, mag_err, flux: 0.0, flux_err: 0.0 }); - } - - // Baseline from first/last 20% of sorted observations - observations.sort_by(|a, b| a.hjd.partial_cmp(&b.hjd).unwrap()); - let n = observations.len(); - let wing = (n as f64 * 0.2).ceil() as usize; - if n > 0 && wing > 0 { - let baseline_flux: f64 = observations[..wing].iter() - .chain(observations[n.saturating_sub(wing)..].iter()) - .map(|o| 10.0f64.powf(-0.4 * (o.mag - 22.0))) - .sum::() / (2 * wing) as f64; - // Normalize all to relative flux - for obs in &mut observations { - let raw = 10.0f64.powf(-0.4 * (obs.mag - 22.0)); - obs.flux = raw / baseline_flux; - obs.flux_err = obs.flux * 0.4 * 2.302585 * obs.mag_err; - } - } - - finalize_event(observations, event_name, "MOA") -} - -/// Shared finalization: sort, compute baseline/peak, convert to flux. -fn finalize_event(mut obs: Vec, name: &str, source: &str) -> MicrolensingEvent { - obs.sort_by(|a, b| a.hjd.partial_cmp(&b.hjd).unwrap()); - let mut mags: Vec = obs.iter().map(|o| o.mag).collect(); - mags.sort_by(|a, b| a.partial_cmp(b).unwrap()); - let baseline_mag = if !mags.is_empty() { mags[mags.len() * 3 / 4] } else { 20.0 }; - let peak_mag = mags.first().copied().unwrap_or(20.0); - - for o in &mut obs { - if o.flux == 0.0 { - o.flux = 10.0f64.powf(-0.4 * (o.mag - baseline_mag)); - o.flux_err = o.flux * 0.4 * 2.302585 * o.mag_err; - } - } - - let t0_est = obs.iter().min_by(|a, b| a.mag.partial_cmp(&b.mag).unwrap()) - .map(|o| o.hjd).unwrap_or(0.0); - let half_mag = (baseline_mag + peak_mag) / 2.0; - let above: Vec<_> = obs.iter().filter(|o| o.mag < half_mag).collect(); - let t_e_est = if above.len() >= 2 { - (above.last().unwrap().hjd - above.first().unwrap().hjd) / 2.0 - } else { 20.0 }; - let peak_flux = 10.0f64.powf(-0.4 * (peak_mag - baseline_mag)); - let u0_est = if peak_flux > 1.5 { (1.0 / peak_flux).max(0.01) } else { 0.5 }; - - MicrolensingEvent { - name: name.to_string(), source: source.to_string(), observations: obs, - baseline_mag, peak_mag, t0_est, t_e_est, u0_est, - } -} - -// --------------------------------------------------------------------------- -// Progressive download manifest -// --------------------------------------------------------------------------- - #[derive(Debug, Clone, Copy)] struct PublishedParams { - t0: f64, t_e: f64, u0: f64, - has_planet: bool, - planet_mass_ratio: Option, + einstein_time: f64, // t_E in days + impact_param: f64, // u_0 + planet_mass_ratio: Option, // q (None = no planet) + planet_separation: Option, // s (None = no planet) } #[derive(Debug, Clone)] struct ManifestEntry { - name: &'static str, - survey: &'static str, - url_template: &'static str, - year: u16, - event_id: &'static str, - published_params: Option, + event_id: String, // e.g. "OGLE-2005-BLG-390" + survey: String, // "ogle" or "moa" + published: PublishedParams, } -struct DownloadManifest { events: Vec } - #[derive(Debug, Clone)] enum DownloadState { Pending, Simulated, #[allow(dead_code)] Downloaded, Failed(String) } - impl std::fmt::Display for DownloadState { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match self { - Self::Pending => write!(f, "PEND"), - Self::Simulated => write!(f, "SIM"), - Self::Downloaded => write!(f, "DL"), - Self::Failed(e) => write!(f, "FAIL:{}", e), + Self::Pending => write!(f, "PEND"), Self::Simulated => write!(f, "SIM"), + Self::Downloaded => write!(f, "DL"), Self::Failed(e) => write!(f, "FAIL:{}", e), } } } -fn build_manifest() -> DownloadManifest { - let ogle = "https://ogle.astrouw.edu.pl/ogle4/ews/{year}/blg-{id}/phot.dat"; - let moa = "https://www.massey.ac.nz/~iabond/moa/alert{year}/fetch.php?path=moa{id}"; - DownloadManifest { events: vec![ - ManifestEntry { name: "OGLE-2005-BLG-390", survey: "OGLE", url_template: ogle, - year: 2005, event_id: "390", - published_params: Some(PublishedParams { t0: 3582.7, t_e: 11.03, u0: 0.359, - has_planet: true, planet_mass_ratio: Some(7.6e-5) }) }, - ManifestEntry { name: "MOA-2011-BLG-262", survey: "MOA", url_template: moa, - year: 2011, event_id: "262", - published_params: Some(PublishedParams { t0: 5700.5, t_e: 3.3, u0: 0.05, - has_planet: true, planet_mass_ratio: Some(4e-4) }) }, - ManifestEntry { name: "OGLE-2016-BLG-1195", survey: "OGLE", url_template: ogle, - year: 2016, event_id: "1195", - published_params: Some(PublishedParams { t0: 7568.0, t_e: 8.2, u0: 0.35, - has_planet: true, planet_mass_ratio: Some(4.5e-5) }) }, - ManifestEntry { name: "MOA-2009-BLG-387", survey: "MOA", url_template: moa, - year: 2009, event_id: "387", - published_params: Some(PublishedParams { t0: 5010.0, t_e: 42.5, u0: 0.012, - has_planet: true, planet_mass_ratio: Some(3.2e-3) }) }, - ManifestEntry { name: "OGLE-2003-BLG-235", survey: "OGLE", url_template: ogle, - year: 2003, event_id: "235", - published_params: Some(PublishedParams { t0: 2838.5, t_e: 61.5, u0: 0.133, - has_planet: true, planet_mass_ratio: Some(3.9e-3) }) }, - ManifestEntry { name: "OGLE-2005-BLG-071", survey: "OGLE", url_template: ogle, - year: 2005, event_id: "071", - published_params: Some(PublishedParams { t0: 3469.1, t_e: 70.9, u0: 0.026, - has_planet: true, planet_mass_ratio: Some(7.0e-3) }) }, - ManifestEntry { name: "MOA-2007-BLG-192", survey: "MOA", url_template: moa, - year: 2007, event_id: "192", - published_params: Some(PublishedParams { t0: 4239.0, t_e: 38.0, u0: 0.29, - has_planet: true, planet_mass_ratio: Some(2.0e-4) }) }, - ManifestEntry { name: "OGLE-2006-BLG-109", survey: "OGLE", url_template: ogle, - year: 2006, event_id: "109", - published_params: Some(PublishedParams { t0: 3831.0, t_e: 127.3, u0: 0.0035, - has_planet: true, planet_mass_ratio: Some(1.4e-3) }) }, - // Non-planet events for comparison - ManifestEntry { name: "OGLE-2018-BLG-0799", survey: "OGLE", url_template: ogle, - year: 2018, event_id: "0799", - published_params: Some(PublishedParams { t0: 8250.0, t_e: 25.0, u0: 0.008, - has_planet: false, planet_mass_ratio: None }) }, - ManifestEntry { name: "MOA-2019-BLG-008", survey: "MOA", url_template: moa, - year: 2019, event_id: "008", - published_params: Some(PublishedParams { t0: 8580.0, t_e: 18.0, u0: 0.15, - has_planet: false, planet_mass_ratio: None }) }, - ManifestEntry { name: "OGLE-2012-BLG-0563", survey: "OGLE", url_template: ogle, - year: 2012, event_id: "0563", - published_params: Some(PublishedParams { t0: 6090.0, t_e: 55.0, u0: 0.045, - has_planet: true, planet_mass_ratio: Some(1.5e-3) }) }, - ManifestEntry { name: "OGLE-2017-BLG-0373", survey: "OGLE", url_template: ogle, - year: 2017, event_id: "0373", - published_params: Some(PublishedParams { t0: 7870.0, t_e: 40.0, u0: 0.12, - has_planet: false, planet_mass_ratio: None }) }, - ManifestEntry { name: "OGLE-2015-BLG-0966", survey: "OGLE", url_template: ogle, - year: 2015, event_id: "0966", - published_params: Some(PublishedParams { t0: 7190.0, t_e: 22.0, u0: 0.21, - has_planet: false, planet_mass_ratio: None }) }, - ManifestEntry { name: "MOA-2013-BLG-220", survey: "MOA", url_template: moa, - year: 2013, event_id: "220", - published_params: Some(PublishedParams { t0: 6440.0, t_e: 60.0, u0: 0.08, - has_planet: false, planet_mass_ratio: None }) }, - ManifestEntry { name: "MOA-2015-BLG-337", survey: "MOA", url_template: moa, - year: 2015, event_id: "337", - published_params: Some(PublishedParams { t0: 7200.0, t_e: 15.0, u0: 0.30, - has_planet: false, planet_mass_ratio: None }) }, - ]} -} - -// --------------------------------------------------------------------------- -// Cross-survey normalization -// --------------------------------------------------------------------------- - -/// Normalize light curves to fractional deviation from baseline: (F - F_base) / F_base. -/// OGLE-IV I-band zero-point ~21.0, MOA-II R-band zero-point ~22.0. -fn normalize_cross_survey(event: &mut MicrolensingEvent) { - let zp = match event.source.as_str() { - "MOA" => 22.0, - _ => 21.0, // OGLE-IV I-band +fn build_manifest() -> Vec { + let e = |id: &str, survey: &str, t_e: f64, u0: f64, q: Option, s: Option| { + ManifestEntry { + event_id: id.into(), survey: survey.into(), + published: PublishedParams { einstein_time: t_e, impact_param: u0, + planet_mass_ratio: q, planet_separation: s }, + } }; - // Compute baseline flux from faintest 50% of observations - let mut mags: Vec = event.observations.iter().map(|o| o.mag).collect(); - mags.sort_by(|a, b| a.partial_cmp(b).unwrap()); - let mid = mags.len() / 2; - let baseline_flux = if mid > 0 { - let bm: f64 = mags[mid..].iter().sum::() / (mags.len() - mid) as f64; - 10.0f64.powf(-0.4 * (bm - zp)) - } else { 1.0 }; - - for obs in &mut event.observations { - let f = 10.0f64.powf(-0.4 * (obs.mag - zp)); - obs.flux = (f - baseline_flux) / baseline_flux; // fractional deviation - obs.flux_err = f * 0.4 * 2.302585 * obs.mag_err / baseline_flux; - } + vec![ + // --- Planetary events --- + // First cold rocky/icy super-Earth + e("OGLE-2005-BLG-390", "ogle", 11.0, 0.359, Some(7.6e-5), Some(1.610)), + // Possible rogue planet with moon + e("MOA-2011-BLG-262", "moa", 3.3, 0.04, Some(3.8e-4), None), + // Ice-line planet, very low mass ratio + e("OGLE-2016-BLG-1195","ogle", 9.9, 0.045, Some(4.2e-5), Some(1.04)), + // First microlensing planet (OGLE+MOA joint) + e("OGLE-2003-BLG-235", "ogle", 61.5, 0.133, Some(0.0039), Some(1.12)), + // Very low-mass host star planet + e("MOA-2007-BLG-192", "moa", 46.0, 0.001, Some(0.0002), Some(1.05)), + // Jupiter/Saturn analog system (two planets) + e("OGLE-2006-BLG-109", "ogle",127.3, 0.0035,Some(1.4e-3), Some(2.3)), + // 3.8 M_Jup planet + e("OGLE-2005-BLG-071", "ogle", 70.9, 0.026, Some(7.0e-3), Some(1.29)), + // Saturn-mass planet + e("OGLE-2012-BLG-0563","ogle", 55.0, 0.045, Some(1.5e-3), Some(1.42)), + // --- Non-planet PSPL comparison events --- + e("OGLE-2018-BLG-0799","ogle", 25.0, 0.008, None, None), + e("MOA-2019-BLG-008", "moa", 18.0, 0.15, None, None), + e("OGLE-2017-BLG-0373","ogle", 40.0, 0.12, None, None), + e("OGLE-2015-BLG-0966","ogle", 22.0, 0.21, None, None), + e("MOA-2013-BLG-220", "moa", 60.0, 0.08, None, None), + ] } -// --------------------------------------------------------------------------- -// Simulation from published parameters (replaces actual download) -// --------------------------------------------------------------------------- +// ---- Paczynski formula & binary-lens perturbation ---- fn pspl_magnification(u: f64) -> f64 { if u < 1e-10 { return 1e10; } @@ -340,61 +104,189 @@ fn pspl_magnification(u: f64) -> f64 { (u2 + 2.0) / (u * (u2 + 4.0).sqrt()) } -fn simulate_from_manifest(entry: &ManifestEntry, rng: &mut u64) -> (MicrolensingEvent, DownloadState) { - let params = match entry.published_params { - Some(p) => p, - None => return (MicrolensingEvent { - name: entry.name.to_string(), source: entry.survey.to_string(), - observations: vec![], baseline_mag: 20.0, peak_mag: 20.0, - t0_est: 0.0, t_e_est: 20.0, u0_est: 0.5, - }, DownloadState::Failed("no published params".into())), - }; - let t0 = 2459000.0 + params.t0 % 1000.0; +/// Binary-lens perturbation factor near the planetary caustic. +/// Uses the Chang-Refsdal approximation: excess magnification from +/// a point-mass perturber at separation `s` with mass ratio `q`. +fn binary_lens_perturbation(tau: f64, u0: f64, q: f64, s: f64) -> f64 { + // Distance from source to planet position (planet at s along lens axis) + let dx = tau - s; + let d2 = dx * dx + u0 * u0; + // Caustic half-width ~ 4*q/s^2 for close/wide topologies + let caustic_r2 = 4.0 * q / (s * s); + if d2 > caustic_r2 * 25.0 { return 0.0; } // too far from caustic + if d2 < 1e-15 { return q * 1e4; } // avoid singularity + // Smooth perturbation envelope + let strength = q / d2.max(q * 0.1); + strength * (-d2 / (8.0 * caustic_r2.max(1e-10))).exp() +} + +// ---- Simulate light curve from published parameters ---- + +fn simulate_from_published(entry: &ManifestEntry, rng: &mut u64) -> (MicrolensingEvent, DownloadState) { + let p = &entry.published; + let is_moa = entry.survey == "moa"; + let t0 = 2459000.0 + (lcg_f64(rng) * 500.0); let baseline_mag = 18.0 + lcg_f64(rng) * 3.0; let f_b = 0.05 + lcg_f64(rng) * 0.3; - let cadence = if entry.survey == "MOA" { 0.25 } else { 0.5 }; - let sigma_floor = if entry.survey == "MOA" { 0.01 } else { 0.005 }; + let cadence = if is_moa { 0.25 } else { 0.5 }; // days between obs + let sigma_floor = if is_moa { 0.01 } else { 0.005 }; + let span = p.einstein_time.max(10.0) * 3.0; // cover +-3 t_E let mut observations = Vec::new(); - let mut t = t0 - 60.0; - while t < t0 + 60.0 { - t += cadence / 24.0 * (0.5 + lcg_f64(rng)); - if lcg_f64(rng) < 0.3 { continue; } + let mut t = t0 - span; + while t < t0 + span { + t += cadence * (0.5 + lcg_f64(rng)); + if lcg_f64(rng) < 0.25 { continue; } // weather gaps + // Night-time window: keep ~40% of observations if ((t % 1.0) - 0.2).abs() > 0.17 { continue; } - let tau = (t - t0) / params.t_e; - let u = (params.u0 * params.u0 + tau * tau).sqrt(); - let mut mag_a = pspl_magnification(u); - if params.has_planet { - let q = params.planet_mass_ratio.unwrap_or(0.0); - let s = 1.0 + lcg_f64(rng) * 0.01; // stable separation - let d2 = (tau - s).powi(2) + params.u0.powi(2); - if d2 < q * 25.0 && d2 > 1e-15 { - mag_a += (q / d2.max(q * 0.1)) * (-d2 / (8.0 * q)).exp() * mag_a; - } + let tau = (t - t0) / p.einstein_time; + let u = (p.impact_param * p.impact_param + tau * tau).sqrt(); + let mut amp = pspl_magnification(u); + + // Add binary-lens planetary perturbation if planet present + if let (Some(q), Some(s)) = (p.planet_mass_ratio, p.planet_separation) { + amp += binary_lens_perturbation(tau, p.impact_param, q, s) * amp; + } else if let Some(q) = p.planet_mass_ratio { + // No separation published (e.g. MOA-2011-BLG-262): use s ~ 1.0 + amp += binary_lens_perturbation(tau, p.impact_param, q, 1.0) * amp; } - let true_flux = mag_a + f_b; + + let true_flux = amp + f_b; let true_mag = baseline_mag - 2.5 * true_flux.log10(); - let sigma = (sigma_floor.powi(2) + 0.003f64.powi(2)).sqrt(); + let sigma = (sigma_floor * sigma_floor + 0.003f64.powi(2)).sqrt(); let obs_mag = true_mag + lcg_normal(rng) * sigma; - observations.push(Observation { - hjd: t, mag: obs_mag, mag_err: sigma, flux: 0.0, flux_err: 0.0, - }); + observations.push(Observation { hjd: t, mag: obs_mag, mag_err: sigma, flux: 0.0, flux_err: 0.0 }); } let peak_mag = observations.iter().map(|o| o.mag).fold(f64::INFINITY, f64::min); let mut evt = MicrolensingEvent { - name: entry.name.to_string(), source: entry.survey.to_string(), + name: entry.event_id.clone(), source: entry.survey.clone(), observations, baseline_mag, peak_mag, - t0_est: t0, t_e_est: params.t_e, u0_est: params.u0, + t0_est: t0, t_e_est: p.einstein_time, u0_est: p.impact_param, }; normalize_cross_survey(&mut evt); (evt, DownloadState::Simulated) } -// --------------------------------------------------------------------------- -// PSPL fitting (compact) -// --------------------------------------------------------------------------- +// ---- Cross-survey normalization ---- + +/// Normalize to fractional deviation (F - F_base) / F_base. +/// Survey-specific zero-points: OGLE I-band = 21.0, MOA R-band = 22.0. +fn normalize_cross_survey(event: &mut MicrolensingEvent) { + let zp = if event.source == "moa" { 22.0 } else { 21.0 }; + // Baseline flux from faintest 50% of observations + let mut mags: Vec = event.observations.iter().map(|o| o.mag).collect(); + mags.sort_by(|a, b| a.partial_cmp(b).unwrap()); + let mid = mags.len() / 2; + let baseline_flux = if mid > 0 { + let bm: f64 = mags[mid..].iter().sum::() / (mags.len() - mid) as f64; + 10.0f64.powf(-0.4 * (bm - zp)) + } else { 1.0 }; + for obs in &mut event.observations { + let f = 10.0f64.powf(-0.4 * (obs.mag - zp)); + obs.flux = (f - baseline_flux) / baseline_flux; + obs.flux_err = f * 0.4 * 2.302585 * obs.mag_err / baseline_flux; + } +} + +// ---- OGLE format parser ---- + +/// Parse OGLE EWS photometry: HJD/JD, magnitude or flux, error. +fn parse_ogle_format(data: &str, event_name: &str) -> MicrolensingEvent { + let mut obs = Vec::new(); + for line in data.lines() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') || line.starts_with('\\') { continue; } + let p: Vec<&str> = line.split_whitespace().collect(); + if p.len() < 3 { continue; } + let (hjd, val, err) = match (p[0].parse::(), p[1].parse::(), p[2].parse::()) { + (Ok(t), Ok(v), Ok(e)) => (t, v, e), _ => continue, + }; + if [hjd, val, err].iter().any(|x| x.is_nan() || x.is_infinite()) { continue; } + let hjd = if hjd > 2400000.0 { hjd - 2450000.0 } else { hjd }; + let (mag, mag_err) = if val > 10.0 { (val, err) } + else if val > 0.0 { + (-2.5 * val.log10() + 22.0, 2.5 / (val * 2.302585) * err) + } else { continue; }; + if mag_err <= 0.0 || mag_err > 1.0 || mag < 5.0 || mag > 25.0 { continue; } + obs.push(Observation { hjd, mag, mag_err, flux: 0.0, flux_err: 0.0 }); + } + finalize_event(obs, event_name, "ogle") +} + +// ---- Enhanced MOA format parser ---- + +/// Parse MOA-II photometry: JD, flux, flux_error columns. +/// Baseline estimated from first and last 20% of time-sorted observations. +fn parse_moa_format(data: &str, event_name: &str) -> MicrolensingEvent { + let mut obs = Vec::new(); + for line in data.lines() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { continue; } + let p: Vec<&str> = line.split_whitespace().collect(); + if p.len() < 3 { continue; } + let (jd, flux, ferr) = match (p[0].parse::(), p[1].parse::(), p[2].parse::()) { + (Ok(t), Ok(f), Ok(e)) => (t, f, e), _ => continue, + }; + if [jd, flux, ferr].iter().any(|x| x.is_nan() || x.is_infinite()) { continue; } + if flux <= 0.0 || ferr <= 0.0 { continue; } + let hjd = if jd > 2400000.0 { jd - 2450000.0 } else { jd }; + // MOA R-band zero-point 22.0 + let mag = -2.5 * flux.log10() + 22.0; + let mag_err = 2.5 / (flux * 2.302585) * ferr; + if mag_err <= 0.0 || mag_err > 1.0 { continue; } + obs.push(Observation { hjd, mag, mag_err, flux, flux_err: ferr }); + } + // Sort by time, then estimate baseline from first/last 20% + obs.sort_by(|a, b| a.hjd.partial_cmp(&b.hjd).unwrap()); + let n = obs.len(); + let wing = (n as f64 * 0.2).ceil() as usize; + if n > 0 && wing > 0 { + let wing_obs: Vec = obs[..wing].iter() + .chain(obs[n.saturating_sub(wing)..].iter()) + .map(|o| o.flux).collect(); + let baseline_flux = wing_obs.iter().sum::() / wing_obs.len() as f64; + if baseline_flux > 0.0 { + for o in &mut obs { + let raw = o.flux; + o.flux = (raw - baseline_flux) / baseline_flux; // fractional deviation + o.flux_err = o.flux_err / baseline_flux; + } + } + } + finalize_event(obs, event_name, "moa") +} + +/// Shared finalization: sort, compute baseline/peak, convert flux where needed. +fn finalize_event(mut obs: Vec, name: &str, source: &str) -> MicrolensingEvent { + obs.sort_by(|a, b| a.hjd.partial_cmp(&b.hjd).unwrap()); + let mut mags: Vec = obs.iter().map(|o| o.mag).collect(); + mags.sort_by(|a, b| a.partial_cmp(b).unwrap()); + let baseline_mag = if !mags.is_empty() { mags[mags.len() * 3 / 4] } else { 20.0 }; + let peak_mag = mags.first().copied().unwrap_or(20.0); + for o in &mut obs { + if o.flux == 0.0 { + o.flux = 10.0f64.powf(-0.4 * (o.mag - baseline_mag)); + o.flux_err = o.flux * 0.4 * 2.302585 * o.mag_err; + } + } + let t0_est = obs.iter().min_by(|a, b| a.mag.partial_cmp(&b.mag).unwrap()) + .map(|o| o.hjd).unwrap_or(0.0); + let half = (baseline_mag + peak_mag) / 2.0; + let above: Vec<_> = obs.iter().filter(|o| o.mag < half).collect(); + let t_e_est = if above.len() >= 2 { + (above.last().unwrap().hjd - above.first().unwrap().hjd) / 2.0 + } else { 20.0 }; + let pf = 10.0f64.powf(-0.4 * (peak_mag - baseline_mag)); + let u0_est = if pf > 1.5 { (1.0 / pf).max(0.01) } else { 0.5 }; + MicrolensingEvent { + name: name.into(), source: source.into(), observations: obs, + baseline_mag, peak_mag, t0_est, t_e_est, u0_est, + } +} + +// ---- PSPL fitting ---- #[derive(Debug, Clone)] struct PSPLFit { t0: f64, u0: f64, t_e: f64, f_s: f64, f_b: f64, chi2: f64 } @@ -422,7 +314,6 @@ fn fit_pspl_linear(obs: &[Observation], t0: f64, u0: f64, t_e: f64) -> Option<(f fn fit_pspl(event: &MicrolensingEvent) -> PSPLFit { let mut best = PSPLFit { t0: 0.0, u0: 0.0, t_e: 0.0, f_s: 1.0, f_b: 0.1, chi2: f64::MAX }; - // Coarse grid for t0_i in 0..20 { let t0 = (event.t0_est - 10.0) + 20.0 * t0_i as f64 / 19.0; for te_i in 0..15 { @@ -436,7 +327,6 @@ fn fit_pspl(event: &MicrolensingEvent) -> PSPLFit { } } } - // Fine refinement around best let b = best.clone(); for dt in -5..=5 { for dte in -5..=5 { for du in -5..=5 { let (t0, t_e, u0) = (b.t0 + dt as f64 * 0.2, b.t_e * (1.0 + dte as f64 * 0.02), b.u0 + du as f64 * 0.005); @@ -448,9 +338,7 @@ fn fit_pspl(event: &MicrolensingEvent) -> PSPLFit { best } -// --------------------------------------------------------------------------- -// Anomaly detection (compact) -// --------------------------------------------------------------------------- +// ---- Anomaly detection ---- fn analyze_residuals(event: &MicrolensingEvent, fit: &PSPLFit) -> Vec<(f64, f64, String, Vec)> { let rchi2 = fit.chi2 / event.observations.len().max(1) as f64; @@ -477,7 +365,6 @@ fn analyze_residuals(event: &MicrolensingEvent, fit: &PSPLFit) -> Vec<(f64, f64, let combined = 0.3 * excess + 0.7 * coherence_z; let class = if combined > 3.0 { if tau.abs() < 0.5 { "planet" } else { "moon-candidate" } } else if combined > 2.0 { "unknown" } else { "noise" }; - // 32-dim embedding let mean_r = resid.iter().sum::() / resid.len() as f64; let var_r = resid.iter().map(|r| (r - mean_r).powi(2)).sum::() / resid.len() as f64; let mut emb = vec![mean_r as f32, var_r.sqrt() as f32, excess as f32, coherence_z as f32, tau as f32]; @@ -494,9 +381,7 @@ fn analyze_residuals(event: &MicrolensingEvent, fit: &PSPLFit) -> Vec<(f64, f64, results } -// --------------------------------------------------------------------------- -// Main pipeline -// --------------------------------------------------------------------------- +// ---- Main pipeline ---- fn main() { println!("=== Real Microlensing Analysis Pipeline ===\n"); @@ -508,33 +393,33 @@ fn main() { dimension: dim as u16, metric: DistanceMetric::Cosine, ..Default::default() }).expect("failed to create store"); - // Step 1: Build manifest and simulate downloads - println!("--- Step 1. Progressive Download Manifest ---\n"); + // Step 1: Build manifest and simulate from published parameters + println!("--- Step 1. Download Manifest ({} real events) ---\n", "13"); let manifest = build_manifest(); let mut rng = 42u64; let mut events = Vec::new(); let mut states = Vec::new(); - for entry in &manifest.events { - let (evt, state) = simulate_from_manifest(entry, &mut rng); - let p = entry.published_params.unwrap(); - let planet_label = if p.has_planet { - match p.planet_mass_ratio { - Some(q) => format!("HAS PLANET (q={:.1e})", q), - None => "HAS PLANET".to_string(), - } - } else { "no planet".to_string() }; - println!(" [{}] {:24} t_E={:5.1}d, u_0={:5.3}, {}", - state, entry.name, p.t_e, p.u0, planet_label); + for entry in &manifest { + let (evt, state) = simulate_from_published(entry, &mut rng); + let p = &entry.published; + let label = match p.planet_mass_ratio { + Some(q) => match p.planet_separation { + Some(s) => format!("PLANET q={:.1e} s={:.2}", q, s), + None => format!("PLANET q={:.1e}", q), + }, + None => "PSPL-only".into(), + }; + println!(" [{}] {:24} t_E={:6.1}d u_0={:5.3} {}", state, entry.event_id, p.einstein_time, p.impact_param, label); events.push(evt); states.push(state); } - // Also parse a synthetic OGLE/MOA data snippet to exercise the parsers + // Exercise the parsers on synthetic snippets let _ogle_test = parse_ogle_format( - "# OGLE-IV EWS photometry\n2453500.0 18.5 0.02\n2453501.0 17.2 0.015\n", "test-ogle"); + "# OGLE-IV EWS\n2453500.0 18.5 0.02\n2453501.0 17.2 0.015\n", "test-ogle"); let _moa_test = parse_moa_format( - "# MOA-II photometry\n2455700.0 500.0 10.0\n2455701.0 800.0 15.0\n", "test-moa"); + "# MOA-II\n2455700.0 500.0 10.0\n2455701.0 800.0 15.0\n2455702.0 520.0 11.0\n", "test-moa"); // Step 2: Fit and analyze println!("\n--- Step 2. PSPL Fit + Anomaly Search ---\n"); @@ -601,7 +486,7 @@ fn main() { println!(" Witness chain: {} entries, {}", verify_witness_chain(&chain).expect("verify").len(), "VALID"); println!("\n=== Summary ===\n"); - println!(" Manifest events: {}", manifest.events.len()); + println!(" Manifest events: {}", manifest.len()); println!(" Simulated: {}", states.iter().filter(|s| matches!(s, DownloadState::Simulated)).count()); println!(" Windows ingested: {}", ingest.accepted); println!(" Anomalies found: {}", discoveries.len());