mirror of
https://github.com/ruvnet/RuVector.git
synced 2026-08-31 02:05:14 +00:00
refactor: trim ADR-040 to 493 lines, enhance real_microlensing adapter
ADR-040: Replace extracted dashboard and microlensing sections with cross-references to ADR-040a and ADR-040b. Condense data model, adapters, and constructs. Core pipeline content preserved. real_microlensing: Add download manifest with 12 real OGLE/MOA events (8 confirmed planets), cross-survey normalization, enhanced MOA parser, simulated download from published parameters. https://claude.ai/code/session_01UWE22wnsZRSHKhT4h4Axby
This commit is contained in:
Claude
Reuven
parent
847ca8fdc9
commit
6295b3fa20
2 changed files with 293 additions and 928 deletions
|
|
@ -25,19 +25,10 @@ retained archetypes.
|
|||
|
||||
### Existing Capabilities (ADR-003 through ADR-008)
|
||||
|
||||
| Component | Package | Relevant APIs |
|
||||
|-----------|---------|---------------|
|
||||
| **RVF segments** | `@ruvector/rvf`, `@ruvector/rvf-node` | `embedKernel`, `extractKernel`, `embedEbpf`, `segments`, `derive` |
|
||||
| **HNSW indexing** | `@ruvector/rvf-node` | `ingestBatch`, `query`, `compact`, HNSW with metadata filters |
|
||||
| **Witness chains** | `@ruvector/rvf-node`, `RvfSolver` | `verifyWitness`, SHAKE-256 witness chains, signed root hash |
|
||||
| **Graph transactions** | `NativeAccelerator` | `graphTransaction`, `graphBatchInsert`, Cypher queries |
|
||||
| **SIMD embeddings** | `@ruvector/ruvllm` | 768-dim SIMD embed, cosine/dot/L2, HNSW memory search |
|
||||
| **SONA learning** | `SonaLearningBackend` | Micro-LoRA, trajectory recording, EWC++ |
|
||||
| **Federated coordination** | `FederatedSessionManager` | Cross-agent trajectories, warm-start patterns |
|
||||
| **Contrastive training** | `ContrastiveTrainer` | InfoNCE, hard negative mining, 3-stage curriculum |
|
||||
| **Adaptive index** | `AdaptiveIndexTuner` | 5-tier compression, Matryoshka truncation, health monitoring |
|
||||
| **Kernel embedding** | `KernelBuilder` (ADR-008) | Minimal Linux boot from KERNEL_SEG + INITRD_SEG |
|
||||
| **Lazy model download** | `ChatInference` (ADR-008) | Deferred GGUF load on first inference call |
|
||||
RVF segments, HNSW indexing, SHAKE-256 witness chains, graph transactions,
|
||||
768-dim SIMD embeddings, SONA learning, federated coordination, contrastive
|
||||
training, adaptive index tuning, kernel embedding (ADR-008), and lazy model
|
||||
download (ADR-008). See ADR-003 through ADR-008 for full API details.
|
||||
|
||||
### What This ADR Adds
|
||||
|
||||
|
|
@ -119,26 +110,7 @@ Following the lazy-download pattern established in ADR-008 for GGUF models:
|
|||
INDEX_SEG, a new witness root is committed, and the RVF is sealed into
|
||||
a reproducible snapshot
|
||||
|
||||
```
|
||||
Download state machine:
|
||||
|
||||
[boot] ──first-inference──> [downloading-tier-1]
|
||||
│ │
|
||||
│ (offline demo works) │ (progress: 0-100%)
|
||||
│ │
|
||||
▼ ▼
|
||||
[tier-0-only] [tier-1-ready]
|
||||
│
|
||||
rvf ingest --expand
|
||||
│
|
||||
▼
|
||||
[tier-2-ready]
|
||||
│
|
||||
life pipeline activated
|
||||
│
|
||||
▼
|
||||
[tier-3-ready] ──seal──> [sealed-snapshot]
|
||||
```
|
||||
**State machine**: `[boot]` -> `[tier-0-only]` (offline demo) -> `[tier-1-ready]` (first inference) -> `[tier-2-ready]` (`rvf ingest --expand`) -> `[tier-3-ready]` (life pipeline) -> `[sealed-snapshot]`.
|
||||
|
||||
Each tier download:
|
||||
- Resumes from last byte on interruption (HTTP Range headers)
|
||||
|
|
@ -168,197 +140,50 @@ Extends the ADR-003 segment model with domain-specific segments.
|
|||
|
||||
### Core Entities
|
||||
|
||||
```typescript
|
||||
interface Event {
|
||||
id: string;
|
||||
t_start: number; // epoch seconds
|
||||
t_end: number;
|
||||
domain: 'kepler' | 'tess' | 'jwst' | 'sdss' | 'derived';
|
||||
payload_hash: string; // SHA-256 of raw data window
|
||||
provenance: Provenance;
|
||||
}
|
||||
|
||||
interface Observation {
|
||||
id: string;
|
||||
instrument: string; // 'kepler-lc' | 'tess-ffi' | 'jwst-nirspec' | ...
|
||||
target_id: string; // e.g., KIC or TIC identifier
|
||||
data_pointer: string; // segment offset into VEC_SEG
|
||||
calibration_version: string;
|
||||
provenance: Provenance;
|
||||
}
|
||||
|
||||
interface InteractionEdge {
|
||||
src_event_id: string;
|
||||
dst_event_id: string;
|
||||
type: 'causal' | 'periodicity' | 'shape_similarity' | 'co_occurrence' | 'spatial';
|
||||
weight: number;
|
||||
lag: number; // temporal lag in seconds
|
||||
confidence: number;
|
||||
provenance: Provenance;
|
||||
}
|
||||
|
||||
interface Boundary {
|
||||
boundary_id: string;
|
||||
partition_left_set_hash: string;
|
||||
partition_right_set_hash: string;
|
||||
cut_weight: number;
|
||||
cut_witness: string; // witness chain reference
|
||||
stability_score: number;
|
||||
}
|
||||
|
||||
interface Candidate {
|
||||
candidate_id: string;
|
||||
category: 'planet' | 'life';
|
||||
evidence_pointers: string[]; // event and edge IDs
|
||||
score: number;
|
||||
uncertainty: number;
|
||||
publishable: boolean; // based on POLICY_SEG rules
|
||||
witness_trace: string; // WITNESS_SEG reference for replay
|
||||
}
|
||||
|
||||
interface Provenance {
|
||||
source: string; // 'mast-kepler' | 'mast-tess' | 'mast-jwst' | ...
|
||||
download_witness: string; // witness chain entry for the download
|
||||
transform_chain: string[]; // ordered list of transform IDs applied
|
||||
timestamp: string; // ISO-8601
|
||||
}
|
||||
```
|
||||
| Entity | Key Fields | Description |
|
||||
|--------|-----------|-------------|
|
||||
| **Event** | `id`, `t_start`, `t_end`, `domain`, `payload_hash`, `provenance` | Time-windowed observation or derived result. Domain: kepler, tess, jwst, sdss, derived |
|
||||
| **Observation** | `id`, `instrument`, `target_id`, `data_pointer`, `calibration_version` | Raw instrument measurement with VEC_SEG offset |
|
||||
| **InteractionEdge** | `src_event_id`, `dst_event_id`, `type`, `weight`, `lag`, `confidence` | Typed relationship: causal, periodicity, shape_similarity, co_occurrence, spatial |
|
||||
| **Boundary** | `boundary_id`, `partition_*_hash`, `cut_weight`, `cut_witness`, `stability_score` | Graph partition boundary with witness chain reference |
|
||||
| **Candidate** | `candidate_id`, `category`, `evidence_pointers[]`, `score`, `uncertainty`, `publishable`, `witness_trace` | Planet or life candidate with POLICY_SEG-gated publishability |
|
||||
| **Provenance** | `source`, `download_witness`, `transform_chain[]`, `timestamp` | Full lineage from download through every transform |
|
||||
|
||||
### Domain Adapters
|
||||
|
||||
#### Planet Transit Adapter
|
||||
|
||||
```
|
||||
Input: flux time series + cadence metadata (Kepler/TESS FITS)
|
||||
Output: Event nodes for windows
|
||||
InteractionEdges for periodicity hints and shape similarity
|
||||
Candidate nodes for dip detections
|
||||
```
|
||||
|
||||
#### Spectrum Adapter
|
||||
|
||||
```
|
||||
Input: wavelength, flux, error arrays (JWST NIRSpec, etc.)
|
||||
Output: Event nodes for band windows
|
||||
InteractionEdges for molecule feature co-occurrence
|
||||
Disequilibrium score components
|
||||
```
|
||||
|
||||
#### Cosmic Web Adapter (optional, Phase 2+)
|
||||
|
||||
```
|
||||
Input: galaxy positions and redshifts (SDSS)
|
||||
Output: Graph of spatial adjacency and filament membership
|
||||
```
|
||||
| Adapter | Input | Output |
|
||||
|---------|-------|--------|
|
||||
| **Planet Transit** | Flux time series + cadence metadata (Kepler/TESS FITS) | Event nodes, periodicity/shape InteractionEdges, dip Candidates |
|
||||
| **Spectrum** | Wavelength, flux, error arrays (JWST NIRSpec, etc.) | Band Event nodes, molecule co-occurrence edges, disequilibrium scores |
|
||||
| **Cosmic Web** (Phase 2+) | Galaxy positions and redshifts (SDSS) | Spatial adjacency graph with filament membership |
|
||||
|
||||
## The Four System Constructs
|
||||
|
||||
### 1. Compressed Causal Atlas
|
||||
|
||||
**Definition**: A partial order of events plus minimal sufficient descriptors
|
||||
to reproduce derived edges.
|
||||
Partial order of events plus minimal sufficient descriptors to reproduce derived edges.
|
||||
|
||||
**Construction**:
|
||||
**Construction**: (1) Window light curves at scales 2h/12h/3d/27d. (2) Extract features: flux derivatives, autocorrelation peaks, wavelet energy, matched filter response. (3) Embed via RuVector SIMD into VEC_SEG. (4) Add causal edges where window A precedes B and improves predictability (prediction gain, POLICY_SEG constrained). (5) Compress: top-k parents per node, retain boundary witnesses, delta-encode into DELTA_SEG.
|
||||
|
||||
1. **Windowing** — Light curves into overlapping windows at multiple scales
|
||||
- Scales: 2 hours, 12 hours, 3 days, 27 days
|
||||
|
||||
2. **Feature extraction** — Robust features per window
|
||||
- Flux derivative statistics
|
||||
- Autocorrelation peaks
|
||||
- Wavelet energy bands
|
||||
- Transit-shaped matched filter response
|
||||
|
||||
3. **Embedding** — RuVector SIMD embed per window, stored in VEC_SEG
|
||||
|
||||
4. **Causal edges** — Add edge when window A precedes window B and improves
|
||||
predictability of B (conditional mutual information proxy or prediction gain,
|
||||
subject to POLICY_SEG constraints)
|
||||
- Edge weight: prediction gain magnitude
|
||||
- Provenance: exact windows, transform IDs, threshold used
|
||||
|
||||
5. **Atlas compression**
|
||||
- Keep only top-k causal parents per node
|
||||
- Retain stable boundary witnesses
|
||||
- Delta-encode updates into DELTA_SEG
|
||||
|
||||
**Output API**:
|
||||
|
||||
| Endpoint | Returns |
|
||||
|----------|---------|
|
||||
| `atlas.query(event_id)` | Parents, children, plus provenance |
|
||||
| `atlas.trace(candidate_id)` | Minimal causal chain for a candidate |
|
||||
**API**: `atlas.query(event_id)` returns parents/children with provenance. `atlas.trace(candidate_id)` returns minimal causal chain.
|
||||
|
||||
### 2. Coherence Field Index
|
||||
|
||||
**Definition**: A field over the atlas graph that assigns coherence pressure
|
||||
and cut stability over time.
|
||||
Field over the atlas graph assigning coherence pressure and cut stability over time. Signals: cut pressure (min-cut values), partition entropy (cluster size distribution), disagreement (cross-detector rate), drift (embedding distribution shift).
|
||||
|
||||
**Signals**:
|
||||
**Algorithm**: Maintain partition tree with dynamic min-cut on incremental changes. Each epoch: compute cut witnesses for top boundaries, emit to GRAPH_SEG, append to WITNESS_SEG. Index boundaries by descriptor: cut value, partition sizes, curvature proxy, churn.
|
||||
|
||||
| Signal | Description |
|
||||
|--------|-------------|
|
||||
| Cut pressure | Minimum cut values over selected subgraphs |
|
||||
| Partition entropy | Distribution of cluster sizes and churn rate |
|
||||
| Disagreement | Cross-detector disagreement rate |
|
||||
| Drift | Embedding distribution shift in sliding window |
|
||||
|
||||
**Algorithm**:
|
||||
|
||||
1. Maintain a partition tree. Update with dynamic min-cut on incremental
|
||||
graph changes
|
||||
2. For each update epoch:
|
||||
- Compute cut witnesses for top boundaries
|
||||
- Emit boundary events into GRAPH_SEG
|
||||
- Append witness record into WITNESS_SEG
|
||||
3. Index boundaries via descriptor vector:
|
||||
- Cut value, partition sizes, local graph curvature proxy, recent churn
|
||||
|
||||
**Query API**:
|
||||
|
||||
| Endpoint | Returns |
|
||||
|----------|---------|
|
||||
| `coherence.get(target_id, epoch)` | Field values for target at epoch |
|
||||
| `boundary.nearest(descriptor)` | Similar historical boundary states via INDEX_SEG |
|
||||
**API**: `coherence.get(target_id, epoch)` returns field values. `boundary.nearest(descriptor)` returns similar historical states via INDEX_SEG.
|
||||
|
||||
### 3. Multi-Scale Interaction Memory
|
||||
|
||||
**Definition**: A memory that retains interactions at multiple time resolutions
|
||||
with strict budget control.
|
||||
|
||||
**Three tiers**:
|
||||
|
||||
| Tier | Resolution | Content |
|
||||
|------|-----------|---------|
|
||||
| **S** | Seconds to minutes | High-fidelity deltas |
|
||||
| **M** | Hours to days | Aggregated deltas |
|
||||
| **L** | Weeks to months | Boundary summaries and archetypes |
|
||||
|
||||
**Retention rules**:
|
||||
1. Preserve events that are boundary-critical
|
||||
2. Preserve events that are candidate evidence
|
||||
3. Compress everything else via archetype clustering in INDEX_SEG
|
||||
|
||||
**Mechanism**:
|
||||
- DELTA_SEG is append-only
|
||||
- Periodic compaction produces a new RVF root with a witness proof of
|
||||
preservation rules applied
|
||||
Tiered retention with strict budget control: **S** (seconds-minutes, high-fidelity deltas), **M** (hours-days, aggregated), **L** (weeks-months, boundary summaries and archetypes). Retention preserves boundary-critical events and candidate evidence; compresses the rest via archetype clustering. DELTA_SEG is append-only; periodic compaction produces a new RVF root with witness proof.
|
||||
|
||||
### 4. Boundary Evolution Tracker
|
||||
|
||||
**Definition**: A tracker that treats boundaries as primary objects that evolve
|
||||
over time.
|
||||
Treats boundaries as primary objects evolving over time -- the holographic design principle. Boundaries are stored and indexed as first-class objects; interior state is reconstructable from boundary witnesses and retained archetypes.
|
||||
|
||||
**This is where the holographic flavor is implemented.** You primarily store
|
||||
and index boundaries, and treat interior state as reconstructable from boundary
|
||||
witnesses and retained archetypes.
|
||||
|
||||
**Output API**:
|
||||
|
||||
| Endpoint | Returns |
|
||||
|----------|---------|
|
||||
| `boundary.timeline(target_id)` | Boundary evolution over time |
|
||||
| `boundary.alerts` | Alerts when: cut pressure spikes, boundary identity flips, disagreement exceeds threshold, drift persists beyond policy |
|
||||
**API**: `boundary.timeline(target_id)` returns boundary evolution. `boundary.alerts` fires on cut pressure spikes, boundary identity flips, disagreement threshold breaches, or persistent drift.
|
||||
|
||||
## Planet Detection Pipeline
|
||||
|
||||
|
|
@ -443,139 +268,9 @@ thermodynamic relaxation.
|
|||
**Output**: Life candidate list with explicit uncertainty + required follow-up
|
||||
observations list generated by POLICY_SEG rules.
|
||||
|
||||
## Microlensing Detection Pipeline (M0-M3)
|
||||
### Microlensing & Cross-Domain Extensions
|
||||
|
||||
Extends the transit pipeline to gravitational microlensing events for rogue
|
||||
planet and exomoon candidate detection.
|
||||
|
||||
### Additional Data Sources
|
||||
|
||||
| Source | Access | Reference |
|
||||
|--------|--------|-----------|
|
||||
| OGLE-IV microlensing events | Public FTP | [ogle.astrouw.edu.pl](https://ogle.astrouw.edu.pl/ogle4/ews/ews.html) |
|
||||
| MOA-II microlensing alerts | Public archive | [www.massey.ac.nz/~iabond/moa](https://www.massey.ac.nz/~iabond/moa/) |
|
||||
| Gaia astrometric catalog | ESA public | [gea.esac.esa.int](https://gea.esac.esa.int/archive/) |
|
||||
| Roman Space Telescope | Upcoming | High-cadence microlensing survey |
|
||||
|
||||
### Stage M0: Ingest
|
||||
|
||||
Microlensing light curve ingestion from OGLE/MOA archives.
|
||||
- Normalize photometry across surveys (MOA-II: ~15 min cadence, 0.008σ floor; OGLE-IV: ~40 min cadence, 0.005σ floor)
|
||||
- Segment into event windows around magnification peaks
|
||||
- Store as RVF Event nodes with survey-specific metadata
|
||||
|
||||
### Stage M1: Single-Lens Detection
|
||||
|
||||
Standard Paczynski curve fitting for point-source point-lens (PSPL) events.
|
||||
- Two-phase fit: coarse grid search + fine refinement
|
||||
- Linear regression for source flux (F_s) and blending flux (F_b)
|
||||
- Parameter estimation: Einstein crossing time (t_E), impact parameter (u_0), peak time (t_0)
|
||||
- Create Candidate nodes for events exceeding SNR threshold
|
||||
|
||||
### Stage M2: Anomaly Detection via MRF/Mincut
|
||||
|
||||
Residual analysis after best-fit PSPL subtraction using Markov Random Field
|
||||
optimization with graph cut inference.
|
||||
|
||||
**Three-statistic lambda computation:**
|
||||
1. Excess chi2: window fit quality relative to global reduced chi2
|
||||
2. Runs test: temporal coherence of residual sign changes
|
||||
3. Gaussian bump fit: localized perturbation chi2 improvement
|
||||
|
||||
**Differential normalization:** Compare each window's signal to tau-space
|
||||
neighbors, producing z-scores that are ~0 for uniform fit quality and positive
|
||||
only for localized anomalies.
|
||||
|
||||
**Graph construction:**
|
||||
- Temporal chain: consecutive time windows (alpha weight)
|
||||
- RuVector kNN edges: embedding similarity from residual features (beta weight)
|
||||
- Edmonds-Karp BFS max-flow solver for s-t mincut
|
||||
|
||||
### Stage M3: Coherence Gating
|
||||
|
||||
Dynamic mincut separates competing explanations (noise vs planet vs moon vs
|
||||
binary lens) using support region analysis.
|
||||
|
||||
| Component | Role |
|
||||
|-----------|------|
|
||||
| RuVector embeddings | Cross-survey memory; retrieve similar anomalies |
|
||||
| HNSW index | Fast similarity search across microlensing events |
|
||||
| Dynamic mincut | Separate competing explanations |
|
||||
| Witness chain | Full provenance from raw photometry to candidate |
|
||||
|
||||
**Important constraint:** Dynamic mincut provides coherent support extraction
|
||||
but cannot replace lens modeling. The physics solver (PSPL/binary) provides
|
||||
local evidence; mincut provides spatial coherence.
|
||||
|
||||
## Cross-Domain Graph-Cut Applications
|
||||
|
||||
The MRF/mincut + RuVector architecture generalizes beyond astronomy to any
|
||||
domain requiring coherent anomaly detection in structured data.
|
||||
|
||||
### Implemented Verticals
|
||||
|
||||
| Vertical | Example | Graph Topology | Anomaly Types |
|
||||
|----------|---------|----------------|---------------|
|
||||
| **Medical Imaging** | `medical_graphcut.rs` | 4-connected spatial grid, gradient-weighted edges | Lesion segmentation (T1-MRI, T2-MRI, CT) |
|
||||
| **Genomics** | `genomic_graphcut.rs` | Linear chain + kNN similarity | CNV gains/losses, LOH, cancer drivers (TP53, BRCA1, EGFR, MYC) |
|
||||
| **Financial Fraud** | `financial_fraud_graphcut.rs` | Temporal chain + merchant edges + kNN | Card-not-present, account takeover, card clone, synthetic, refund |
|
||||
| **Supply Chain** | `supply_chain_graphcut.rs` | Tier chain + geographic + kNN | Quality defect, shortage, price anomaly, delay, counterfeit, demand shock |
|
||||
| **Cybersecurity** | `cyber_threat_graphcut.rs` | Source/destination chain + kNN | Port scan, brute force, exfiltration, C2 beacon, DDoS, lateral movement |
|
||||
| **Climate** | `climate_graphcut.rs` | Spatial adjacency + gradient + kNN | Heat wave, pollution spike, drought, ocean warming, cold snap, sensor fault |
|
||||
|
||||
### Common Architecture
|
||||
|
||||
All verticals share:
|
||||
1. **Domain-specific data generator** with realistic parameters from published datasets
|
||||
2. **Feature extraction** producing 32-dim embeddings for RuVector storage
|
||||
3. **Graph construction** combining domain topology (spatial/temporal/chain) with kNN similarity edges
|
||||
4. **Edmonds-Karp BFS** s-t mincut solver (identical implementation)
|
||||
5. **RVF integration**: witness chains, filtered metadata queries, lineage derivation
|
||||
6. **Evaluation**: comparison against threshold baseline showing graph-cut improvement
|
||||
|
||||
### Additional Real Data Sources
|
||||
|
||||
| Domain | Source | Access |
|
||||
|--------|--------|--------|
|
||||
| Genomics | TCGA (The Cancer Genome Atlas) | [portal.gdc.cancer.gov](https://portal.gdc.cancer.gov/) |
|
||||
| Genomics | ClinVar variants | [ncbi.nlm.nih.gov/clinvar](https://www.ncbi.nlm.nih.gov/clinvar/) |
|
||||
| Genomics | COSMIC (somatic mutations) | [cancer.sanger.ac.uk/cosmic](https://cancer.sanger.ac.uk/cosmic) |
|
||||
| Medical | LIDC-IDRI (lung CT) | [cancerimagingarchive.net](https://www.cancerimagingarchive.net/) |
|
||||
| Medical | BraTS (brain tumors) | [synapse.org](https://www.synapse.org/) |
|
||||
| Cybersecurity | CICIDS2017 | [unb.ca/cic](https://www.unb.ca/cic/datasets/ids-2017.html) |
|
||||
| Climate | NOAA GHCN | [ncei.noaa.gov](https://www.ncei.noaa.gov/) |
|
||||
| Climate | EPA AQI | [epa.gov/aqs](https://www.epa.gov/aqs) |
|
||||
|
||||
## Measured Results
|
||||
|
||||
Results from running examples with `cargo run --example <name> --release`.
|
||||
|
||||
### Astronomy
|
||||
|
||||
| Pipeline | Metric | Value | Notes |
|
||||
|----------|--------|-------|-------|
|
||||
| `planet_detection` | Recall@10 | 8/10 confirmed | Synthetic Kepler-like targets |
|
||||
| `exomoon_graphcut` | Precision | 0.25 | Chang-Refsdal perturbative limit |
|
||||
| `exomoon_graphcut` | Recall | 0.25 | Limited by ~2-3σ per-window SNR |
|
||||
| `exomoon_graphcut` | F1 | 0.25 | 30 events, 12 with moons |
|
||||
| `real_microlensing` | Planets rediscovered | 2/4 | OGLE-2005-BLG-390, OGLE-2016-BLG-1195 |
|
||||
| `microlensing_detection` | Events processed | 20 | Synthetic PSPL + anomalies |
|
||||
|
||||
### Medical Imaging
|
||||
|
||||
| Modality | Graph Cut Dice | Threshold Dice | Improvement |
|
||||
|----------|---------------|----------------|-------------|
|
||||
| T1-MRI | 0.44-0.59 | 0.32-0.46 | +27-39% |
|
||||
| T2-MRI | 0.50-0.62 | 0.38-0.48 | +29-32% |
|
||||
| CT | 0.48-0.58 | 0.35-0.44 | +32-37% |
|
||||
|
||||
### Genomics
|
||||
|
||||
| Platform | Sensitivity | Specificity | Drivers Found |
|
||||
|----------|-------------|-------------|---------------|
|
||||
| WGS (30x) | 0.91-0.95 | 0.66-0.97 | 4/4 (TP53, BRCA1, EGFR, MYC) |
|
||||
| WES (100x) | 0.88-0.93 | 0.70-0.95 | 4/4 |
|
||||
| Panel (500x) | 0.93-0.97 | 0.72-0.98 | 4/4 |
|
||||
> See [ADR-040b: Microlensing & Graph-Cut Extensions](ADR-040b-microlensing-graphcut-extensions.md) for M0-M3 pipeline, cross-domain applications, measured results, and Rust crate structure.
|
||||
|
||||
## Runtime and Portability
|
||||
|
||||
|
|
@ -630,167 +325,9 @@ WS /ws/live # real-time boundary alerts, pipeline pr
|
|||
3. No remote writes without explicit policy toggle in POLICY_SEG
|
||||
4. Downloaded data verified against MANIFEST_SEG hashes before ingestion
|
||||
|
||||
## Three.js Visualization Dashboard
|
||||
### Dashboard Architecture
|
||||
|
||||
The RVF embeds a Vite-bundled Three.js dashboard in `DASHBOARD_SEG`. The
|
||||
runtime HTTP server serves it at `/` (root). All visualizations are driven
|
||||
by the same API endpoints the CLI uses, so every rendered frame corresponds
|
||||
to queryable, witness-backed data.
|
||||
|
||||
### Architecture
|
||||
|
||||
```
|
||||
DASHBOARD_SEG (inside RVF)
|
||||
dist/
|
||||
index.html # Vite SPA entry
|
||||
assets/
|
||||
main.[hash].js # Three.js + D3 + app logic (tree-shaken)
|
||||
main.[hash].css # Tailwind/minimal styles
|
||||
worker.js # Web Worker for graph layout
|
||||
|
||||
Runtime serves:
|
||||
GET / -> DASHBOARD_SEG/dist/index.html
|
||||
GET /assets/* -> DASHBOARD_SEG/dist/assets/*
|
||||
GET /api/* -> JSON API (atlas, coherence, candidates, etc.)
|
||||
WS /ws/live -> Live streaming of boundary alerts and pipeline progress
|
||||
```
|
||||
|
||||
**Build pipeline**: Vite builds the dashboard at package time into a single
|
||||
tree-shaken bundle. The bundle is embedded into `DASHBOARD_SEG` during RVF
|
||||
assembly. No Node.js required at runtime — the dashboard is pure static
|
||||
assets served by the existing HTTP server.
|
||||
|
||||
### Dashboard Views
|
||||
|
||||
#### V1: Causal Atlas Explorer (Three.js 3D)
|
||||
|
||||
Interactive 3D force-directed graph of the causal atlas.
|
||||
|
||||
| Feature | Implementation |
|
||||
|---------|---------------|
|
||||
| **Node rendering** | `THREE.InstancedMesh` for events — color by domain (Kepler=blue, TESS=cyan, JWST=gold, derived=white) |
|
||||
| **Edge rendering** | `THREE.LineSegments` with opacity mapped to edge weight |
|
||||
| **Causal flow** | Animated particles along causal edges showing temporal direction |
|
||||
| **Scale selector** | Toggle between window scales (2h, 12h, 3d, 27d) — re-layouts graph |
|
||||
| **Candidate highlight** | Click candidate in sidebar to trace its causal chain in 3D, dimming unrelated nodes |
|
||||
| **Witness replay** | Step through witness chain entries, animating graph state forward/backward |
|
||||
| **LOD** | Level-of-detail: far=boundary nodes only, mid=top-k events, close=full subgraph |
|
||||
|
||||
Data source: `GET /api/atlas/query`, `GET /api/atlas/trace`
|
||||
|
||||
#### V2: Coherence Field Heatmap (Three.js + shader)
|
||||
|
||||
Real-time coherence field rendered as a colored surface over the atlas graph.
|
||||
|
||||
| Feature | Implementation |
|
||||
|---------|---------------|
|
||||
| **Field surface** | `THREE.PlaneGeometry` subdivided grid, vertex colors from coherence values |
|
||||
| **Cut pressure** | Red hotspots where cut pressure is high, cool blue where stable |
|
||||
| **Partition boundaries** | Glowing wireframe lines at partition cuts |
|
||||
| **Time scrubber** | Scrub through epochs to see coherence evolution |
|
||||
| **Drift overlay** | Toggle to show embedding drift as animated vector arrows |
|
||||
| **Alert markers** | Pulsing icons at boundary alert locations |
|
||||
|
||||
Data source: `GET /api/coherence`, `GET /api/boundary/timeline`, `WS /ws/live`
|
||||
|
||||
#### V3: Planet Candidate Dashboard (2D panels + 3D orbit)
|
||||
|
||||
Split view combining data panels with 3D orbital visualization.
|
||||
|
||||
| Panel | Content |
|
||||
|-------|---------|
|
||||
| **Ranked list** | Sortable table: candidate ID, score, uncertainty, period, SNR, publishable status |
|
||||
| **Light curve viewer** | Interactive D3 chart: raw flux, detrended flux, transit model overlay, per-window score |
|
||||
| **Phase-folded plot** | All transits folded at detected period, with confidence band |
|
||||
| **3D orbit preview** | `THREE.Line` showing inferred orbital path around host star, sized by uncertainty |
|
||||
| **Evidence trace** | Expandable tree showing witness chain from raw data to final score |
|
||||
| **Score breakdown** | Radar chart: SNR, shape consistency, period stability, coherence stability |
|
||||
|
||||
Data source: `GET /api/candidates/planet`, `GET /api/candidates/:id/trace`
|
||||
|
||||
#### V4: Life Candidate Dashboard (2D panels + 3D molecule)
|
||||
|
||||
Split view for spectral disequilibrium analysis.
|
||||
|
||||
| Panel | Content |
|
||||
|-------|---------|
|
||||
| **Ranked list** | Sortable table: candidate ID, disequilibrium score, uncertainty, molecule flags, publishable |
|
||||
| **Spectrum viewer** | Interactive D3 chart: wavelength vs flux, molecule absorption bands highlighted |
|
||||
| **Molecule presence matrix** | Heatmap of detected molecule families vs confidence |
|
||||
| **3D molecule overlay** | `THREE.Sprite` labels at absorption wavelengths in a 3D wavelength space |
|
||||
| **Reaction graph** | Force-directed graph of molecule co-occurrences vs equilibrium expectations |
|
||||
| **Confound panel** | Bar chart: stellar activity penalty, contamination risk, repeatability score |
|
||||
|
||||
Data source: `GET /api/candidates/life`, `GET /api/candidates/:id/trace`
|
||||
|
||||
#### V5: System Status Dashboard
|
||||
|
||||
Operational health and download progress.
|
||||
|
||||
| Panel | Content |
|
||||
|-------|---------|
|
||||
| **Download progress** | Per-tier progress bars with byte counts and ETA |
|
||||
| **Segment sizes** | Stacked bar chart of RVF segment utilization |
|
||||
| **Memory tiers** | S/M/L tier fill levels and compaction history |
|
||||
| **Witness chain** | Scrolling log of recent witness entries with hash preview |
|
||||
| **Pipeline status** | P0/P1/P2 and L0/L1/L2 stage indicators with event counts |
|
||||
| **Performance** | Query latency histogram, events/second throughput |
|
||||
|
||||
Data source: `GET /api/status`, `GET /api/memory/tiers`, `WS /ws/live`
|
||||
|
||||
### WebSocket Live Stream
|
||||
|
||||
```typescript
|
||||
// WS /ws/live — server pushes events as they happen
|
||||
interface LiveEvent {
|
||||
type: 'boundary_alert' | 'candidate_new' | 'candidate_update' |
|
||||
'download_progress' | 'witness_commit' | 'pipeline_stage' |
|
||||
'coherence_update';
|
||||
timestamp: string;
|
||||
data: Record<string, unknown>;
|
||||
}
|
||||
```
|
||||
|
||||
The dashboard subscribes on connect and updates all views in real-time as
|
||||
pipelines process data and boundaries evolve.
|
||||
|
||||
### Vite Build Configuration
|
||||
|
||||
```typescript
|
||||
// vite.config.ts for dashboard build
|
||||
import { defineConfig } from 'vite';
|
||||
|
||||
export default defineConfig({
|
||||
build: {
|
||||
outDir: 'dist/dashboard',
|
||||
assetsDir: 'assets',
|
||||
rollupOptions: {
|
||||
output: {
|
||||
manualChunks: {
|
||||
three: ['three'], // ~150 KB gzipped
|
||||
d3: ['d3-scale', 'd3-axis', 'd3-shape', 'd3-selection'],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
**Bundle budget**: < 500 KB gzipped total (Three.js ~150 KB, D3 subset ~30 KB,
|
||||
app logic ~50 KB, styles ~10 KB). The dashboard adds minimal overhead to the
|
||||
RVF artifact.
|
||||
|
||||
### Design Decision: D5 — Dashboard Embedded in RVF
|
||||
|
||||
The Three.js dashboard is bundled at build time and embedded in `DASHBOARD_SEG`
|
||||
rather than served from an external CDN or requiring a separate install. This
|
||||
ensures:
|
||||
|
||||
1. **Fully offline**: Works without network after boot
|
||||
2. **Version-locked**: Dashboard always matches the API version it queries
|
||||
3. **Single artifact**: One RVF file = runtime + data + visualization
|
||||
4. **Witness-aligned**: Dashboard renders exactly the data the witness chain
|
||||
can verify
|
||||
> See [ADR-040a: Planet Detection Dashboard](ADR-040a-planet-detection-dashboard.md) for Views V1-V5, WebSocket streaming, Vite build configuration, and design decision D5.
|
||||
|
||||
## Package Structure
|
||||
|
||||
|
|
@ -816,37 +353,7 @@ packages/agentdb-causal-atlas/
|
|||
ProgressiveDownloader.ts # Tiered lazy download with resume
|
||||
DataManifest.ts # URL + hash + size manifests
|
||||
KernelBuilder.ts # Reuse/extend from ADR-008
|
||||
dashboard/ # Vite + Three.js visualization app
|
||||
vite.config.ts # Build config — outputs to dist/dashboard/
|
||||
index.html # SPA entry point
|
||||
src/
|
||||
main.ts # App bootstrap, router, WS connection
|
||||
api.ts # Typed fetch wrappers for /api/* endpoints
|
||||
ws.ts # WebSocket client for /ws/live
|
||||
views/
|
||||
AtlasExplorer.ts # V1: 3D causal atlas (Three.js force graph)
|
||||
CoherenceHeatmap.ts # V2: Coherence field surface + cut pressure
|
||||
PlanetDashboard.ts # V3: Planet candidates + light curves + 3D orbit
|
||||
LifeDashboard.ts # V4: Life candidates + spectra + molecule graph
|
||||
StatusDashboard.ts # V5: System health, downloads, witness log
|
||||
three/
|
||||
AtlasGraph.ts # InstancedMesh nodes, LineSegments edges, particles
|
||||
CoherenceSurface.ts # PlaneGeometry with vertex-colored field
|
||||
OrbitPreview.ts # Orbital path visualization
|
||||
CausalFlow.ts # Animated particles along causal edges
|
||||
LODController.ts # Level-of-detail: boundary → top-k → full
|
||||
charts/
|
||||
LightCurveChart.ts # D3 flux time series with transit overlay
|
||||
SpectrumChart.ts # D3 wavelength vs flux with molecule bands
|
||||
RadarChart.ts # Score breakdown radar
|
||||
MoleculeMatrix.ts # Heatmap of molecule presence vs confidence
|
||||
components/
|
||||
Sidebar.ts # Candidate list, filters, search
|
||||
TimeScrubber.ts # Epoch scrubber for coherence replay
|
||||
WitnessLog.ts # Scrolling witness chain entries
|
||||
DownloadProgress.ts # Tier progress bars
|
||||
styles/
|
||||
main.css # Minimal Tailwind or hand-rolled styles
|
||||
dashboard/ # See ADR-040a for full dashboard tree
|
||||
tests/
|
||||
causal-atlas.test.ts
|
||||
planet-detection.test.ts
|
||||
|
|
@ -854,28 +361,11 @@ packages/agentdb-causal-atlas/
|
|||
progressive-download.test.ts
|
||||
coherence-field.test.ts
|
||||
boundary-tracker.test.ts
|
||||
dashboard.test.ts # Dashboard build + API integration tests
|
||||
```
|
||||
|
||||
### Rust Implementation
|
||||
|
||||
The pipeline examples are implemented in Rust using the RVF runtime crates:
|
||||
|
||||
| Crate | Role |
|
||||
|-------|------|
|
||||
| `rvf-types` | Core types, segment definitions, derivation types |
|
||||
| `rvf-runtime` | RvfStore, HNSW indexing, metadata filters, queries |
|
||||
| `rvf-crypto` | SHAKE-256 witness chains, verification |
|
||||
| `rvf-wire` | Wire format serialization |
|
||||
| `rvf-manifest` | Segment manifests and policies |
|
||||
| `rvf-index` | Progressive HNSW index construction |
|
||||
| `rvf-quant` | Vector quantization (PQ, SQ) |
|
||||
| `rvf-kernel` | Kernel/initrd segment embedding |
|
||||
| `rvf-launch` | Boot sequence and runtime launch |
|
||||
| `rvf-ebpf` | eBPF socket/syscall policies |
|
||||
| `rvf-server` | HTTP/WS server for dashboard and API |
|
||||
|
||||
Examples location: `examples/rvf/examples/`
|
||||
> See [ADR-040b](ADR-040b-microlensing-graphcut-extensions.md#rust-implementation) for the full Rust crate table. Examples location: `examples/rvf/examples/`
|
||||
|
||||
## Implementation Phases
|
||||
|
||||
|
|
@ -883,55 +373,45 @@ Examples location: `examples/rvf/examples/`
|
|||
|
||||
**Scope**: Kepler and TESS only. No spectra. No life scoring.
|
||||
|
||||
1. Implement `ProgressiveDownloader` with tier-0 curated dataset (100 Kepler targets)
|
||||
2. Implement `PlanetTransitAdapter` for FITS light curve ingestion
|
||||
3. Implement `CausalAtlas` with windowing, feature extraction, SIMD embedding
|
||||
4. Implement `PlanetDetection` pipeline (P0-P2)
|
||||
5. Implement `WITNESS_SEG` with SHAKE-256 chain
|
||||
1. `ProgressiveDownloader` with tier-0 curated dataset (100 Kepler targets)
|
||||
2. `PlanetTransitAdapter` for FITS light curve ingestion
|
||||
3. `CausalAtlas` with windowing, feature extraction, SIMD embedding
|
||||
4. `PlanetDetection` pipeline (P0-P2)
|
||||
5. `WITNESS_SEG` with SHAKE-256 chain
|
||||
6. CLI: `rvf run`, `rvf query planet list`, `rvf trace`
|
||||
7. HTTP: `/api/candidates/planet`, `/api/atlas/trace`
|
||||
8. Dashboard: Vite scaffold, V1 Atlas Explorer (Three.js 3D graph), V3 Planet
|
||||
Dashboard (ranked list + light curve chart), V5 Status Dashboard (download
|
||||
progress + witness log). Embedded in `DASHBOARD_SEG`, served at `/`
|
||||
9. WebSocket `/ws/live` for real-time pipeline progress
|
||||
8. Dashboard shell: V1, V3, V5 views (see ADR-040a), WebSocket `/ws/live`
|
||||
|
||||
**Acceptance**: 1,000 Kepler targets, top-100 ranked list includes >= 80
|
||||
confirmed planets, every item replays to same score and witness root on two
|
||||
machines. Dashboard renders atlas graph and candidate list in browser.
|
||||
machines.
|
||||
|
||||
### Phase 2: Coherence Field + Boundary Tracker + Dashboard V2 (v0.2)
|
||||
### Phase 2: Coherence Field + Boundary Tracker (v0.2)
|
||||
|
||||
1. Implement `CoherenceField` with dynamic min-cut, partition entropy
|
||||
2. Implement `BoundaryTracker` with timeline and alerts
|
||||
3. Implement `MultiScaleMemory` with S/M/L tiers and budget control
|
||||
4. Add coherence gating to planet pipeline
|
||||
1. `CoherenceField` with dynamic min-cut, partition entropy
|
||||
2. `BoundaryTracker` with timeline and alerts
|
||||
3. `MultiScaleMemory` with S/M/L tiers and budget control
|
||||
4. Coherence gating added to planet pipeline
|
||||
5. HTTP: `/api/coherence`, `/api/boundary/*`, `/api/memory/tiers`
|
||||
6. Dashboard: V2 Coherence Heatmap (Three.js field surface + cut pressure
|
||||
overlay + time scrubber), boundary alert markers via WebSocket
|
||||
6. Dashboard V2 Coherence Heatmap (see ADR-040a)
|
||||
|
||||
### Phase 3: Life Candidate Pipeline + Dashboard V4 (v0.3)
|
||||
### Phase 3: Life Candidate Pipeline (v0.3)
|
||||
|
||||
1. Implement `SpectrumAdapter` for JWST/archive spectral data
|
||||
2. Implement `LifeCandidate` pipeline (L0-L2)
|
||||
3. Implement disequilibrium scoring with reaction plausibility graph
|
||||
4. Tier-3 progressive download for spectral data
|
||||
5. CLI: `rvf query life list`
|
||||
6. HTTP: `/api/candidates/life`
|
||||
7. Dashboard: V4 Life Dashboard (spectrum viewer + molecule presence matrix
|
||||
+ reaction graph + confound panel)
|
||||
1. `SpectrumAdapter` for JWST/archive spectral data
|
||||
2. `LifeCandidate` pipeline (L0-L2) with disequilibrium scoring
|
||||
3. Tier-3 progressive download for spectral data
|
||||
4. CLI: `rvf query life list`; HTTP: `/api/candidates/life`
|
||||
5. Dashboard V4 Life Dashboard (see ADR-040a)
|
||||
|
||||
**Acceptance**: Published spectra with known atmospheric detections vs nulls,
|
||||
AUC > 0.8, every score includes confound penalties and provenance trace.
|
||||
Dashboard renders spectrum analysis in browser.
|
||||
**Acceptance**: AUC > 0.8 on published spectra with known atmospheric
|
||||
detections vs nulls, every score includes confound penalties and provenance.
|
||||
|
||||
### Phase 4: Cosmic Web + Full Integration (v0.4)
|
||||
|
||||
1. `CosmicWebAdapter` for SDSS spatial graph
|
||||
2. Cross-domain coherence (planet candidates enriched by large-scale context)
|
||||
3. Dashboard: 3D cosmic web view, cross-domain candidate linking
|
||||
4. Full offline demo with sealed RVF snapshot
|
||||
5. `rvf ingest --expand` for tier-2 bulk download
|
||||
6. Dashboard polish: LOD optimization, mobile-responsive layout, dark/light theme
|
||||
3. Full offline demo with sealed RVF snapshot
|
||||
4. `rvf ingest --expand` for tier-2 bulk download
|
||||
|
||||
## Evaluation Plan
|
||||
|
||||
|
|
|
|||
|
|
@ -1,12 +1,6 @@
|
|||
//! Real Microlensing Data Analysis Pipeline
|
||||
//!
|
||||
//! Downloads and analyzes real microlensing light curves from public surveys:
|
||||
//! - OGLE Early Warning System (EWS)
|
||||
//! - MOA Alerts
|
||||
//!
|
||||
//! Features progressive download manifest with real OGLE/MOA event parameters,
|
||||
//! cross-survey normalization, and anomaly detection via graph-cut residual analysis.
|
||||
//!
|
||||
//! Real Microlensing Data Analysis Pipeline — simulates OGLE/MOA light curves
|
||||
//! from published event parameters, with cross-survey normalization, binary-lens
|
||||
//! planetary perturbations, and anomaly detection via graph-cut residual analysis.
|
||||
//! Run: cargo run --example real_microlensing --release
|
||||
|
||||
use rvf_runtime::{
|
||||
|
|
@ -22,317 +16,87 @@ const FIELD_SOURCE: u16 = 0;
|
|||
const FIELD_EVENT: u16 = 1;
|
||||
const FIELD_ANOMALY_TYPE: u16 = 2;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// LCG deterministic random
|
||||
// ---------------------------------------------------------------------------
|
||||
fn lcg_f64(state: &mut u64) -> f64 {
|
||||
*state = state.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
|
||||
((*state >> 33) as f64) / (u32::MAX as f64)
|
||||
}
|
||||
|
||||
fn lcg_normal(state: &mut u64) -> f64 {
|
||||
let u1 = lcg_f64(state).max(1e-15);
|
||||
let u2 = lcg_f64(state);
|
||||
const LCG_MUL: u64 = 6364136223846793005;
|
||||
const LCG_INC: u64 = 1442695040888963407;
|
||||
fn lcg_f64(s: &mut u64) -> f64 { *s = s.wrapping_mul(LCG_MUL).wrapping_add(LCG_INC); ((*s >> 33) as f64) / (u32::MAX as f64) }
|
||||
fn lcg_normal(s: &mut u64) -> f64 {
|
||||
let (u1, u2) = (lcg_f64(s).max(1e-15), lcg_f64(s));
|
||||
(-2.0 * u1.ln()).sqrt() * (2.0 * std::f64::consts::PI * u2).cos()
|
||||
}
|
||||
|
||||
fn random_vector(dim: usize, seed: u64) -> Vec<f32> {
|
||||
let mut v = Vec::with_capacity(dim);
|
||||
let mut x = seed.wrapping_add(1);
|
||||
for _ in 0..dim {
|
||||
x = x.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
|
||||
v.push(((x >> 33) as f32) / (u32::MAX as f32) - 0.5);
|
||||
}
|
||||
v
|
||||
(0..dim).map(|_| { x = x.wrapping_mul(LCG_MUL).wrapping_add(LCG_INC); ((x >> 33) as f32) / (u32::MAX as f32) - 0.5 }).collect()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Photometric data types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct Observation {
|
||||
hjd: f64, mag: f64, mag_err: f64, flux: f64, flux_err: f64,
|
||||
}
|
||||
|
||||
struct Observation { hjd: f64, mag: f64, mag_err: f64, flux: f64, flux_err: f64 }
|
||||
#[derive(Debug, Clone)]
|
||||
struct MicrolensingEvent {
|
||||
name: String,
|
||||
source: String,
|
||||
observations: Vec<Observation>,
|
||||
baseline_mag: f64,
|
||||
peak_mag: f64,
|
||||
t0_est: f64,
|
||||
t_e_est: f64,
|
||||
u0_est: f64,
|
||||
name: String, source: String, observations: Vec<Observation>,
|
||||
baseline_mag: f64, peak_mag: f64, t0_est: f64, t_e_est: f64, u0_est: f64,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Enhanced OGLE format parser
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Parse OGLE EWS photometry. Handles HJD and JD time formats, magnitude and
|
||||
/// flux columns (detected by value range: mag > 10, flux < 100), comment lines,
|
||||
/// and malformed/NaN/inf values gracefully.
|
||||
fn parse_ogle_format(data: &str, event_name: &str) -> MicrolensingEvent {
|
||||
let mut observations = Vec::new();
|
||||
|
||||
for line in data.lines() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() || line.starts_with('#') || line.starts_with("\\") { continue; }
|
||||
|
||||
let parts: Vec<&str> = line.split_whitespace().collect();
|
||||
if parts.len() < 3 { continue; }
|
||||
|
||||
let (hjd, val, err) = match (parts[0].parse::<f64>(), parts[1].parse::<f64>(), parts[2].parse::<f64>()) {
|
||||
(Ok(t), Ok(v), Ok(e)) => (t, v, e),
|
||||
_ => continue,
|
||||
};
|
||||
|
||||
// Skip NaN/inf values
|
||||
if hjd.is_nan() || hjd.is_infinite() || val.is_nan() || val.is_infinite()
|
||||
|| err.is_nan() || err.is_infinite() { continue; }
|
||||
|
||||
// JD -> HJD conversion: subtract 2450000 if time looks like full JD
|
||||
let hjd = if hjd > 2400000.0 { hjd - 2450000.0 } else { hjd };
|
||||
|
||||
// Detect magnitude vs flux by value range: magnitudes > 10, flux < 100
|
||||
let (mag, mag_err) = if val > 10.0 {
|
||||
(val, err) // already magnitude
|
||||
} else if val > 0.0 {
|
||||
// Flux column: convert to magnitude using arbitrary zero-point 22.0
|
||||
let m = -2.5 * val.log10() + 22.0;
|
||||
let me = 2.5 / (val * 2.302585) * err;
|
||||
(m, me)
|
||||
} else { continue; };
|
||||
|
||||
if mag_err <= 0.0 || mag_err > 1.0 || mag < 5.0 || mag > 25.0 { continue; }
|
||||
|
||||
observations.push(Observation { hjd, mag, mag_err, flux: 0.0, flux_err: 0.0 });
|
||||
}
|
||||
|
||||
finalize_event(observations, event_name, "OGLE")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MOA format parser
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Parse MOA-II photometry. MOA column order: JD, flux, flux_error (no magnitude).
|
||||
/// Normalizes to relative flux using baseline estimated from first/last 20% of data.
|
||||
fn parse_moa_format(data: &str, event_name: &str) -> MicrolensingEvent {
|
||||
let mut observations = Vec::new();
|
||||
|
||||
for line in data.lines() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() || line.starts_with('#') { continue; }
|
||||
|
||||
let parts: Vec<&str> = line.split_whitespace().collect();
|
||||
if parts.len() < 3 { continue; }
|
||||
|
||||
let (jd, flux, flux_err) = match (parts[0].parse::<f64>(), parts[1].parse::<f64>(), parts[2].parse::<f64>()) {
|
||||
(Ok(t), Ok(f), Ok(e)) => (t, f, e),
|
||||
_ => continue,
|
||||
};
|
||||
|
||||
if jd.is_nan() || flux.is_nan() || flux_err.is_nan() { continue; }
|
||||
if flux.is_infinite() || flux_err.is_infinite() { continue; }
|
||||
if flux <= 0.0 || flux_err <= 0.0 { continue; }
|
||||
|
||||
let hjd = if jd > 2400000.0 { jd - 2450000.0 } else { jd };
|
||||
let mag = -2.5 * flux.log10() + 22.0;
|
||||
let mag_err = 2.5 / (flux * 2.302585) * flux_err;
|
||||
|
||||
if mag_err <= 0.0 || mag_err > 1.0 { continue; }
|
||||
observations.push(Observation { hjd, mag, mag_err, flux: 0.0, flux_err: 0.0 });
|
||||
}
|
||||
|
||||
// Baseline from first/last 20% of sorted observations
|
||||
observations.sort_by(|a, b| a.hjd.partial_cmp(&b.hjd).unwrap());
|
||||
let n = observations.len();
|
||||
let wing = (n as f64 * 0.2).ceil() as usize;
|
||||
if n > 0 && wing > 0 {
|
||||
let baseline_flux: f64 = observations[..wing].iter()
|
||||
.chain(observations[n.saturating_sub(wing)..].iter())
|
||||
.map(|o| 10.0f64.powf(-0.4 * (o.mag - 22.0)))
|
||||
.sum::<f64>() / (2 * wing) as f64;
|
||||
// Normalize all to relative flux
|
||||
for obs in &mut observations {
|
||||
let raw = 10.0f64.powf(-0.4 * (obs.mag - 22.0));
|
||||
obs.flux = raw / baseline_flux;
|
||||
obs.flux_err = obs.flux * 0.4 * 2.302585 * obs.mag_err;
|
||||
}
|
||||
}
|
||||
|
||||
finalize_event(observations, event_name, "MOA")
|
||||
}
|
||||
|
||||
/// Shared finalization: sort, compute baseline/peak, convert to flux.
|
||||
fn finalize_event(mut obs: Vec<Observation>, name: &str, source: &str) -> MicrolensingEvent {
|
||||
obs.sort_by(|a, b| a.hjd.partial_cmp(&b.hjd).unwrap());
|
||||
let mut mags: Vec<f64> = obs.iter().map(|o| o.mag).collect();
|
||||
mags.sort_by(|a, b| a.partial_cmp(b).unwrap());
|
||||
let baseline_mag = if !mags.is_empty() { mags[mags.len() * 3 / 4] } else { 20.0 };
|
||||
let peak_mag = mags.first().copied().unwrap_or(20.0);
|
||||
|
||||
for o in &mut obs {
|
||||
if o.flux == 0.0 {
|
||||
o.flux = 10.0f64.powf(-0.4 * (o.mag - baseline_mag));
|
||||
o.flux_err = o.flux * 0.4 * 2.302585 * o.mag_err;
|
||||
}
|
||||
}
|
||||
|
||||
let t0_est = obs.iter().min_by(|a, b| a.mag.partial_cmp(&b.mag).unwrap())
|
||||
.map(|o| o.hjd).unwrap_or(0.0);
|
||||
let half_mag = (baseline_mag + peak_mag) / 2.0;
|
||||
let above: Vec<_> = obs.iter().filter(|o| o.mag < half_mag).collect();
|
||||
let t_e_est = if above.len() >= 2 {
|
||||
(above.last().unwrap().hjd - above.first().unwrap().hjd) / 2.0
|
||||
} else { 20.0 };
|
||||
let peak_flux = 10.0f64.powf(-0.4 * (peak_mag - baseline_mag));
|
||||
let u0_est = if peak_flux > 1.5 { (1.0 / peak_flux).max(0.01) } else { 0.5 };
|
||||
|
||||
MicrolensingEvent {
|
||||
name: name.to_string(), source: source.to_string(), observations: obs,
|
||||
baseline_mag, peak_mag, t0_est, t_e_est, u0_est,
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Progressive download manifest
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct PublishedParams {
|
||||
t0: f64, t_e: f64, u0: f64,
|
||||
has_planet: bool,
|
||||
planet_mass_ratio: Option<f64>,
|
||||
einstein_time: f64, // t_E in days
|
||||
impact_param: f64, // u_0
|
||||
planet_mass_ratio: Option<f64>, // q (None = no planet)
|
||||
planet_separation: Option<f64>, // s (None = no planet)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct ManifestEntry {
|
||||
name: &'static str,
|
||||
survey: &'static str,
|
||||
url_template: &'static str,
|
||||
year: u16,
|
||||
event_id: &'static str,
|
||||
published_params: Option<PublishedParams>,
|
||||
event_id: String, // e.g. "OGLE-2005-BLG-390"
|
||||
survey: String, // "ogle" or "moa"
|
||||
published: PublishedParams,
|
||||
}
|
||||
|
||||
struct DownloadManifest { events: Vec<ManifestEntry> }
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
enum DownloadState { Pending, Simulated, #[allow(dead_code)] Downloaded, Failed(String) }
|
||||
|
||||
impl std::fmt::Display for DownloadState {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::Pending => write!(f, "PEND"),
|
||||
Self::Simulated => write!(f, "SIM"),
|
||||
Self::Downloaded => write!(f, "DL"),
|
||||
Self::Failed(e) => write!(f, "FAIL:{}", e),
|
||||
Self::Pending => write!(f, "PEND"), Self::Simulated => write!(f, "SIM"),
|
||||
Self::Downloaded => write!(f, "DL"), Self::Failed(e) => write!(f, "FAIL:{}", e),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn build_manifest() -> DownloadManifest {
|
||||
let ogle = "https://ogle.astrouw.edu.pl/ogle4/ews/{year}/blg-{id}/phot.dat";
|
||||
let moa = "https://www.massey.ac.nz/~iabond/moa/alert{year}/fetch.php?path=moa{id}";
|
||||
DownloadManifest { events: vec![
|
||||
ManifestEntry { name: "OGLE-2005-BLG-390", survey: "OGLE", url_template: ogle,
|
||||
year: 2005, event_id: "390",
|
||||
published_params: Some(PublishedParams { t0: 3582.7, t_e: 11.03, u0: 0.359,
|
||||
has_planet: true, planet_mass_ratio: Some(7.6e-5) }) },
|
||||
ManifestEntry { name: "MOA-2011-BLG-262", survey: "MOA", url_template: moa,
|
||||
year: 2011, event_id: "262",
|
||||
published_params: Some(PublishedParams { t0: 5700.5, t_e: 3.3, u0: 0.05,
|
||||
has_planet: true, planet_mass_ratio: Some(4e-4) }) },
|
||||
ManifestEntry { name: "OGLE-2016-BLG-1195", survey: "OGLE", url_template: ogle,
|
||||
year: 2016, event_id: "1195",
|
||||
published_params: Some(PublishedParams { t0: 7568.0, t_e: 8.2, u0: 0.35,
|
||||
has_planet: true, planet_mass_ratio: Some(4.5e-5) }) },
|
||||
ManifestEntry { name: "MOA-2009-BLG-387", survey: "MOA", url_template: moa,
|
||||
year: 2009, event_id: "387",
|
||||
published_params: Some(PublishedParams { t0: 5010.0, t_e: 42.5, u0: 0.012,
|
||||
has_planet: true, planet_mass_ratio: Some(3.2e-3) }) },
|
||||
ManifestEntry { name: "OGLE-2003-BLG-235", survey: "OGLE", url_template: ogle,
|
||||
year: 2003, event_id: "235",
|
||||
published_params: Some(PublishedParams { t0: 2838.5, t_e: 61.5, u0: 0.133,
|
||||
has_planet: true, planet_mass_ratio: Some(3.9e-3) }) },
|
||||
ManifestEntry { name: "OGLE-2005-BLG-071", survey: "OGLE", url_template: ogle,
|
||||
year: 2005, event_id: "071",
|
||||
published_params: Some(PublishedParams { t0: 3469.1, t_e: 70.9, u0: 0.026,
|
||||
has_planet: true, planet_mass_ratio: Some(7.0e-3) }) },
|
||||
ManifestEntry { name: "MOA-2007-BLG-192", survey: "MOA", url_template: moa,
|
||||
year: 2007, event_id: "192",
|
||||
published_params: Some(PublishedParams { t0: 4239.0, t_e: 38.0, u0: 0.29,
|
||||
has_planet: true, planet_mass_ratio: Some(2.0e-4) }) },
|
||||
ManifestEntry { name: "OGLE-2006-BLG-109", survey: "OGLE", url_template: ogle,
|
||||
year: 2006, event_id: "109",
|
||||
published_params: Some(PublishedParams { t0: 3831.0, t_e: 127.3, u0: 0.0035,
|
||||
has_planet: true, planet_mass_ratio: Some(1.4e-3) }) },
|
||||
// Non-planet events for comparison
|
||||
ManifestEntry { name: "OGLE-2018-BLG-0799", survey: "OGLE", url_template: ogle,
|
||||
year: 2018, event_id: "0799",
|
||||
published_params: Some(PublishedParams { t0: 8250.0, t_e: 25.0, u0: 0.008,
|
||||
has_planet: false, planet_mass_ratio: None }) },
|
||||
ManifestEntry { name: "MOA-2019-BLG-008", survey: "MOA", url_template: moa,
|
||||
year: 2019, event_id: "008",
|
||||
published_params: Some(PublishedParams { t0: 8580.0, t_e: 18.0, u0: 0.15,
|
||||
has_planet: false, planet_mass_ratio: None }) },
|
||||
ManifestEntry { name: "OGLE-2012-BLG-0563", survey: "OGLE", url_template: ogle,
|
||||
year: 2012, event_id: "0563",
|
||||
published_params: Some(PublishedParams { t0: 6090.0, t_e: 55.0, u0: 0.045,
|
||||
has_planet: true, planet_mass_ratio: Some(1.5e-3) }) },
|
||||
ManifestEntry { name: "OGLE-2017-BLG-0373", survey: "OGLE", url_template: ogle,
|
||||
year: 2017, event_id: "0373",
|
||||
published_params: Some(PublishedParams { t0: 7870.0, t_e: 40.0, u0: 0.12,
|
||||
has_planet: false, planet_mass_ratio: None }) },
|
||||
ManifestEntry { name: "OGLE-2015-BLG-0966", survey: "OGLE", url_template: ogle,
|
||||
year: 2015, event_id: "0966",
|
||||
published_params: Some(PublishedParams { t0: 7190.0, t_e: 22.0, u0: 0.21,
|
||||
has_planet: false, planet_mass_ratio: None }) },
|
||||
ManifestEntry { name: "MOA-2013-BLG-220", survey: "MOA", url_template: moa,
|
||||
year: 2013, event_id: "220",
|
||||
published_params: Some(PublishedParams { t0: 6440.0, t_e: 60.0, u0: 0.08,
|
||||
has_planet: false, planet_mass_ratio: None }) },
|
||||
ManifestEntry { name: "MOA-2015-BLG-337", survey: "MOA", url_template: moa,
|
||||
year: 2015, event_id: "337",
|
||||
published_params: Some(PublishedParams { t0: 7200.0, t_e: 15.0, u0: 0.30,
|
||||
has_planet: false, planet_mass_ratio: None }) },
|
||||
]}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Cross-survey normalization
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Normalize light curves to fractional deviation from baseline: (F - F_base) / F_base.
|
||||
/// OGLE-IV I-band zero-point ~21.0, MOA-II R-band zero-point ~22.0.
|
||||
fn normalize_cross_survey(event: &mut MicrolensingEvent) {
|
||||
let zp = match event.source.as_str() {
|
||||
"MOA" => 22.0,
|
||||
_ => 21.0, // OGLE-IV I-band
|
||||
fn build_manifest() -> Vec<ManifestEntry> {
|
||||
let e = |id: &str, survey: &str, t_e: f64, u0: f64, q: Option<f64>, s: Option<f64>| {
|
||||
ManifestEntry {
|
||||
event_id: id.into(), survey: survey.into(),
|
||||
published: PublishedParams { einstein_time: t_e, impact_param: u0,
|
||||
planet_mass_ratio: q, planet_separation: s },
|
||||
}
|
||||
};
|
||||
// Compute baseline flux from faintest 50% of observations
|
||||
let mut mags: Vec<f64> = event.observations.iter().map(|o| o.mag).collect();
|
||||
mags.sort_by(|a, b| a.partial_cmp(b).unwrap());
|
||||
let mid = mags.len() / 2;
|
||||
let baseline_flux = if mid > 0 {
|
||||
let bm: f64 = mags[mid..].iter().sum::<f64>() / (mags.len() - mid) as f64;
|
||||
10.0f64.powf(-0.4 * (bm - zp))
|
||||
} else { 1.0 };
|
||||
|
||||
for obs in &mut event.observations {
|
||||
let f = 10.0f64.powf(-0.4 * (obs.mag - zp));
|
||||
obs.flux = (f - baseline_flux) / baseline_flux; // fractional deviation
|
||||
obs.flux_err = f * 0.4 * 2.302585 * obs.mag_err / baseline_flux;
|
||||
}
|
||||
vec![
|
||||
// --- Planetary events ---
|
||||
// First cold rocky/icy super-Earth
|
||||
e("OGLE-2005-BLG-390", "ogle", 11.0, 0.359, Some(7.6e-5), Some(1.610)),
|
||||
// Possible rogue planet with moon
|
||||
e("MOA-2011-BLG-262", "moa", 3.3, 0.04, Some(3.8e-4), None),
|
||||
// Ice-line planet, very low mass ratio
|
||||
e("OGLE-2016-BLG-1195","ogle", 9.9, 0.045, Some(4.2e-5), Some(1.04)),
|
||||
// First microlensing planet (OGLE+MOA joint)
|
||||
e("OGLE-2003-BLG-235", "ogle", 61.5, 0.133, Some(0.0039), Some(1.12)),
|
||||
// Very low-mass host star planet
|
||||
e("MOA-2007-BLG-192", "moa", 46.0, 0.001, Some(0.0002), Some(1.05)),
|
||||
// Jupiter/Saturn analog system (two planets)
|
||||
e("OGLE-2006-BLG-109", "ogle",127.3, 0.0035,Some(1.4e-3), Some(2.3)),
|
||||
// 3.8 M_Jup planet
|
||||
e("OGLE-2005-BLG-071", "ogle", 70.9, 0.026, Some(7.0e-3), Some(1.29)),
|
||||
// Saturn-mass planet
|
||||
e("OGLE-2012-BLG-0563","ogle", 55.0, 0.045, Some(1.5e-3), Some(1.42)),
|
||||
// --- Non-planet PSPL comparison events ---
|
||||
e("OGLE-2018-BLG-0799","ogle", 25.0, 0.008, None, None),
|
||||
e("MOA-2019-BLG-008", "moa", 18.0, 0.15, None, None),
|
||||
e("OGLE-2017-BLG-0373","ogle", 40.0, 0.12, None, None),
|
||||
e("OGLE-2015-BLG-0966","ogle", 22.0, 0.21, None, None),
|
||||
e("MOA-2013-BLG-220", "moa", 60.0, 0.08, None, None),
|
||||
]
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Simulation from published parameters (replaces actual download)
|
||||
// ---------------------------------------------------------------------------
|
||||
// ---- Paczynski formula & binary-lens perturbation ----
|
||||
|
||||
fn pspl_magnification(u: f64) -> f64 {
|
||||
if u < 1e-10 { return 1e10; }
|
||||
|
|
@ -340,61 +104,189 @@ fn pspl_magnification(u: f64) -> f64 {
|
|||
(u2 + 2.0) / (u * (u2 + 4.0).sqrt())
|
||||
}
|
||||
|
||||
fn simulate_from_manifest(entry: &ManifestEntry, rng: &mut u64) -> (MicrolensingEvent, DownloadState) {
|
||||
let params = match entry.published_params {
|
||||
Some(p) => p,
|
||||
None => return (MicrolensingEvent {
|
||||
name: entry.name.to_string(), source: entry.survey.to_string(),
|
||||
observations: vec![], baseline_mag: 20.0, peak_mag: 20.0,
|
||||
t0_est: 0.0, t_e_est: 20.0, u0_est: 0.5,
|
||||
}, DownloadState::Failed("no published params".into())),
|
||||
};
|
||||
let t0 = 2459000.0 + params.t0 % 1000.0;
|
||||
/// Binary-lens perturbation factor near the planetary caustic.
|
||||
/// Uses the Chang-Refsdal approximation: excess magnification from
|
||||
/// a point-mass perturber at separation `s` with mass ratio `q`.
|
||||
fn binary_lens_perturbation(tau: f64, u0: f64, q: f64, s: f64) -> f64 {
|
||||
// Distance from source to planet position (planet at s along lens axis)
|
||||
let dx = tau - s;
|
||||
let d2 = dx * dx + u0 * u0;
|
||||
// Caustic half-width ~ 4*q/s^2 for close/wide topologies
|
||||
let caustic_r2 = 4.0 * q / (s * s);
|
||||
if d2 > caustic_r2 * 25.0 { return 0.0; } // too far from caustic
|
||||
if d2 < 1e-15 { return q * 1e4; } // avoid singularity
|
||||
// Smooth perturbation envelope
|
||||
let strength = q / d2.max(q * 0.1);
|
||||
strength * (-d2 / (8.0 * caustic_r2.max(1e-10))).exp()
|
||||
}
|
||||
|
||||
// ---- Simulate light curve from published parameters ----
|
||||
|
||||
fn simulate_from_published(entry: &ManifestEntry, rng: &mut u64) -> (MicrolensingEvent, DownloadState) {
|
||||
let p = &entry.published;
|
||||
let is_moa = entry.survey == "moa";
|
||||
let t0 = 2459000.0 + (lcg_f64(rng) * 500.0);
|
||||
let baseline_mag = 18.0 + lcg_f64(rng) * 3.0;
|
||||
let f_b = 0.05 + lcg_f64(rng) * 0.3;
|
||||
let cadence = if entry.survey == "MOA" { 0.25 } else { 0.5 };
|
||||
let sigma_floor = if entry.survey == "MOA" { 0.01 } else { 0.005 };
|
||||
let cadence = if is_moa { 0.25 } else { 0.5 }; // days between obs
|
||||
let sigma_floor = if is_moa { 0.01 } else { 0.005 };
|
||||
let span = p.einstein_time.max(10.0) * 3.0; // cover +-3 t_E
|
||||
|
||||
let mut observations = Vec::new();
|
||||
let mut t = t0 - 60.0;
|
||||
while t < t0 + 60.0 {
|
||||
t += cadence / 24.0 * (0.5 + lcg_f64(rng));
|
||||
if lcg_f64(rng) < 0.3 { continue; }
|
||||
let mut t = t0 - span;
|
||||
while t < t0 + span {
|
||||
t += cadence * (0.5 + lcg_f64(rng));
|
||||
if lcg_f64(rng) < 0.25 { continue; } // weather gaps
|
||||
// Night-time window: keep ~40% of observations
|
||||
if ((t % 1.0) - 0.2).abs() > 0.17 { continue; }
|
||||
|
||||
let tau = (t - t0) / params.t_e;
|
||||
let u = (params.u0 * params.u0 + tau * tau).sqrt();
|
||||
let mut mag_a = pspl_magnification(u);
|
||||
if params.has_planet {
|
||||
let q = params.planet_mass_ratio.unwrap_or(0.0);
|
||||
let s = 1.0 + lcg_f64(rng) * 0.01; // stable separation
|
||||
let d2 = (tau - s).powi(2) + params.u0.powi(2);
|
||||
if d2 < q * 25.0 && d2 > 1e-15 {
|
||||
mag_a += (q / d2.max(q * 0.1)) * (-d2 / (8.0 * q)).exp() * mag_a;
|
||||
}
|
||||
let tau = (t - t0) / p.einstein_time;
|
||||
let u = (p.impact_param * p.impact_param + tau * tau).sqrt();
|
||||
let mut amp = pspl_magnification(u);
|
||||
|
||||
// Add binary-lens planetary perturbation if planet present
|
||||
if let (Some(q), Some(s)) = (p.planet_mass_ratio, p.planet_separation) {
|
||||
amp += binary_lens_perturbation(tau, p.impact_param, q, s) * amp;
|
||||
} else if let Some(q) = p.planet_mass_ratio {
|
||||
// No separation published (e.g. MOA-2011-BLG-262): use s ~ 1.0
|
||||
amp += binary_lens_perturbation(tau, p.impact_param, q, 1.0) * amp;
|
||||
}
|
||||
let true_flux = mag_a + f_b;
|
||||
|
||||
let true_flux = amp + f_b;
|
||||
let true_mag = baseline_mag - 2.5 * true_flux.log10();
|
||||
let sigma = (sigma_floor.powi(2) + 0.003f64.powi(2)).sqrt();
|
||||
let sigma = (sigma_floor * sigma_floor + 0.003f64.powi(2)).sqrt();
|
||||
let obs_mag = true_mag + lcg_normal(rng) * sigma;
|
||||
observations.push(Observation {
|
||||
hjd: t, mag: obs_mag, mag_err: sigma, flux: 0.0, flux_err: 0.0,
|
||||
});
|
||||
observations.push(Observation { hjd: t, mag: obs_mag, mag_err: sigma, flux: 0.0, flux_err: 0.0 });
|
||||
}
|
||||
|
||||
let peak_mag = observations.iter().map(|o| o.mag).fold(f64::INFINITY, f64::min);
|
||||
let mut evt = MicrolensingEvent {
|
||||
name: entry.name.to_string(), source: entry.survey.to_string(),
|
||||
name: entry.event_id.clone(), source: entry.survey.clone(),
|
||||
observations, baseline_mag, peak_mag,
|
||||
t0_est: t0, t_e_est: params.t_e, u0_est: params.u0,
|
||||
t0_est: t0, t_e_est: p.einstein_time, u0_est: p.impact_param,
|
||||
};
|
||||
normalize_cross_survey(&mut evt);
|
||||
(evt, DownloadState::Simulated)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// PSPL fitting (compact)
|
||||
// ---------------------------------------------------------------------------
|
||||
// ---- Cross-survey normalization ----
|
||||
|
||||
/// Normalize to fractional deviation (F - F_base) / F_base.
|
||||
/// Survey-specific zero-points: OGLE I-band = 21.0, MOA R-band = 22.0.
|
||||
fn normalize_cross_survey(event: &mut MicrolensingEvent) {
|
||||
let zp = if event.source == "moa" { 22.0 } else { 21.0 };
|
||||
// Baseline flux from faintest 50% of observations
|
||||
let mut mags: Vec<f64> = event.observations.iter().map(|o| o.mag).collect();
|
||||
mags.sort_by(|a, b| a.partial_cmp(b).unwrap());
|
||||
let mid = mags.len() / 2;
|
||||
let baseline_flux = if mid > 0 {
|
||||
let bm: f64 = mags[mid..].iter().sum::<f64>() / (mags.len() - mid) as f64;
|
||||
10.0f64.powf(-0.4 * (bm - zp))
|
||||
} else { 1.0 };
|
||||
for obs in &mut event.observations {
|
||||
let f = 10.0f64.powf(-0.4 * (obs.mag - zp));
|
||||
obs.flux = (f - baseline_flux) / baseline_flux;
|
||||
obs.flux_err = f * 0.4 * 2.302585 * obs.mag_err / baseline_flux;
|
||||
}
|
||||
}
|
||||
|
||||
// ---- OGLE format parser ----
|
||||
|
||||
/// Parse OGLE EWS photometry: HJD/JD, magnitude or flux, error.
|
||||
fn parse_ogle_format(data: &str, event_name: &str) -> MicrolensingEvent {
|
||||
let mut obs = Vec::new();
|
||||
for line in data.lines() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() || line.starts_with('#') || line.starts_with('\\') { continue; }
|
||||
let p: Vec<&str> = line.split_whitespace().collect();
|
||||
if p.len() < 3 { continue; }
|
||||
let (hjd, val, err) = match (p[0].parse::<f64>(), p[1].parse::<f64>(), p[2].parse::<f64>()) {
|
||||
(Ok(t), Ok(v), Ok(e)) => (t, v, e), _ => continue,
|
||||
};
|
||||
if [hjd, val, err].iter().any(|x| x.is_nan() || x.is_infinite()) { continue; }
|
||||
let hjd = if hjd > 2400000.0 { hjd - 2450000.0 } else { hjd };
|
||||
let (mag, mag_err) = if val > 10.0 { (val, err) }
|
||||
else if val > 0.0 {
|
||||
(-2.5 * val.log10() + 22.0, 2.5 / (val * 2.302585) * err)
|
||||
} else { continue; };
|
||||
if mag_err <= 0.0 || mag_err > 1.0 || mag < 5.0 || mag > 25.0 { continue; }
|
||||
obs.push(Observation { hjd, mag, mag_err, flux: 0.0, flux_err: 0.0 });
|
||||
}
|
||||
finalize_event(obs, event_name, "ogle")
|
||||
}
|
||||
|
||||
// ---- Enhanced MOA format parser ----
|
||||
|
||||
/// Parse MOA-II photometry: JD, flux, flux_error columns.
|
||||
/// Baseline estimated from first and last 20% of time-sorted observations.
|
||||
fn parse_moa_format(data: &str, event_name: &str) -> MicrolensingEvent {
|
||||
let mut obs = Vec::new();
|
||||
for line in data.lines() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() || line.starts_with('#') { continue; }
|
||||
let p: Vec<&str> = line.split_whitespace().collect();
|
||||
if p.len() < 3 { continue; }
|
||||
let (jd, flux, ferr) = match (p[0].parse::<f64>(), p[1].parse::<f64>(), p[2].parse::<f64>()) {
|
||||
(Ok(t), Ok(f), Ok(e)) => (t, f, e), _ => continue,
|
||||
};
|
||||
if [jd, flux, ferr].iter().any(|x| x.is_nan() || x.is_infinite()) { continue; }
|
||||
if flux <= 0.0 || ferr <= 0.0 { continue; }
|
||||
let hjd = if jd > 2400000.0 { jd - 2450000.0 } else { jd };
|
||||
// MOA R-band zero-point 22.0
|
||||
let mag = -2.5 * flux.log10() + 22.0;
|
||||
let mag_err = 2.5 / (flux * 2.302585) * ferr;
|
||||
if mag_err <= 0.0 || mag_err > 1.0 { continue; }
|
||||
obs.push(Observation { hjd, mag, mag_err, flux, flux_err: ferr });
|
||||
}
|
||||
// Sort by time, then estimate baseline from first/last 20%
|
||||
obs.sort_by(|a, b| a.hjd.partial_cmp(&b.hjd).unwrap());
|
||||
let n = obs.len();
|
||||
let wing = (n as f64 * 0.2).ceil() as usize;
|
||||
if n > 0 && wing > 0 {
|
||||
let wing_obs: Vec<f64> = obs[..wing].iter()
|
||||
.chain(obs[n.saturating_sub(wing)..].iter())
|
||||
.map(|o| o.flux).collect();
|
||||
let baseline_flux = wing_obs.iter().sum::<f64>() / wing_obs.len() as f64;
|
||||
if baseline_flux > 0.0 {
|
||||
for o in &mut obs {
|
||||
let raw = o.flux;
|
||||
o.flux = (raw - baseline_flux) / baseline_flux; // fractional deviation
|
||||
o.flux_err = o.flux_err / baseline_flux;
|
||||
}
|
||||
}
|
||||
}
|
||||
finalize_event(obs, event_name, "moa")
|
||||
}
|
||||
|
||||
/// Shared finalization: sort, compute baseline/peak, convert flux where needed.
|
||||
fn finalize_event(mut obs: Vec<Observation>, name: &str, source: &str) -> MicrolensingEvent {
|
||||
obs.sort_by(|a, b| a.hjd.partial_cmp(&b.hjd).unwrap());
|
||||
let mut mags: Vec<f64> = obs.iter().map(|o| o.mag).collect();
|
||||
mags.sort_by(|a, b| a.partial_cmp(b).unwrap());
|
||||
let baseline_mag = if !mags.is_empty() { mags[mags.len() * 3 / 4] } else { 20.0 };
|
||||
let peak_mag = mags.first().copied().unwrap_or(20.0);
|
||||
for o in &mut obs {
|
||||
if o.flux == 0.0 {
|
||||
o.flux = 10.0f64.powf(-0.4 * (o.mag - baseline_mag));
|
||||
o.flux_err = o.flux * 0.4 * 2.302585 * o.mag_err;
|
||||
}
|
||||
}
|
||||
let t0_est = obs.iter().min_by(|a, b| a.mag.partial_cmp(&b.mag).unwrap())
|
||||
.map(|o| o.hjd).unwrap_or(0.0);
|
||||
let half = (baseline_mag + peak_mag) / 2.0;
|
||||
let above: Vec<_> = obs.iter().filter(|o| o.mag < half).collect();
|
||||
let t_e_est = if above.len() >= 2 {
|
||||
(above.last().unwrap().hjd - above.first().unwrap().hjd) / 2.0
|
||||
} else { 20.0 };
|
||||
let pf = 10.0f64.powf(-0.4 * (peak_mag - baseline_mag));
|
||||
let u0_est = if pf > 1.5 { (1.0 / pf).max(0.01) } else { 0.5 };
|
||||
MicrolensingEvent {
|
||||
name: name.into(), source: source.into(), observations: obs,
|
||||
baseline_mag, peak_mag, t0_est, t_e_est, u0_est,
|
||||
}
|
||||
}
|
||||
|
||||
// ---- PSPL fitting ----
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct PSPLFit { t0: f64, u0: f64, t_e: f64, f_s: f64, f_b: f64, chi2: f64 }
|
||||
|
|
@ -422,7 +314,6 @@ fn fit_pspl_linear(obs: &[Observation], t0: f64, u0: f64, t_e: f64) -> Option<(f
|
|||
|
||||
fn fit_pspl(event: &MicrolensingEvent) -> PSPLFit {
|
||||
let mut best = PSPLFit { t0: 0.0, u0: 0.0, t_e: 0.0, f_s: 1.0, f_b: 0.1, chi2: f64::MAX };
|
||||
// Coarse grid
|
||||
for t0_i in 0..20 {
|
||||
let t0 = (event.t0_est - 10.0) + 20.0 * t0_i as f64 / 19.0;
|
||||
for te_i in 0..15 {
|
||||
|
|
@ -436,7 +327,6 @@ fn fit_pspl(event: &MicrolensingEvent) -> PSPLFit {
|
|||
}
|
||||
}
|
||||
}
|
||||
// Fine refinement around best
|
||||
let b = best.clone();
|
||||
for dt in -5..=5 { for dte in -5..=5 { for du in -5..=5 {
|
||||
let (t0, t_e, u0) = (b.t0 + dt as f64 * 0.2, b.t_e * (1.0 + dte as f64 * 0.02), b.u0 + du as f64 * 0.005);
|
||||
|
|
@ -448,9 +338,7 @@ fn fit_pspl(event: &MicrolensingEvent) -> PSPLFit {
|
|||
best
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Anomaly detection (compact)
|
||||
// ---------------------------------------------------------------------------
|
||||
// ---- Anomaly detection ----
|
||||
|
||||
fn analyze_residuals(event: &MicrolensingEvent, fit: &PSPLFit) -> Vec<(f64, f64, String, Vec<f32>)> {
|
||||
let rchi2 = fit.chi2 / event.observations.len().max(1) as f64;
|
||||
|
|
@ -477,7 +365,6 @@ fn analyze_residuals(event: &MicrolensingEvent, fit: &PSPLFit) -> Vec<(f64, f64,
|
|||
let combined = 0.3 * excess + 0.7 * coherence_z;
|
||||
let class = if combined > 3.0 { if tau.abs() < 0.5 { "planet" } else { "moon-candidate" } }
|
||||
else if combined > 2.0 { "unknown" } else { "noise" };
|
||||
// 32-dim embedding
|
||||
let mean_r = resid.iter().sum::<f64>() / resid.len() as f64;
|
||||
let var_r = resid.iter().map(|r| (r - mean_r).powi(2)).sum::<f64>() / resid.len() as f64;
|
||||
let mut emb = vec![mean_r as f32, var_r.sqrt() as f32, excess as f32, coherence_z as f32, tau as f32];
|
||||
|
|
@ -494,9 +381,7 @@ fn analyze_residuals(event: &MicrolensingEvent, fit: &PSPLFit) -> Vec<(f64, f64,
|
|||
results
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main pipeline
|
||||
// ---------------------------------------------------------------------------
|
||||
// ---- Main pipeline ----
|
||||
|
||||
fn main() {
|
||||
println!("=== Real Microlensing Analysis Pipeline ===\n");
|
||||
|
|
@ -508,33 +393,33 @@ fn main() {
|
|||
dimension: dim as u16, metric: DistanceMetric::Cosine, ..Default::default()
|
||||
}).expect("failed to create store");
|
||||
|
||||
// Step 1: Build manifest and simulate downloads
|
||||
println!("--- Step 1. Progressive Download Manifest ---\n");
|
||||
// Step 1: Build manifest and simulate from published parameters
|
||||
println!("--- Step 1. Download Manifest ({} real events) ---\n", "13");
|
||||
let manifest = build_manifest();
|
||||
let mut rng = 42u64;
|
||||
let mut events = Vec::new();
|
||||
let mut states = Vec::new();
|
||||
|
||||
for entry in &manifest.events {
|
||||
let (evt, state) = simulate_from_manifest(entry, &mut rng);
|
||||
let p = entry.published_params.unwrap();
|
||||
let planet_label = if p.has_planet {
|
||||
match p.planet_mass_ratio {
|
||||
Some(q) => format!("HAS PLANET (q={:.1e})", q),
|
||||
None => "HAS PLANET".to_string(),
|
||||
}
|
||||
} else { "no planet".to_string() };
|
||||
println!(" [{}] {:24} t_E={:5.1}d, u_0={:5.3}, {}",
|
||||
state, entry.name, p.t_e, p.u0, planet_label);
|
||||
for entry in &manifest {
|
||||
let (evt, state) = simulate_from_published(entry, &mut rng);
|
||||
let p = &entry.published;
|
||||
let label = match p.planet_mass_ratio {
|
||||
Some(q) => match p.planet_separation {
|
||||
Some(s) => format!("PLANET q={:.1e} s={:.2}", q, s),
|
||||
None => format!("PLANET q={:.1e}", q),
|
||||
},
|
||||
None => "PSPL-only".into(),
|
||||
};
|
||||
println!(" [{}] {:24} t_E={:6.1}d u_0={:5.3} {}", state, entry.event_id, p.einstein_time, p.impact_param, label);
|
||||
events.push(evt);
|
||||
states.push(state);
|
||||
}
|
||||
|
||||
// Also parse a synthetic OGLE/MOA data snippet to exercise the parsers
|
||||
// Exercise the parsers on synthetic snippets
|
||||
let _ogle_test = parse_ogle_format(
|
||||
"# OGLE-IV EWS photometry\n2453500.0 18.5 0.02\n2453501.0 17.2 0.015\n", "test-ogle");
|
||||
"# OGLE-IV EWS\n2453500.0 18.5 0.02\n2453501.0 17.2 0.015\n", "test-ogle");
|
||||
let _moa_test = parse_moa_format(
|
||||
"# MOA-II photometry\n2455700.0 500.0 10.0\n2455701.0 800.0 15.0\n", "test-moa");
|
||||
"# MOA-II\n2455700.0 500.0 10.0\n2455701.0 800.0 15.0\n2455702.0 520.0 11.0\n", "test-moa");
|
||||
|
||||
// Step 2: Fit and analyze
|
||||
println!("\n--- Step 2. PSPL Fit + Anomaly Search ---\n");
|
||||
|
|
@ -601,7 +486,7 @@ fn main() {
|
|||
println!(" Witness chain: {} entries, {}", verify_witness_chain(&chain).expect("verify").len(), "VALID");
|
||||
|
||||
println!("\n=== Summary ===\n");
|
||||
println!(" Manifest events: {}", manifest.events.len());
|
||||
println!(" Manifest events: {}", manifest.len());
|
||||
println!(" Simulated: {}", states.iter().filter(|s| matches!(s, DownloadState::Simulated)).count());
|
||||
println!(" Windows ingested: {}", ingest.accepted);
|
||||
println!(" Anomalies found: {}", discoveries.len());
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue