mirror of
https://github.com/ruvnet/RuView.git
synced 2026-04-28 05:59:32 +00:00
472 lines
16 KiB
JavaScript
472 lines
16 KiB
JavaScript
/**
|
|
* WiFi-DensePose — Dual-Modal Pose Estimation Demo
|
|
*
|
|
* Main orchestration: video capture → CNN embedding → CSI processing → fusion → rendering
|
|
*/
|
|
|
|
import { VideoCapture } from './video-capture.js?v=13';
|
|
import { CsiSimulator } from './csi-simulator.js?v=13';
|
|
import { CnnEmbedder } from './cnn-embedder.js?v=13';
|
|
import { FusionEngine } from './fusion-engine.js?v=13';
|
|
import { PoseDecoder } from './pose-decoder.js?v=13';
|
|
import { CanvasRenderer } from './canvas-renderer.js?v=13';
|
|
|
|
// === State ===
|
|
let mode = 'dual'; // 'dual' | 'video' | 'csi'
|
|
let isRunning = false;
|
|
let isPaused = false;
|
|
let startTime = 0;
|
|
let frameCount = 0;
|
|
let fps = 0;
|
|
let lastFpsTime = 0;
|
|
let confidenceThreshold = 0.3;
|
|
|
|
// Latency tracking
|
|
const latency = { video: 0, csi: 0, fusion: 0, total: 0 };
|
|
|
|
// === Components ===
|
|
const videoCapture = new VideoCapture(document.getElementById('webcam'));
|
|
const csiSimulator = new CsiSimulator({ subcarriers: 52, timeWindow: 56 });
|
|
const visualCnn = new CnnEmbedder({ inputSize: 56, embeddingDim: 128, seed: 42 });
|
|
const csiCnn = new CnnEmbedder({ inputSize: 56, embeddingDim: 128, seed: 137 });
|
|
const fusionEngine = new FusionEngine(128);
|
|
const poseDecoder = new PoseDecoder(128);
|
|
const renderer = new CanvasRenderer();
|
|
|
|
// === Canvas Elements ===
|
|
const skeletonCanvas = document.getElementById('skeleton-canvas');
|
|
const skeletonCtx = skeletonCanvas.getContext('2d');
|
|
const csiCanvas = document.getElementById('csi-canvas');
|
|
const csiCtx = csiCanvas.getContext('2d');
|
|
const embeddingCanvas = document.getElementById('embedding-canvas');
|
|
const embeddingCtx = embeddingCanvas.getContext('2d');
|
|
|
|
// === UI Elements ===
|
|
const modeSelect = document.getElementById('mode-select');
|
|
const statusDot = document.getElementById('status-dot');
|
|
const statusLabel = document.getElementById('status-label');
|
|
const fpsDisplay = document.getElementById('fps-display');
|
|
const cameraPrompt = document.getElementById('camera-prompt');
|
|
const startCameraBtn = document.getElementById('start-camera-btn');
|
|
const pauseBtn = document.getElementById('pause-btn');
|
|
const confSlider = document.getElementById('confidence-slider');
|
|
const confValue = document.getElementById('confidence-value');
|
|
const wsUrlInput = document.getElementById('ws-url');
|
|
const connectWsBtn = document.getElementById('connect-ws-btn');
|
|
|
|
// Fusion bar elements
|
|
const videoBar = document.getElementById('video-bar');
|
|
const csiBar = document.getElementById('csi-bar');
|
|
const fusedBar = document.getElementById('fused-bar');
|
|
const videoBarVal = document.getElementById('video-bar-val');
|
|
const csiBarVal = document.getElementById('csi-bar-val');
|
|
const fusedBarVal = document.getElementById('fused-bar-val');
|
|
|
|
// Latency elements
|
|
const latVideoEl = document.getElementById('lat-video');
|
|
const latCsiEl = document.getElementById('lat-csi');
|
|
const latFusionEl = document.getElementById('lat-fusion');
|
|
const latTotalEl = document.getElementById('lat-total');
|
|
|
|
// Cross-modal similarity
|
|
const crossModalEl = document.getElementById('cross-modal-sim');
|
|
|
|
// RSSI elements
|
|
const rssiBarEl = document.getElementById('rssi-bar');
|
|
const rssiValueEl = document.getElementById('rssi-value');
|
|
const rssiQualityEl = document.getElementById('rssi-quality');
|
|
const rssiSparkCanvas = document.getElementById('rssi-sparkline');
|
|
const rssiSparkCtx = rssiSparkCanvas ? rssiSparkCanvas.getContext('2d') : null;
|
|
const rssiHistory = [];
|
|
const RSSI_HISTORY_MAX = 80;
|
|
|
|
// === Initialize ===
|
|
function init() {
|
|
console.log(`[PoseFusion] init() v4 — CsiSimulator=${CsiSimulator.VERSION || 'OLD'}, starting...`);
|
|
resizeCanvases();
|
|
console.log(`[PoseFusion] canvases: skeleton=${skeletonCanvas.width}x${skeletonCanvas.height}, csi=${csiCanvas.width}x${csiCanvas.height}, emb=${embeddingCanvas.width}x${embeddingCanvas.height}`);
|
|
window.addEventListener('resize', resizeCanvases);
|
|
|
|
// Mode change
|
|
modeSelect.addEventListener('change', (e) => {
|
|
mode = e.target.value;
|
|
updateModeUI();
|
|
});
|
|
|
|
// Camera start
|
|
startCameraBtn.addEventListener('click', startCamera);
|
|
|
|
// Pause
|
|
pauseBtn.addEventListener('click', () => {
|
|
isPaused = !isPaused;
|
|
pauseBtn.textContent = isPaused ? '▶ Resume' : '⏸ Pause';
|
|
pauseBtn.classList.toggle('active', isPaused);
|
|
});
|
|
|
|
// Confidence slider
|
|
confSlider.addEventListener('input', (e) => {
|
|
confidenceThreshold = parseFloat(e.target.value);
|
|
confValue.textContent = confidenceThreshold.toFixed(2);
|
|
});
|
|
|
|
// WebSocket connect
|
|
connectWsBtn.addEventListener('click', async () => {
|
|
const url = wsUrlInput.value.trim();
|
|
if (!url) return;
|
|
connectWsBtn.textContent = 'Connecting...';
|
|
const ok = await csiSimulator.connectLive(url);
|
|
connectWsBtn.textContent = ok ? '✓ Connected' : 'Connect';
|
|
if (ok) {
|
|
connectWsBtn.classList.add('active');
|
|
}
|
|
});
|
|
|
|
// Try to load RuVector Attention WASM embedders (non-blocking)
|
|
const wasmBase = new URL('../pkg/ruvector-attention', import.meta.url).href;
|
|
visualCnn.tryLoadWasm(wasmBase).then((ok) => {
|
|
// Share the WASM module with FusionEngine for cosine_similarity, normalize, etc.
|
|
if (visualCnn.rvModule) fusionEngine.setWasmModule(visualCnn.rvModule);
|
|
// Update footer backend label
|
|
const backendEl = document.getElementById('cnn-backend');
|
|
if (backendEl) {
|
|
backendEl.textContent = ok && visualCnn.useRuVector
|
|
? `RuVector WASM v${visualCnn.rvModule.version()} — 6 attention mechanisms`
|
|
: 'ruvector-cnn (JS fallback)';
|
|
}
|
|
});
|
|
csiCnn.tryLoadWasm(wasmBase);
|
|
|
|
// Auto-connect to local sensing server WebSocket if available
|
|
const defaultWsUrl = 'ws://localhost:8765/ws/sensing';
|
|
if (wsUrlInput) wsUrlInput.value = defaultWsUrl;
|
|
csiSimulator.connectLive(defaultWsUrl).then(ok => {
|
|
if (ok && connectWsBtn) {
|
|
connectWsBtn.textContent = '✓ Live ESP32';
|
|
connectWsBtn.classList.add('active');
|
|
statusLabel.textContent = 'LIVE CSI';
|
|
statusDot.classList.remove('offline');
|
|
}
|
|
});
|
|
|
|
// Auto-start camera for video/dual modes
|
|
updateModeUI();
|
|
startTime = performance.now() / 1000;
|
|
isRunning = true;
|
|
requestAnimationFrame(mainLoop);
|
|
}
|
|
|
|
async function startCamera() {
|
|
cameraPrompt.style.display = 'none';
|
|
const ok = await videoCapture.start();
|
|
if (ok) {
|
|
statusDot.classList.remove('offline');
|
|
statusLabel.textContent = 'LIVE';
|
|
resizeCanvases();
|
|
} else {
|
|
cameraPrompt.style.display = 'flex';
|
|
cameraPrompt.querySelector('p').textContent = 'Camera access denied. Try CSI-only mode.';
|
|
}
|
|
}
|
|
|
|
function updateModeUI() {
|
|
const needsVideo = mode !== 'csi';
|
|
|
|
// Show/hide camera prompt
|
|
if (needsVideo && !videoCapture.isActive) {
|
|
cameraPrompt.style.display = 'flex';
|
|
} else {
|
|
cameraPrompt.style.display = 'none';
|
|
}
|
|
|
|
// Update mode label in both the overlay and the camera prompt
|
|
const labelMap = { dual: 'DUAL FUSION', video: 'VIDEO ONLY', csi: 'CSI ONLY' };
|
|
const modeLabel = document.getElementById('mode-label');
|
|
const promptLabel = document.getElementById('prompt-mode-label');
|
|
if (modeLabel) modeLabel.textContent = labelMap[mode] || mode;
|
|
if (promptLabel) promptLabel.textContent = labelMap[mode] || mode;
|
|
}
|
|
|
|
function resizeCanvases() {
|
|
const videoPanel = document.querySelector('.video-panel');
|
|
if (videoPanel) {
|
|
const rect = videoPanel.getBoundingClientRect();
|
|
skeletonCanvas.width = rect.width;
|
|
skeletonCanvas.height = rect.height;
|
|
}
|
|
|
|
// CSI canvas (min 200px width)
|
|
csiCanvas.width = Math.max(200, csiCanvas.parentElement.clientWidth);
|
|
csiCanvas.height = 120;
|
|
|
|
// Embedding canvas (min 200px width)
|
|
embeddingCanvas.width = Math.max(200, embeddingCanvas.parentElement.clientWidth);
|
|
embeddingCanvas.height = 140;
|
|
}
|
|
|
|
// === Main Loop ===
|
|
let _loopErrorShown = false;
|
|
let _diagDone = false;
|
|
function mainLoop(timestamp) {
|
|
if (!isRunning) return;
|
|
requestAnimationFrame(mainLoop);
|
|
|
|
if (isPaused) return;
|
|
|
|
try {
|
|
const elapsed = performance.now() / 1000 - startTime;
|
|
const totalStart = performance.now();
|
|
|
|
// --- Video Pipeline ---
|
|
let videoEmb = null;
|
|
let motionRegion = null;
|
|
if (mode !== 'csi' && videoCapture.isActive) {
|
|
const t0 = performance.now();
|
|
const frame = videoCapture.captureFrame(56, 56);
|
|
if (frame) {
|
|
videoEmb = visualCnn.extract(frame.rgb, frame.width, frame.height);
|
|
motionRegion = videoCapture.detectMotionRegion(56, 56);
|
|
|
|
// Feed motion to CSI simulator for correlated demo data
|
|
// When detected=false, CSI simulator handles through-wall persistence
|
|
csiSimulator.updatePersonState(
|
|
motionRegion.detected ? 1.0 : 0,
|
|
motionRegion.detected ? motionRegion.x + motionRegion.w / 2 : 0.5,
|
|
motionRegion.detected ? motionRegion.y + motionRegion.h / 2 : 0.5,
|
|
frame.motion
|
|
);
|
|
|
|
fusionEngine.updateConfidence(
|
|
frame.brightness, frame.motion,
|
|
0, csiSimulator.isLive || mode === 'dual'
|
|
);
|
|
}
|
|
latency.video = performance.now() - t0;
|
|
}
|
|
|
|
// --- CSI Pipeline ---
|
|
let csiEmb = null;
|
|
if (mode !== 'video') {
|
|
const t0 = performance.now();
|
|
const csiFrame = csiSimulator.nextFrame(elapsed);
|
|
const pseudoImage = csiSimulator.buildPseudoImage(56);
|
|
csiEmb = csiCnn.extract(pseudoImage, 56, 56);
|
|
|
|
fusionEngine.updateConfidence(
|
|
videoCapture.brightnessScore,
|
|
videoCapture.motionScore,
|
|
csiFrame.snr,
|
|
true
|
|
);
|
|
|
|
// Draw CSI heatmap
|
|
const heatmap = csiSimulator.getHeatmapData();
|
|
renderer.drawCsiHeatmap(csiCtx, heatmap, csiCanvas.width, csiCanvas.height);
|
|
|
|
latency.csi = performance.now() - t0;
|
|
}
|
|
|
|
// --- Fusion ---
|
|
const t0f = performance.now();
|
|
const fusedEmb = fusionEngine.fuse(videoEmb, csiEmb, mode);
|
|
latency.fusion = performance.now() - t0f;
|
|
|
|
// --- Pose Decode ---
|
|
// For CSI-only mode, generate a synthetic motion region from CSI energy
|
|
if (mode === 'csi' && (!motionRegion || !motionRegion.detected)) {
|
|
const csiPresence = csiSimulator.personPresence;
|
|
if (csiPresence > 0.1) {
|
|
motionRegion = {
|
|
detected: true,
|
|
x: 0.25, y: 0.15, w: 0.5, h: 0.7,
|
|
coverage: csiPresence,
|
|
motionGrid: null,
|
|
gridCols: 10,
|
|
gridRows: 8
|
|
};
|
|
}
|
|
}
|
|
|
|
// CSI state for through-wall tracking
|
|
const csiState = {
|
|
csiPresence: csiSimulator.personPresence,
|
|
isLive: csiSimulator.isLive
|
|
};
|
|
|
|
const keypoints = poseDecoder.decode(fusedEmb, motionRegion, elapsed, csiState);
|
|
|
|
// --- Render Skeleton ---
|
|
const labelMap = { dual: 'DUAL FUSION', video: 'VIDEO ONLY', csi: 'CSI ONLY' };
|
|
renderer.drawSkeleton(skeletonCtx, keypoints, skeletonCanvas.width, skeletonCanvas.height, {
|
|
minConfidence: confidenceThreshold,
|
|
color: mode === 'csi' ? 'amber' : 'green',
|
|
label: labelMap[mode]
|
|
});
|
|
|
|
// --- Render Embedding Space ---
|
|
const embPoints = fusionEngine.getEmbeddingPoints();
|
|
renderer.drawEmbeddingSpace(embeddingCtx, embPoints, embeddingCanvas.width, embeddingCanvas.height);
|
|
|
|
// --- Update UI ---
|
|
latency.total = performance.now() - totalStart;
|
|
|
|
// FPS
|
|
frameCount++;
|
|
if (timestamp - lastFpsTime > 500) {
|
|
fps = Math.round(frameCount * 1000 / (timestamp - lastFpsTime));
|
|
lastFpsTime = timestamp;
|
|
frameCount = 0;
|
|
fpsDisplay.textContent = `${fps} FPS`;
|
|
}
|
|
|
|
// Fusion bars
|
|
const vc = fusionEngine.videoConfidence;
|
|
const cc = fusionEngine.csiConfidence;
|
|
const fc = fusionEngine.fusedConfidence;
|
|
videoBar.style.width = `${vc * 100}%`;
|
|
csiBar.style.width = `${cc * 100}%`;
|
|
fusedBar.style.width = `${fc * 100}%`;
|
|
videoBarVal.textContent = `${Math.round(vc * 100)}%`;
|
|
csiBarVal.textContent = `${Math.round(cc * 100)}%`;
|
|
fusedBarVal.textContent = `${Math.round(fc * 100)}%`;
|
|
|
|
// Latency
|
|
latVideoEl.textContent = `${latency.video.toFixed(1)}ms`;
|
|
latCsiEl.textContent = `${latency.csi.toFixed(1)}ms`;
|
|
latFusionEl.textContent = `${latency.fusion.toFixed(1)}ms`;
|
|
latTotalEl.textContent = `${latency.total.toFixed(1)}ms`;
|
|
|
|
// Cross-modal similarity
|
|
const sim = fusionEngine.getCrossModalSimilarity();
|
|
crossModalEl.textContent = sim.toFixed(3);
|
|
|
|
// RuVector attention pipeline stats
|
|
const rvStats = poseDecoder.attentionStats;
|
|
const rvEnergyEl = document.getElementById('rv-energy');
|
|
const rvRefineEl = document.getElementById('rv-refine');
|
|
const rvImpactEl = document.getElementById('rv-impact');
|
|
if (rvEnergyEl) rvEnergyEl.textContent = (rvStats.energy || 0).toFixed(2);
|
|
if (rvRefineEl) rvRefineEl.textContent = ((rvStats.refinementMag || 0) * 1000).toFixed(1) + 'px';
|
|
if (rvImpactEl) {
|
|
const impact = Math.min(100, (rvStats.refinementMag || 0) * 5000);
|
|
rvImpactEl.textContent = impact.toFixed(0) + '%';
|
|
}
|
|
// Pulse the pipeline stages when active
|
|
if (visualCnn.useRuVector && rvStats.energy > 0.1) {
|
|
document.querySelectorAll('.rv-stage').forEach(el => el.classList.add('active'));
|
|
}
|
|
|
|
// RSSI update
|
|
updateRssi(csiSimulator.rssiDbm);
|
|
|
|
// One-time diagnostic
|
|
if (!_diagDone) {
|
|
_diagDone = true;
|
|
console.log(`[PoseFusion] frame 1 OK — mode=${mode}, csi.bufLen=${csiSimulator.amplitudeBuffer.length}, embPts=${embPoints?.fused?.length ?? 0}, rssi=${(csiSimulator.rssiDbm ?? -99).toFixed(1)}`);
|
|
}
|
|
|
|
} catch (err) {
|
|
if (!_loopErrorShown) {
|
|
_loopErrorShown = true;
|
|
console.error('[MainLoop]', err);
|
|
// Show error visually on page
|
|
const errDiv = document.createElement('div');
|
|
errDiv.style.cssText = 'position:fixed;bottom:60px;left:24px;right:24px;background:rgba(255,48,64,0.95);color:#fff;padding:12px 16px;border-radius:8px;font:12px/1.4 "JetBrains Mono",monospace;z-index:9999;max-height:120px;overflow:auto';
|
|
errDiv.textContent = `[MainLoop Error] ${err.message}\n${err.stack?.split('\n').slice(0,3).join('\n')}`;
|
|
document.body.appendChild(errDiv);
|
|
}
|
|
}
|
|
}
|
|
|
|
// === RSSI Visualization ===
|
|
function updateRssi(dbm) {
|
|
if (!rssiBarEl) return;
|
|
|
|
// Clamp to typical WiFi range: -100 (worst) to -30 (best)
|
|
const clamped = Math.max(-100, Math.min(-30, dbm));
|
|
const pct = ((clamped + 100) / 70) * 100; // 0-100%
|
|
|
|
rssiBarEl.style.width = `${pct}%`;
|
|
rssiValueEl.textContent = `${Math.round(clamped)} dBm`;
|
|
|
|
// Quality label
|
|
let quality;
|
|
if (clamped > -50) quality = 'Excellent';
|
|
else if (clamped > -60) quality = 'Good';
|
|
else if (clamped > -70) quality = 'Fair';
|
|
else if (clamped > -80) quality = 'Weak';
|
|
else quality = 'Poor';
|
|
rssiQualityEl.textContent = quality;
|
|
|
|
// Color the dBm value based on quality
|
|
if (clamped > -60) rssiValueEl.style.color = 'var(--green-glow)';
|
|
else if (clamped > -75) rssiValueEl.style.color = 'var(--amber)';
|
|
else rssiValueEl.style.color = 'var(--red-alert)';
|
|
|
|
// Sparkline history
|
|
rssiHistory.push(clamped);
|
|
if (rssiHistory.length > RSSI_HISTORY_MAX) rssiHistory.shift();
|
|
drawRssiSparkline();
|
|
}
|
|
|
|
function drawRssiSparkline() {
|
|
if (!rssiSparkCtx || rssiHistory.length < 2) return;
|
|
const w = rssiSparkCanvas.width;
|
|
const h = rssiSparkCanvas.height;
|
|
const ctx = rssiSparkCtx;
|
|
|
|
ctx.clearRect(0, 0, w, h);
|
|
|
|
// Draw signal strength line
|
|
const len = rssiHistory.length;
|
|
const step = w / (RSSI_HISTORY_MAX - 1);
|
|
|
|
// Gradient fill under line
|
|
const grad = ctx.createLinearGradient(0, 0, 0, h);
|
|
grad.addColorStop(0, 'rgba(0,210,120,0.3)');
|
|
grad.addColorStop(1, 'rgba(0,210,120,0)');
|
|
|
|
ctx.beginPath();
|
|
for (let i = 0; i < len; i++) {
|
|
const x = (RSSI_HISTORY_MAX - len + i) * step;
|
|
const y = h - ((rssiHistory[i] + 100) / 70) * h;
|
|
if (i === 0) ctx.moveTo(x, y);
|
|
else ctx.lineTo(x, y);
|
|
}
|
|
// Fill area
|
|
const lastX = (RSSI_HISTORY_MAX - 1) * step;
|
|
const firstX = (RSSI_HISTORY_MAX - len) * step;
|
|
ctx.lineTo(lastX, h);
|
|
ctx.lineTo(firstX, h);
|
|
ctx.closePath();
|
|
ctx.fillStyle = grad;
|
|
ctx.fill();
|
|
|
|
// Draw line on top
|
|
ctx.beginPath();
|
|
for (let i = 0; i < len; i++) {
|
|
const x = (RSSI_HISTORY_MAX - len + i) * step;
|
|
const y = h - ((rssiHistory[i] + 100) / 70) * h;
|
|
if (i === 0) ctx.moveTo(x, y);
|
|
else ctx.lineTo(x, y);
|
|
}
|
|
ctx.strokeStyle = '#00d878';
|
|
ctx.lineWidth = 1.5;
|
|
ctx.stroke();
|
|
|
|
// Pulsing dot at latest value
|
|
const latestX = lastX;
|
|
const latestY = h - ((rssiHistory[len - 1] + 100) / 70) * h;
|
|
const pulse = 0.5 + 0.5 * Math.sin(performance.now() / 300);
|
|
ctx.beginPath();
|
|
ctx.arc(latestX, latestY, 2 + pulse, 0, Math.PI * 2);
|
|
ctx.fillStyle = '#00d878';
|
|
ctx.fill();
|
|
ctx.beginPath();
|
|
ctx.arc(latestX, latestY, 4 + pulse * 2, 0, Math.PI * 2);
|
|
ctx.strokeStyle = `rgba(0,216,120,${0.3 + pulse * 0.3})`;
|
|
ctx.lineWidth = 1;
|
|
ctx.stroke();
|
|
}
|
|
|
|
// Boot
|
|
document.addEventListener('DOMContentLoaded', init);
|