mirror of
https://github.com/ruvnet/RuVector.git
synced 2026-05-29 19:33:34 +00:00
feat(npm): add @ruvector/raft, @ruvector/replication, @ruvector/scipix packages (#129)
- @ruvector/raft: Raft consensus implementation for distributed systems - Leader election and log replication - Fault-tolerant state machine - Configurable election timeouts and heartbeats - @ruvector/replication: Data replication and synchronization - Multi-node replica sets with primary/secondary roles - Vector clocks for conflict detection - Sync modes: synchronous, asynchronous, semi-sync - Automatic failover with configurable policies - @ruvector/scipix: OCR client for scientific documents - LaTeX and MathML extraction from equations - Batch processing support - Multiple output formats (LaTeX, MathML, AsciiMath, Text) All packages built with TypeScript, fully typed, ready for npm publish. Co-authored-by: Reuven <cohen@ruv-mac-mini.local> Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
rUv
GitHub
parent
02cde18353
commit
2d2c23fceb
19 changed files with 2807 additions and 0 deletions
66
npm/packages/raft/package.json
Normal file
66
npm/packages/raft/package.json
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
{
|
||||
"name": "@ruvector/raft",
|
||||
"version": "0.1.0",
|
||||
"description": "Raft consensus implementation for distributed systems - leader election, log replication, and fault tolerance",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"exports": {
|
||||
".": {
|
||||
"import": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
}
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"prepublishOnly": "npm run build",
|
||||
"test": "node --test test/*.test.js",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"clean": "rm -rf dist"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.19.30",
|
||||
"typescript": "^5.9.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"eventemitter3": "^5.0.4"
|
||||
},
|
||||
"keywords": [
|
||||
"raft",
|
||||
"consensus",
|
||||
"distributed-systems",
|
||||
"leader-election",
|
||||
"log-replication",
|
||||
"fault-tolerance",
|
||||
"distributed-consensus",
|
||||
"ruvector",
|
||||
"cluster"
|
||||
],
|
||||
"author": "rUv Team <team@ruv.io>",
|
||||
"license": "MIT OR Apache-2.0",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/ruvnet/ruvector.git",
|
||||
"directory": "npm/packages/raft"
|
||||
},
|
||||
"homepage": "https://github.com/ruvnet/ruvector/tree/main/crates/ruvector-raft",
|
||||
"bugs": {
|
||||
"url": "https://github.com/ruvnet/ruvector/issues"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 18"
|
||||
},
|
||||
"publishConfig": {
|
||||
"registry": "https://registry.npmjs.org/",
|
||||
"access": "public"
|
||||
},
|
||||
"files": [
|
||||
"dist",
|
||||
"README.md"
|
||||
]
|
||||
}
|
||||
78
npm/packages/raft/src/index.ts
Normal file
78
npm/packages/raft/src/index.ts
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
/**
|
||||
* @ruvector/raft - Raft Consensus Implementation
|
||||
*
|
||||
* A TypeScript implementation of the Raft consensus algorithm for
|
||||
* distributed systems, providing leader election, log replication,
|
||||
* and fault tolerance.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* import { RaftNode, RaftTransport, NodeState } from '@ruvector/raft';
|
||||
*
|
||||
* // Create a Raft node
|
||||
* const node = new RaftNode({
|
||||
* nodeId: 'node-1',
|
||||
* peers: ['node-2', 'node-3'],
|
||||
* electionTimeout: [150, 300],
|
||||
* heartbeatInterval: 50,
|
||||
* maxEntriesPerRequest: 100,
|
||||
* });
|
||||
*
|
||||
* // Set up transport for RPC communication
|
||||
* node.setTransport(myTransport);
|
||||
*
|
||||
* // Set up state machine for applying commands
|
||||
* node.setStateMachine(myStateMachine);
|
||||
*
|
||||
* // Listen for events
|
||||
* node.on('stateChange', (event) => {
|
||||
* console.log(`State changed: ${event.previousState} -> ${event.newState}`);
|
||||
* });
|
||||
*
|
||||
* node.on('leaderElected', (event) => {
|
||||
* console.log(`New leader: ${event.leaderId} in term ${event.term}`);
|
||||
* });
|
||||
*
|
||||
* // Start the node
|
||||
* node.start();
|
||||
*
|
||||
* // Propose a command (only works if leader)
|
||||
* if (node.isLeader) {
|
||||
* await node.propose({ type: 'SET', key: 'foo', value: 'bar' });
|
||||
* }
|
||||
* ```
|
||||
*
|
||||
* @packageDocumentation
|
||||
*/
|
||||
|
||||
// Types
|
||||
export {
|
||||
NodeId,
|
||||
Term,
|
||||
LogIndex,
|
||||
NodeState,
|
||||
LogEntry,
|
||||
PersistentState,
|
||||
VolatileState,
|
||||
LeaderState,
|
||||
RaftNodeConfig,
|
||||
RequestVoteRequest,
|
||||
RequestVoteResponse,
|
||||
AppendEntriesRequest,
|
||||
AppendEntriesResponse,
|
||||
RaftError,
|
||||
RaftErrorCode,
|
||||
RaftEvent,
|
||||
StateChangeEvent,
|
||||
LeaderElectedEvent,
|
||||
LogCommittedEvent,
|
||||
} from './types.js';
|
||||
|
||||
// Log
|
||||
export { RaftLog } from './log.js';
|
||||
|
||||
// State
|
||||
export { RaftState } from './state.js';
|
||||
|
||||
// Node
|
||||
export { RaftNode, RaftTransport, StateMachine } from './node.js';
|
||||
135
npm/packages/raft/src/log.ts
Normal file
135
npm/packages/raft/src/log.ts
Normal file
|
|
@ -0,0 +1,135 @@
|
|||
/**
|
||||
* Raft Log Implementation
|
||||
* Manages the replicated log with persistence support
|
||||
*/
|
||||
|
||||
import { LogEntry, LogIndex, Term } from './types.js';
|
||||
|
||||
/** In-memory log storage with optional persistence callback */
|
||||
export class RaftLog<T = unknown> {
|
||||
private entries: LogEntry<T>[] = [];
|
||||
private persistCallback?: (entries: LogEntry<T>[]) => Promise<void>;
|
||||
|
||||
constructor(options?: { onPersist?: (entries: LogEntry<T>[]) => Promise<void> }) {
|
||||
this.persistCallback = options?.onPersist;
|
||||
}
|
||||
|
||||
/** Get the last log index */
|
||||
get lastIndex(): LogIndex {
|
||||
return this.entries.length > 0 ? this.entries[this.entries.length - 1].index : 0;
|
||||
}
|
||||
|
||||
/** Get the last log term */
|
||||
get lastTerm(): Term {
|
||||
return this.entries.length > 0 ? this.entries[this.entries.length - 1].term : 0;
|
||||
}
|
||||
|
||||
/** Get log length */
|
||||
get length(): number {
|
||||
return this.entries.length;
|
||||
}
|
||||
|
||||
/** Get entry at index */
|
||||
get(index: LogIndex): LogEntry<T> | undefined {
|
||||
return this.entries.find((e) => e.index === index);
|
||||
}
|
||||
|
||||
/** Get term at index */
|
||||
termAt(index: LogIndex): Term | undefined {
|
||||
if (index === 0) return 0;
|
||||
const entry = this.get(index);
|
||||
return entry?.term;
|
||||
}
|
||||
|
||||
/** Append entries to log */
|
||||
async append(entries: LogEntry<T>[]): Promise<void> {
|
||||
if (entries.length === 0) return;
|
||||
|
||||
// Find where to start appending (handle conflicting entries)
|
||||
for (const entry of entries) {
|
||||
const existing = this.get(entry.index);
|
||||
if (existing) {
|
||||
if (existing.term !== entry.term) {
|
||||
// Conflict: delete this and all following entries
|
||||
this.truncateFrom(entry.index);
|
||||
} else {
|
||||
// Same entry, skip
|
||||
continue;
|
||||
}
|
||||
}
|
||||
this.entries.push(entry);
|
||||
}
|
||||
|
||||
// Sort by index to maintain order
|
||||
this.entries.sort((a, b) => a.index - b.index);
|
||||
|
||||
if (this.persistCallback) {
|
||||
await this.persistCallback(this.entries);
|
||||
}
|
||||
}
|
||||
|
||||
/** Append a single command, returning the new entry */
|
||||
async appendCommand(term: Term, command: T): Promise<LogEntry<T>> {
|
||||
const entry: LogEntry<T> = {
|
||||
term,
|
||||
index: this.lastIndex + 1,
|
||||
command,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
await this.append([entry]);
|
||||
return entry;
|
||||
}
|
||||
|
||||
/** Get entries starting from index */
|
||||
getFrom(startIndex: LogIndex, maxCount?: number): LogEntry<T>[] {
|
||||
const result: LogEntry<T>[] = [];
|
||||
for (const entry of this.entries) {
|
||||
if (entry.index >= startIndex) {
|
||||
result.push(entry);
|
||||
if (maxCount && result.length >= maxCount) break;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Get entries in range [start, end] */
|
||||
getRange(startIndex: LogIndex, endIndex: LogIndex): LogEntry<T>[] {
|
||||
return this.entries.filter((e) => e.index >= startIndex && e.index <= endIndex);
|
||||
}
|
||||
|
||||
/** Truncate log from index (remove index and all following) */
|
||||
truncateFrom(index: LogIndex): void {
|
||||
this.entries = this.entries.filter((e) => e.index < index);
|
||||
}
|
||||
|
||||
/** Check if log is at least as up-to-date as given term/index */
|
||||
isUpToDate(lastLogTerm: Term, lastLogIndex: LogIndex): boolean {
|
||||
if (this.lastTerm !== lastLogTerm) {
|
||||
return this.lastTerm > lastLogTerm;
|
||||
}
|
||||
return this.lastIndex >= lastLogIndex;
|
||||
}
|
||||
|
||||
/** Check if log contains entry at index with matching term */
|
||||
containsEntry(index: LogIndex, term: Term): boolean {
|
||||
if (index === 0) return true;
|
||||
const entry = this.get(index);
|
||||
return entry?.term === term;
|
||||
}
|
||||
|
||||
/** Get all entries */
|
||||
getAll(): LogEntry<T>[] {
|
||||
return [...this.entries];
|
||||
}
|
||||
|
||||
/** Clear all entries */
|
||||
clear(): void {
|
||||
this.entries = [];
|
||||
}
|
||||
|
||||
/** Load entries from storage */
|
||||
load(entries: LogEntry<T>[]): void {
|
||||
this.entries = [...entries];
|
||||
this.entries.sort((a, b) => a.index - b.index);
|
||||
}
|
||||
}
|
||||
435
npm/packages/raft/src/node.ts
Normal file
435
npm/packages/raft/src/node.ts
Normal file
|
|
@ -0,0 +1,435 @@
|
|||
/**
|
||||
* Raft Node Implementation
|
||||
* Core Raft consensus algorithm implementation
|
||||
*/
|
||||
|
||||
import EventEmitter from 'eventemitter3';
|
||||
import {
|
||||
NodeId,
|
||||
Term,
|
||||
LogIndex,
|
||||
NodeState,
|
||||
RaftNodeConfig,
|
||||
RequestVoteRequest,
|
||||
RequestVoteResponse,
|
||||
AppendEntriesRequest,
|
||||
AppendEntriesResponse,
|
||||
LogEntry,
|
||||
RaftError,
|
||||
RaftEvent,
|
||||
StateChangeEvent,
|
||||
LeaderElectedEvent,
|
||||
LogCommittedEvent,
|
||||
PersistentState,
|
||||
} from './types.js';
|
||||
import { RaftState } from './state.js';
|
||||
|
||||
/** Transport interface for sending RPCs to peers */
|
||||
export interface RaftTransport<T = unknown> {
|
||||
/** Send RequestVote RPC to a peer */
|
||||
requestVote(peerId: NodeId, request: RequestVoteRequest): Promise<RequestVoteResponse>;
|
||||
/** Send AppendEntries RPC to a peer */
|
||||
appendEntries(peerId: NodeId, request: AppendEntriesRequest<T>): Promise<AppendEntriesResponse>;
|
||||
}
|
||||
|
||||
/** State machine interface for applying committed entries */
|
||||
export interface StateMachine<T = unknown, R = void> {
|
||||
/** Apply a committed command to the state machine */
|
||||
apply(command: T): Promise<R>;
|
||||
}
|
||||
|
||||
/** Default configuration values */
|
||||
const DEFAULT_CONFIG: Partial<RaftNodeConfig> = {
|
||||
electionTimeout: [150, 300],
|
||||
heartbeatInterval: 50,
|
||||
maxEntriesPerRequest: 100,
|
||||
};
|
||||
|
||||
/** Raft consensus node */
|
||||
export class RaftNode<T = unknown, R = void> extends EventEmitter {
|
||||
private readonly config: Required<RaftNodeConfig>;
|
||||
private readonly state: RaftState<T>;
|
||||
private nodeState: NodeState = NodeState.Follower;
|
||||
private leaderId: NodeId | null = null;
|
||||
private transport: RaftTransport<T> | null = null;
|
||||
private stateMachine: StateMachine<T, R> | null = null;
|
||||
|
||||
private electionTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private heartbeatTimer: ReturnType<typeof setInterval> | null = null;
|
||||
private running = false;
|
||||
|
||||
constructor(config: RaftNodeConfig) {
|
||||
super();
|
||||
this.config = { ...DEFAULT_CONFIG, ...config } as Required<RaftNodeConfig>;
|
||||
this.state = new RaftState<T>(config.nodeId, config.peers);
|
||||
}
|
||||
|
||||
/** Get node ID */
|
||||
get nodeId(): NodeId {
|
||||
return this.config.nodeId;
|
||||
}
|
||||
|
||||
/** Get current state */
|
||||
get currentState(): NodeState {
|
||||
return this.nodeState;
|
||||
}
|
||||
|
||||
/** Get current term */
|
||||
get currentTerm(): Term {
|
||||
return this.state.currentTerm;
|
||||
}
|
||||
|
||||
/** Get current leader ID */
|
||||
get leader(): NodeId | null {
|
||||
return this.leaderId;
|
||||
}
|
||||
|
||||
/** Check if this node is the leader */
|
||||
get isLeader(): boolean {
|
||||
return this.nodeState === NodeState.Leader;
|
||||
}
|
||||
|
||||
/** Get commit index */
|
||||
get commitIndex(): LogIndex {
|
||||
return this.state.commitIndex;
|
||||
}
|
||||
|
||||
/** Set transport for RPC communication */
|
||||
setTransport(transport: RaftTransport<T>): void {
|
||||
this.transport = transport;
|
||||
}
|
||||
|
||||
/** Set state machine for applying commands */
|
||||
setStateMachine(stateMachine: StateMachine<T, R>): void {
|
||||
this.stateMachine = stateMachine;
|
||||
}
|
||||
|
||||
/** Start the Raft node */
|
||||
start(): void {
|
||||
if (this.running) return;
|
||||
this.running = true;
|
||||
this.resetElectionTimer();
|
||||
}
|
||||
|
||||
/** Stop the Raft node */
|
||||
stop(): void {
|
||||
this.running = false;
|
||||
this.clearTimers();
|
||||
}
|
||||
|
||||
/** Propose a command to be replicated (only works if leader) */
|
||||
async propose(command: T): Promise<LogEntry<T>> {
|
||||
if (this.nodeState !== NodeState.Leader) {
|
||||
throw RaftError.notLeader();
|
||||
}
|
||||
|
||||
const entry = await this.state.log.appendCommand(this.state.currentTerm, command);
|
||||
this.emit(RaftEvent.LogAppended, entry);
|
||||
|
||||
// Immediately replicate to followers
|
||||
await this.replicateToFollowers();
|
||||
|
||||
return entry;
|
||||
}
|
||||
|
||||
/** Handle RequestVote RPC from a candidate */
|
||||
async handleRequestVote(request: RequestVoteRequest): Promise<RequestVoteResponse> {
|
||||
// If request term is higher, update term and become follower
|
||||
if (request.term > this.state.currentTerm) {
|
||||
await this.state.setTerm(request.term);
|
||||
this.transitionTo(NodeState.Follower);
|
||||
}
|
||||
|
||||
// Deny vote if request term is less than current term
|
||||
if (request.term < this.state.currentTerm) {
|
||||
return { term: this.state.currentTerm, voteGranted: false };
|
||||
}
|
||||
|
||||
// Check if we can vote for this candidate
|
||||
const canVote =
|
||||
(this.state.votedFor === null || this.state.votedFor === request.candidateId) &&
|
||||
this.state.log.isUpToDate(request.lastLogTerm, request.lastLogIndex);
|
||||
|
||||
if (canVote) {
|
||||
await this.state.vote(request.term, request.candidateId);
|
||||
this.resetElectionTimer();
|
||||
this.emit(RaftEvent.VoteGranted, { candidateId: request.candidateId, term: request.term });
|
||||
return { term: this.state.currentTerm, voteGranted: true };
|
||||
}
|
||||
|
||||
return { term: this.state.currentTerm, voteGranted: false };
|
||||
}
|
||||
|
||||
/** Handle AppendEntries RPC from leader */
|
||||
async handleAppendEntries(request: AppendEntriesRequest<T>): Promise<AppendEntriesResponse> {
|
||||
// If request term is higher, update term
|
||||
if (request.term > this.state.currentTerm) {
|
||||
await this.state.setTerm(request.term);
|
||||
this.transitionTo(NodeState.Follower);
|
||||
}
|
||||
|
||||
// Reject if term is less than current term
|
||||
if (request.term < this.state.currentTerm) {
|
||||
return { term: this.state.currentTerm, success: false };
|
||||
}
|
||||
|
||||
// Valid leader - reset election timer
|
||||
this.leaderId = request.leaderId;
|
||||
this.resetElectionTimer();
|
||||
|
||||
// If not follower, become follower
|
||||
if (this.nodeState !== NodeState.Follower) {
|
||||
this.transitionTo(NodeState.Follower);
|
||||
}
|
||||
|
||||
this.emit(RaftEvent.Heartbeat, { leaderId: request.leaderId, term: request.term });
|
||||
|
||||
// Check if log contains entry at prevLogIndex with prevLogTerm
|
||||
if (request.prevLogIndex > 0 && !this.state.log.containsEntry(request.prevLogIndex, request.prevLogTerm)) {
|
||||
return { term: this.state.currentTerm, success: false };
|
||||
}
|
||||
|
||||
// Append entries
|
||||
if (request.entries.length > 0) {
|
||||
await this.state.log.append(request.entries);
|
||||
}
|
||||
|
||||
// Update commit index
|
||||
if (request.leaderCommit > this.state.commitIndex) {
|
||||
this.state.setCommitIndex(
|
||||
Math.min(request.leaderCommit, this.state.log.lastIndex),
|
||||
);
|
||||
await this.applyCommitted();
|
||||
}
|
||||
|
||||
return {
|
||||
term: this.state.currentTerm,
|
||||
success: true,
|
||||
matchIndex: this.state.log.lastIndex,
|
||||
};
|
||||
}
|
||||
|
||||
/** Load persistent state */
|
||||
loadState(state: PersistentState<T>): void {
|
||||
this.state.loadPersistentState(state);
|
||||
}
|
||||
|
||||
/** Get current persistent state */
|
||||
getState(): PersistentState<T> {
|
||||
return this.state.getPersistentState();
|
||||
}
|
||||
|
||||
// Private methods
|
||||
|
||||
private transitionTo(newState: NodeState): void {
|
||||
const previousState = this.nodeState;
|
||||
if (previousState === newState) return;
|
||||
|
||||
this.nodeState = newState;
|
||||
this.clearTimers();
|
||||
|
||||
if (newState === NodeState.Leader) {
|
||||
this.state.initLeaderState();
|
||||
this.leaderId = this.config.nodeId;
|
||||
this.startHeartbeat();
|
||||
this.emit(RaftEvent.LeaderElected, {
|
||||
leaderId: this.config.nodeId,
|
||||
term: this.state.currentTerm,
|
||||
} as LeaderElectedEvent);
|
||||
} else {
|
||||
this.state.clearLeaderState();
|
||||
if (newState === NodeState.Follower) {
|
||||
this.leaderId = null;
|
||||
this.resetElectionTimer();
|
||||
}
|
||||
}
|
||||
|
||||
this.emit(RaftEvent.StateChange, {
|
||||
previousState,
|
||||
newState,
|
||||
term: this.state.currentTerm,
|
||||
} as StateChangeEvent);
|
||||
}
|
||||
|
||||
private getRandomElectionTimeout(): number {
|
||||
const [min, max] = this.config.electionTimeout;
|
||||
return min + Math.random() * (max - min);
|
||||
}
|
||||
|
||||
private resetElectionTimer(): void {
|
||||
if (this.electionTimer) {
|
||||
clearTimeout(this.electionTimer);
|
||||
}
|
||||
if (!this.running) return;
|
||||
|
||||
this.electionTimer = setTimeout(() => {
|
||||
this.startElection();
|
||||
}, this.getRandomElectionTimeout());
|
||||
}
|
||||
|
||||
private clearTimers(): void {
|
||||
if (this.electionTimer) {
|
||||
clearTimeout(this.electionTimer);
|
||||
this.electionTimer = null;
|
||||
}
|
||||
if (this.heartbeatTimer) {
|
||||
clearInterval(this.heartbeatTimer);
|
||||
this.heartbeatTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
private async startElection(): Promise<void> {
|
||||
if (!this.running) return;
|
||||
|
||||
// Increment term and become candidate
|
||||
await this.state.setTerm(this.state.currentTerm + 1);
|
||||
await this.state.vote(this.state.currentTerm, this.config.nodeId);
|
||||
this.transitionTo(NodeState.Candidate);
|
||||
|
||||
this.emit(RaftEvent.VoteRequested, {
|
||||
term: this.state.currentTerm,
|
||||
candidateId: this.config.nodeId,
|
||||
});
|
||||
|
||||
// Start with 1 vote (self)
|
||||
let votesReceived = 1;
|
||||
const majority = Math.floor((this.config.peers.length + 1) / 2) + 1;
|
||||
|
||||
// Request votes from all peers
|
||||
if (!this.transport) {
|
||||
this.resetElectionTimer();
|
||||
return;
|
||||
}
|
||||
|
||||
const votePromises = this.config.peers.map(async (peerId) => {
|
||||
try {
|
||||
const response = await this.transport!.requestVote(peerId, {
|
||||
term: this.state.currentTerm,
|
||||
candidateId: this.config.nodeId,
|
||||
lastLogIndex: this.state.log.lastIndex,
|
||||
lastLogTerm: this.state.log.lastTerm,
|
||||
});
|
||||
|
||||
// If response term is higher, become follower
|
||||
if (response.term > this.state.currentTerm) {
|
||||
await this.state.setTerm(response.term);
|
||||
this.transitionTo(NodeState.Follower);
|
||||
return;
|
||||
}
|
||||
|
||||
if (response.voteGranted && this.nodeState === NodeState.Candidate) {
|
||||
votesReceived++;
|
||||
if (votesReceived >= majority) {
|
||||
this.transitionTo(NodeState.Leader);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Peer unavailable, continue
|
||||
}
|
||||
});
|
||||
|
||||
await Promise.allSettled(votePromises);
|
||||
|
||||
// If still candidate, restart election timer
|
||||
if (this.nodeState === NodeState.Candidate) {
|
||||
this.resetElectionTimer();
|
||||
}
|
||||
}
|
||||
|
||||
private startHeartbeat(): void {
|
||||
if (this.heartbeatTimer) {
|
||||
clearInterval(this.heartbeatTimer);
|
||||
}
|
||||
|
||||
// Send immediate heartbeat
|
||||
this.replicateToFollowers();
|
||||
|
||||
// Start periodic heartbeat
|
||||
this.heartbeatTimer = setInterval(() => {
|
||||
if (this.nodeState === NodeState.Leader) {
|
||||
this.replicateToFollowers();
|
||||
}
|
||||
}, this.config.heartbeatInterval);
|
||||
}
|
||||
|
||||
private async replicateToFollowers(): Promise<void> {
|
||||
if (!this.transport || this.nodeState !== NodeState.Leader) return;
|
||||
|
||||
const replicationPromises = this.config.peers.map(async (peerId) => {
|
||||
await this.replicateToPeer(peerId);
|
||||
});
|
||||
|
||||
await Promise.allSettled(replicationPromises);
|
||||
|
||||
// Update commit index if majority have replicated
|
||||
if (this.state.updateCommitIndex()) {
|
||||
this.emit(RaftEvent.LogCommitted, {
|
||||
index: this.state.commitIndex,
|
||||
term: this.state.currentTerm,
|
||||
} as LogCommittedEvent);
|
||||
await this.applyCommitted();
|
||||
}
|
||||
}
|
||||
|
||||
private async replicateToPeer(peerId: NodeId): Promise<void> {
|
||||
if (!this.transport || this.nodeState !== NodeState.Leader) return;
|
||||
|
||||
const nextIndex = this.state.getNextIndex(peerId);
|
||||
const prevLogIndex = nextIndex - 1;
|
||||
const prevLogTerm = this.state.log.termAt(prevLogIndex) ?? 0;
|
||||
const entries = this.state.log.getFrom(nextIndex, this.config.maxEntriesPerRequest);
|
||||
|
||||
try {
|
||||
const response = await this.transport.appendEntries(peerId, {
|
||||
term: this.state.currentTerm,
|
||||
leaderId: this.config.nodeId,
|
||||
prevLogIndex,
|
||||
prevLogTerm,
|
||||
entries,
|
||||
leaderCommit: this.state.commitIndex,
|
||||
});
|
||||
|
||||
if (response.term > this.state.currentTerm) {
|
||||
await this.state.setTerm(response.term);
|
||||
this.transitionTo(NodeState.Follower);
|
||||
return;
|
||||
}
|
||||
|
||||
if (response.success) {
|
||||
if (response.matchIndex !== undefined) {
|
||||
this.state.setNextIndex(peerId, response.matchIndex + 1);
|
||||
this.state.setMatchIndex(peerId, response.matchIndex);
|
||||
} else if (entries.length > 0) {
|
||||
const lastEntry = entries[entries.length - 1];
|
||||
this.state.setNextIndex(peerId, lastEntry.index + 1);
|
||||
this.state.setMatchIndex(peerId, lastEntry.index);
|
||||
}
|
||||
} else {
|
||||
// Decrement nextIndex and retry
|
||||
this.state.setNextIndex(peerId, nextIndex - 1);
|
||||
}
|
||||
} catch {
|
||||
// Peer unavailable, will retry on next heartbeat
|
||||
}
|
||||
}
|
||||
|
||||
private async applyCommitted(): Promise<void> {
|
||||
while (this.state.lastApplied < this.state.commitIndex) {
|
||||
const nextIndex = this.state.lastApplied + 1;
|
||||
const entry = this.state.log.get(nextIndex);
|
||||
|
||||
if (entry && this.stateMachine) {
|
||||
try {
|
||||
await this.stateMachine.apply(entry.command);
|
||||
this.state.setLastApplied(nextIndex);
|
||||
this.emit(RaftEvent.LogApplied, entry);
|
||||
} catch (error) {
|
||||
this.emit(RaftEvent.Error, error);
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
this.state.setLastApplied(nextIndex);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
200
npm/packages/raft/src/state.ts
Normal file
200
npm/packages/raft/src/state.ts
Normal file
|
|
@ -0,0 +1,200 @@
|
|||
/**
|
||||
* Raft State Management
|
||||
* Manages persistent and volatile state for Raft consensus
|
||||
*/
|
||||
|
||||
import type {
|
||||
NodeId,
|
||||
Term,
|
||||
LogIndex,
|
||||
PersistentState,
|
||||
VolatileState,
|
||||
LeaderState,
|
||||
LogEntry,
|
||||
} from './types.js';
|
||||
import { RaftLog } from './log.js';
|
||||
|
||||
/** State manager for a Raft node */
|
||||
export class RaftState<T = unknown> {
|
||||
private _currentTerm: Term = 0;
|
||||
private _votedFor: NodeId | null = null;
|
||||
private _commitIndex: LogIndex = 0;
|
||||
private _lastApplied: LogIndex = 0;
|
||||
private _leaderState: LeaderState | null = null;
|
||||
|
||||
public readonly log: RaftLog<T>;
|
||||
|
||||
constructor(
|
||||
private readonly nodeId: NodeId,
|
||||
private readonly peers: NodeId[],
|
||||
options?: {
|
||||
onPersist?: (state: PersistentState<T>) => Promise<void>;
|
||||
onLogPersist?: (entries: LogEntry<T>[]) => Promise<void>;
|
||||
},
|
||||
) {
|
||||
this.log = new RaftLog({ onPersist: options?.onLogPersist });
|
||||
this.persistCallback = options?.onPersist;
|
||||
}
|
||||
|
||||
private persistCallback?: (state: PersistentState<T>) => Promise<void>;
|
||||
|
||||
/** Get current term */
|
||||
get currentTerm(): Term {
|
||||
return this._currentTerm;
|
||||
}
|
||||
|
||||
/** Get voted for */
|
||||
get votedFor(): NodeId | null {
|
||||
return this._votedFor;
|
||||
}
|
||||
|
||||
/** Get commit index */
|
||||
get commitIndex(): LogIndex {
|
||||
return this._commitIndex;
|
||||
}
|
||||
|
||||
/** Get last applied */
|
||||
get lastApplied(): LogIndex {
|
||||
return this._lastApplied;
|
||||
}
|
||||
|
||||
/** Get leader state (null if not leader) */
|
||||
get leaderState(): LeaderState | null {
|
||||
return this._leaderState;
|
||||
}
|
||||
|
||||
/** Update term (with persistence) */
|
||||
async setTerm(term: Term): Promise<void> {
|
||||
if (term > this._currentTerm) {
|
||||
this._currentTerm = term;
|
||||
this._votedFor = null;
|
||||
await this.persist();
|
||||
}
|
||||
}
|
||||
|
||||
/** Record vote (with persistence) */
|
||||
async vote(term: Term, candidateId: NodeId): Promise<void> {
|
||||
this._currentTerm = term;
|
||||
this._votedFor = candidateId;
|
||||
await this.persist();
|
||||
}
|
||||
|
||||
/** Update commit index */
|
||||
setCommitIndex(index: LogIndex): void {
|
||||
if (index > this._commitIndex) {
|
||||
this._commitIndex = index;
|
||||
}
|
||||
}
|
||||
|
||||
/** Update last applied */
|
||||
setLastApplied(index: LogIndex): void {
|
||||
if (index > this._lastApplied) {
|
||||
this._lastApplied = index;
|
||||
}
|
||||
}
|
||||
|
||||
/** Initialize leader state */
|
||||
initLeaderState(): void {
|
||||
const nextIndex = new Map<NodeId, LogIndex>();
|
||||
const matchIndex = new Map<NodeId, LogIndex>();
|
||||
|
||||
for (const peer of this.peers) {
|
||||
// Initialize nextIndex to leader's last log index + 1
|
||||
nextIndex.set(peer, this.log.lastIndex + 1);
|
||||
// Initialize matchIndex to 0
|
||||
matchIndex.set(peer, 0);
|
||||
}
|
||||
|
||||
this._leaderState = { nextIndex, matchIndex };
|
||||
}
|
||||
|
||||
/** Clear leader state */
|
||||
clearLeaderState(): void {
|
||||
this._leaderState = null;
|
||||
}
|
||||
|
||||
/** Update nextIndex for a peer */
|
||||
setNextIndex(peerId: NodeId, index: LogIndex): void {
|
||||
if (this._leaderState) {
|
||||
this._leaderState.nextIndex.set(peerId, Math.max(1, index));
|
||||
}
|
||||
}
|
||||
|
||||
/** Update matchIndex for a peer */
|
||||
setMatchIndex(peerId: NodeId, index: LogIndex): void {
|
||||
if (this._leaderState) {
|
||||
this._leaderState.matchIndex.set(peerId, index);
|
||||
}
|
||||
}
|
||||
|
||||
/** Get nextIndex for a peer */
|
||||
getNextIndex(peerId: NodeId): LogIndex {
|
||||
return this._leaderState?.nextIndex.get(peerId) ?? this.log.lastIndex + 1;
|
||||
}
|
||||
|
||||
/** Get matchIndex for a peer */
|
||||
getMatchIndex(peerId: NodeId): LogIndex {
|
||||
return this._leaderState?.matchIndex.get(peerId) ?? 0;
|
||||
}
|
||||
|
||||
/** Update commit index based on match indices (for leader) */
|
||||
updateCommitIndex(): boolean {
|
||||
if (!this._leaderState) return false;
|
||||
|
||||
// Find the highest index N such that a majority have matchIndex >= N
|
||||
// and log[N].term == currentTerm
|
||||
const matchIndices = Array.from(this._leaderState.matchIndex.values());
|
||||
matchIndices.push(this.log.lastIndex); // Include self
|
||||
matchIndices.sort((a, b) => b - a); // Sort descending
|
||||
|
||||
const majority = Math.floor((this.peers.length + 1) / 2) + 1;
|
||||
|
||||
for (const index of matchIndices) {
|
||||
if (index <= this._commitIndex) break;
|
||||
|
||||
const term = this.log.termAt(index);
|
||||
if (term === this._currentTerm) {
|
||||
// Count how many have this index or higher
|
||||
const count =
|
||||
matchIndices.filter((m) => m >= index).length + 1; // +1 for self
|
||||
if (count >= majority) {
|
||||
this._commitIndex = index;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Get persistent state */
|
||||
getPersistentState(): PersistentState<T> {
|
||||
return {
|
||||
currentTerm: this._currentTerm,
|
||||
votedFor: this._votedFor,
|
||||
log: this.log.getAll(),
|
||||
};
|
||||
}
|
||||
|
||||
/** Get volatile state */
|
||||
getVolatileState(): VolatileState {
|
||||
return {
|
||||
commitIndex: this._commitIndex,
|
||||
lastApplied: this._lastApplied,
|
||||
};
|
||||
}
|
||||
|
||||
/** Load persistent state */
|
||||
loadPersistentState(state: PersistentState<T>): void {
|
||||
this._currentTerm = state.currentTerm;
|
||||
this._votedFor = state.votedFor;
|
||||
this.log.load(state.log);
|
||||
}
|
||||
|
||||
/** Persist state */
|
||||
private async persist(): Promise<void> {
|
||||
if (this.persistCallback) {
|
||||
await this.persistCallback(this.getPersistentState());
|
||||
}
|
||||
}
|
||||
}
|
||||
189
npm/packages/raft/src/types.ts
Normal file
189
npm/packages/raft/src/types.ts
Normal file
|
|
@ -0,0 +1,189 @@
|
|||
/**
|
||||
* Raft Consensus Types
|
||||
* Based on the Raft paper specification
|
||||
*/
|
||||
|
||||
/** Unique identifier for a node in the cluster */
|
||||
export type NodeId = string;
|
||||
|
||||
/** Monotonically increasing term number */
|
||||
export type Term = number;
|
||||
|
||||
/** Index into the replicated log */
|
||||
export type LogIndex = number;
|
||||
|
||||
/** Possible states of a Raft node */
|
||||
export enum NodeState {
|
||||
Follower = 'follower',
|
||||
Candidate = 'candidate',
|
||||
Leader = 'leader',
|
||||
}
|
||||
|
||||
/** Entry in the replicated log */
|
||||
export interface LogEntry<T = unknown> {
|
||||
/** Term when entry was received by leader */
|
||||
term: Term;
|
||||
/** Index in the log */
|
||||
index: LogIndex;
|
||||
/** Command to be applied to state machine */
|
||||
command: T;
|
||||
/** Timestamp when entry was created */
|
||||
timestamp: number;
|
||||
}
|
||||
|
||||
/** Persistent state on all servers (updated on stable storage before responding to RPCs) */
|
||||
export interface PersistentState<T = unknown> {
|
||||
/** Latest term server has seen */
|
||||
currentTerm: Term;
|
||||
/** CandidateId that received vote in current term (or null if none) */
|
||||
votedFor: NodeId | null;
|
||||
/** Log entries */
|
||||
log: LogEntry<T>[];
|
||||
}
|
||||
|
||||
/** Volatile state on all servers */
|
||||
export interface VolatileState {
|
||||
/** Index of highest log entry known to be committed */
|
||||
commitIndex: LogIndex;
|
||||
/** Index of highest log entry applied to state machine */
|
||||
lastApplied: LogIndex;
|
||||
}
|
||||
|
||||
/** Volatile state on leaders (reinitialized after election) */
|
||||
export interface LeaderState {
|
||||
/** For each server, index of the next log entry to send to that server */
|
||||
nextIndex: Map<NodeId, LogIndex>;
|
||||
/** For each server, index of highest log entry known to be replicated on server */
|
||||
matchIndex: Map<NodeId, LogIndex>;
|
||||
}
|
||||
|
||||
/** Configuration for a Raft node */
|
||||
export interface RaftNodeConfig {
|
||||
/** Unique identifier for this node */
|
||||
nodeId: NodeId;
|
||||
/** List of all node IDs in the cluster */
|
||||
peers: NodeId[];
|
||||
/** Election timeout range in milliseconds [min, max] */
|
||||
electionTimeout: [number, number];
|
||||
/** Heartbeat interval in milliseconds */
|
||||
heartbeatInterval: number;
|
||||
/** Maximum entries per AppendEntries RPC */
|
||||
maxEntriesPerRequest: number;
|
||||
}
|
||||
|
||||
/** Request for RequestVote RPC */
|
||||
export interface RequestVoteRequest {
|
||||
/** Candidate's term */
|
||||
term: Term;
|
||||
/** Candidate requesting vote */
|
||||
candidateId: NodeId;
|
||||
/** Index of candidate's last log entry */
|
||||
lastLogIndex: LogIndex;
|
||||
/** Term of candidate's last log entry */
|
||||
lastLogTerm: Term;
|
||||
}
|
||||
|
||||
/** Response for RequestVote RPC */
|
||||
export interface RequestVoteResponse {
|
||||
/** Current term, for candidate to update itself */
|
||||
term: Term;
|
||||
/** True means candidate received vote */
|
||||
voteGranted: boolean;
|
||||
}
|
||||
|
||||
/** Request for AppendEntries RPC */
|
||||
export interface AppendEntriesRequest<T = unknown> {
|
||||
/** Leader's term */
|
||||
term: Term;
|
||||
/** So follower can redirect clients */
|
||||
leaderId: NodeId;
|
||||
/** Index of log entry immediately preceding new ones */
|
||||
prevLogIndex: LogIndex;
|
||||
/** Term of prevLogIndex entry */
|
||||
prevLogTerm: Term;
|
||||
/** Log entries to store (empty for heartbeat) */
|
||||
entries: LogEntry<T>[];
|
||||
/** Leader's commitIndex */
|
||||
leaderCommit: LogIndex;
|
||||
}
|
||||
|
||||
/** Response for AppendEntries RPC */
|
||||
export interface AppendEntriesResponse {
|
||||
/** Current term, for leader to update itself */
|
||||
term: Term;
|
||||
/** True if follower contained entry matching prevLogIndex and prevLogTerm */
|
||||
success: boolean;
|
||||
/** Hint for next index to try (optimization) */
|
||||
matchIndex?: LogIndex;
|
||||
}
|
||||
|
||||
/** Raft error types */
|
||||
export class RaftError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public readonly code: RaftErrorCode,
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'RaftError';
|
||||
}
|
||||
|
||||
static notLeader(): RaftError {
|
||||
return new RaftError('Node is not the leader', RaftErrorCode.NotLeader);
|
||||
}
|
||||
|
||||
static noLeader(): RaftError {
|
||||
return new RaftError('No leader available', RaftErrorCode.NoLeader);
|
||||
}
|
||||
|
||||
static electionTimeout(): RaftError {
|
||||
return new RaftError('Election timeout', RaftErrorCode.ElectionTimeout);
|
||||
}
|
||||
|
||||
static logInconsistency(): RaftError {
|
||||
return new RaftError('Log inconsistency detected', RaftErrorCode.LogInconsistency);
|
||||
}
|
||||
}
|
||||
|
||||
export enum RaftErrorCode {
|
||||
NotLeader = 'NOT_LEADER',
|
||||
NoLeader = 'NO_LEADER',
|
||||
InvalidTerm = 'INVALID_TERM',
|
||||
InvalidLogIndex = 'INVALID_LOG_INDEX',
|
||||
ElectionTimeout = 'ELECTION_TIMEOUT',
|
||||
LogInconsistency = 'LOG_INCONSISTENCY',
|
||||
SnapshotFailed = 'SNAPSHOT_FAILED',
|
||||
ConfigError = 'CONFIG_ERROR',
|
||||
Internal = 'INTERNAL',
|
||||
}
|
||||
|
||||
/** Event types emitted by RaftNode */
|
||||
export enum RaftEvent {
|
||||
StateChange = 'stateChange',
|
||||
LeaderElected = 'leaderElected',
|
||||
LogAppended = 'logAppended',
|
||||
LogCommitted = 'logCommitted',
|
||||
LogApplied = 'logApplied',
|
||||
VoteRequested = 'voteRequested',
|
||||
VoteGranted = 'voteGranted',
|
||||
Heartbeat = 'heartbeat',
|
||||
Error = 'error',
|
||||
}
|
||||
|
||||
/** State change event data */
|
||||
export interface StateChangeEvent {
|
||||
previousState: NodeState;
|
||||
newState: NodeState;
|
||||
term: Term;
|
||||
}
|
||||
|
||||
/** Leader elected event data */
|
||||
export interface LeaderElectedEvent {
|
||||
leaderId: NodeId;
|
||||
term: Term;
|
||||
}
|
||||
|
||||
/** Log committed event data */
|
||||
export interface LogCommittedEvent {
|
||||
index: LogIndex;
|
||||
term: Term;
|
||||
}
|
||||
19
npm/packages/raft/tsconfig.json
Normal file
19
npm/packages/raft/tsconfig.json
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"sourceMap": true,
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "dist", "test"]
|
||||
}
|
||||
67
npm/packages/replication/package.json
Normal file
67
npm/packages/replication/package.json
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
{
|
||||
"name": "@ruvector/replication",
|
||||
"version": "0.1.0",
|
||||
"description": "Data replication and synchronization - multi-node replicas, conflict resolution with vector clocks, change data capture, and automatic failover",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"exports": {
|
||||
".": {
|
||||
"import": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
}
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"prepublishOnly": "npm run build",
|
||||
"test": "node --test test/*.test.js",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"clean": "rm -rf dist"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.19.30",
|
||||
"typescript": "^5.9.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"eventemitter3": "^5.0.4"
|
||||
},
|
||||
"keywords": [
|
||||
"replication",
|
||||
"synchronization",
|
||||
"distributed-systems",
|
||||
"conflict-resolution",
|
||||
"vector-clock",
|
||||
"crdt",
|
||||
"change-data-capture",
|
||||
"failover",
|
||||
"high-availability",
|
||||
"ruvector"
|
||||
],
|
||||
"author": "rUv Team <team@ruv.io>",
|
||||
"license": "MIT OR Apache-2.0",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/ruvnet/ruvector.git",
|
||||
"directory": "npm/packages/replication"
|
||||
},
|
||||
"homepage": "https://github.com/ruvnet/ruvector/tree/main/crates/ruvector-replication",
|
||||
"bugs": {
|
||||
"url": "https://github.com/ruvnet/ruvector/issues"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 18"
|
||||
},
|
||||
"publishConfig": {
|
||||
"registry": "https://registry.npmjs.org/",
|
||||
"access": "public"
|
||||
},
|
||||
"files": [
|
||||
"dist",
|
||||
"README.md"
|
||||
]
|
||||
}
|
||||
83
npm/packages/replication/src/index.ts
Normal file
83
npm/packages/replication/src/index.ts
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
/**
|
||||
* @ruvector/replication - Data Replication and Synchronization
|
||||
*
|
||||
* A TypeScript implementation of data replication capabilities including:
|
||||
* - Multi-node replica management
|
||||
* - Synchronous, asynchronous, and semi-synchronous replication modes
|
||||
* - Conflict resolution with vector clocks
|
||||
* - Change data capture and streaming
|
||||
* - Automatic failover and split-brain prevention
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* import {
|
||||
* ReplicaSet,
|
||||
* ReplicaRole,
|
||||
* SyncManager,
|
||||
* ReplicationLog,
|
||||
* SyncMode,
|
||||
* ChangeOperation,
|
||||
* } from '@ruvector/replication';
|
||||
*
|
||||
* // Create a replica set
|
||||
* const replicaSet = new ReplicaSet('my-cluster');
|
||||
*
|
||||
* // Add replicas
|
||||
* replicaSet.addReplica('replica-1', '192.168.1.10:9001', ReplicaRole.Primary);
|
||||
* replicaSet.addReplica('replica-2', '192.168.1.11:9001', ReplicaRole.Secondary);
|
||||
* replicaSet.addReplica('replica-3', '192.168.1.12:9001', ReplicaRole.Secondary);
|
||||
*
|
||||
* // Create replication log and sync manager
|
||||
* const log = new ReplicationLog('replica-1');
|
||||
* const syncManager = new SyncManager(replicaSet, log);
|
||||
*
|
||||
* // Configure semi-sync replication
|
||||
* syncManager.setSyncMode(SyncMode.SemiSync, 1);
|
||||
*
|
||||
* // Listen for events
|
||||
* syncManager.on('changeReceived', (change) => {
|
||||
* console.log(`Change: ${change.operation} on ${change.key}`);
|
||||
* });
|
||||
*
|
||||
* // Record a change
|
||||
* await syncManager.recordChange('user:123', ChangeOperation.Update, { name: 'Alice' });
|
||||
* ```
|
||||
*
|
||||
* @packageDocumentation
|
||||
*/
|
||||
|
||||
// Types
|
||||
export {
|
||||
ReplicaId,
|
||||
LogicalClock,
|
||||
ReplicaRole,
|
||||
ReplicaStatus,
|
||||
SyncMode,
|
||||
HealthStatus,
|
||||
Replica,
|
||||
ChangeOperation,
|
||||
ChangeEvent,
|
||||
VectorClockValue,
|
||||
LogEntry,
|
||||
FailoverPolicy,
|
||||
ReplicationError,
|
||||
ReplicationErrorCode,
|
||||
ReplicationEvent,
|
||||
ReplicaSetConfig,
|
||||
SyncConfig,
|
||||
} from './types.js';
|
||||
|
||||
// Vector Clock
|
||||
export {
|
||||
VectorClock,
|
||||
VectorClockComparison,
|
||||
ConflictResolver,
|
||||
LastWriteWins,
|
||||
MergeFunction,
|
||||
} from './vector-clock.js';
|
||||
|
||||
// Replica Set
|
||||
export { ReplicaSet } from './replica-set.js';
|
||||
|
||||
// Sync Manager
|
||||
export { SyncManager, ReplicationLog } from './sync-manager.js';
|
||||
254
npm/packages/replication/src/replica-set.ts
Normal file
254
npm/packages/replication/src/replica-set.ts
Normal file
|
|
@ -0,0 +1,254 @@
|
|||
/**
|
||||
* Replica Set Management
|
||||
* Manages a set of replicas for distributed data storage
|
||||
*/
|
||||
|
||||
import EventEmitter from 'eventemitter3';
|
||||
import {
|
||||
type Replica,
|
||||
type ReplicaId,
|
||||
type ReplicaSetConfig,
|
||||
ReplicaRole,
|
||||
ReplicaStatus,
|
||||
ReplicationError,
|
||||
ReplicationEvent,
|
||||
FailoverPolicy,
|
||||
} from './types.js';
|
||||
|
||||
/** Default configuration */
|
||||
const DEFAULT_CONFIG: ReplicaSetConfig = {
|
||||
name: 'default',
|
||||
minQuorum: 2,
|
||||
heartbeatInterval: 1000,
|
||||
healthCheckTimeout: 5000,
|
||||
failoverPolicy: FailoverPolicy.Automatic,
|
||||
};
|
||||
|
||||
/** Manages a set of replicas */
|
||||
export class ReplicaSet extends EventEmitter {
|
||||
private replicas: Map<ReplicaId, Replica> = new Map();
|
||||
private config: ReplicaSetConfig;
|
||||
private heartbeatTimer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
constructor(name: string, config?: Partial<ReplicaSetConfig>) {
|
||||
super();
|
||||
this.config = { ...DEFAULT_CONFIG, name, ...config };
|
||||
}
|
||||
|
||||
/** Get replica set name */
|
||||
get name(): string {
|
||||
return this.config.name;
|
||||
}
|
||||
|
||||
/** Get the primary replica */
|
||||
get primary(): Replica | undefined {
|
||||
for (const replica of this.replicas.values()) {
|
||||
if (replica.role === ReplicaRole.Primary && replica.status === ReplicaStatus.Active) {
|
||||
return replica;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Get all secondary replicas */
|
||||
get secondaries(): Replica[] {
|
||||
return Array.from(this.replicas.values()).filter(
|
||||
(r) => r.role === ReplicaRole.Secondary && r.status === ReplicaStatus.Active,
|
||||
);
|
||||
}
|
||||
|
||||
/** Get all active replicas */
|
||||
get activeReplicas(): Replica[] {
|
||||
return Array.from(this.replicas.values()).filter((r) => r.status === ReplicaStatus.Active);
|
||||
}
|
||||
|
||||
/** Get replica count */
|
||||
get size(): number {
|
||||
return this.replicas.size;
|
||||
}
|
||||
|
||||
/** Check if quorum is met */
|
||||
get hasQuorum(): boolean {
|
||||
const activeCount = this.activeReplicas.length;
|
||||
return activeCount >= this.config.minQuorum;
|
||||
}
|
||||
|
||||
/** Add a replica to the set */
|
||||
addReplica(id: ReplicaId, address: string, role: ReplicaRole): Replica {
|
||||
if (this.replicas.has(id)) {
|
||||
throw new Error(`Replica ${id} already exists`);
|
||||
}
|
||||
|
||||
// Check if adding a primary when one exists
|
||||
if (role === ReplicaRole.Primary && this.primary) {
|
||||
throw new Error('Primary already exists in replica set');
|
||||
}
|
||||
|
||||
const replica: Replica = {
|
||||
id,
|
||||
address,
|
||||
role,
|
||||
status: ReplicaStatus.Active,
|
||||
lastSeen: Date.now(),
|
||||
lag: 0,
|
||||
};
|
||||
|
||||
this.replicas.set(id, replica);
|
||||
this.emit(ReplicationEvent.ReplicaAdded, replica);
|
||||
|
||||
return replica;
|
||||
}
|
||||
|
||||
/** Remove a replica from the set */
|
||||
removeReplica(id: ReplicaId): boolean {
|
||||
const replica = this.replicas.get(id);
|
||||
if (!replica) return false;
|
||||
|
||||
this.replicas.delete(id);
|
||||
this.emit(ReplicationEvent.ReplicaRemoved, replica);
|
||||
|
||||
// If primary was removed, trigger failover
|
||||
if (replica.role === ReplicaRole.Primary && this.config.failoverPolicy === FailoverPolicy.Automatic) {
|
||||
this.triggerFailover();
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Get a replica by ID */
|
||||
getReplica(id: ReplicaId): Replica | undefined {
|
||||
return this.replicas.get(id);
|
||||
}
|
||||
|
||||
/** Update replica status */
|
||||
updateStatus(id: ReplicaId, status: ReplicaStatus): void {
|
||||
const replica = this.replicas.get(id);
|
||||
if (!replica) {
|
||||
throw ReplicationError.replicaNotFound(id);
|
||||
}
|
||||
|
||||
const previousStatus = replica.status;
|
||||
replica.status = status;
|
||||
replica.lastSeen = Date.now();
|
||||
|
||||
if (previousStatus !== status) {
|
||||
this.emit(ReplicationEvent.ReplicaStatusChanged, {
|
||||
replica,
|
||||
previousStatus,
|
||||
newStatus: status,
|
||||
});
|
||||
|
||||
// Check for failover conditions
|
||||
if (
|
||||
replica.role === ReplicaRole.Primary &&
|
||||
status === ReplicaStatus.Failed &&
|
||||
this.config.failoverPolicy === FailoverPolicy.Automatic
|
||||
) {
|
||||
this.triggerFailover();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Update replica lag */
|
||||
updateLag(id: ReplicaId, lag: number): void {
|
||||
const replica = this.replicas.get(id);
|
||||
if (replica) {
|
||||
replica.lag = lag;
|
||||
replica.lastSeen = Date.now();
|
||||
}
|
||||
}
|
||||
|
||||
/** Promote a secondary to primary */
|
||||
promote(id: ReplicaId): void {
|
||||
const replica = this.replicas.get(id);
|
||||
if (!replica) {
|
||||
throw ReplicationError.replicaNotFound(id);
|
||||
}
|
||||
|
||||
if (replica.role === ReplicaRole.Primary) {
|
||||
return; // Already primary
|
||||
}
|
||||
|
||||
// Demote current primary
|
||||
const currentPrimary = this.primary;
|
||||
if (currentPrimary) {
|
||||
currentPrimary.role = ReplicaRole.Secondary;
|
||||
}
|
||||
|
||||
// Promote new primary
|
||||
replica.role = ReplicaRole.Primary;
|
||||
this.emit(ReplicationEvent.PrimaryChanged, {
|
||||
previousPrimary: currentPrimary?.id,
|
||||
newPrimary: id,
|
||||
});
|
||||
}
|
||||
|
||||
/** Trigger automatic failover */
|
||||
private triggerFailover(): void {
|
||||
this.emit(ReplicationEvent.FailoverStarted, {});
|
||||
|
||||
// Find the best candidate (lowest lag, active secondary)
|
||||
const candidates = this.secondaries
|
||||
.filter((r) => r.status === ReplicaStatus.Active)
|
||||
.sort((a, b) => a.lag - b.lag);
|
||||
|
||||
if (candidates.length === 0) {
|
||||
this.emit(ReplicationEvent.Error, ReplicationError.noPrimary());
|
||||
return;
|
||||
}
|
||||
|
||||
const newPrimary = candidates[0];
|
||||
this.promote(newPrimary.id);
|
||||
|
||||
this.emit(ReplicationEvent.FailoverCompleted, { newPrimary: newPrimary.id });
|
||||
}
|
||||
|
||||
/** Start heartbeat monitoring */
|
||||
startHeartbeat(): void {
|
||||
if (this.heartbeatTimer) return;
|
||||
|
||||
this.heartbeatTimer = setInterval(() => {
|
||||
const now = Date.now();
|
||||
for (const replica of this.replicas.values()) {
|
||||
if (now - replica.lastSeen > this.config.healthCheckTimeout) {
|
||||
if (replica.status === ReplicaStatus.Active) {
|
||||
this.updateStatus(replica.id, ReplicaStatus.Offline);
|
||||
}
|
||||
}
|
||||
}
|
||||
}, this.config.heartbeatInterval);
|
||||
}
|
||||
|
||||
/** Stop heartbeat monitoring */
|
||||
stopHeartbeat(): void {
|
||||
if (this.heartbeatTimer) {
|
||||
clearInterval(this.heartbeatTimer);
|
||||
this.heartbeatTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Get all replicas */
|
||||
getAllReplicas(): Replica[] {
|
||||
return Array.from(this.replicas.values());
|
||||
}
|
||||
|
||||
/** Get replica set stats */
|
||||
getStats(): {
|
||||
total: number;
|
||||
active: number;
|
||||
syncing: number;
|
||||
offline: number;
|
||||
failed: number;
|
||||
hasQuorum: boolean;
|
||||
} {
|
||||
const replicas = Array.from(this.replicas.values());
|
||||
return {
|
||||
total: replicas.length,
|
||||
active: replicas.filter((r) => r.status === ReplicaStatus.Active).length,
|
||||
syncing: replicas.filter((r) => r.status === ReplicaStatus.Syncing).length,
|
||||
offline: replicas.filter((r) => r.status === ReplicaStatus.Offline).length,
|
||||
failed: replicas.filter((r) => r.status === ReplicaStatus.Failed).length,
|
||||
hasQuorum: this.hasQuorum,
|
||||
};
|
||||
}
|
||||
}
|
||||
260
npm/packages/replication/src/sync-manager.ts
Normal file
260
npm/packages/replication/src/sync-manager.ts
Normal file
|
|
@ -0,0 +1,260 @@
|
|||
/**
|
||||
* Sync Manager Implementation
|
||||
* Manages data synchronization across replicas
|
||||
*/
|
||||
|
||||
import EventEmitter from 'eventemitter3';
|
||||
import {
|
||||
type ReplicaId,
|
||||
type SyncConfig,
|
||||
type LogEntry,
|
||||
type ChangeEvent,
|
||||
SyncMode,
|
||||
ReplicationError,
|
||||
ReplicationEvent,
|
||||
ChangeOperation,
|
||||
} from './types.js';
|
||||
import { VectorClock, type ConflictResolver, LastWriteWins } from './vector-clock.js';
|
||||
import type { ReplicaSet } from './replica-set.js';
|
||||
|
||||
/** Default sync configuration */
|
||||
const DEFAULT_SYNC_CONFIG: SyncConfig = {
|
||||
mode: SyncMode.Asynchronous,
|
||||
batchSize: 100,
|
||||
maxLag: 5000,
|
||||
};
|
||||
|
||||
/** Replication log for tracking changes */
|
||||
export class ReplicationLog<T = unknown> {
|
||||
private entries: LogEntry<T>[] = [];
|
||||
private sequence = 0;
|
||||
private readonly replicaId: ReplicaId;
|
||||
private vectorClock: VectorClock;
|
||||
|
||||
constructor(replicaId: ReplicaId) {
|
||||
this.replicaId = replicaId;
|
||||
this.vectorClock = new VectorClock();
|
||||
}
|
||||
|
||||
/** Get the current sequence number */
|
||||
get currentSequence(): number {
|
||||
return this.sequence;
|
||||
}
|
||||
|
||||
/** Get the current vector clock */
|
||||
get clock(): VectorClock {
|
||||
return this.vectorClock.clone();
|
||||
}
|
||||
|
||||
/** Append an entry to the log */
|
||||
append(data: T): LogEntry<T> {
|
||||
this.sequence++;
|
||||
this.vectorClock.increment(this.replicaId);
|
||||
|
||||
const entry: LogEntry<T> = {
|
||||
id: `${this.replicaId}-${this.sequence}`,
|
||||
sequence: this.sequence,
|
||||
data,
|
||||
timestamp: Date.now(),
|
||||
vectorClock: this.vectorClock.getValue(),
|
||||
};
|
||||
|
||||
this.entries.push(entry);
|
||||
return entry;
|
||||
}
|
||||
|
||||
/** Get entries since a sequence number */
|
||||
getEntriesSince(sequence: number, limit?: number): LogEntry<T>[] {
|
||||
const filtered = this.entries.filter((e) => e.sequence > sequence);
|
||||
return limit ? filtered.slice(0, limit) : filtered;
|
||||
}
|
||||
|
||||
/** Get entry by ID */
|
||||
getEntry(id: string): LogEntry<T> | undefined {
|
||||
return this.entries.find((e) => e.id === id);
|
||||
}
|
||||
|
||||
/** Get all entries */
|
||||
getAllEntries(): LogEntry<T>[] {
|
||||
return [...this.entries];
|
||||
}
|
||||
|
||||
/** Apply entries from another replica */
|
||||
applyEntries(entries: LogEntry<T>[]): void {
|
||||
for (const entry of entries) {
|
||||
const entryClock = new VectorClock(entry.vectorClock);
|
||||
this.vectorClock.merge(entryClock);
|
||||
}
|
||||
// Note: In a real implementation, entries would be merged properly
|
||||
}
|
||||
|
||||
/** Clear the log */
|
||||
clear(): void {
|
||||
this.entries = [];
|
||||
this.sequence = 0;
|
||||
this.vectorClock = new VectorClock();
|
||||
}
|
||||
}
|
||||
|
||||
/** Manages synchronization across replicas */
|
||||
export class SyncManager<T = unknown> extends EventEmitter {
|
||||
private readonly replicaSet: ReplicaSet;
|
||||
private readonly log: ReplicationLog<T>;
|
||||
private config: SyncConfig;
|
||||
private conflictResolver: ConflictResolver<T>;
|
||||
private pendingChanges: ChangeEvent<T>[] = [];
|
||||
private syncTimer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
constructor(
|
||||
replicaSet: ReplicaSet,
|
||||
log: ReplicationLog<T>,
|
||||
config?: Partial<SyncConfig>,
|
||||
) {
|
||||
super();
|
||||
this.replicaSet = replicaSet;
|
||||
this.log = log;
|
||||
this.config = { ...DEFAULT_SYNC_CONFIG, ...config };
|
||||
// Default to timestamp-based resolution
|
||||
this.conflictResolver = new LastWriteWins() as unknown as ConflictResolver<T>;
|
||||
}
|
||||
|
||||
/** Set sync mode */
|
||||
setSyncMode(mode: SyncMode, minReplicas?: number): void {
|
||||
this.config.mode = mode;
|
||||
if (minReplicas !== undefined) {
|
||||
this.config.minReplicas = minReplicas;
|
||||
}
|
||||
}
|
||||
|
||||
/** Set custom conflict resolver */
|
||||
setConflictResolver(resolver: ConflictResolver<T>): void {
|
||||
this.conflictResolver = resolver;
|
||||
}
|
||||
|
||||
/** Record a change for replication */
|
||||
async recordChange(
|
||||
key: string,
|
||||
operation: ChangeOperation,
|
||||
value?: T,
|
||||
previousValue?: T,
|
||||
): Promise<void> {
|
||||
const primary = this.replicaSet.primary;
|
||||
if (!primary) {
|
||||
throw ReplicationError.noPrimary();
|
||||
}
|
||||
|
||||
const entry = this.log.append({ key, operation, value, previousValue } as unknown as T);
|
||||
|
||||
const change: ChangeEvent<T> = {
|
||||
id: entry.id,
|
||||
operation,
|
||||
key,
|
||||
value,
|
||||
previousValue,
|
||||
timestamp: entry.timestamp,
|
||||
sourceReplica: primary.id,
|
||||
vectorClock: entry.vectorClock,
|
||||
};
|
||||
|
||||
this.emit(ReplicationEvent.ChangeReceived, change);
|
||||
|
||||
// Handle based on sync mode
|
||||
switch (this.config.mode) {
|
||||
case SyncMode.Synchronous:
|
||||
await this.syncAll(change);
|
||||
break;
|
||||
case SyncMode.SemiSync:
|
||||
await this.syncMinimum(change);
|
||||
break;
|
||||
case SyncMode.Asynchronous:
|
||||
this.pendingChanges.push(change);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/** Sync a change to all replicas */
|
||||
private async syncAll(change: ChangeEvent<T>): Promise<void> {
|
||||
const secondaries = this.replicaSet.secondaries;
|
||||
if (secondaries.length === 0) return;
|
||||
|
||||
this.emit(ReplicationEvent.SyncStarted, { replicas: secondaries.map((r) => r.id) });
|
||||
|
||||
// In a real implementation, this would send to all replicas
|
||||
// For now, we just emit the completion event
|
||||
this.emit(ReplicationEvent.SyncCompleted, { change, replicas: secondaries.map((r) => r.id) });
|
||||
}
|
||||
|
||||
/** Sync to minimum number of replicas (semi-sync) */
|
||||
private async syncMinimum(change: ChangeEvent<T>): Promise<void> {
|
||||
const minReplicas = this.config.minReplicas ?? 1;
|
||||
const secondaries = this.replicaSet.secondaries;
|
||||
|
||||
if (secondaries.length < minReplicas) {
|
||||
throw ReplicationError.quorumNotMet(minReplicas, secondaries.length);
|
||||
}
|
||||
|
||||
// Sync to minimum number of replicas
|
||||
const targetReplicas = secondaries.slice(0, minReplicas);
|
||||
this.emit(ReplicationEvent.SyncStarted, { replicas: targetReplicas.map((r) => r.id) });
|
||||
|
||||
// In a real implementation, this would wait for acknowledgments
|
||||
this.emit(ReplicationEvent.SyncCompleted, { change, replicas: targetReplicas.map((r) => r.id) });
|
||||
}
|
||||
|
||||
/** Start background sync for async mode */
|
||||
startBackgroundSync(interval: number = 1000): void {
|
||||
if (this.syncTimer) return;
|
||||
|
||||
this.syncTimer = setInterval(async () => {
|
||||
if (this.pendingChanges.length > 0) {
|
||||
const batch = this.pendingChanges.splice(0, this.config.batchSize);
|
||||
for (const change of batch) {
|
||||
await this.syncAll(change);
|
||||
}
|
||||
}
|
||||
}, interval);
|
||||
}
|
||||
|
||||
/** Stop background sync */
|
||||
stopBackgroundSync(): void {
|
||||
if (this.syncTimer) {
|
||||
clearInterval(this.syncTimer);
|
||||
this.syncTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Resolve a conflict between local and remote values */
|
||||
resolveConflict(
|
||||
local: T,
|
||||
remote: T,
|
||||
localClock: VectorClock,
|
||||
remoteClock: VectorClock,
|
||||
): T {
|
||||
// Check for causal relationship
|
||||
if (localClock.happensBefore(remoteClock)) {
|
||||
return remote; // Remote is newer
|
||||
} else if (localClock.happensAfter(remoteClock)) {
|
||||
return local; // Local is newer
|
||||
}
|
||||
|
||||
// Concurrent - need conflict resolution
|
||||
this.emit(ReplicationEvent.ConflictDetected, { local, remote });
|
||||
const resolved = this.conflictResolver.resolve(local, remote, localClock, remoteClock);
|
||||
this.emit(ReplicationEvent.ConflictResolved, { local, remote, resolved });
|
||||
|
||||
return resolved;
|
||||
}
|
||||
|
||||
/** Get sync statistics */
|
||||
getStats(): {
|
||||
pendingChanges: number;
|
||||
lastSequence: number;
|
||||
syncMode: SyncMode;
|
||||
} {
|
||||
return {
|
||||
pendingChanges: this.pendingChanges.length,
|
||||
lastSequence: this.log.currentSequence,
|
||||
syncMode: this.config.mode,
|
||||
};
|
||||
}
|
||||
}
|
||||
205
npm/packages/replication/src/types.ts
Normal file
205
npm/packages/replication/src/types.ts
Normal file
|
|
@ -0,0 +1,205 @@
|
|||
/**
|
||||
* Replication Types
|
||||
* Data replication and synchronization types
|
||||
*/
|
||||
|
||||
import type EventEmitter from 'eventemitter3';
|
||||
|
||||
/** Unique identifier for a replica */
|
||||
export type ReplicaId = string;
|
||||
|
||||
/** Logical timestamp for ordering events */
|
||||
export type LogicalClock = number;
|
||||
|
||||
/** Role of a replica in the set */
|
||||
export enum ReplicaRole {
|
||||
Primary = 'primary',
|
||||
Secondary = 'secondary',
|
||||
Arbiter = 'arbiter',
|
||||
}
|
||||
|
||||
/** Status of a replica */
|
||||
export enum ReplicaStatus {
|
||||
Active = 'active',
|
||||
Syncing = 'syncing',
|
||||
Offline = 'offline',
|
||||
Failed = 'failed',
|
||||
}
|
||||
|
||||
/** Synchronization mode */
|
||||
export enum SyncMode {
|
||||
/** All replicas must confirm before commit */
|
||||
Synchronous = 'synchronous',
|
||||
/** Commit immediately, replicate in background */
|
||||
Asynchronous = 'asynchronous',
|
||||
/** Wait for minimum number of replicas */
|
||||
SemiSync = 'semi-sync',
|
||||
}
|
||||
|
||||
/** Health status of a replica */
|
||||
export enum HealthStatus {
|
||||
Healthy = 'healthy',
|
||||
Degraded = 'degraded',
|
||||
Unhealthy = 'unhealthy',
|
||||
Unknown = 'unknown',
|
||||
}
|
||||
|
||||
/** Replica information */
|
||||
export interface Replica {
|
||||
id: ReplicaId;
|
||||
address: string;
|
||||
role: ReplicaRole;
|
||||
status: ReplicaStatus;
|
||||
lastSeen: number;
|
||||
lag: number; // Replication lag in milliseconds
|
||||
}
|
||||
|
||||
/** Change operation type */
|
||||
export enum ChangeOperation {
|
||||
Insert = 'insert',
|
||||
Update = 'update',
|
||||
Delete = 'delete',
|
||||
}
|
||||
|
||||
/** Change event for CDC */
|
||||
export interface ChangeEvent<T = unknown> {
|
||||
/** Unique event ID */
|
||||
id: string;
|
||||
/** Operation type */
|
||||
operation: ChangeOperation;
|
||||
/** Affected key/path */
|
||||
key: string;
|
||||
/** New value (for insert/update) */
|
||||
value?: T;
|
||||
/** Previous value (for update/delete) */
|
||||
previousValue?: T;
|
||||
/** Timestamp of the change */
|
||||
timestamp: number;
|
||||
/** Source replica */
|
||||
sourceReplica: ReplicaId;
|
||||
/** Vector clock for ordering */
|
||||
vectorClock: VectorClockValue;
|
||||
}
|
||||
|
||||
/** Vector clock entry for a single node */
|
||||
export type VectorClockValue = Map<ReplicaId, LogicalClock>;
|
||||
|
||||
/** Log entry for replication */
|
||||
export interface LogEntry<T = unknown> {
|
||||
/** Unique entry ID */
|
||||
id: string;
|
||||
/** Sequence number */
|
||||
sequence: number;
|
||||
/** Operation data */
|
||||
data: T;
|
||||
/** Timestamp */
|
||||
timestamp: number;
|
||||
/** Vector clock */
|
||||
vectorClock: VectorClockValue;
|
||||
}
|
||||
|
||||
/** Failover policy */
|
||||
export enum FailoverPolicy {
|
||||
/** Automatic failover with quorum */
|
||||
Automatic = 'automatic',
|
||||
/** Manual intervention required */
|
||||
Manual = 'manual',
|
||||
/** Priority-based failover */
|
||||
Priority = 'priority',
|
||||
}
|
||||
|
||||
/** Replication error types */
|
||||
export class ReplicationError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public readonly code: ReplicationErrorCode,
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'ReplicationError';
|
||||
}
|
||||
|
||||
static replicaNotFound(id: string): ReplicationError {
|
||||
return new ReplicationError(`Replica not found: ${id}`, ReplicationErrorCode.ReplicaNotFound);
|
||||
}
|
||||
|
||||
static noPrimary(): ReplicationError {
|
||||
return new ReplicationError('No primary replica available', ReplicationErrorCode.NoPrimary);
|
||||
}
|
||||
|
||||
static timeout(operation: string): ReplicationError {
|
||||
return new ReplicationError(`Replication timeout: ${operation}`, ReplicationErrorCode.Timeout);
|
||||
}
|
||||
|
||||
static quorumNotMet(needed: number, available: number): ReplicationError {
|
||||
return new ReplicationError(
|
||||
`Quorum not met: needed ${needed}, got ${available}`,
|
||||
ReplicationErrorCode.QuorumNotMet,
|
||||
);
|
||||
}
|
||||
|
||||
static splitBrain(): ReplicationError {
|
||||
return new ReplicationError('Split-brain detected', ReplicationErrorCode.SplitBrain);
|
||||
}
|
||||
|
||||
static conflictResolution(reason: string): ReplicationError {
|
||||
return new ReplicationError(
|
||||
`Conflict resolution failed: ${reason}`,
|
||||
ReplicationErrorCode.ConflictResolution,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export enum ReplicationErrorCode {
|
||||
ReplicaNotFound = 'REPLICA_NOT_FOUND',
|
||||
NoPrimary = 'NO_PRIMARY',
|
||||
Timeout = 'TIMEOUT',
|
||||
SyncFailed = 'SYNC_FAILED',
|
||||
ConflictResolution = 'CONFLICT_RESOLUTION',
|
||||
FailoverFailed = 'FAILOVER_FAILED',
|
||||
Network = 'NETWORK',
|
||||
QuorumNotMet = 'QUORUM_NOT_MET',
|
||||
SplitBrain = 'SPLIT_BRAIN',
|
||||
InvalidState = 'INVALID_STATE',
|
||||
}
|
||||
|
||||
/** Events emitted by replication components */
|
||||
export enum ReplicationEvent {
|
||||
ReplicaAdded = 'replicaAdded',
|
||||
ReplicaRemoved = 'replicaRemoved',
|
||||
ReplicaStatusChanged = 'replicaStatusChanged',
|
||||
PrimaryChanged = 'primaryChanged',
|
||||
ChangeReceived = 'changeReceived',
|
||||
SyncStarted = 'syncStarted',
|
||||
SyncCompleted = 'syncCompleted',
|
||||
ConflictDetected = 'conflictDetected',
|
||||
ConflictResolved = 'conflictResolved',
|
||||
FailoverStarted = 'failoverStarted',
|
||||
FailoverCompleted = 'failoverCompleted',
|
||||
Error = 'error',
|
||||
}
|
||||
|
||||
/** Configuration for replica set */
|
||||
export interface ReplicaSetConfig {
|
||||
/** Cluster name */
|
||||
name: string;
|
||||
/** Minimum replicas for quorum */
|
||||
minQuorum: number;
|
||||
/** Heartbeat interval in milliseconds */
|
||||
heartbeatInterval: number;
|
||||
/** Timeout for health checks in milliseconds */
|
||||
healthCheckTimeout: number;
|
||||
/** Failover policy */
|
||||
failoverPolicy: FailoverPolicy;
|
||||
}
|
||||
|
||||
/** Configuration for sync manager */
|
||||
export interface SyncConfig {
|
||||
/** Sync mode */
|
||||
mode: SyncMode;
|
||||
/** Minimum replicas for semi-sync */
|
||||
minReplicas?: number;
|
||||
/** Batch size for streaming changes */
|
||||
batchSize: number;
|
||||
/** Maximum lag before triggering catchup */
|
||||
maxLag: number;
|
||||
}
|
||||
148
npm/packages/replication/src/vector-clock.ts
Normal file
148
npm/packages/replication/src/vector-clock.ts
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
/**
|
||||
* Vector Clock Implementation
|
||||
* For conflict detection and resolution in distributed systems
|
||||
*/
|
||||
|
||||
import type { ReplicaId, LogicalClock, VectorClockValue } from './types.js';
|
||||
|
||||
/** Comparison result between vector clocks */
|
||||
export enum VectorClockComparison {
|
||||
/** First happens before second */
|
||||
Before = 'before',
|
||||
/** First happens after second */
|
||||
After = 'after',
|
||||
/** Clocks are concurrent (no causal relationship) */
|
||||
Concurrent = 'concurrent',
|
||||
/** Clocks are equal */
|
||||
Equal = 'equal',
|
||||
}
|
||||
|
||||
/** Vector clock for tracking causality in distributed systems */
|
||||
export class VectorClock {
|
||||
private clock: Map<ReplicaId, LogicalClock>;
|
||||
|
||||
constructor(initial?: VectorClockValue | Map<ReplicaId, LogicalClock>) {
|
||||
this.clock = new Map(initial);
|
||||
}
|
||||
|
||||
/** Get the clock value for a replica */
|
||||
get(replicaId: ReplicaId): LogicalClock {
|
||||
return this.clock.get(replicaId) ?? 0;
|
||||
}
|
||||
|
||||
/** Increment the clock for a replica */
|
||||
increment(replicaId: ReplicaId): void {
|
||||
const current = this.get(replicaId);
|
||||
this.clock.set(replicaId, current + 1);
|
||||
}
|
||||
|
||||
/** Update with a received clock (merge) */
|
||||
merge(other: VectorClock): void {
|
||||
for (const [replicaId, otherTime] of other.clock) {
|
||||
const myTime = this.get(replicaId);
|
||||
this.clock.set(replicaId, Math.max(myTime, otherTime));
|
||||
}
|
||||
}
|
||||
|
||||
/** Create a copy of this clock */
|
||||
clone(): VectorClock {
|
||||
return new VectorClock(new Map(this.clock));
|
||||
}
|
||||
|
||||
/** Get the clock value as a Map */
|
||||
getValue(): VectorClockValue {
|
||||
return new Map(this.clock);
|
||||
}
|
||||
|
||||
/** Compare two vector clocks */
|
||||
compare(other: VectorClock): VectorClockComparison {
|
||||
let isLess = false;
|
||||
let isGreater = false;
|
||||
|
||||
// Get all unique replica IDs
|
||||
const allReplicas = new Set([...this.clock.keys(), ...other.clock.keys()]);
|
||||
|
||||
for (const replicaId of allReplicas) {
|
||||
const myTime = this.get(replicaId);
|
||||
const otherTime = other.get(replicaId);
|
||||
|
||||
if (myTime < otherTime) {
|
||||
isLess = true;
|
||||
} else if (myTime > otherTime) {
|
||||
isGreater = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (isLess && isGreater) {
|
||||
return VectorClockComparison.Concurrent;
|
||||
} else if (isLess) {
|
||||
return VectorClockComparison.Before;
|
||||
} else if (isGreater) {
|
||||
return VectorClockComparison.After;
|
||||
} else {
|
||||
return VectorClockComparison.Equal;
|
||||
}
|
||||
}
|
||||
|
||||
/** Check if this clock happens before another */
|
||||
happensBefore(other: VectorClock): boolean {
|
||||
return this.compare(other) === VectorClockComparison.Before;
|
||||
}
|
||||
|
||||
/** Check if this clock happens after another */
|
||||
happensAfter(other: VectorClock): boolean {
|
||||
return this.compare(other) === VectorClockComparison.After;
|
||||
}
|
||||
|
||||
/** Check if clocks are concurrent (no causal relationship) */
|
||||
isConcurrent(other: VectorClock): boolean {
|
||||
return this.compare(other) === VectorClockComparison.Concurrent;
|
||||
}
|
||||
|
||||
/** Serialize to JSON */
|
||||
toJSON(): Record<string, number> {
|
||||
const obj: Record<string, number> = {};
|
||||
for (const [key, value] of this.clock) {
|
||||
obj[key] = value;
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
/** Create from JSON */
|
||||
static fromJSON(json: Record<string, number>): VectorClock {
|
||||
const clock = new VectorClock();
|
||||
for (const [key, value] of Object.entries(json)) {
|
||||
clock.clock.set(key, value);
|
||||
}
|
||||
return clock;
|
||||
}
|
||||
|
||||
/** Create a new vector clock with a single entry */
|
||||
static single(replicaId: ReplicaId, time: LogicalClock = 1): VectorClock {
|
||||
const clock = new VectorClock();
|
||||
clock.clock.set(replicaId, time);
|
||||
return clock;
|
||||
}
|
||||
}
|
||||
|
||||
/** Conflict resolver interface */
|
||||
export interface ConflictResolver<T> {
|
||||
/** Resolve a conflict between two values */
|
||||
resolve(local: T, remote: T, localClock: VectorClock, remoteClock: VectorClock): T;
|
||||
}
|
||||
|
||||
/** Last-write-wins conflict resolver */
|
||||
export class LastWriteWins<T extends { timestamp: number }> implements ConflictResolver<T> {
|
||||
resolve(local: T, remote: T): T {
|
||||
return local.timestamp >= remote.timestamp ? local : remote;
|
||||
}
|
||||
}
|
||||
|
||||
/** Custom merge function conflict resolver */
|
||||
export class MergeFunction<T> implements ConflictResolver<T> {
|
||||
constructor(private mergeFn: (local: T, remote: T) => T) {}
|
||||
|
||||
resolve(local: T, remote: T): T {
|
||||
return this.mergeFn(local, remote);
|
||||
}
|
||||
}
|
||||
19
npm/packages/replication/tsconfig.json
Normal file
19
npm/packages/replication/tsconfig.json
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"sourceMap": true,
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "dist", "test"]
|
||||
}
|
||||
64
npm/packages/scipix/package.json
Normal file
64
npm/packages/scipix/package.json
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
{
|
||||
"name": "@ruvector/scipix",
|
||||
"version": "0.1.0",
|
||||
"description": "OCR client for scientific documents - extract LaTeX, MathML from equations, research papers, and technical diagrams",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"exports": {
|
||||
".": {
|
||||
"import": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
},
|
||||
"require": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
}
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"prepublishOnly": "npm run build",
|
||||
"test": "node --test test/*.test.js",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"clean": "rm -rf dist"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.19.30",
|
||||
"typescript": "^5.9.3"
|
||||
},
|
||||
"keywords": [
|
||||
"ocr",
|
||||
"latex",
|
||||
"mathml",
|
||||
"scientific-computing",
|
||||
"image-recognition",
|
||||
"math-ocr",
|
||||
"equation-recognition",
|
||||
"document-processing",
|
||||
"ruvector",
|
||||
"pdf-extraction"
|
||||
],
|
||||
"author": "rUv Team <team@ruv.io>",
|
||||
"license": "MIT OR Apache-2.0",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/ruvnet/ruvector.git",
|
||||
"directory": "npm/packages/scipix"
|
||||
},
|
||||
"homepage": "https://github.com/ruvnet/ruvector/tree/main/examples/scipix",
|
||||
"bugs": {
|
||||
"url": "https://github.com/ruvnet/ruvector/issues"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 18"
|
||||
},
|
||||
"publishConfig": {
|
||||
"registry": "https://registry.npmjs.org/",
|
||||
"access": "public"
|
||||
},
|
||||
"files": [
|
||||
"dist",
|
||||
"README.md"
|
||||
]
|
||||
}
|
||||
272
npm/packages/scipix/src/client.ts
Normal file
272
npm/packages/scipix/src/client.ts
Normal file
|
|
@ -0,0 +1,272 @@
|
|||
/**
|
||||
* SciPix OCR Client
|
||||
* Client for interacting with SciPix OCR API
|
||||
*/
|
||||
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { extname } from 'node:path';
|
||||
import {
|
||||
type SciPixConfig,
|
||||
type OCROptions,
|
||||
type OCRResult,
|
||||
type BatchOCRRequest,
|
||||
type BatchOCRResult,
|
||||
type HealthStatus,
|
||||
SciPixError,
|
||||
SciPixErrorCode,
|
||||
OutputFormat,
|
||||
ImageType,
|
||||
} from './types.js';
|
||||
|
||||
/** Default configuration */
|
||||
const DEFAULT_CONFIG: Required<Omit<SciPixConfig, 'apiKey'>> = {
|
||||
baseUrl: 'http://localhost:8080',
|
||||
timeout: 30000,
|
||||
maxRetries: 3,
|
||||
defaultOptions: {
|
||||
formats: [OutputFormat.LaTeX, OutputFormat.Text],
|
||||
detectEquations: true,
|
||||
preprocess: true,
|
||||
},
|
||||
};
|
||||
|
||||
/** SciPix OCR Client */
|
||||
export class SciPixClient {
|
||||
private config: Required<Omit<SciPixConfig, 'apiKey'>> & { apiKey?: string };
|
||||
|
||||
constructor(config?: SciPixConfig) {
|
||||
this.config = { ...DEFAULT_CONFIG, ...config };
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform OCR on an image
|
||||
* @param image - Image data as Buffer, base64 string, or file path
|
||||
* @param options - OCR options
|
||||
*/
|
||||
async ocr(image: Buffer | string, options?: OCROptions): Promise<OCRResult> {
|
||||
const imageData = await this.prepareImage(image);
|
||||
const mergedOptions = { ...this.config.defaultOptions, ...options };
|
||||
|
||||
const response = await this.request('/api/v1/ocr', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({
|
||||
image: imageData.base64,
|
||||
imageType: imageData.type,
|
||||
options: mergedOptions,
|
||||
}),
|
||||
});
|
||||
|
||||
return response as OCRResult;
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform OCR on a file
|
||||
* @param filePath - Path to the image file
|
||||
* @param options - OCR options
|
||||
*/
|
||||
async ocrFile(filePath: string, options?: OCROptions): Promise<OCRResult> {
|
||||
const buffer = await readFile(filePath);
|
||||
return this.ocr(buffer, options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform batch OCR on multiple images
|
||||
* @param request - Batch OCR request
|
||||
*/
|
||||
async batchOcr(request: BatchOCRRequest): Promise<BatchOCRResult> {
|
||||
const response = await this.request('/api/v1/ocr/batch', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(request),
|
||||
});
|
||||
|
||||
return response as BatchOCRResult;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract LaTeX from an equation image
|
||||
* @param image - Image data
|
||||
*/
|
||||
async extractLatex(image: Buffer | string): Promise<string> {
|
||||
const result = await this.ocr(image, {
|
||||
formats: [OutputFormat.LaTeX],
|
||||
detectEquations: true,
|
||||
});
|
||||
|
||||
return result.latex ?? result.text;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract MathML from an equation image
|
||||
* @param image - Image data
|
||||
*/
|
||||
async extractMathML(image: Buffer | string): Promise<string> {
|
||||
const result = await this.ocr(image, {
|
||||
formats: [OutputFormat.MathML],
|
||||
detectEquations: true,
|
||||
});
|
||||
|
||||
return result.mathml ?? '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Check API health status
|
||||
*/
|
||||
async health(): Promise<HealthStatus> {
|
||||
const response = await this.request('/api/v1/health', {
|
||||
method: 'GET',
|
||||
});
|
||||
|
||||
return response as HealthStatus;
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepare image for API request
|
||||
*/
|
||||
private async prepareImage(
|
||||
image: Buffer | string,
|
||||
): Promise<{ base64: string; type: ImageType }> {
|
||||
let buffer: Buffer;
|
||||
let type: ImageType = ImageType.PNG;
|
||||
|
||||
if (Buffer.isBuffer(image)) {
|
||||
buffer = image;
|
||||
type = this.detectImageType(buffer);
|
||||
} else if (image.startsWith('data:')) {
|
||||
// Base64 data URL
|
||||
const match = image.match(/^data:image\/(\w+);base64,(.+)$/);
|
||||
if (!match) {
|
||||
throw SciPixError.invalidImage('Invalid data URL format');
|
||||
}
|
||||
type = this.parseImageType(match[1]);
|
||||
return { base64: match[2], type };
|
||||
} else if (image.startsWith('/') || image.includes(':\\')) {
|
||||
// File path
|
||||
buffer = await readFile(image);
|
||||
type = this.getTypeFromExtension(extname(image));
|
||||
} else {
|
||||
// Assume base64 string
|
||||
return { base64: image, type: ImageType.PNG };
|
||||
}
|
||||
|
||||
return {
|
||||
base64: buffer.toString('base64'),
|
||||
type,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect image type from buffer magic bytes
|
||||
*/
|
||||
private detectImageType(buffer: Buffer): ImageType {
|
||||
if (buffer[0] === 0x89 && buffer[1] === 0x50) return ImageType.PNG;
|
||||
if (buffer[0] === 0xff && buffer[1] === 0xd8) return ImageType.JPEG;
|
||||
if (buffer[0] === 0x52 && buffer[1] === 0x49) return ImageType.WebP;
|
||||
if (buffer[0] === 0x25 && buffer[1] === 0x50) return ImageType.PDF;
|
||||
if (buffer[0] === 0x49 && buffer[1] === 0x49) return ImageType.TIFF;
|
||||
if (buffer[0] === 0x4d && buffer[1] === 0x4d) return ImageType.TIFF;
|
||||
if (buffer[0] === 0x42 && buffer[1] === 0x4d) return ImageType.BMP;
|
||||
return ImageType.PNG; // Default
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse image type from MIME type
|
||||
*/
|
||||
private parseImageType(mimeType: string): ImageType {
|
||||
switch (mimeType.toLowerCase()) {
|
||||
case 'png':
|
||||
return ImageType.PNG;
|
||||
case 'jpeg':
|
||||
case 'jpg':
|
||||
return ImageType.JPEG;
|
||||
case 'webp':
|
||||
return ImageType.WebP;
|
||||
case 'pdf':
|
||||
return ImageType.PDF;
|
||||
case 'tiff':
|
||||
case 'tif':
|
||||
return ImageType.TIFF;
|
||||
case 'bmp':
|
||||
return ImageType.BMP;
|
||||
default:
|
||||
return ImageType.PNG;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get image type from file extension
|
||||
*/
|
||||
private getTypeFromExtension(ext: string): ImageType {
|
||||
return this.parseImageType(ext.slice(1));
|
||||
}
|
||||
|
||||
/**
|
||||
* Make HTTP request to API
|
||||
*/
|
||||
private async request(
|
||||
path: string,
|
||||
options: RequestInit,
|
||||
retries = 0,
|
||||
): Promise<unknown> {
|
||||
const url = `${this.config.baseUrl}${path}`;
|
||||
const headers: Record<string, string> = {
|
||||
'Content-Type': 'application/json',
|
||||
};
|
||||
|
||||
if (this.config.apiKey) {
|
||||
headers['Authorization'] = `Bearer ${this.config.apiKey}`;
|
||||
}
|
||||
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), this.config.timeout);
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
...options,
|
||||
headers: { ...headers, ...options.headers },
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await response.text();
|
||||
|
||||
if (response.status === 401) {
|
||||
throw new SciPixError('Unauthorized', SciPixErrorCode.Unauthorized, 401);
|
||||
}
|
||||
if (response.status === 429) {
|
||||
throw new SciPixError('Rate limited', SciPixErrorCode.RateLimited, 429);
|
||||
}
|
||||
|
||||
throw SciPixError.serverError(error, response.status);
|
||||
}
|
||||
|
||||
return await response.json();
|
||||
} catch (error) {
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
if (error instanceof SciPixError) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
if ((error as Error).name === 'AbortError') {
|
||||
throw SciPixError.timeout();
|
||||
}
|
||||
|
||||
// Retry on network errors
|
||||
if (retries < this.config.maxRetries) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000 * (retries + 1)));
|
||||
return this.request(path, options, retries + 1);
|
||||
}
|
||||
|
||||
throw SciPixError.networkError((error as Error).message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a SciPix client with default configuration
|
||||
*/
|
||||
export function createClient(config?: SciPixConfig): SciPixClient {
|
||||
return new SciPixClient(config);
|
||||
}
|
||||
64
npm/packages/scipix/src/index.ts
Normal file
64
npm/packages/scipix/src/index.ts
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
/**
|
||||
* @ruvector/scipix - OCR Client for Scientific Documents
|
||||
*
|
||||
* A TypeScript client for the SciPix OCR API, enabling extraction of
|
||||
* LaTeX, MathML, and text from scientific images, equations, and documents.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* import { SciPixClient, OutputFormat } from '@ruvector/scipix';
|
||||
*
|
||||
* // Create client
|
||||
* const client = new SciPixClient({
|
||||
* baseUrl: 'http://localhost:8080',
|
||||
* apiKey: 'your-api-key',
|
||||
* });
|
||||
*
|
||||
* // OCR an image file
|
||||
* const result = await client.ocrFile('./equation.png', {
|
||||
* formats: [OutputFormat.LaTeX, OutputFormat.MathML],
|
||||
* detectEquations: true,
|
||||
* });
|
||||
*
|
||||
* console.log('LaTeX:', result.latex);
|
||||
* console.log('Confidence:', result.confidence);
|
||||
*
|
||||
* // Quick LaTeX extraction
|
||||
* const latex = await client.extractLatex('./math.png');
|
||||
* console.log('Extracted LaTeX:', latex);
|
||||
*
|
||||
* // Batch processing
|
||||
* const batchResult = await client.batchOcr({
|
||||
* images: [
|
||||
* { source: 'base64...', id: 'eq1' },
|
||||
* { source: 'base64...', id: 'eq2' },
|
||||
* ],
|
||||
* defaultOptions: { formats: [OutputFormat.LaTeX] },
|
||||
* });
|
||||
*
|
||||
* console.log(`Processed ${batchResult.successful}/${batchResult.totalImages} images`);
|
||||
* ```
|
||||
*
|
||||
* @packageDocumentation
|
||||
*/
|
||||
|
||||
// Types
|
||||
export {
|
||||
OutputFormat,
|
||||
ImageType,
|
||||
ConfidenceLevel,
|
||||
ContentType,
|
||||
BoundingBox,
|
||||
OCRRegion,
|
||||
OCRResult,
|
||||
OCROptions,
|
||||
BatchOCRRequest,
|
||||
BatchOCRResult,
|
||||
SciPixConfig,
|
||||
HealthStatus,
|
||||
SciPixError,
|
||||
SciPixErrorCode,
|
||||
} from './types.js';
|
||||
|
||||
// Client
|
||||
export { SciPixClient, createClient } from './client.js';
|
||||
230
npm/packages/scipix/src/types.ts
Normal file
230
npm/packages/scipix/src/types.ts
Normal file
|
|
@ -0,0 +1,230 @@
|
|||
/**
|
||||
* SciPix OCR Types
|
||||
* Types for scientific document OCR and equation recognition
|
||||
*/
|
||||
|
||||
/** Supported output formats for OCR results */
|
||||
export enum OutputFormat {
|
||||
/** LaTeX mathematical notation */
|
||||
LaTeX = 'latex',
|
||||
/** MathML markup language */
|
||||
MathML = 'mathml',
|
||||
/** ASCII math notation */
|
||||
AsciiMath = 'asciimath',
|
||||
/** Plain text */
|
||||
Text = 'text',
|
||||
/** Structured JSON with metadata */
|
||||
JSON = 'json',
|
||||
}
|
||||
|
||||
/** Supported image input types */
|
||||
export enum ImageType {
|
||||
PNG = 'png',
|
||||
JPEG = 'jpeg',
|
||||
WebP = 'webp',
|
||||
PDF = 'pdf',
|
||||
TIFF = 'tiff',
|
||||
BMP = 'bmp',
|
||||
}
|
||||
|
||||
/** OCR confidence level */
|
||||
export enum ConfidenceLevel {
|
||||
High = 'high',
|
||||
Medium = 'medium',
|
||||
Low = 'low',
|
||||
}
|
||||
|
||||
/** Type of content detected in the image */
|
||||
export enum ContentType {
|
||||
/** Mathematical equation */
|
||||
Equation = 'equation',
|
||||
/** Text content */
|
||||
Text = 'text',
|
||||
/** Table structure */
|
||||
Table = 'table',
|
||||
/** Diagram or chart */
|
||||
Diagram = 'diagram',
|
||||
/** Mixed content */
|
||||
Mixed = 'mixed',
|
||||
}
|
||||
|
||||
/** Bounding box for detected regions */
|
||||
export interface BoundingBox {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
/** Single OCR result region */
|
||||
export interface OCRRegion {
|
||||
/** Unique identifier for this region */
|
||||
id: string;
|
||||
/** Bounding box of the detected region */
|
||||
bbox: BoundingBox;
|
||||
/** Type of content detected */
|
||||
contentType: ContentType;
|
||||
/** Raw text content */
|
||||
text: string;
|
||||
/** LaTeX representation (if applicable) */
|
||||
latex?: string;
|
||||
/** MathML representation (if applicable) */
|
||||
mathml?: string;
|
||||
/** Confidence score (0-1) */
|
||||
confidence: number;
|
||||
/** Confidence level */
|
||||
confidenceLevel: ConfidenceLevel;
|
||||
}
|
||||
|
||||
/** Complete OCR result */
|
||||
export interface OCRResult {
|
||||
/** Unique result identifier */
|
||||
id: string;
|
||||
/** Original image dimensions */
|
||||
imageDimensions: {
|
||||
width: number;
|
||||
height: number;
|
||||
};
|
||||
/** All detected regions */
|
||||
regions: OCRRegion[];
|
||||
/** Combined text output */
|
||||
text: string;
|
||||
/** Combined LaTeX output (if requested) */
|
||||
latex?: string;
|
||||
/** Combined MathML output (if requested) */
|
||||
mathml?: string;
|
||||
/** Processing time in milliseconds */
|
||||
processingTime: number;
|
||||
/** Model version used */
|
||||
modelVersion: string;
|
||||
/** Overall confidence */
|
||||
confidence: number;
|
||||
/** Metadata */
|
||||
metadata: {
|
||||
imageType: ImageType;
|
||||
hasEquations: boolean;
|
||||
hasTables: boolean;
|
||||
hasDiagrams: boolean;
|
||||
pageCount?: number;
|
||||
};
|
||||
}
|
||||
|
||||
/** OCR request options */
|
||||
export interface OCROptions {
|
||||
/** Desired output formats */
|
||||
formats?: OutputFormat[];
|
||||
/** Language hints for OCR */
|
||||
languages?: string[];
|
||||
/** Enable equation detection */
|
||||
detectEquations?: boolean;
|
||||
/** Enable table detection */
|
||||
detectTables?: boolean;
|
||||
/** Enable diagram detection */
|
||||
detectDiagrams?: boolean;
|
||||
/** Minimum confidence threshold (0-1) */
|
||||
minConfidence?: number;
|
||||
/** Enable preprocessing (deskew, denoise) */
|
||||
preprocess?: boolean;
|
||||
/** DPI hint for scanned documents */
|
||||
dpi?: number;
|
||||
/** Specific pages to process (for PDFs) */
|
||||
pages?: number[];
|
||||
}
|
||||
|
||||
/** Batch OCR request */
|
||||
export interface BatchOCRRequest {
|
||||
/** Array of image URLs or base64 data */
|
||||
images: Array<{
|
||||
/** URL or base64 data */
|
||||
source: string;
|
||||
/** Optional identifier */
|
||||
id?: string;
|
||||
/** Per-image options */
|
||||
options?: OCROptions;
|
||||
}>;
|
||||
/** Default options for all images */
|
||||
defaultOptions?: OCROptions;
|
||||
}
|
||||
|
||||
/** Batch OCR result */
|
||||
export interface BatchOCRResult {
|
||||
/** Total images processed */
|
||||
totalImages: number;
|
||||
/** Successful results */
|
||||
successful: number;
|
||||
/** Failed results */
|
||||
failed: number;
|
||||
/** Individual results */
|
||||
results: Array<{
|
||||
id: string;
|
||||
success: boolean;
|
||||
result?: OCRResult;
|
||||
error?: string;
|
||||
}>;
|
||||
/** Total processing time */
|
||||
totalProcessingTime: number;
|
||||
}
|
||||
|
||||
/** SciPix client configuration */
|
||||
export interface SciPixConfig {
|
||||
/** API base URL */
|
||||
baseUrl?: string;
|
||||
/** API key for authentication */
|
||||
apiKey?: string;
|
||||
/** Request timeout in milliseconds */
|
||||
timeout?: number;
|
||||
/** Maximum retries for failed requests */
|
||||
maxRetries?: number;
|
||||
/** Default OCR options */
|
||||
defaultOptions?: OCROptions;
|
||||
}
|
||||
|
||||
/** Health check response */
|
||||
export interface HealthStatus {
|
||||
status: 'healthy' | 'degraded' | 'unhealthy';
|
||||
version: string;
|
||||
models: {
|
||||
name: string;
|
||||
loaded: boolean;
|
||||
version: string;
|
||||
}[];
|
||||
uptime: number;
|
||||
}
|
||||
|
||||
/** Error types */
|
||||
export class SciPixError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public readonly code: SciPixErrorCode,
|
||||
public readonly statusCode?: number,
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'SciPixError';
|
||||
}
|
||||
|
||||
static networkError(message: string): SciPixError {
|
||||
return new SciPixError(message, SciPixErrorCode.Network);
|
||||
}
|
||||
|
||||
static serverError(message: string, statusCode: number): SciPixError {
|
||||
return new SciPixError(message, SciPixErrorCode.Server, statusCode);
|
||||
}
|
||||
|
||||
static invalidImage(message: string): SciPixError {
|
||||
return new SciPixError(message, SciPixErrorCode.InvalidImage);
|
||||
}
|
||||
|
||||
static timeout(): SciPixError {
|
||||
return new SciPixError('Request timed out', SciPixErrorCode.Timeout);
|
||||
}
|
||||
}
|
||||
|
||||
export enum SciPixErrorCode {
|
||||
Network = 'NETWORK',
|
||||
Server = 'SERVER',
|
||||
InvalidImage = 'INVALID_IMAGE',
|
||||
Timeout = 'TIMEOUT',
|
||||
InvalidConfig = 'INVALID_CONFIG',
|
||||
Unauthorized = 'UNAUTHORIZED',
|
||||
RateLimited = 'RATE_LIMITED',
|
||||
}
|
||||
19
npm/packages/scipix/tsconfig.json
Normal file
19
npm/packages/scipix/tsconfig.json
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"sourceMap": true,
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "dist", "test"]
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue