refactor(agent-core-v2): reroute blob store backend from host fs to storage layer

- Add IBlobStorage role token for pluggable blob backends
- Make BlobStoreService depend on IBlobStorage instead of IHostFileSystem
- Remove host filesystem dependency from WireRecordService
- Seed IBlobStorage in local and server-v2 composition roots
- Add BlobStoreService tests
This commit is contained in:
haozhe.yang 2026-06-29 05:12:14 +08:00
parent eb3752c74f
commit 0875c23014
9 changed files with 171 additions and 52 deletions

View file

@ -0,0 +1,6 @@
---
"@moonshot-ai/agent-core-v2": patch
"@moonshot-ai/server-v2": patch
---
Reroute the blob store backend from the host filesystem to the pluggable storage layer, so server-only deployments no longer require a local filesystem implementation.

View file

@ -6,7 +6,11 @@ export const BLOBREF_PROTOCOL = 'blobref:';
export const MISSING_MEDIA_PLACEHOLDER = '[media missing]';
export interface BlobStoreServiceOptions {
readonly blobsDir?: string;
/**
* Storage scope used to namespace blob keys in the `IBlobStorage` backend.
* Defaults to `'blobs'`.
*/
readonly storageScope?: string;
readonly threshold?: number;
readonly maxCacheSize?: number;
}

View file

@ -1,10 +1,8 @@
import {
createHash } from 'node:crypto';
import { join } from 'pathe';
import { createHash } from 'node:crypto';
import type { ContentPart } from '@moonshot-ai/kosong';
import { InstantiationType } from '#/_base/di/extensions';
import { LifecycleScope, registerScopedService } from '#/_base/di/scope';
import { IHostFileSystem } from '#/hostFs';
import { IBlobStorage, type IStorageService } from '#/storage';
import {
BLOBREF_PROTOCOL,
@ -15,10 +13,11 @@ import {
const DEFAULT_THRESHOLD = 4096;
const DEFAULT_MAX_CACHE_SIZE = 50 * 1024 * 1024;
const DEFAULT_STORAGE_SCOPE = 'blobs';
const DATA_URI_HEADER_RE = /^data:([^;]+);base64,/;
export class BlobStoreService implements IBlobStoreService {
private readonly blobsDir: string | undefined;
private readonly storageScope: string;
private readonly threshold: number;
private readonly maxCacheSize: number;
private readonly cache = new Map<string, Buffer>();
@ -26,10 +25,10 @@ export class BlobStoreService implements IBlobStoreService {
private currentCacheSize = 0;
constructor(
@IBlobStorage private readonly storage: IStorageService,
options: BlobStoreServiceOptions = {},
@IHostFileSystem private readonly hostFs: IHostFileSystem,
) {
this.blobsDir = options.blobsDir;
this.storageScope = options.storageScope ?? DEFAULT_STORAGE_SCOPE;
this.threshold = options.threshold ?? DEFAULT_THRESHOLD;
this.maxCacheSize = options.maxCacheSize ?? DEFAULT_MAX_CACHE_SIZE;
}
@ -39,8 +38,6 @@ export class BlobStoreService implements IBlobStoreService {
}
async offloadParts(parts: readonly ContentPart[]): Promise<readonly ContentPart[]> {
if (this.blobsDir === undefined) return parts;
let changed = false;
const out: ContentPart[] = [];
for (const part of parts) {
@ -52,8 +49,6 @@ export class BlobStoreService implements IBlobStoreService {
}
async rehydrateParts(parts: readonly ContentPart[]): Promise<readonly ContentPart[]> {
if (this.blobsDir === undefined) return parts;
let changed = false;
const out: ContentPart[] = [];
for (const part of parts) {
@ -120,9 +115,8 @@ export class BlobStoreService implements IBlobStoreService {
this.cache.set(hash, cached);
return cached;
}
if (this.blobsDir === undefined) return undefined;
const payload = await this.hostFs.readBytes(join(this.blobsDir, hash)).catch(() => undefined);
const payload = await this.storage.read(this.storageScope, hash).catch(() => undefined);
if (payload !== undefined) {
this.setCache(hash, Buffer.from(payload));
}
@ -143,14 +137,9 @@ export class BlobStoreService implements IBlobStoreService {
}
private async writeBlob(mimeType: string, base64Payload: string): Promise<string> {
const blobsDir = this.blobsDir;
if (blobsDir === undefined) return `data:${mimeType};base64,${base64Payload}`;
await this.hostFs.mkdir(blobsDir, { recursive: true });
const hash = createHash('sha256').update(base64Payload, 'utf8').digest('hex');
const blobPath = join(blobsDir, hash);
const binary = Buffer.from(base64Payload, 'base64');
await this.hostFs.createExclusive(blobPath, binary);
await this.storage.write(this.storageScope, hash, binary, { atomic: true });
this.setCache(hash, binary);
return `${BLOBREF_PROTOCOL}${mimeType};${hash}`;
}

View file

@ -21,7 +21,7 @@ import type { Environment } from '@moonshot-ai/kaos';
import { createDecorator, type ServiceIdentifier } from '#/_base/di/instantiation';
import { createCoreScope, type Scope, type ScopeSeed } from '#/_base/di/scope';
import { FileStorageService, IStorageService } from '#/storage';
import { FileStorageService, IBlobStorage, IStorageService } from '#/storage';
export interface IBootstrapOptions {
readonly homeDir: string;
@ -102,7 +102,10 @@ export function bootstrap(input: BootstrapInput = {}, extraSeeds: ScopeSeed = []
}
function storageSeed(options: IBootstrapOptions): ScopeSeed {
return [[IStorageService as ServiceIdentifier<unknown>, new FileStorageService(options.homeDir)]];
return [
[IStorageService as ServiceIdentifier<unknown>, new FileStorageService(options.homeDir)],
[IBlobStorage as ServiceIdentifier<unknown>, new FileStorageService(options.homeDir)],
];
}
export function resolveKimiHome(

View file

@ -25,6 +25,7 @@ import { Emitter, type Event } from '#/_base/event';
import {
IAppendLogStorage,
IAtomicDocumentStorage,
IBlobStorage,
IStorageService,
type StorageAppendOptions,
type StorageWriteOptions,
@ -167,3 +168,11 @@ registerScopedService(
InstantiationType.Delayed,
'storage',
);
registerScopedService(
LifecycleScope.Core,
IBlobStorage,
InMemoryStorageService,
InstantiationType.Delayed,
'storage',
);

View file

@ -66,3 +66,15 @@ export const IAppendLogStorage: ServiceIdentifier<IStorageService> =
export const IAtomicDocumentStorage: ServiceIdentifier<IStorageService> =
createDecorator<IStorageService>('atomicDocumentStorage');
/**
* `IBlobStorage` role token for the blob-store backend.
*
* Like `IAppendLogStorage` and `IAtomicDocumentStorage`, this is the same
* `IStorageService` interface under a distinct identity so the composition root
* can route large, content-addressed blob objects to a dedicated backend
* (e.g., S3 in a server-only deployment) while other storage roles use a
* different backend.
*/
export const IBlobStorage: ServiceIdentifier<IStorageService> =
createDecorator<IStorageService>('blobStorage');

View file

@ -1,22 +1,13 @@
import { InstantiationType } from '#/_base/di/extensions';
import { LifecycleScope, registerScopedService } from '#/_base/di/scope';
import {
join
} from 'pathe';
import { createHash } from 'node:crypto';
import { InstantiationType } from '#/_base/di/extensions';
import { LifecycleScope, registerScopedService } from '#/_base/di/scope';
import {
Disposable,
toDisposable,
} from "#/_base/di";
import {
IBlobStoreService,
type BlobStoreServiceOptions,
} from '#/blobStore';
import { IHostFileSystem } from '#/hostFs';
import { IBlobStoreService } from '#/blobStore';
import { IAppendLogStore } from '#/storage';
import { BlobStoreService } from '../blobStore/blobStoreService';
import { OrderedHookSlot } from '../hooks';
import type { WireRecord, WireRecordMap } from '../wireRecord';
import {
@ -63,10 +54,9 @@ export class WireRecordService extends Disposable implements IWireRecord {
private readonly options: WireRecordServiceOptions = {},
@IBlobStoreService injectedBlobStore?: IBlobStoreService,
@IAppendLogStore private readonly log?: IAppendLogStore,
@IHostFileSystem private readonly hostFs?: IHostFileSystem,
) {
super();
this.blobStore = this.resolveBlobStore(options, injectedBlobStore);
this.blobStore = options.blobStore ?? injectedBlobStore;
this.persistKey = options.homedir === undefined ? undefined : hashKey(options.homedir);
if (this.log !== undefined && this.persistKey !== undefined) {
this._register(this.log.acquire('wire', this.persistKey));
@ -324,19 +314,6 @@ export class WireRecordService extends Disposable implements IWireRecord {
}
return current;
}
private resolveBlobStore(
options: WireRecordServiceOptions,
injectedBlobStore: IBlobStoreService | undefined,
): IBlobStoreService | undefined {
if (options.blobStore !== undefined) return options.blobStore;
if (options.homedir === undefined || this.hostFs === undefined) return injectedBlobStore;
const blobOptions: BlobStoreServiceOptions = {
blobsDir: join(options.homedir, 'blobs'),
};
return new BlobStoreService(blobOptions, this.hostFs);
}
}
async function* toAsyncIterable<T>(

View file

@ -0,0 +1,117 @@
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import type { ContentPart } from '@moonshot-ai/kosong';
import { LifecycleScope } from '#/_base/di/scope';
import { createScopedTestHost, stubPair } from '#/_base/di/test';
import { IBlobStoreService } from '#/blobStore';
import { IBlobStorage, InMemoryStorageService } from '#/storage';
function makeLargeDataUri(mimeType = 'image/png'): { uri: string; payload: string } {
// The default threshold is 4096 base64 characters.
const payload = 'A'.repeat(5000);
return { uri: `data:${mimeType};base64,${payload}`, payload };
}
function makeSmallDataUri(mimeType = 'image/png'): { uri: string; payload: string } {
const payload = 'AQID';
return { uri: `data:${mimeType};base64,${payload}`, payload };
}
describe('BlobStoreService', () => {
let host: ReturnType<typeof createScopedTestHost>;
beforeEach(() => {
host = createScopedTestHost([stubPair(IBlobStorage, new InMemoryStorageService())]);
});
afterEach(() => {
host.dispose();
});
function getBlobStore(): IBlobStoreService {
const agent = host.core.createChild(LifecycleScope.Agent, 'test-agent');
return agent.accessor.get(IBlobStoreService);
}
it('leaves small data URIs unchanged', async () => {
const store = getBlobStore();
const { uri } = makeSmallDataUri();
const parts: ContentPart[] = [{ type: 'image_url', imageUrl: { url: uri } }];
const result = await store.offloadParts(parts);
expect(result).toBe(parts);
expect((result[0]! as { imageUrl: { url: string } }).imageUrl.url).toBe(uri);
});
it('offloads large data URIs to blobref URIs', async () => {
const store = getBlobStore();
const { uri, payload } = makeLargeDataUri();
const parts: ContentPart[] = [{ type: 'image_url', imageUrl: { url: uri } }];
const result = await store.offloadParts(parts);
expect(result).not.toBe(parts);
const newUrl = (result[0]! as { imageUrl: { url: string } }).imageUrl.url;
expect(store.isBlobRef(newUrl)).toBe(true);
expect(newUrl.startsWith('blobref:image/png;')).toBe(true);
const backend = host.core.accessor.get(IBlobStorage);
const keys = await backend.list('blobs');
expect(keys).toHaveLength(1);
expect(Buffer.from((await backend.read('blobs', keys[0]!))!).toString('base64')).toBe(payload);
});
it('rehydrates blobref URIs back to data URIs', async () => {
const store = getBlobStore();
const { uri } = makeLargeDataUri();
const parts: ContentPart[] = [{ type: 'image_url', imageUrl: { url: uri } }];
const offloaded = await store.offloadParts(parts);
const rehydrated = await store.rehydrateParts(offloaded);
expect((rehydrated[0]! as { imageUrl: { url: string } }).imageUrl.url).toBe(uri);
});
it('rehydrates only blobref URLs and leaves other URLs alone', async () => {
const store = getBlobStore();
const { uri: largeUri } = makeLargeDataUri();
const { uri: smallUri } = makeSmallDataUri();
const parts: ContentPart[] = [
{ type: 'image_url', imageUrl: { url: largeUri } },
{ type: 'audio_url', audioUrl: { url: smallUri } },
];
const offloaded = await store.offloadParts(parts);
expect(store.isBlobRef((offloaded[0]! as { imageUrl: { url: string } }).imageUrl.url)).toBe(true);
expect((offloaded[1]! as { audioUrl: { url: string } }).audioUrl.url).toBe(smallUri);
const rehydrated = await store.rehydrateParts(offloaded);
expect((rehydrated[0]! as { imageUrl: { url: string } }).imageUrl.url).toBe(largeUri);
expect((rehydrated[1]! as { audioUrl: { url: string } }).audioUrl.url).toBe(smallUri);
});
it('replaces missing blobs with a placeholder', async () => {
const store = getBlobStore();
const parts: ContentPart[] = [
{ type: 'image_url', imageUrl: { url: 'blobref:image/png;deadbeef' } },
];
const result = await store.rehydrateParts(parts);
expect((result[0]! as { imageUrl: { url: string } }).imageUrl.url).toBe('[media missing]');
});
it('is idempotent across offload/rehydrate cycles', async () => {
const store = getBlobStore();
const { uri } = makeLargeDataUri();
const parts: ContentPart[] = [{ type: 'image_url', imageUrl: { url: uri } }];
const first = await store.offloadParts(parts);
const second = await store.offloadParts(first);
expect((second[0]! as { imageUrl: { url: string } }).imageUrl.url).toBe(
(first[0]! as { imageUrl: { url: string } }).imageUrl.url,
);
});
});

View file

@ -12,6 +12,7 @@ import {
FileStorageService,
IAppendLogStorage,
IAtomicDocumentStorage,
IBlobStorage,
logSeed,
resolveConfigPath,
resolveKimiHome,
@ -61,6 +62,7 @@ function durableStorageSeeds(homeDir: string): ScopeSeed {
return [
[IAtomicDocumentStorage as ServiceIdentifier<unknown>, new FileStorageService(homeDir)],
[IAppendLogStorage as ServiceIdentifier<unknown>, new FileStorageService(homeDir)],
[IBlobStorage as ServiceIdentifier<unknown>, new FileStorageService(homeDir)],
];
}
@ -71,9 +73,9 @@ export async function startServer(opts: ServerStartOptions = {}): Promise<Runnin
// route that creates a session (e.g. POST /sessions) would otherwise fail to
// instantiate the Session scope. Resolve it from env + homeDir like the CLI.
const logging = resolveLoggingConfig({ homeDir, env: process.env });
// `IAtomicDocumentStorage` / `IAppendLogStorage` default to in-memory; seed
// file-backed stores rooted at homeDir so session metadata (and later wire
// records) persist to disk where `FileSessionIndex` reads them.
// `IAtomicDocumentStorage` / `IAppendLogStorage` / `IBlobStorage` default to
// in-memory; seed file-backed stores rooted at homeDir so session metadata,
// wire records, and blobs persist to disk where `FileSessionIndex` reads them.
const { core } = bootstrap({ homeDir, configPath }, [
...logSeed(logging),
...durableStorageSeeds(homeDir),