From 4c2c3956fe71d963aeba707da79fd66bd8325b75 Mon Sep 17 00:00:00 2001 From: _Kerman Date: Sat, 4 Jul 2026 13:49:22 +0800 Subject: [PATCH] refactor: generalize loop error recovery hook --- .../agent-core-v2/docs/di-scope-domains.puml | 2 +- .../agent-core-v2/docs/di-scope-domains.svg | 2 +- .../fullCompaction/fullCompactionService.ts | 20 ++++--- .../agent-core-v2/src/agent/loop/errors.ts | 10 +++- packages/agent-core-v2/src/agent/loop/loop.ts | 12 +++- .../src/agent/loop/loopService.ts | 32 ++++------ .../agent-core-v2/test/loop/basic.test.ts | 60 +++++++++++++++++++ packages/agent-core-v2/test/turn/stubs.ts | 2 +- 8 files changed, 104 insertions(+), 36 deletions(-) diff --git a/packages/agent-core-v2/docs/di-scope-domains.puml b/packages/agent-core-v2/docs/di-scope-domains.puml index 08070aef2..f2ef59abf 100644 --- a/packages/agent-core-v2/docs/di-scope-domains.puml +++ b/packages/agent-core-v2/docs/di-scope-domains.puml @@ -352,7 +352,7 @@ contextMemory ..> wireRecord #16A085 : context.splice cron ..> wireRecord #16A085 : cron.add / delete / cursor todo ..> wireRecord #16A085 : todo.set fullCompaction ..> wireRecord #16A085 : full_compaction.begin/cancel/complete -fullCompaction ..> loop #16A085 : hooks.onContextOverflow +fullCompaction ..> loop #16A085 : hooks.onError permissionRules ..> wireRecord #16A085 : permission.rules.add / record_approval_result skill ..> wireRecord #16A085 : skill.activate goal ..> wireRecord #16A085 : goal.create/update/clear diff --git a/packages/agent-core-v2/docs/di-scope-domains.svg b/packages/agent-core-v2/docs/di-scope-domains.svg index 8fdae5a57..b562fcc0d 100644 --- a/packages/agent-core-v2/docs/di-scope-domains.svg +++ b/packages/agent-core-v2/docs/di-scope-domains.svg @@ -1 +1 @@ -App scope (process-wide)Session scope (per session)Agent scope (per agent)bootstrapAppIBootstrapServicelogAppILogServiceILogWriterServicetelemetryAppITelemetryServiceeventAppIEventServicestorageAppIStorageServiceIAppendLogStoreIAtomicDocumentStoreIAtomicTomlDocumentStorefilestoreAppIFileStoregatewayAppIRestGatewayIWSGatewaysessionLifecycleAppISessionLifecycleServicesessionIndexAppISessionIndexhostFsAppIHostFileSystemworkspaceRegistryAppIWorkspaceRegistryhostFolderBrowserAppIHostFolderBrowserhostEnvironmentAppIHostEnvironmenthostProcessAppIHostProcessServiceauthAppIOAuthServiceIAuthSummaryServiceIWebSearchProviderServicewebAppIWebFetchServiceproviderAppIProviderServiceflagAppIFlagServiceIFlagRegistryconfigAppIConfigRegistryIConfigServicepluginAppIPluginServicechatProviderAppIChatProviderFactorymodelAppIModelServicemodelCatalogAppIModelCatalogServiceglobalSkillCatalogAppIGlobalSkillCatalogISkillCatalogStoresessionContextSessionISessionContext (seed)sessionMetadataSessionISessionMetadatasessionActivitySessionISessionActivityagentLifecycleSessionIAgentLifecycleServiceinteractionSessionIInteractionServiceworkspaceContextSessionIWorkspaceContextsessionLogSessionISessionLogServiceILogWriterServicesessionSkillCatalogSessionISessionSkillCatalogexecContextSessionIExecContext (seed)agentFsSessionIAgentFileSystemIFsServiceapprovalSessionIApprovalServicequestionSessionIQuestionServiceprocessSessionIProcessRunnerIProcessterminalAppIHostTerminalServicesessionTerminalSessionISessionTerminalServicemodelProviderSessionIModelProvider (seed)todoSessionISessionTodoServicewireRecordAgentIAgentWireRecordService (event hub)recordAgentIAgentRecordServiceblobStoreAgentIAgentBlobStoreServicecontextMemoryAgentIAgentContextMemoryServicecontextProjectorAgentIAgentContextProjectorServicecontextInjectorAgentIAgentContextInjectorServicecontextSizeAgentIAgentContextSizeServicesystemReminderAgentIAgentSystemReminderServiceprofileAgentIAgentProfileServicepromptAgentIAgentPromptServiceturnAgentIAgentTurnServiceloopAgentIAgentLoopServicellmRequesterAgentIAgentLLMRequesterServicetoolRegistryAgentIAgentToolRegistryServicetoolExecutorAgentIAgentToolExecutorServicetoolStoreAgentIAgentToolStoreServicetoolDedupAgentIAgentToolDedupeServicepermissionGateAgentIAgentPermissionGatepermissionModeAgentIAgentPermissionModeServicepermissionPolicyAgentIAgentPermissionPolicyServicepermissionRulesAgentIAgentPermissionRulesServiceplanAgentIAgentPlanServicegoalAgentIAgentGoalServiceskillAgentIAgentSkillServicequestionToolsAgentIAgentQuestionToolsServiceuserToolAgentIAgentUserToolServicetaskAgentIAgentTaskServicecronAgentIAgentCronServiceswarmAgentIAgentSwarmServicemcpAgentIAgentMcpServicefullCompactionAgentIAgentFullCompactionServicemicroCompactionAgentIAgentMicroCompactionServiceexternalHooksAgentIAgentExternalHooksServiceusageAgentIAgentUsageServicerpcAgentIAgentRPCServicefileToolsAgentRead/Write/Grep/Glob toolseditAgentEditToolFileEditServiceshellToolsAgentIAgentShellToolsServicescopeContextAgentIAgentScopeContext (seed)environmenteventSinkonDidChangeConfigurationcontext.splicehooks.onSplicedcontext_size.measuredconfig.update / tools.set_active_toolshooks.onSplicedtools.update_storepermission.set_modepermission.rules.add / record_approval_resultplan_mode.enter/cancel/exitgoal.create/update/clearturn lifecycle hooksstep/usage hooksskill.activatetools.register_/unregister_user_tooltask.started/terminatedhooks.onSplicedcron.add / delete / cursorswarm_mode.enter/exitfull_compaction.begin/cancel/completehooks.onContextOverflowmicro_compaction.apply / full_compaction.completeusage.recordtodo.sethook.result / goal.updatedhooks.onEndedonWillExecuteTool / onDidExecuteToolapproval hooksprompt/end hooksstop hookcompaction hooksnotification hookSubagentStart/Stop (mirrorAgentRun)hooks.onResumeEndedScope = node color      App (process-wide)      Session (per session)      Agent (per agent)Edgessolid: DI injection (ctor @IX)dashed: event-driven (subscribe/emit)direction: consumer ---> providerNotesGenerated from `node scripts/dep-graph.mjs` output;`_base` / seed / options deps are omitted. \ No newline at end of file +App scope (process-wide)Session scope (per session)Agent scope (per agent)bootstrapAppIBootstrapServicelogAppILogServiceILogWriterServicetelemetryAppITelemetryServiceeventAppIEventServicestorageAppIStorageServiceIAppendLogStoreIAtomicDocumentStoreIAtomicTomlDocumentStorefilestoreAppIFileStoregatewayAppIRestGatewayIWSGatewaysessionLifecycleAppISessionLifecycleServicesessionIndexAppISessionIndexhostFsAppIHostFileSystemworkspaceRegistryAppIWorkspaceRegistryhostFolderBrowserAppIHostFolderBrowserhostEnvironmentAppIHostEnvironmenthostProcessAppIHostProcessServiceauthAppIOAuthServiceIAuthSummaryServiceIWebSearchProviderServicewebAppIWebFetchServiceproviderAppIProviderServiceflagAppIFlagServiceIFlagRegistryconfigAppIConfigRegistryIConfigServicepluginAppIPluginServicechatProviderAppIChatProviderFactorymodelAppIModelServicemodelCatalogAppIModelCatalogServiceglobalSkillCatalogAppIGlobalSkillCatalogISkillCatalogStoresessionContextSessionISessionContext (seed)sessionMetadataSessionISessionMetadatasessionActivitySessionISessionActivityagentLifecycleSessionIAgentLifecycleServiceinteractionSessionIInteractionServiceworkspaceContextSessionIWorkspaceContextsessionLogSessionISessionLogServiceILogWriterServicesessionSkillCatalogSessionISessionSkillCatalogexecContextSessionIExecContext (seed)agentFsSessionIAgentFileSystemIFsServiceapprovalSessionIApprovalServicequestionSessionIQuestionServiceprocessSessionIProcessRunnerIProcessterminalAppIHostTerminalServicesessionTerminalSessionISessionTerminalServicemodelProviderSessionIModelProvider (seed)todoSessionISessionTodoServicewireRecordAgentIAgentWireRecordService (event hub)recordAgentIAgentRecordServiceblobStoreAgentIAgentBlobStoreServicecontextMemoryAgentIAgentContextMemoryServicecontextProjectorAgentIAgentContextProjectorServicecontextInjectorAgentIAgentContextInjectorServicecontextSizeAgentIAgentContextSizeServicesystemReminderAgentIAgentSystemReminderServiceprofileAgentIAgentProfileServicepromptAgentIAgentPromptServiceturnAgentIAgentTurnServiceloopAgentIAgentLoopServicellmRequesterAgentIAgentLLMRequesterServicetoolRegistryAgentIAgentToolRegistryServicetoolExecutorAgentIAgentToolExecutorServicetoolStoreAgentIAgentToolStoreServicetoolDedupAgentIAgentToolDedupeServicepermissionGateAgentIAgentPermissionGatepermissionModeAgentIAgentPermissionModeServicepermissionPolicyAgentIAgentPermissionPolicyServicepermissionRulesAgentIAgentPermissionRulesServiceplanAgentIAgentPlanServicegoalAgentIAgentGoalServiceskillAgentIAgentSkillServicequestionToolsAgentIAgentQuestionToolsServiceuserToolAgentIAgentUserToolServicetaskAgentIAgentTaskServicecronAgentIAgentCronServiceswarmAgentIAgentSwarmServicemcpAgentIAgentMcpServicefullCompactionAgentIAgentFullCompactionServicemicroCompactionAgentIAgentMicroCompactionServiceexternalHooksAgentIAgentExternalHooksServiceusageAgentIAgentUsageServicerpcAgentIAgentRPCServicefileToolsAgentRead/Write/Grep/Glob toolseditAgentEditToolFileEditServiceshellToolsAgentIAgentShellToolsServicescopeContextAgentIAgentScopeContext (seed)environmenteventSinkonDidChangeConfigurationcontext.splicehooks.onSplicedcontext_size.measuredconfig.update / tools.set_active_toolshooks.onSplicedtools.update_storepermission.set_modepermission.rules.add / record_approval_resultplan_mode.enter/cancel/exitgoal.create/update/clearturn lifecycle hooksstep/usage hooksskill.activatetools.register_/unregister_user_tooltask.started/terminatedhooks.onSplicedcron.add / delete / cursorswarm_mode.enter/exitfull_compaction.begin/cancel/completehooks.onErrormicro_compaction.apply / full_compaction.completeusage.recordtodo.sethook.result / goal.updatedhooks.onEndedonWillExecuteTool / onDidExecuteToolapproval hooksprompt/end hooksstop hookcompaction hooksnotification hookSubagentStart/Stop (mirrorAgentRun)hooks.onResumeEndedScope = node color      App (process-wide)      Session (per session)      Agent (per agent)Edgessolid: DI injection (ctor @IX)dashed: event-driven (subscribe/emit)direction: consumer ---> providerNotesGenerated from `node scripts/dep-graph.mjs` output;`_base` / seed / options deps are omitted. \ No newline at end of file diff --git a/packages/agent-core-v2/src/agent/fullCompaction/fullCompactionService.ts b/packages/agent-core-v2/src/agent/fullCompaction/fullCompactionService.ts index 2498664a0..b2a681a28 100644 --- a/packages/agent-core-v2/src/agent/fullCompaction/fullCompactionService.ts +++ b/packages/agent-core-v2/src/agent/fullCompaction/fullCompactionService.ts @@ -14,8 +14,8 @@ import { sleepForRetry, type LLMRequestFinish, } from '#/agent/llmRequester'; -import { IAgentLoopService, type TurnContextOverflowContext } from '#/agent/loop'; -import { isAbortError } from '#/agent/loop/errors'; +import { IAgentLoopService, type TurnErrorContext } from '#/agent/loop'; +import { isAbortError, isContextOverflowError } from '#/agent/loop/errors'; import { IAgentProfileService } from '#/agent/profile'; import { IAgentRecordService } from '#/agent/record'; import { IAgentTurnService } from '#/agent/turn'; @@ -141,8 +141,8 @@ export class AgentFullCompactionService extends Disposable implements IAgentFull }), ); this._register( - loopService.hooks.onContextOverflow.register('full-compaction', async (ctx, next) => { - await this.onContextOverflow(ctx, next); + loopService.hooks.onError.register('full-compaction', async (ctx, next) => { + await this.onLoopError(ctx, next); }), ); this._register( @@ -241,10 +241,14 @@ export class AgentFullCompactionService extends Disposable implements IAgentFull this.consecutiveOverflowCompactions = 0; } - private async onContextOverflow( - context: TurnContextOverflowContext, + private async onLoopError( + context: TurnErrorContext, next: () => Promise, ): Promise { + if (!isContextOverflowError(context.error)) { + await next(); + return; + } this.consecutiveOverflowCompactions += 1; const maxAttempts = this.strategy.maxOverflowCompactionAttempts; if (this.consecutiveOverflowCompactions > maxAttempts) { @@ -259,7 +263,7 @@ export class AgentFullCompactionService extends Disposable implements IAgentFull await next(); return; } - context.handled = true; + context.retry = true; await this.block(context.signal, context.turnId); } @@ -615,7 +619,7 @@ function isTodoItem(value: unknown): value is TodoItem { export { AgentFullCompactionService as FullCompaction }; // Construct eagerly (not delayed): the service registers turn and loop hooks -// (onLaunched / beforeStep / afterStep / onContextOverflow) that drive auto +// (onLaunched / beforeStep / afterStep / onError) that drive auto // compaction. With delayed instantiation the eager `accessor.get(IAgentFullCompactionService)` // only realizes a proxy, so the hooks would not register until the first RPC — // after turns have already run without the auto-compaction gate. diff --git a/packages/agent-core-v2/src/agent/loop/errors.ts b/packages/agent-core-v2/src/agent/loop/errors.ts index fd43ad0de..28e7dab56 100644 --- a/packages/agent-core-v2/src/agent/loop/errors.ts +++ b/packages/agent-core-v2/src/agent/loop/errors.ts @@ -2,7 +2,8 @@ * `loop` domain error codes and loop-local error helpers. */ -import { KimiError, registerErrorDomain, type ErrorDomain } from '#/_base/errors'; +import { KimiError, isKimiError, registerErrorDomain, type ErrorDomain } from '#/_base/errors'; +import { APIContextOverflowError } from '#/app/llmProtocol'; export const LoopErrors = { codes: { @@ -42,6 +43,13 @@ export function isMaxStepsExceededError(error: unknown): boolean { return error instanceof KimiError && error.code === LoopErrors.codes.LOOP_MAX_STEPS_EXCEEDED; } +export function isContextOverflowError(error: unknown): boolean { + return ( + error instanceof APIContextOverflowError || + (isKimiError(error) && error.code === LoopErrors.codes.CONTEXT_OVERFLOW) + ); +} + export function isAbortError(err: unknown): boolean { if (err instanceof Error) { return err.name === 'AbortError'; diff --git a/packages/agent-core-v2/src/agent/loop/loop.ts b/packages/agent-core-v2/src/agent/loop/loop.ts index 570baf6e3..b2c55cacc 100644 --- a/packages/agent-core-v2/src/agent/loop/loop.ts +++ b/packages/agent-core-v2/src/agent/loop/loop.ts @@ -16,11 +16,17 @@ export interface TurnAfterStepContext extends TurnBeforeStepContext { continue: boolean; } -export interface TurnContextOverflowContext { +export interface TurnErrorContext { readonly turnId: number; + /** The currently executing step, or undefined for turn-level failures. */ + readonly step?: number; readonly signal: AbortSignal; readonly error: unknown; - handled: boolean; + /** + * Set to true only after a handler has changed state enough for the loop to + * retry. Handlers that do not recognize the error must call next(). + */ + retry: boolean; } export interface RunTurnOptions { @@ -34,7 +40,7 @@ export interface IAgentLoopService { readonly hooks: Hooks<{ beforeStep: TurnBeforeStepContext; afterStep: TurnAfterStepContext; - onContextOverflow: TurnContextOverflowContext; + onError: TurnErrorContext; }>; runTurn(turnId: number, options?: RunTurnOptions): Promise; } diff --git a/packages/agent-core-v2/src/agent/loop/loopService.ts b/packages/agent-core-v2/src/agent/loop/loopService.ts index 7b1920bea..c0846a929 100644 --- a/packages/agent-core-v2/src/agent/loop/loopService.ts +++ b/packages/agent-core-v2/src/agent/loop/loopService.ts @@ -9,7 +9,6 @@ import type { ToolResult } from '#/agent/tool'; import { IAgentToolExecutorService } from '#/agent/toolExecutor'; import { IConfigService } from '#/app/config'; import { - APIContextOverflowError, createToolMessage, type ContentPart, type FinishReason, @@ -17,7 +16,7 @@ import { type TokenUsage, } from '#/app/llmProtocol'; import { ILogService } from '#/app/log'; -import { ErrorCodes, KimiError, isKimiError } from '#/errors'; +import { ErrorCodes, KimiError } from '#/errors'; import { OrderedHookSlot } from '#/hooks'; import { IAgentContextMemoryService, newMessageId, type ContextMessage } from '../contextMemory'; @@ -47,7 +46,7 @@ export class AgentLoopService implements IAgentLoopService { readonly hooks: IAgentLoopService['hooks'] = { beforeStep: new OrderedHookSlot(), afterStep: new OrderedHookSlot(), - onContextOverflow: new OrderedHookSlot(), + onError: new OrderedHookSlot(), }; constructor( @@ -119,17 +118,15 @@ export class AgentLoopService implements IAgentLoopService { const reason: LoopInterruptReason = isMaxStepsExceededError(error) ? 'max_steps' : 'error'; this.emitStepInterrupted(turnId, activeStep, reason, errorMessage(error)); - if (isContextOverflowError(error)) { - const context = { turnId, signal, error, handled: false }; - try { - await this.hooks.onContextOverflow.run(context); - } catch (hookError) { - return { reason: 'failed', error: hookError, steps }; - } - if (context.handled) { - activeStep = undefined; - continue; - } + const context = { turnId, step: activeStep, signal, error, retry: false }; + try { + await this.hooks.onError.run(context); + } catch (hookError) { + return { reason: 'failed', error: hookError, steps }; + } + if (context.retry) { + activeStep = undefined; + continue; } return { reason: 'failed', error, steps }; } @@ -359,13 +356,6 @@ export class AgentLoopService implements IAgentLoopService { } } -function isContextOverflowError(error: unknown): boolean { - return ( - error instanceof APIContextOverflowError || - (isKimiError(error) && error.code === ErrorCodes.CONTEXT_OVERFLOW) - ); -} - function toolResultOutputForModel(result: ToolResult): string | ContentPart[] { const output = result.output; if (typeof output === 'string') { diff --git a/packages/agent-core-v2/test/loop/basic.test.ts b/packages/agent-core-v2/test/loop/basic.test.ts index d5fcd5806..cb6dbac57 100644 --- a/packages/agent-core-v2/test/loop/basic.test.ts +++ b/packages/agent-core-v2/test/loop/basic.test.ts @@ -98,6 +98,66 @@ describe('Agent loop', () => { }); }); + it('lets onError recover a non-context loop error by retrying', async () => { + profile.update({ activeToolNames: [] }); + const seenErrors: Array<{ readonly step: number | undefined; readonly message: string }> = []; + + loop.hooks.onError.register('test-recover-generate-error', async (hookCtx, next) => { + seenErrors.push({ + step: hookCtx.step, + message: hookCtx.error instanceof Error ? hookCtx.error.message : String(hookCtx.error), + }); + if (seenErrors.length === 1) { + ctx.mockNextResponse({ type: 'text', text: 'Recovered.' }); + hookCtx.retry = true; + return; + } + await next(); + }); + + await ctx.rpc.prompt({ input: [{ type: 'text', text: 'Hello' }] }); + await ctx.untilTurnEnd(); + + expect(seenErrors).toEqual([ + { step: 1, message: 'Unexpected generate call #1' }, + ]); + expect(ctx.allEvents).toContainEqual( + expect.objectContaining({ + event: 'turn.ended', + args: expect.objectContaining({ reason: 'completed' }), + }), + ); + }); + + it('does not run onError for aborted turns', async () => { + let called = false; + loop.hooks.onError.register('test-abort-not-recoverable', async (_hookCtx, next) => { + called = true; + await next(); + }); + const controller = new AbortController(); + controller.abort(new Error('stop')); + + const result = await loop.runTurn(0, { signal: controller.signal }); + + expect(result.reason).toBe('cancelled'); + expect(called).toBe(false); + }); + + it('fails with the onError handler error when recovery throws', async () => { + const recoveryError = new Error('recovery failed'); + loop.hooks.onError.register('test-throw-recovery-error', async () => { + throw recoveryError; + }); + + const result = await loop.runTurn(0); + + expect(result.reason).toBe('failed'); + if (result.reason === 'failed') { + expect(result.error).toBe(recoveryError); + } + }); + it('runs an agent turn through registered tool approval and execution', async () => { const lookupCall: ToolCall = { type: 'function', diff --git a/packages/agent-core-v2/test/turn/stubs.ts b/packages/agent-core-v2/test/turn/stubs.ts index 5d953251d..458dd6c81 100644 --- a/packages/agent-core-v2/test/turn/stubs.ts +++ b/packages/agent-core-v2/test/turn/stubs.ts @@ -52,7 +52,7 @@ function makeAgentLoopHookSlots(): IAgentLoopService['hooks'] { return createHooks([ 'beforeStep', 'afterStep', - 'onContextOverflow', + 'onError', ]) as IAgentLoopService['hooks']; }