refactor(export): clean up unnecessary fields and simplify data structure

Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com>
This commit is contained in:
mingholy.lmh 2026-03-19 14:02:42 +08:00
parent 8e221a3606
commit 9060663f60
10 changed files with 135 additions and 145 deletions

View file

@ -264,22 +264,36 @@ function extractTaskToolTokens(record: ChatRecord): number {
/**
* Calculate token statistics from ChatRecords.
* Aggregates usageMetadata from assistant records and TaskTool executionSummary to get total token usage.
* Uses the last assistant record that has both totalTokenCount and contextWindowSize for calculating context usage percent.
*/
function calculateTokenStats(
records: ChatRecord[],
contextWindowSize?: number,
): { totalTokens: number; contextUsagePercent?: number } {
function calculateTokenStats(records: ChatRecord[]): {
totalTokens: number;
contextUsagePercent?: number;
contextWindowSize?: number;
} {
let totalTokens = 0;
let lastTotalTokens = 0;
// Track the last assistant record that has BOTH totalTokenCount and contextWindowSize
// to ensure the percentage calculation uses values from the same record
let lastValidRecord: {
totalTokenCount: number;
contextWindowSize: number;
} | null = null;
// Aggregate usageMetadata from all assistant records
// Use last available totalTokenCount for context usage calculation
for (const record of records) {
if (record.type === 'assistant' && record.usageMetadata) {
totalTokens += record.usageMetadata.totalTokenCount ?? 0;
// Use the last available totalTokenCount for context usage calculation
if (record.usageMetadata.totalTokenCount !== undefined) {
lastTotalTokens = record.usageMetadata.totalTokenCount;
if (record.type === 'assistant') {
if (record.usageMetadata) {
totalTokens += record.usageMetadata.totalTokenCount ?? 0;
}
// Only update lastValidRecord when BOTH values are present in the same record
if (
record.usageMetadata?.totalTokenCount !== undefined &&
record.contextWindowSize !== undefined
) {
lastValidRecord = {
totalTokenCount: record.usageMetadata.totalTokenCount,
contextWindowSize: record.contextWindowSize,
};
}
}
@ -290,17 +304,29 @@ function calculateTokenStats(
}
}
// Use last totalTokenCount for context usage calculation
// Use last valid record's values for context usage calculation
// This represents how much of the context window is being used by the total tokens
if (contextWindowSize && lastTotalTokens > 0) {
const percent = (lastTotalTokens / contextWindowSize) * 100;
if (lastValidRecord) {
const percent =
(lastValidRecord.totalTokenCount / lastValidRecord.contextWindowSize) *
100;
return {
totalTokens,
contextUsagePercent: Math.round(percent * 10) / 10,
contextWindowSize: lastValidRecord.contextWindowSize,
};
}
return { totalTokens };
// Fallback: return the contextWindowSize from the last assistant record even if no valid pair found
// (for display purposes only, without percentage)
const lastAssistantRecord = [...records]
.reverse()
.find((r) => r.type === 'assistant' && r.contextWindowSize !== undefined);
return {
totalTokens,
contextWindowSize: lastAssistantRecord?.contextWindowSize,
};
}
/**
@ -343,25 +369,12 @@ async function extractMetadata(
// Count user prompts
const promptCount = messages.filter((m) => m.type === 'user').length;
// Get context window size
const contentGenConfig = config.getContentGeneratorConfig?.();
const contextWindowSize = contentGenConfig?.contextWindowSize;
// Calculate file stats from original ChatRecords
const fileStats = calculateFileStats(messages);
// Calculate token stats from original ChatRecords
const tokenStats = calculateTokenStats(messages, contextWindowSize);
// Extract the last response_id from assistant records (for request tracking)
let requestId: string | undefined;
for (let i = messages.length - 1; i >= 0; i--) {
const record = messages[i];
if (record.type === 'assistant' && record.response_id) {
requestId = record.response_id;
break;
}
}
// contextWindowSize is retrieved from the last assistant record for accuracy
const tokenStats = calculateTokenStats(messages);
return {
sessionId,
@ -374,13 +387,12 @@ async function extractMetadata(
channel,
promptCount,
contextUsagePercent: tokenStats.contextUsagePercent,
contextWindowSize,
contextWindowSize: tokenStats.contextWindowSize,
totalTokens: tokenStats.totalTokens,
filesWritten: fileStats.writtenFilePaths.size,
linesAdded: fileStats.linesAdded,
linesRemoved: fileStats.linesRemoved,
uniqueFiles: Array.from(fileStats.writtenFilePaths),
requestId,
};
}

View file

@ -64,9 +64,6 @@ export function toJsonl(sessionData: ExportSessionData): string {
if (sourceMetadata?.uniqueFiles && sourceMetadata.uniqueFiles.length > 0) {
metadata['uniqueFiles'] = sourceMetadata.uniqueFiles;
}
if (sourceMetadata?.requestId) {
metadata['requestId'] = sourceMetadata.requestId;
}
lines.push(JSON.stringify(metadata));

View file

@ -21,11 +21,6 @@ export function toMarkdown(sessionData: ExportSessionData): string {
`- **Exported**: ${sanitizeText(metadata?.exportTime ?? new Date().toISOString())}`,
);
// Add requestId if available
if (metadata?.requestId) {
lines.push(`- **Request ID**: \`${sanitizeText(metadata.requestId)}\``);
}
lines.push('');
// Add context info
@ -101,9 +96,6 @@ export function toMarkdown(sessionData: ExportSessionData): string {
lines.push(formatMessageContent(message));
} else if (message.type === 'assistant') {
lines.push('## Assistant\n');
if (message.response_id) {
lines.push(`*Response ID: \`${sanitizeText(message.response_id)}\`*\n`);
}
lines.push(formatMessageContent(message));
} else if (message.type === 'tool_call') {
lines.push(formatToolCall(message));

View file

@ -28,7 +28,7 @@ export function normalizeSessionData(
}
});
// Build index of assistant messages by uuid for response_id mapping
// Build index of assistant messages by uuid for usageMetadata merging
const assistantMessageIndexByUuid = new Map<string, number>();
normalized.forEach((message, index) => {
if (message.type === 'assistant') {
@ -66,17 +66,6 @@ export function normalizeSessionData(
mergeToolCallData(existingMessage.toolCall, toolCallMessage.toolCall);
}
// Merge response_id from assistant records
for (const record of originalRecords) {
if (record.type !== 'assistant') continue;
if (!record.response_id) continue;
const existingIndex = assistantMessageIndexByUuid.get(record.uuid);
if (existingIndex !== undefined) {
normalized[existingIndex].response_id = record.response_id;
}
}
// Merge usageMetadata from assistant records
for (const record of originalRecords) {
if (record.type !== 'assistant') continue;

View file

@ -27,9 +27,6 @@ export interface ExportMessage {
/** Model used for assistant messages */
model?: string;
/** Response ID from the LLM API for telemetry/tracing correlation */
response_id?: string;
/** Token usage for this message (mainly for assistant messages) */
usageMetadata?: GenerateContentResponseUsageMetadata;
@ -88,8 +85,6 @@ export interface ExportMetadata {
linesRemoved?: number;
/** Unique files referenced in the session (written files only) */
uniqueFiles: string[];
/** Last response ID from the LLM API (request ID) */
requestId?: string;
}
/**