From 8884d60c01a4a6314bb793c1d87ece4a9dba3605 Mon Sep 17 00:00:00 2001 From: miroyong Date: Sun, 2 Aug 2026 22:26:21 -0400 Subject: [PATCH 1/2] feat(cli): prefix-cache-preserving compaction (Reasonix-style) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports Reasonix's prefix-cache strategy to the CLI's auto-compaction: - compactChatHistory now pins the cache-stable prefix (system message, first user turn when small, prior digests) verbatim and keeps a token-budgeted recent tail, splicing the new digest in the middle instead of collapsing history to [system, summary]. The prompt prefix stays byte-identical across turns so providers with automatic prompt caching (DeepSeek, OpenAI) serve cache hits on subsequent requests. - No-op when the conversation already fits within pinned prefix + tail, so the summarizer isn't invoked needlessly. - getHistoryForLLM returns the full history: after compaction the stored history already IS the compacted layout, so trimming before the compaction index would drop the cache-stable prefix. - SystemMessageService memoizes the constructed system message per (mode, additionalRules, format, headless), since it is re-fetched on every streaming iteration and its construction re-reads AGENTS.md / runs git status — any change cold-starts the provider's prompt cache. - Compaction digest prompt rewritten as a structured briefing (Standing facts, Goal, Decisions, Files & code, Commands, Errors, Pending & next step) mirroring Reasonix's summarySystemPrompt. Generated with [Continue](https://continue.dev) Co-Authored-By: Continue --- .../cli/src/compaction.infiniteLoop.test.ts | 155 ++-- extensions/cli/src/compaction.test.ts | 681 +++++------------- extensions/cli/src/compaction.ts | 192 ++++- .../src/services/ChatHistoryService.test.ts | 34 +- .../cli/src/services/ChatHistoryService.ts | 21 +- .../src/services/SystemMessageService.test.ts | 58 ++ .../cli/src/services/SystemMessageService.ts | 17 + 7 files changed, 532 insertions(+), 626 deletions(-) diff --git a/extensions/cli/src/compaction.infiniteLoop.test.ts b/extensions/cli/src/compaction.infiniteLoop.test.ts index d31734d6d81..c1eab1f94bd 100644 --- a/extensions/cli/src/compaction.infiniteLoop.test.ts +++ b/extensions/cli/src/compaction.infiniteLoop.test.ts @@ -1,7 +1,7 @@ import { ModelConfig } from "@continuedev/config-yaml"; import { BaseLlmApi } from "@continuedev/openai-adapters"; import { convertToUnifiedHistory } from "core/util/messageConversion.js"; -import { describe, expect, it, vi } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; import { compactChatHistory } from "./compaction.js"; import { streamChatResponse } from "./stream/streamChatResponse.js"; @@ -12,7 +12,10 @@ vi.mock("./stream/streamChatResponse.js", () => ({ })); vi.mock("./util/tokenizer.js", () => ({ + countChatHistoryItemTokens: vi.fn(), countChatHistoryTokens: vi.fn(), + countToolDefinitionTokens: vi.fn(), + countTotalInputTokens: vi.fn(), getModelContextLimit: vi.fn(), getModelMaxTokens: vi.fn(), })); @@ -30,17 +33,51 @@ describe("compaction infinite loop prevention", () => { const mockLlmApi = {} as BaseLlmApi; - it("should not loop infinitely when pruning doesn't reduce history size", async () => { - const { countChatHistoryTokens, getModelContextLimit } = await import( - "./util/tokenizer.js" - ); + // Keep tokenizer/stream spy call histories isolated per test so exact + // call-count assertions (e.g. "pruned until it fits") are per-test. + beforeEach(() => { + vi.clearAllMocks(); + }); + + /** + * A history large enough that the pinned prefix + recent tail do not cover + * it all, so compactChatHistory reaches the pruning loop instead of taking + * the no-op (everything-fits) early return. + */ + const buildLargeHistory = () => { + const history = convertToUnifiedHistory([ + { role: "system", content: "System message" }, + { role: "user", content: "Initial task" }, + ]); + for (let i = 0; i < 20; i++) { + history.push({ + message: { role: "assistant", content: `Assistant response ${i}` }, + contextItems: [], + }); + history.push({ + message: { role: "user", content: `User followup ${i}` }, + contextItems: [], + }); + } + return history; + }; + + const setupDefaultMocks = async () => { + const { + countChatHistoryItemTokens, + countChatHistoryTokens, + getModelContextLimit, + getModelMaxTokens, + } = await import("./util/tokenizer.js"); const mockStreamResponse = vi.mocked(streamChatResponse); - const mockCountTokens = vi.mocked(countChatHistoryTokens); - const mockGetContextLimit = vi.mocked(getModelContextLimit); - // Setup mocks - mockGetContextLimit.mockReturnValue(4000); - mockCountTokens.mockReturnValue(5000); // Always too big + vi.mocked(getModelContextLimit).mockReturnValue(4000); + vi.mocked(getModelMaxTokens).mockReturnValue(1000); + // Per-message token estimate: keep the tail budget (~12K) from covering + // the whole 40+ message history, so the fold region is non-empty. + vi.mocked(countChatHistoryItemTokens).mockReturnValue(1000); + vi.mocked(countChatHistoryTokens).mockReturnValue(5000); + mockStreamResponse.mockImplementation( async (history, model, api, controller, callbacks) => { callbacks?.onContent?.("Summary"); @@ -49,91 +86,65 @@ describe("compaction infinite loop prevention", () => { }, ); - // History that can't be pruned further (only system message) - const history = convertToUnifiedHistory([ - { role: "system", content: "System message" }, - ]); + return { + mockStreamResponse, + mockCountHistoryTokens: vi.mocked(countChatHistoryTokens), + mockGetContextLimit: vi.mocked(getModelContextLimit), + }; + }; + + it("should not loop infinitely when pruning doesn't reduce history size", async () => { + const { mockCountHistoryTokens } = await setupDefaultMocks(); + + // Token count is always over the available-for-input budget, so the + // pruning loop must keep pruning and eventually exit (when the history is + // exhausted) instead of hanging. + const history = buildLargeHistory(); - // This should not hang - it should break out of the loop const result = await compactChatHistory(history, mockModel, mockLlmApi); - // Should complete successfully even though token count is still too high expect(result.compactedHistory).toBeDefined(); - expect(mockCountTokens).toHaveBeenCalled(); + expect(mockCountHistoryTokens).toHaveBeenCalled(); }); it("should not loop infinitely with history ending in assistant message", async () => { - const { countChatHistoryTokens, getModelContextLimit } = await import( - "./util/tokenizer.js" - ); - const mockStreamResponse = vi.mocked(streamChatResponse); - const mockCountTokens = vi.mocked(countChatHistoryTokens); - const mockGetContextLimit = vi.mocked(getModelContextLimit); - - // Setup mocks - mockGetContextLimit.mockReturnValue(4000); - mockCountTokens.mockReturnValue(5000); // Always too big - mockStreamResponse.mockImplementation( - async (history, model, api, controller, callbacks) => { - callbacks?.onContent?.("Summary"); - callbacks?.onContentComplete?.("Summary"); - return "Summary"; - }, - ); - - // History that ends with assistant - pruning won't change it - const history = convertToUnifiedHistory([ - { role: "system", content: "System message" }, - { role: "user", content: "Hello" }, - { role: "assistant", content: "Hi there" }, - ]); + await setupDefaultMocks(); + + // Ends with an assistant message (after a user turn); pruneLastMessage + // must remove the pair and the loop must terminate. + const history = buildLargeHistory(); + history.push({ + message: { role: "user", content: "Last user turn" }, + contextItems: [], + }); + history.push({ + message: { role: "assistant", content: "Last assistant turn" }, + contextItems: [], + }); - // This should not hang const result = await compactChatHistory(history, mockModel, mockLlmApi); expect(result.compactedHistory).toBeDefined(); }); it("should successfully prune when pruning actually reduces size", async () => { - const { countChatHistoryTokens, getModelContextLimit } = await import( - "./util/tokenizer.js" - ); - const mockStreamResponse = vi.mocked(streamChatResponse); - const mockCountTokens = vi.mocked(countChatHistoryTokens); - const mockGetContextLimit = vi.mocked(getModelContextLimit); - - // Setup mocks - mockGetContextLimit.mockReturnValue(4000); + const { mockCountHistoryTokens } = await setupDefaultMocks(); - // Mock token counting to show reduction after pruning + // Available for input = 4000 - 1000 - 700 (prompt) = 2300. First check is + // over budget, second is still over, third fits -> loop exits. let callCount = 0; - mockCountTokens.mockImplementation(() => { + mockCountHistoryTokens.mockImplementation(() => { callCount++; if (callCount === 1) return 5000; // Initial too big - if (callCount === 2) return 3000; // After pruning, fits - return 2000; // Subsequent calls + if (callCount === 2) return 3000; // After first prune, still too big + return 2000; // Subsequent calls fit }); - mockStreamResponse.mockImplementation( - async (history, model, api, controller, callbacks) => { - callbacks?.onContent?.("Summary"); - callbacks?.onContentComplete?.("Summary"); - return "Summary"; - }, - ); - - // History that can be successfully pruned - const history = convertToUnifiedHistory([ - { role: "system", content: "System message" }, - { role: "user", content: "Hello" }, - { role: "assistant", content: "Hi there" }, - { role: "user", content: "Another question" }, - ]); + const history = buildLargeHistory(); const result = await compactChatHistory(history, mockModel, mockLlmApi); expect(result.compactedHistory).toBeDefined(); - // The function will call countTokens multiple times during the process - expect(mockCountTokens).toHaveBeenCalled(); + expect(mockCountHistoryTokens).toHaveBeenCalledTimes(3); }); }); diff --git a/extensions/cli/src/compaction.test.ts b/extensions/cli/src/compaction.test.ts index 6c3d6df4e0b..fde618b7ed6 100644 --- a/extensions/cli/src/compaction.test.ts +++ b/extensions/cli/src/compaction.test.ts @@ -2,7 +2,7 @@ import { ModelConfig } from "@continuedev/config-yaml"; import { BaseLlmApi } from "@continuedev/openai-adapters"; import type { ChatHistoryItem } from "core/index.js"; import { convertToUnifiedHistory } from "core/util/messageConversion.js"; -import { describe, expect, it, vi } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; import { compactChatHistory, @@ -23,14 +23,57 @@ describe("compaction", () => { model: "test-model", } as ModelConfig; - const claudeModel: ModelConfig = { - name: "claude-3-5-sonnet", - provider: "anthropic", - model: "claude-3-5-sonnet-20241022", - } as ModelConfig; - const mockLlmApi = {} as BaseLlmApi; + // Keep the streamChatResponse spy's call history isolated per test; mock + // implementations set via stubSummaryStream persist across tests. + beforeEach(() => { + vi.clearAllMocks(); + }); + + /** + * Builds a history large enough that a compaction has a real fold region: + * a pinned prefix (system + small first user turn) followed by many turns + * with large tool results, so the middle exceeds the recent-tail budget. + */ + const buildLargeHistory = (): ChatHistoryItem[] => { + const history = convertToUnifiedHistory([ + { role: "system", content: "System message" }, + { role: "user", content: "Initial task: implement feature X" }, + ]); + for (let i = 0; i < 40; i++) { + history.push({ + message: { role: "assistant", content: `Assistant response ${i}` }, + contextItems: [], + }); + history.push({ + message: { + role: "tool", + content: "x".repeat(4000), + tool_call_id: `tool-${i}`, + }, + contextItems: [], + }); + history.push({ + message: { role: "user", content: `User followup ${i}` }, + contextItems: [], + }); + } + return history; + }; + + const stubSummaryStream = (mockContent = "This is a summary") => { + const mockStreamResponse = vi.mocked(streamChatResponse); + mockStreamResponse.mockImplementation( + async (history, model, api, controller, callbacks) => { + callbacks?.onContent?.(mockContent); + callbacks?.onContentComplete?.(mockContent); + return mockContent; + }, + ); + return mockContent; + }; + describe("findCompactionIndex", () => { it("should find compaction marker in chat history", () => { const history = convertToUnifiedHistory([ @@ -155,7 +198,7 @@ describe("compaction", () => { expect(result).toEqual(history); }); - it("should return compacted history with system message", () => { + it("should return the full compacted history, preserving the pinned prefix", () => { const history = convertToUnifiedHistory([ { role: "system", content: "System message" }, { role: "user", content: "Hello" }, @@ -166,231 +209,128 @@ describe("compaction", () => { }, { role: "user", content: "Another message" }, ]); + history[3].conversationSummary = "This is a summary"; + // The stored history already IS the compacted history (pinned prefix + + // summary + recent tail), so it must be sent in full — trimming before + // the compaction index would drop the cache-stable prefix. const result = getHistoryForLLM(history, 3); - expect(result).toEqual([ - { - message: { role: "system", content: "System message" }, - contextItems: [], - }, - { - message: { - role: "assistant", - content: `\nThis is a summary`, - }, - contextItems: [], - }, - { - message: { role: "user", content: "Another message" }, - contextItems: [], - }, - ]); - }); - - it("should return compacted history without system message when compaction is at index 0", () => { - const history = convertToUnifiedHistory([ - { - role: "assistant", - content: `\nThis is a summary`, - }, - { role: "user", content: "Another message" }, - ]); - - const result = getHistoryForLLM(history, 0); expect(result).toEqual(history); + expect(result).toHaveLength(5); }); - it("should return full history when compactionIndex is out of bounds", () => { + it("should return full history for out-of-bounds, negative, empty, and system-only cases", () => { const history = convertToUnifiedHistory([ { role: "system", content: "System message" }, { role: "user", content: "Hello" }, ]); - const result = getHistoryForLLM(history, 10); - expect(result).toEqual(history); - }); - - it("should handle negative compactionIndex", () => { - const history = convertToUnifiedHistory([ - { role: "system", content: "System message" }, - { role: "user", content: "Hello" }, - ]); - - const result = getHistoryForLLM(history, -1); - // Negative index with slice means "from the end", so -1 gives us only the last message - // Since compactionIndex is not > 0, system message is not included - expect(result).toEqual([ - { - message: { role: "user", content: "Hello" }, - contextItems: [], - }, - ]); - }); - - it("should handle empty history", () => { - const history = convertToUnifiedHistory([]); - const result = getHistoryForLLM(history, 0); - expect(result).toEqual([]); + expect(getHistoryForLLM(history, 10)).toEqual(history); + expect(getHistoryForLLM(history, -1)).toEqual(history); + expect(getHistoryForLLM([], 0)).toEqual([]); + expect( + getHistoryForLLM(convertToUnifiedHistory([{ role: "system", content: "System" }]), 0), + ).toEqual(convertToUnifiedHistory([{ role: "system", content: "System" }])); }); + }); - it("should handle history with only system message", () => { - const history = convertToUnifiedHistory([ - { role: "system", content: "System message" }, - ]); + describe("compactChatHistory", () => { + it("should preserve pinned prefix and recent tail when compacting a large history", async () => { + const mockContent = stubSummaryStream("Structured summary"); + const history = buildLargeHistory(); - const result = getHistoryForLLM(history, 0); - expect(result).toEqual(history); - }); + const result = await compactChatHistory(history, mockModel, mockLlmApi); - it("should handle history without system message but with compaction", () => { - const history = convertToUnifiedHistory([ - { role: "user", content: "First message" }, - { role: "assistant", content: `\nSummary` }, - { role: "user", content: "New message" }, - ]); + // Pinned prefix: system message + first user turn kept verbatim + expect(result.compactedHistory[0]).toEqual(history[0]); + expect(result.compactedHistory[1]).toEqual(history[1]); - const result = getHistoryForLLM(history, 1); - expect(result).toEqual([ - { - message: { - role: "assistant", - content: `\nSummary`, - }, - contextItems: [], - }, - { - message: { role: "user", content: "New message" }, - contextItems: [], - }, - ]); - }); + // compactionIndex points at the new summary + expect(result.compactedHistory[result.compactionIndex].conversationSummary).toBe( + mockContent, + ); + // The summary is spliced in the middle: not first, not last + expect(result.compactionIndex).toBeGreaterThan(0); + expect(result.compactionIndex).toBeLessThan( + result.compactedHistory.length - 1, + ); - it("should include system message when first message is not system", () => { - const history = convertToUnifiedHistory([ - { role: "user", content: "First user message" }, - { role: "system", content: "System message in wrong position" }, - { role: "assistant", content: `\nSummary` }, - { role: "user", content: "New message" }, - ]); + // Recent tail preserved verbatim + expect( + result.compactedHistory[result.compactedHistory.length - 1], + ).toEqual(history[history.length - 1]); - // Since system message is not at index 0, it's not included - const result = getHistoryForLLM(history, 2); - expect(result).toEqual([ - { - message: { - role: "assistant", - content: `\nSummary`, - }, - contextItems: [], - }, - { - message: { role: "user", content: "New message" }, - contextItems: [], - }, - ]); + // Compaction actually reduced the message count + expect(result.compactedHistory.length).toBeLessThan(history.length); }); - it("should preserve message order from compaction point", () => { + it("should return history unchanged (no-op) when everything fits", async () => { + const mockStreamResponse = vi.mocked(streamChatResponse); const history = convertToUnifiedHistory([ { role: "system", content: "System" }, - { role: "user", content: "Old 1" }, - { role: "assistant", content: "Old 2" }, - { role: "assistant", content: `\nSummary` }, - { role: "user", content: "New 1" }, - { role: "assistant", content: "New 2" }, - { role: "user", content: "New 3" }, + { role: "user", content: "Hello" }, + { role: "assistant", content: "Hi" }, ]); - const result = getHistoryForLLM(history, 3); - expect(result).toEqual([ - { - message: { role: "system", content: "System" }, - contextItems: [], - }, - { - message: { - role: "assistant", - content: `\nSummary`, - }, - contextItems: [], - }, - { - message: { role: "user", content: "New 1" }, - contextItems: [], - }, - { - message: { role: "assistant", content: "New 2" }, - contextItems: [], - }, - { - message: { role: "user", content: "New 3" }, - contextItems: [], - }, - ]); + const result = await compactChatHistory(history, mockModel, mockLlmApi); + + expect(result.compactedHistory).toEqual(history); + expect(result.compactionContent).toBe(""); + // Nothing worth folding — the summarizer must not be called + expect(mockStreamResponse).not.toHaveBeenCalled(); }); - }); - describe("compactChatHistory", () => { - it("should compact chat history successfully", async () => { - const mockStreamResponse = vi.mocked(streamChatResponse); - const mockContent = "This is a summary of the conversation"; + it("should handle history without system message", async () => { + const mockContent = stubSummaryStream("Summary without system"); + const history = buildLargeHistory().slice(1); // drop the system message - mockStreamResponse.mockImplementation( - async (history, model, api, controller, callbacks) => { - callbacks?.onContent?.(mockContent); - callbacks?.onContentComplete?.(mockContent); - return mockContent; - }, + const result = await compactChatHistory(history, mockModel, mockLlmApi); + + // First user turn pinned verbatim + expect(result.compactedHistory[0]).toEqual(history[0]); + expect(result.compactedHistory[result.compactionIndex].conversationSummary).toBe( + mockContent, ); + expect(result.compactionIndex).toBe(1); + }); - const history = convertToUnifiedHistory([ - { role: "system", content: "System message" }, - { role: "user", content: "Hello" }, - { role: "assistant", content: "Hi there" }, - ]); + it("should keep prior digests verbatim in the pinned prefix", async () => { + stubSummaryStream("New summary"); + const history = buildLargeHistory(); + const priorDigest: ChatHistoryItem = { + message: { role: "assistant", content: "Earlier digest" }, + contextItems: [], + conversationSummary: "Earlier digest", + }; + history.splice(2, 0, priorDigest); const result = await compactChatHistory(history, mockModel, mockLlmApi); - expect(result.compactedHistory).toHaveLength(2); - expect(result.compactedHistory[0]).toEqual({ - message: { - role: "system", - content: "System message", - }, - contextItems: [], - }); - expect(result.compactedHistory[1]).toEqual({ - message: { - role: "assistant", - content: mockContent, - }, - contextItems: [], - conversationSummary: mockContent, - }); - expect(result.compactionIndex).toBe(1); - expect(result.compactionContent).toBe(mockContent); + // The prior digest must survive before the new summary + const priorIdx = result.compactedHistory.findIndex( + (item) => item.conversationSummary === "Earlier digest", + ); + const newIdx = result.compactedHistory.findIndex( + (item) => item.conversationSummary === "New summary", + ); + expect(priorIdx).toBeGreaterThanOrEqual(0); + expect(newIdx).toBeGreaterThan(priorIdx); }); - it("should handle callbacks correctly", async () => { - const mockStreamResponse = vi.mocked(streamChatResponse); - const mockContent = "Summary content"; + it("should call callbacks correctly", async () => { const onStreamContent = vi.fn(); const onStreamComplete = vi.fn(); - + const mockStreamResponse = vi.mocked(streamChatResponse); mockStreamResponse.mockImplementation( async (history, model, api, controller, callbacks) => { callbacks?.onContent?.("Summary "); callbacks?.onContent?.("content"); - callbacks?.onContentComplete?.(mockContent); - return mockContent; + callbacks?.onContentComplete?.("Summary content"); + return "Summary content"; }, ); - const history = convertToUnifiedHistory([ - { role: "user", content: "Hello" }, - ]); - - await compactChatHistory(history, mockModel, mockLlmApi, { + await compactChatHistory(buildLargeHistory(), mockModel, mockLlmApi, { callbacks: { onStreamContent, onStreamComplete, @@ -409,12 +349,8 @@ describe("compaction", () => { mockStreamResponse.mockRejectedValue(mockError); - const history = convertToUnifiedHistory([ - { role: "user", content: "Hello" }, - ]); - await expect( - compactChatHistory(history, mockModel, mockLlmApi, { + compactChatHistory(buildLargeHistory(), mockModel, mockLlmApi, { callbacks: { onError }, }), ).rejects.toThrow("Stream failed"); @@ -422,55 +358,6 @@ describe("compaction", () => { expect(onError).toHaveBeenCalledWith(mockError); }); - it("should handle history with only system message", async () => { - const mockStreamResponse = vi.mocked(streamChatResponse); - const mockContent = "Summary of system setup"; - - mockStreamResponse.mockImplementation( - async (history, model, api, controller, callbacks) => { - callbacks?.onContent?.(mockContent); - callbacks?.onContentComplete?.(mockContent); - return mockContent; - }, - ); - - const history = convertToUnifiedHistory([ - { role: "system", content: "You are a helpful assistant" }, - ]); - - const result = await compactChatHistory(history, mockModel, mockLlmApi); - - expect(result.compactedHistory).toHaveLength(2); - expect(result.compactedHistory[0].message).toEqual({ - role: "system", - content: "You are a helpful assistant", - }); - expect(result.compactedHistory[1].message.content).toContain( - "Summary of system setup", - ); - }); - - it("should handle empty content from stream", async () => { - const mockStreamResponse = vi.mocked(streamChatResponse); - - mockStreamResponse.mockImplementation( - async (history, model, api, controller, callbacks) => { - callbacks?.onContent?.(""); - callbacks?.onContentComplete?.(""); - return ""; - }, - ); - - const history = convertToUnifiedHistory([ - { role: "user", content: "Hello" }, - ]); - - const result = await compactChatHistory(history, mockModel, mockLlmApi); - - expect(result.compactionContent).toBe(""); - expect(result.compactedHistory[0].message.content).toBe(""); - }); - it("should correctly construct prompt for compaction", async () => { const mockStreamResponse = vi.mocked(streamChatResponse); let capturedHistory: ChatHistoryItem[] = []; @@ -484,305 +371,81 @@ describe("compaction", () => { }, ); - const history = convertToUnifiedHistory([ - { role: "system", content: "System" }, - { role: "user", content: "Hello" }, - { role: "assistant", content: "Hi" }, - ]); - - await compactChatHistory(history, mockModel, mockLlmApi); + await compactChatHistory(buildLargeHistory(), mockModel, mockLlmApi); - // Should have original history plus the compaction prompt - expect(capturedHistory).toHaveLength(4); - expect(capturedHistory[3].message).toEqual({ - role: "user", - content: expect.stringContaining("provide a concise summary"), - }); + // The compaction prompt is appended to the history sent to the summarizer + const lastMessage = capturedHistory[capturedHistory.length - 1]; + expect(lastMessage.message.role).toBe("user"); + expect(lastMessage.message.content).toContain("## Standing facts"); + expect(lastMessage.message.content).toContain("## Pending & next step"); }); - it("should handle history with tool calls and mixed message types", async () => { - const mockStreamResponse = vi.mocked(streamChatResponse); - const mockContent = "Summary including tool usage"; - - mockStreamResponse.mockImplementation( - async (history, model, api, controller, callbacks) => { - callbacks?.onContent?.(mockContent); - callbacks?.onContentComplete?.(mockContent); - return mockContent; - }, - ); - - const history = convertToUnifiedHistory([ - { role: "system", content: "System" }, - { role: "user", content: "Do something" }, - { role: "assistant", content: "I'll help", tool_calls: [{} as any] }, - { role: "tool", content: "Tool result", tool_call_id: "123" }, - { role: "assistant", content: "Done" }, - ]); - - const result = await compactChatHistory(history, mockModel, mockLlmApi); - - expect(result.compactedHistory).toHaveLength(2); - expect(result.compactionContent).toBe(mockContent); - }); - - it("should handle very long chat histories", async () => { + it("should handle empty content from stream", async () => { const mockStreamResponse = vi.mocked(streamChatResponse); - mockStreamResponse.mockImplementation( async (history, model, api, controller, callbacks) => { - callbacks?.onContent?.("Long summary"); - callbacks?.onContentComplete?.("Long summary"); - return "Long summary"; + callbacks?.onContent?.(""); + callbacks?.onContentComplete?.(""); + return ""; }, ); - // Create a very long history - const history = convertToUnifiedHistory([ - { role: "system", content: "System" }, - ]); - - for (let i = 0; i < 100; i++) { - history.push({ - message: { role: "user", content: `User message ${i}` }, - contextItems: [], - }); - history.push({ - message: { role: "assistant", content: `Assistant response ${i}` }, - contextItems: [], - }); - } - - const result = await compactChatHistory(history, mockModel, mockLlmApi); - - expect(result.compactedHistory).toHaveLength(2); // System + compaction - expect(result.compactionIndex).toBe(1); - }); - - it("should handle history without system message", async () => { - const mockStreamResponse = vi.mocked(streamChatResponse); - const mockContent = "Summary without system"; - - mockStreamResponse.mockImplementation( - async (history, model, api, controller, callbacks) => { - callbacks?.onContent?.(mockContent); - callbacks?.onContentComplete?.(mockContent); - return mockContent; - }, + const result = await compactChatHistory( + buildLargeHistory(), + mockModel, + mockLlmApi, ); - const history = convertToUnifiedHistory([ - { role: "user", content: "Hello" }, - { role: "assistant", content: "Hi" }, - ]); - - const result = await compactChatHistory(history, mockModel, mockLlmApi); - - expect(result.compactedHistory).toHaveLength(1); // Only compaction - expect(result.compactedHistory[0].message.role).toBe("assistant"); - expect(result.compactionIndex).toBe(0); - }); - }); - - describe("property-based tests", () => { - it("getHistoryForLLM should always return a subset of the original history", () => { - const testCases: Array<{ - history: ChatHistoryItem[]; - compactionIndex: number | null; - }> = [ - { - history: convertToUnifiedHistory([ - { role: "system", content: "System" }, - { role: "user", content: "User 1" }, - { role: "assistant", content: "Assistant 1" }, - ]), - compactionIndex: 1, - }, - { - history: convertToUnifiedHistory([ - { role: "user", content: "User 1" }, - { role: "assistant", content: "Assistant 1" }, - { role: "user", content: "User 2" }, - ]), - compactionIndex: 2, - }, - { - history: [], - compactionIndex: null, - }, - ]; - - testCases.forEach(({ history, compactionIndex }) => { - const result = getHistoryForLLM(history, compactionIndex); - - // Every message in result should exist in original history - result.forEach((msg) => { - expect(history).toContainEqual(msg); - }); - - // Result length should be <= original length - expect(result.length).toBeLessThanOrEqual(history.length); - }); - }); - - it("getHistoryForLLM should always include system message if it exists at index 0", () => { - const systemMessage: ChatHistoryItem = { - message: { - role: "system", - content: "System", - }, - contextItems: [], - }; - const testCases: Array<{ - history: ChatHistoryItem[]; - compactionIndex: number | null; - }> = [ - { - history: convertToUnifiedHistory([ - { role: "system", content: "System" }, - { role: "user", content: "User 1" }, - { role: "assistant", content: `\nSummary` }, - ]), - compactionIndex: 2, - }, - { - history: convertToUnifiedHistory([ - { role: "system", content: "System" }, - { role: "assistant", content: `\nSummary` }, - ]), - compactionIndex: 1, - }, - ]; - - testCases.forEach(({ history, compactionIndex }) => { - const result = getHistoryForLLM(history, compactionIndex); - - if ( - history[0]?.message?.role === "system" && - compactionIndex !== null && - compactionIndex > 0 - ) { - expect(result[0]).toEqual(history[0]); - } - }); - }); - - it("compaction should always reduce message count for non-trivial histories", async () => { - const mockStreamResponse = vi.mocked(streamChatResponse); - - mockStreamResponse.mockImplementation( - async (history, model, api, controller, callbacks) => { - callbacks?.onContent?.("Summary"); - callbacks?.onContentComplete?.("Summary"); - return "Summary"; - }, - ); - - const testHistories: ChatHistoryItem[][] = [ - convertToUnifiedHistory([ - { role: "system", content: "System" }, - { role: "user", content: "User 1" }, - { role: "assistant", content: "Assistant 1" }, - { role: "user", content: "User 2" }, - { role: "assistant", content: "Assistant 2" }, - ]), - convertToUnifiedHistory([ - { role: "user", content: "User 1" }, - { role: "assistant", content: "Assistant 1" }, - { role: "user", content: "User 2" }, - ]), - ]; - - for (const history of testHistories) { - const result = await compactChatHistory(history, mockModel, mockLlmApi); - - // Compacted history should be shorter than original - // (system + compaction) or just compaction - expect(result.compactedHistory.length).toBeLessThanOrEqual(2); - expect(result.compactedHistory.length).toBeLessThan(history.length); - } + expect(result.compactionContent).toBe(""); + expect(result.compactedHistory[result.compactionIndex].message.content).toBe(""); }); }); describe("invariant tests", () => { it("compactionIndex should always point to a message with conversationSummary", async () => { - const mockStreamResponse = vi.mocked(streamChatResponse); + stubSummaryStream("Summary content"); - mockStreamResponse.mockImplementation( - async (history, model, api, controller, callbacks) => { - callbacks?.onContent?.("Summary content"); - callbacks?.onContentComplete?.("Summary content"); - return "Summary content"; - }, + const result = await compactChatHistory( + buildLargeHistory(), + mockModel, + mockLlmApi, ); - const histories: ChatHistoryItem[][] = [ - convertToUnifiedHistory([ - { role: "system", content: "System" }, - { role: "user", content: "Hello" }, - ]), - convertToUnifiedHistory([ - { role: "user", content: "Hello" }, - { role: "assistant", content: "Hi" }, - ]), - convertToUnifiedHistory([{ role: "system", content: "System" }]), - ]; - - for (const history of histories) { - const result = await compactChatHistory(history, mockModel, mockLlmApi); - - // The message at compactionIndex should have a conversationSummary - const compactionMessage = - result.compactedHistory[result.compactionIndex]; - expect(compactionMessage.message.role).toBe("assistant"); - expect(compactionMessage.conversationSummary).toBeDefined(); - } + const compactionMessage = + result.compactedHistory[result.compactionIndex]; + expect(compactionMessage.message.role).toBe("assistant"); + expect(compactionMessage.conversationSummary).toBeDefined(); }); it("system message should always be preserved in the same position", async () => { - const mockStreamResponse = vi.mocked(streamChatResponse); - - mockStreamResponse.mockImplementation( - async (history, model, api, controller, callbacks) => { - callbacks?.onContent?.("Summary"); - callbacks?.onContentComplete?.("Summary"); - return "Summary"; - }, - ); - - const history = convertToUnifiedHistory([ - { role: "system", content: "You are helpful" }, - { role: "user", content: "Hello" }, - { role: "assistant", content: "Hi" }, - ]); + stubSummaryStream("Summary"); + const history = buildLargeHistory(); const result = await compactChatHistory(history, mockModel, mockLlmApi); - // System message should still be at index 0 expect(result.compactedHistory[0]).toEqual(history[0]); }); it("findCompactionIndex should be consistent with compactChatHistory result", async () => { - const mockStreamResponse = vi.mocked(streamChatResponse); + stubSummaryStream("Summary"); - mockStreamResponse.mockImplementation( - async (history, model, api, controller, callbacks) => { - callbacks?.onContent?.("Summary"); - callbacks?.onContentComplete?.("Summary"); - return "Summary"; - }, + const result = await compactChatHistory( + buildLargeHistory(), + mockModel, + mockLlmApi, ); - const history = convertToUnifiedHistory([ - { role: "user", content: "Hello" }, - { role: "assistant", content: "Hi" }, - ]); + const foundIndex = findCompactionIndex(result.compactedHistory); + expect(foundIndex).toBe(result.compactionIndex); + }); + + it("compaction should always reduce message count for a large history", async () => { + stubSummaryStream("Summary"); + const history = buildLargeHistory(); const result = await compactChatHistory(history, mockModel, mockLlmApi); - // findCompactionIndex should find the same index - const foundIndex = findCompactionIndex(result.compactedHistory); - expect(foundIndex).toBe(result.compactionIndex); + expect(result.compactedHistory.length).toBeLessThan(history.length); }); }); }); diff --git a/extensions/cli/src/compaction.ts b/extensions/cli/src/compaction.ts index 4132bb97fd9..b32080858c7 100644 --- a/extensions/cli/src/compaction.ts +++ b/extensions/cli/src/compaction.ts @@ -8,6 +8,7 @@ import { streamChatResponse } from "./stream/streamChatResponse.js"; import { StreamCallbacks } from "./stream/streamChatResponse.types.js"; import { logger } from "./util/logger.js"; import { + countChatHistoryItemTokens, countChatHistoryTokens, countToolDefinitionTokens, countTotalInputTokens, @@ -37,10 +38,123 @@ export interface CompactionOptions { systemMessageTokens?: number; } -const COMPACTION_PROMPT = - "Please provide a concise summary of our conversation so far, capturing the key context, decisions made, and current state. Format this as a single comprehensive message that preserves all important information needed to continue our work. You do not need to recap the system message, as this will remain. Make sure it is clear what the current stream of work was at the very end prior to compaction so that you can continue exactly where you left off without missing any information."; +// A structured briefing prompt (inspired by Reasonix's summarySystemPrompt) so +// the compaction digest is a dependable briefing the model can resume from: +// the section layout mirrors what a coding agent needs mid-task — the goal +// verbatim, concrete file/code state, and an explicit next step. +const COMPACTION_PROMPT = `You are compacting the earlier part of a coding agent's conversation to save context. +The agent keeps your summary alongside the user's own turns (kept verbatim) and the recent tail; your job is to fold the assistant/tool work into a briefing it can resume from. +Write under these exact headings, omitting a heading only if it has no content: -const COMPACTION_PROMPT_TOKENS = 150; // rough generous token count of ^ +## Standing facts & constraints +Everything the user stated that still governs the work — names, paths, IDs, versions, tokens, preferences, and hard "never do X" rules — in their own words. Be exhaustive; this is the durable contract, so prefer over- to under-including. + +## Goal +The user's request and intent. + +## Decisions & rationale +Key choices made so far and why — so they are not re-litigated or reversed. + +## Files & code +Files read or modified, with the specific facts that matter: signatures, line locations, data shapes, and exact edits applied. Be concrete; this is what lets the agent act without re-reading everything. + +## Commands & outcomes +Commands run (builds, tests, git) and their relevant results — what passed, what failed, and the error text that matters. + +## Errors & fixes +Problems hit and how they were resolved (or not), so the same dead ends are not repeated. + +## Pending & next step +What is still in progress or unstarted, and the single most concrete next action to take. + +Rules: be terse — bullet points and fragments, not prose. Preserve identifiers, paths, and numbers exactly. Do NOT invent anything not present in the messages; if something is unknown, leave it out rather than guessing.`; + +const COMPACTION_PROMPT_TOKENS = 700; // rough generous token count of ^ + +// Token budget for the verbatim recent tail kept after a compaction. Bounded so +// a few large tool outputs cannot keep the tail above the auto-compaction +// trigger and re-fire compaction on every turn. +const RECENT_TAIL_TOKEN_BUDGET = 12_000; +// Never keep fewer recent messages than this after a compaction. +const MIN_RECENT_TAIL_MESSAGES = 2; +// Ceiling on pinning the first user turn verbatim; larger first turns (pasted +// content) stay foldable so pinning never starves the context window. +const MAX_PINNED_FIRST_USER_TOKENS = 1_500; + +/** + * Returns the number of leading messages that a compaction must preserve + * verbatim: the system prompt, the first user turn (its task + stated facts) + * when it is small enough to be a brief, and any prior compaction digests. + * This keeps the prompt prefix byte-identical across turns — the invariant + * that lets providers with automatic prompt caching (DeepSeek, OpenAI) serve + * cache hits on subsequent requests. It also guarantees a fold never + * summarizes the user's stated facts away, and a later fold never + * re-summarizes an earlier digest into nothing. + */ +function getPinnedPrefixLength( + chatHistory: ChatHistoryItem[], + model: ModelConfig, +): number { + let i = 0; + if (chatHistory.length > 0 && chatHistory[0].message.role === "system") { + i++; + } + if ( + i < chatHistory.length && + chatHistory[i].message.role === "user" && + chatHistory[i].conversationSummary === undefined && + countChatHistoryItemTokens(chatHistory[i], model) <= + MAX_PINNED_FIRST_USER_TOKENS + ) { + i++; + } + while ( + i < chatHistory.length && + chatHistory[i].conversationSummary !== undefined + ) { + i++; + } + return i; +} + +/** + * Walks newest → oldest, growing the verbatim recent tail until the next + * message would push its token estimate past RECENT_TAIL_TOKEN_BUDGET (but + * never below MIN_RECENT_TAIL_MESSAGES messages), then aligns the boundary + * back off any tool result so the tail never begins with an orphan whose + * assistant tool_calls were folded away. + */ +function getRecentTailStart( + chatHistory: ChatHistoryItem[], + pinnedLength: number, + model: ModelConfig, +): number { + let start = chatHistory.length; + let acc = 0; + // Walk from the newest message down to (and including) the first message + // after the pinned prefix, so a conversation that entirely fits within + // pinned prefix + tail budget is recognized as a no-op (`start` reaches + // `pinnedLength` and the fold region is empty) instead of being folded. + for (let i = chatHistory.length - 1; i >= pinnedLength; i--) { + const tokens = countChatHistoryItemTokens(chatHistory[i], model); + if ( + chatHistory.length - i > MIN_RECENT_TAIL_MESSAGES && + acc + tokens > RECENT_TAIL_TOKEN_BUDGET + ) { + break; + } + acc += tokens; + start = i; + } + while ( + start > pinnedLength && + start < chatHistory.length && + chatHistory[start].message.role === "tool" + ) { + start--; + } + return start; +} /** * Compacts a chat history into a summarized form @@ -57,6 +171,29 @@ export async function compactChatHistory( options?: CompactionOptions, ): Promise { const { callbacks, abortController, systemMessageTokens = 0 } = options || {}; + + // Prefix-aware layout (Reasonix-style): pin the cache-stable prefix and keep + // a token-budgeted recent tail, so compaction splices a summary in the middle + // instead of collapsing the whole history into [system, summary]. Keeping the + // prompt prefix byte-identical across turns is what lets providers with + // automatic prompt caching (DeepSeek, OpenAI) serve cache hits on subsequent + // requests — and it stops a fold from ever summarizing away the user's first + // turn or an earlier digest. + const pinnedLength = getPinnedPrefixLength(chatHistory, model); + const tailStart = getRecentTailStart(chatHistory, pinnedLength, model); + + // Nothing worth folding: the whole conversation already fits in the pinned + // prefix + recent tail, so leave it untouched. Producing a summary here would + // only add tokens without reducing what is sent. + if (tailStart <= pinnedLength) { + const index = findCompactionIndex(chatHistory); + return { + compactedHistory: [...chatHistory], + compactionContent: "", + compactionIndex: index ?? pinnedLength, + }; + } + // Create a prompt to summarize the conversation const compactionPrompt: ChatHistoryItem = { message: { @@ -138,9 +275,6 @@ export async function compactChatHistory( ); // Create the compacted history with a special marker - const systemMessage = chatHistory.find( - (item) => item.message.role === "system", - ); const compactionMessage: ChatHistoryItem = { message: { role: "assistant", @@ -150,14 +284,19 @@ export async function compactChatHistory( conversationSummary: compactionContent, }; - const compactedHistory: ChatHistoryItem[] = systemMessage - ? [systemMessage, compactionMessage] - : [compactionMessage]; + // Splice the new digest between the pinned prefix and the recent tail so + // the cache-stable prefix (system, first user turn, prior digests) and the + // verbatim recent tail survive the compaction. + const compactedHistory: ChatHistoryItem[] = [ + ...chatHistory.slice(0, pinnedLength), + compactionMessage, + ...chatHistory.slice(tailStart), + ]; return { compactedHistory, compactionContent, - compactionIndex: systemMessage ? 1 : 0, + compactionIndex: pinnedLength, }; } catch (error) { logger.error("Compaction failed", error); @@ -180,12 +319,6 @@ export function findCompactionIndex( return compactedIndex === -1 ? null : compactedIndex; } -/** - * Gets the history to send to the LLM, taking compaction into account - * @param fullHistory The complete chat history - * @param compactionIndex The index of the compaction message, if any - * @returns The history to send to the LLM - */ /** * Prunes chat history by removing messages from the end while ensuring * the history ends with either an assistant message or a tool result message @@ -219,22 +352,23 @@ export function pruneLastMessage( return chatHistory.slice(0, -1); } +/** + * Gets the history to send to the LLM, taking compaction into account. + * + * After compaction the stored history already IS the compacted history + * (pinned prefix + summary + recent tail), so it is sent in full: trimming + * anything before the compaction index would drop the cache-stable pinned + * prefix (system message, first user turn, prior digests) that subsequent + * requests depend on for prompt-cache hits. + * @param fullHistory The complete chat history + * @param _compactionIndex The index of the compaction message, if any + * @returns The history to send to the LLM + */ export function getHistoryForLLM( fullHistory: ChatHistoryItem[], - compactionIndex: number | null, + _compactionIndex: number | null, ): ChatHistoryItem[] { - if (compactionIndex === null || compactionIndex >= fullHistory.length) { - return fullHistory; - } - - // Include system message (if at index 0) and everything from compaction index forward - const systemMessage = - fullHistory[0]?.message?.role === "system" ? fullHistory[0] : null; - const messagesFromCompaction = fullHistory.slice(compactionIndex); - - return systemMessage && compactionIndex > 0 - ? [systemMessage, ...messagesFromCompaction] - : messagesFromCompaction; + return fullHistory; } /** diff --git a/extensions/cli/src/services/ChatHistoryService.test.ts b/extensions/cli/src/services/ChatHistoryService.test.ts index dcd4cf4d9ca..25ba619b970 100644 --- a/extensions/cli/src/services/ChatHistoryService.test.ts +++ b/extensions/cli/src/services/ChatHistoryService.test.ts @@ -358,7 +358,12 @@ describe("ChatHistoryService", () => { expect(historyForLLM).toEqual(fullHistory); }); - it("should return history after compaction index", () => { + it("should return full history after compaction (prefix-preserving)", () => { + // After compaction the stored history already IS the compacted history + // (pinned cache-stable prefix + summary + recent tail). Trimming it here + // would drop the pinned system message / first user turn / prior digests + // that subsequent requests depend on for prompt-cache hits, so the full + // history must be sent to the LLM. const history: ChatHistoryItem[] = [ { message: { role: "user", content: "Old message" }, @@ -379,9 +384,30 @@ describe("ChatHistoryService", () => { const historyForLLM = service.getHistoryForLLM(); - expect(historyForLLM).toHaveLength(2); // Compacted message + new message - expect(historyForLLM[0].message.content).toBe("[Compacted] Summary"); - expect(historyForLLM[1].message.content).toBe("New message"); + expect(historyForLLM).toHaveLength(3); // Full history, prefix preserved + expect(historyForLLM[0].message.content).toBe("Old message"); + expect(historyForLLM[1].message.content).toBe("[Compacted] Summary"); + expect(historyForLLM[2].message.content).toBe("New message"); + }); + + it("should ignore compactionIndex argument (full history always returned)", () => { + const history: ChatHistoryItem[] = [ + { + message: { role: "system", content: "[Compacted] Summary" }, + contextItems: [], + }, + { + message: { role: "user", content: "New message" }, + contextItems: [], + }, + ]; + + service.setHistory(history); + service.setCompactionIndex(0); + + const historyForLLM = service.getHistoryForLLM(0); + + expect(historyForLLM).toHaveLength(2); }); }); diff --git a/extensions/cli/src/services/ChatHistoryService.ts b/extensions/cli/src/services/ChatHistoryService.ts index b4a1c56a72f..b8be320bf20 100644 --- a/extensions/cli/src/services/ChatHistoryService.ts +++ b/extensions/cli/src/services/ChatHistoryService.ts @@ -408,19 +408,16 @@ export class ChatHistoryService extends BaseService { } /** - * Get history for LLM (considering compaction) + * Get history for LLM (considering compaction). + * + * After compaction the stored history already IS the compacted history + * (pinned prefix + summary + recent tail), so it is sent in full: trimming + * anything before the compaction index would drop the cache-stable pinned + * prefix (system message, first user turn, prior digests) that subsequent + * requests depend on for prompt-cache hits. */ - getHistoryForLLM(compactionIndex?: number | null): ChatHistoryItem[] { - const index = compactionIndex ?? this.currentState.compactionIndex; - const full = this.currentState.history; - if (index === null || index === undefined || index >= full.length) { - return this.getHistory(); - } - const systemMessage = full[0]?.message?.role === "system" ? full[0] : null; - const messagesFromCompaction = full.slice(index); - return systemMessage && index > 0 - ? [systemMessage, ...messagesFromCompaction] - : messagesFromCompaction; + getHistoryForLLM(_compactionIndex?: number | null): ChatHistoryItem[] { + return this.getHistory(); } /** diff --git a/extensions/cli/src/services/SystemMessageService.test.ts b/extensions/cli/src/services/SystemMessageService.test.ts index d925c88b112..a7b3c07e2cb 100644 --- a/extensions/cli/src/services/SystemMessageService.test.ts +++ b/extensions/cli/src/services/SystemMessageService.test.ts @@ -85,6 +85,64 @@ describe("SystemMessageService", () => { }); }); + describe("system message cache (prefix-preserving)", () => { + it("should construct the system message only once per identical inputs", async () => { + const config = { + additionalRules: ["rule1"], + format: "json" as const, + headless: true, + }; + + constructSystemMessageMock.mockResolvedValue("Cached system message"); + + await service.initialize(config); + + const first = await service.getSystemMessage("normal"); + const second = await service.getSystemMessage("normal"); + const third = await service.getSystemMessage("normal"); + + expect(constructSystemMessageMock).toHaveBeenCalledTimes(1); + expect(first).toBe("Cached system message"); + expect(second).toBe(first); + expect(third).toBe(first); + }); + + it("should re-construct when mode changes", async () => { + constructSystemMessageMock.mockResolvedValue("System message"); + + await service.initialize({ + additionalRules: ["rule1"], + format: "json" as const, + headless: true, + }); + + await service.getSystemMessage("normal"); + await service.getSystemMessage("plan"); + + expect(constructSystemMessageMock).toHaveBeenCalledTimes(2); + }); + + it("should re-construct when config changes via updateConfig", async () => { + constructSystemMessageMock.mockResolvedValue("System message"); + + await service.initialize({ + additionalRules: ["rule1"], + format: "json" as const, + headless: true, + }); + + await service.getSystemMessage("normal"); + await service.getSystemMessage("normal"); + + expect(constructSystemMessageMock).toHaveBeenCalledTimes(1); + + service.updateConfig({ additionalRules: ["rule2"] }); + await service.getSystemMessage("normal"); + + expect(constructSystemMessageMock).toHaveBeenCalledTimes(2); + }); + }); + describe("updateConfig", () => { it("should update configuration partially", async () => { await service.initialize({ diff --git a/extensions/cli/src/services/SystemMessageService.ts b/extensions/cli/src/services/SystemMessageService.ts index 44ae3098c4e..2fb9d429f4f 100644 --- a/extensions/cli/src/services/SystemMessageService.ts +++ b/extensions/cli/src/services/SystemMessageService.ts @@ -16,6 +16,15 @@ export interface SystemMessageServiceState { * Provides fresh system messages that reflect current mode and configuration */ export class SystemMessageService extends BaseService { + // Cache of constructed system messages keyed by the inputs that affect them. + // The system message is re-fetched on every streaming iteration and its + // construction re-reads AGENTS.md/CLAUDE.md files and re-runs `git status`, + // so any change made by the agent invalidates the whole prompt prefix and + // cold-starts the provider's prompt cache. Memoizing per (mode, rules, + // format, headless) keeps the prefix byte-identical across turns — the same + // invariant Reasonix enforces by building its system prompt once at boot. + private systemMessageCache = new Map(); + constructor() { super("SystemMessageService", {}); } @@ -49,6 +58,12 @@ export class SystemMessageService extends BaseService public async getSystemMessage(currentMode: PermissionMode): Promise { const { additionalRules, format, headless } = this.currentState; + const cacheKey = JSON.stringify([currentMode, additionalRules, format, headless]); + const cached = this.systemMessageCache.get(cacheKey); + if (cached !== undefined) { + return cached; + } + const systemMessage = await constructSystemMessage( currentMode, additionalRules, @@ -56,6 +71,8 @@ export class SystemMessageService extends BaseService headless, ); + this.systemMessageCache.set(cacheKey, systemMessage); + logger.debug("Generated fresh system message", { mode: currentMode, messageLength: systemMessage.length, From 801e5d1dc13fe629bbbb6f2f294a17a2c53dac4d Mon Sep 17 00:00:00 2001 From: miroyong Date: Sun, 2 Aug 2026 22:30:51 -0400 Subject: [PATCH 2/2] style(cli): apply prettier formatting to changed files Generated with [Continue](https://continue.dev) Co-Authored-By: Continue --- extensions/cli/src/compaction.test.ts | 28 +++++++++++-------- .../cli/src/services/SystemMessageService.ts | 7 ++++- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/extensions/cli/src/compaction.test.ts b/extensions/cli/src/compaction.test.ts index fde618b7ed6..dcc80cb4473 100644 --- a/extensions/cli/src/compaction.test.ts +++ b/extensions/cli/src/compaction.test.ts @@ -229,8 +229,13 @@ describe("compaction", () => { expect(getHistoryForLLM(history, -1)).toEqual(history); expect(getHistoryForLLM([], 0)).toEqual([]); expect( - getHistoryForLLM(convertToUnifiedHistory([{ role: "system", content: "System" }]), 0), - ).toEqual(convertToUnifiedHistory([{ role: "system", content: "System" }])); + getHistoryForLLM( + convertToUnifiedHistory([{ role: "system", content: "System" }]), + 0, + ), + ).toEqual( + convertToUnifiedHistory([{ role: "system", content: "System" }]), + ); }); }); @@ -246,9 +251,9 @@ describe("compaction", () => { expect(result.compactedHistory[1]).toEqual(history[1]); // compactionIndex points at the new summary - expect(result.compactedHistory[result.compactionIndex].conversationSummary).toBe( - mockContent, - ); + expect( + result.compactedHistory[result.compactionIndex].conversationSummary, + ).toBe(mockContent); // The summary is spliced in the middle: not first, not last expect(result.compactionIndex).toBeGreaterThan(0); expect(result.compactionIndex).toBeLessThan( @@ -288,9 +293,9 @@ describe("compaction", () => { // First user turn pinned verbatim expect(result.compactedHistory[0]).toEqual(history[0]); - expect(result.compactedHistory[result.compactionIndex].conversationSummary).toBe( - mockContent, - ); + expect( + result.compactedHistory[result.compactionIndex].conversationSummary, + ).toBe(mockContent); expect(result.compactionIndex).toBe(1); }); @@ -397,7 +402,9 @@ describe("compaction", () => { ); expect(result.compactionContent).toBe(""); - expect(result.compactedHistory[result.compactionIndex].message.content).toBe(""); + expect( + result.compactedHistory[result.compactionIndex].message.content, + ).toBe(""); }); }); @@ -411,8 +418,7 @@ describe("compaction", () => { mockLlmApi, ); - const compactionMessage = - result.compactedHistory[result.compactionIndex]; + const compactionMessage = result.compactedHistory[result.compactionIndex]; expect(compactionMessage.message.role).toBe("assistant"); expect(compactionMessage.conversationSummary).toBeDefined(); }); diff --git a/extensions/cli/src/services/SystemMessageService.ts b/extensions/cli/src/services/SystemMessageService.ts index 2fb9d429f4f..74d29c6d5da 100644 --- a/extensions/cli/src/services/SystemMessageService.ts +++ b/extensions/cli/src/services/SystemMessageService.ts @@ -58,7 +58,12 @@ export class SystemMessageService extends BaseService public async getSystemMessage(currentMode: PermissionMode): Promise { const { additionalRules, format, headless } = this.currentState; - const cacheKey = JSON.stringify([currentMode, additionalRules, format, headless]); + const cacheKey = JSON.stringify([ + currentMode, + additionalRules, + format, + headless, + ]); const cached = this.systemMessageCache.get(cacheKey); if (cached !== undefined) { return cached;