authorgravatar for julius0216@outlook.comJulius Marminge <julius0216@outlook.com> 2026-02-26 20:24:43-08:00
committergravatar for jmarminge@gmail.comJulius Marminge <jmarminge@gmail.com> 2026-02-26 20:43:22-08:00
log0e9677965cca02d925244e7814a245a194bd42de
tree397284b90e32a18d01897ea9079b4112c70c26cd
parentcc29ceea80d6f02181e4ce7dce2812c07190ebe8

Spill full buffered assistant text when buffer exceeds cap

- Change provider runtime ingestion to flush the entire buffered assistant text once it crosses the max size - Remove partial-retention logic and its keep-chars constant to simplify memory safety behavior

1 files changed, 3 insertions(+), 7 deletions(-)

apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts+3-7
......@@ -31,7 +31,6 @@ const BUFFERED_MESSAGE_TEXT_BY_MESSAGE_ID_TTL = Duration.minutes(120);
3131const BUFFERED_ASSISTANT_MESSAGE_SPILLED_CACHE_CAPACITY = 20_000;
3232const BUFFERED_ASSISTANT_MESSAGE_SPILLED_TTL = Duration.minutes(120);
3333const MAX_BUFFERED_ASSISTANT_CHARS = 24_000;
34const BUFFERED_ASSISTANT_KEEP_CHARS = 8_000;
3534
3635type TurnStartRequestedDomainEvent = Extract<
3736 OrchestrationEvent,
......@@ -245,12 +244,9 @@ const make = Effect.gen(function* () {
245244 return "";
246245 }
247246
248 // Safety valve: spill oldest text as a persisted assistant delta to cap memory.
249 const spillChars = Math.max(1, nextText.length - BUFFERED_ASSISTANT_KEEP_CHARS);
250 const spillChunk = nextText.slice(0, spillChars);
251 const retainedTail = nextText.slice(spillChars);
252 yield* Cache.set(bufferedAssistantTextByMessageId, messageId, retainedTail);
253 return spillChunk;
247 // Safety valve: spill full buffered text as a persisted assistant delta to cap memory.
248 yield* Cache.invalidate(bufferedAssistantTextByMessageId, messageId);
249 return nextText;
254250 }),
255251 ),
256252 );