authorclover caruso <git@paperclover.net> 2026-10-06 23:11:12-07:00
committerclover caruso <git@paperclover.net> 2026-10-09 11:21:14-07:00
loge4ffc9282a8149a9b8a2cacee98111ddddd8d95c
treeddbb9ea81ffde92bfe8609d12cb37a8edd1b6527
parentf6b5e343eae39fdf5f4152ae188612773c47c8d4
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU (~clover)

Accelerate local inference with MTP and bounded prefix caching

7 files changed, 489 insertions(+), 7 deletions(-)

service/local-ai/build/Dockerfile created+13
...@@ -0,0 +1,13 @@
1ARG RUNTIME=ghcr.io/ggml-org/llama.cpp@sha256:e8318a7b3988f9ca57b28c03486569e60def08f2b7b060550ebfa046e175e6cb
2FROM ${RUNTIME} AS build
3RUN rm -f /etc/apt/sources.list.d/* && printf 'deb https://archive.ubuntu.com/ubuntu noble main universe\ndeb https://archive.ubuntu.com/ubuntu noble-updates main universe\ndeb https://security.ubuntu.com/ubuntu noble-security main universe\n' > /etc/apt/sources.list && apt-get -o Acquire::https::Timeout=30 update && apt-get install -y --no-install-recommends cmake g++ make patch libssl-dev
4WORKDIR /src
5RUN python3 -c 'import hashlib,urllib.request; data=urllib.request.urlopen("https://api.github.com/repos/ggml-org/llama.cpp/tarball/11fe02151f79c41d0d4af7da708755d73b9c0da6").read(); assert hashlib.sha256(data).hexdigest()=="965db483b5e27c4917bac5bf83e0d2d533099049f8d5234c75e93c969d76443a"; open("source.tar.gz","wb").write(data)' && tar -xzf source.tar.gz --strip-components=1 && rm source.tar.gz
6COPY prefix-cache.patch /tmp/prefix-cache.patch
7RUN patch -p1 < /tmp/prefix-cache.patch
8COPY extract-ui.cpp /tmp/extract-ui.cpp
9RUN python3 -c 'import pathlib,subprocess; symbols=subprocess.check_output(["nm","-S","/app/libllama-server-impl.so"],text=True); size=int(next(line.split()[1] for line in symbols.splitlines() if line.split()[-1]=="_ZL8g_assets"),16); header=pathlib.Path("tools/ui/ui.h.in").read_text().replace("#cmakedefine LLAMA_UI_HAS_ASSETS 1","#define LLAMA_UI_HAS_ASSETS 1").replace("@N_ASSETS@",str(size)+"/sizeof(llama_ui_asset)"); pathlib.Path("/tmp/ui.h").write_text(header)' && g++ -std=c++17 -I/tmp /tmp/extract-ui.cpp -L/app -Wl,-rpath,/app -lllama-server-impl -o /tmp/extract-ui && LD_LIBRARY_PATH=/app /tmp/extract-ui && python3 -c 'import gzip,pathlib; [p.write_bytes(gzip.decompress(p.read_bytes())) for p in pathlib.Path("tools/ui/dist").rglob("*") if p.is_file()]'
10RUN cmake -S . -B /build -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DGGML_CUDA=OFF -DGGML_CPU=OFF -DGGML_BACKEND_DL=ON -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_UI=OFF -DLLAMA_USE_PREBUILT_UI=OFF -DLLAMA_BUILD_NUMBER=11382 -DLLAMA_BUILD_COMMIT=11fe02151 && cmake --build /build --target llama-ui-assets && ln -s /app/libllama.so* /app/libllama-common.so* /app/libmtmd.so* /app/libggml*.so* /build/bin/ && cmake --build /build --target server-context/fast cpp-httplib/fast llama-common-base/fast llama-ui/fast -j 4 && cmake --build /build --target llama-server-impl/fast -j 4
11
12FROM ${RUNTIME}
13COPY --from=build /build/bin/libllama-server-impl.so /app/libllama-server-impl.so
service/local-ai/build/extract-ui.cpp created+18
...@@ -0,0 +1,18 @@
1#include "ui.h"
2#include <filesystem>
3#include <fstream>
4
5int main() {
6 if (!llama_ui_use_gzip()) {
7 return 1;
8 }
9 for (const auto & asset : llama_ui_get_assets()) {
10 const auto path = std::filesystem::path("tools/ui/dist") / asset.name;
11 std::filesystem::create_directories(path.parent_path());
12 std::ofstream output(path, std::ios::binary);
13 output.write(reinterpret_cast<const char *>(asset.data), asset.size);
14 if (!output) {
15 return 1;
16 }
17 }
18}
service/local-ai/build/prefix-cache.patch created+237
...@@ -0,0 +1,237 @@
1--- a/tools/server/server-task.h
2+++ b/tools/server/server-task.h
3@@ -650,12 +650,13 @@
4 };
5
6 struct server_prompt_cache {
7- server_prompt_cache(int32_t limit_size_mib, size_t limit_tokens) {
8+ server_prompt_cache(int32_t limit_size_mib, size_t limit_tokens, bool keep_on_load = false) : keep_on_load(keep_on_load) {
9 this->limit_size = 1024ull*1024ull*(limit_size_mib < 0 ? 0 : limit_size_mib);
10 this->limit_tokens = limit_tokens;
11 }
12
13 std::list<server_prompt_cache_state> states;
14+ const bool keep_on_load;
15
16 // in bytes, 0 = no limit
17 size_t limit_size = 0;
18--- a/tools/server/server-task.cpp
19+++ b/tools/server/server-task.cpp
20@@ -1226,6 +1226,7 @@
21 {"event", "response.reasoning_text.delta"},
22 {"data", json {
23 {"type", "response.reasoning_text.delta"},
24+ {"content_index", 0},
25 {"delta", diff.reasoning_content_delta},
26 {"item_id", oai_resp_reasoning_id},
27 }},
28@@ -1724,7 +1725,10 @@
29 for (auto it = states.begin(); it != states.end(); ++it) {
30 const int cur_lcp_len = it->prompt.tokens.get_common_prefix(prompt.tokens);
31
32- if (cur_lcp_len == (int) prompt.tokens.size()) {
33+ if (cur_lcp_len == (int) prompt.tokens.size() && (!keep_on_load || it->prompt.tokens.size() == prompt.tokens.size())) {
34+ if (keep_on_load) {
35+ states.splice(states.end(), states, it);
36+ }
37 SRV_TRC("%s", " - prompt is already in the cache, skipping\n");
38 return nullptr;
39 }
40@@ -1749,7 +1753,7 @@
41 for (auto it = states.begin(); it != states.end();) {
42 const int len = it->prompt.tokens.get_common_prefix(prompt.tokens);
43
44- if (len == (int) it->prompt.tokens.size()) {
45+ if (!keep_on_load && len == (int) it->prompt.tokens.size()) {
46 SRV_TRC(" - removing obsolete cached prompt with length %d\n", len);
47
48 it = states.erase(it);
49@@ -1820,12 +1824,18 @@
50
51 SRV_TRC(" - prompt with length %7zu, lcp = %7d, f_keep = %.3f, f_sim = %.3f\n", it->prompt.tokens.size(), lcp_cur, f_keep_cur, f_sim_cur);
52
53+ // Only complete pinned prefixes contain a usable recurrent state.
54+ if (keep_on_load && lcp_cur != (int) it->prompt.tokens.size()) {
55+ continue;
56+ }
57+
58 // don't trash large prompts
59 if (f_keep_cur < 0.25f) {
60 continue;
61 }
62
63- if (f_keep_best < f_keep_cur && f_sim_best < f_sim_cur) {
64+ if ((keep_on_load || f_keep_best < f_keep_cur) &&
65+ (f_sim_best < f_sim_cur || (keep_on_load && f_sim_best == f_sim_cur && f_keep_best < f_keep_cur))) {
66 f_keep_best = f_keep_cur;
67 f_sim_best = f_sim_cur;
68
69@@ -1847,8 +1857,10 @@
70 return false;
71 }
72
73- data.clear();
74- data.shrink_to_fit();
75+ if (!keep_on_load) {
76+ data.clear();
77+ data.shrink_to_fit();
78+ }
79 }
80
81 {
82@@ -1865,14 +1877,20 @@
83 return false;
84 }
85
86- data.clear();
87- data.shrink_to_fit();
88+ if (!keep_on_load) {
89+ data.clear();
90+ data.shrink_to_fit();
91+ }
92 }
93 }
94
95- prompt = std::move(it_best->prompt);
96-
97- states.erase(it_best);
98+ if (keep_on_load) {
99+ prompt = it_best->prompt.clone();
100+ states.splice(states.end(), states, it_best);
101+ } else {
102+ prompt = std::move(it_best->prompt);
103+ states.erase(it_best);
104+ }
105 }
106
107 return true;
108--- a/tools/server/server-context.cpp
109+++ b/tools/server/server-context.cpp
110@@ -954,6 +954,7 @@
111 int n_empty_consecutive = 0;
112
113 std::unique_ptr<server_prompt_cache> prompt_cache;
114+ std::unique_ptr<server_prompt_cache> prefix_cache;
115
116 server_metrics metrics;
117
118@@ -1424,6 +1425,7 @@
119 SRV_TRC("%s", "use `--cache-ram 0` to disable the prompt cache\n");
120
121 prompt_cache = std::make_unique<server_prompt_cache>(params_base.cache_ram_mib, n_ctx);
122+ prefix_cache = std::make_unique<server_prompt_cache>(2048, n_ctx, true);
123 } else {
124 SRV_TRC("%s", "prompt cache is disabled - use `--cache-ram N` to enable it\n");
125 }
126@@ -1721,6 +1723,11 @@
127
128 SRV_TRC("prompt cache update took %.2f ms\n", (ggml_time_us() - t_start) / 1000.0);
129 }
130+ }
131+
132+ if (ret && prefix_cache && task.type == SERVER_TASK_TYPE_COMPLETION &&
133+ !ret->prompt_load(*prefix_cache, task.tokens)) {
134+ ret->prompt_clear();
135 }
136
137 return ret;
138@@ -3091,7 +3098,7 @@
139 // add generated tokens to cache
140 // ref: https://github.com/ggml-org/llama.cpp/pull/16818#discussion_r2473269481
141 {
142- GGML_ASSERT(!slot.prompt.tokens.has_mtmd);
143+ GGML_ASSERT(!slot.prompt.tokens.find_next_media_chunk(0).first);
144
145 llama_tokens new_tokens = slot.prompt.tokens.get_tokens(); // copy
146 for (size_t i = n_keep + n_discard; i < new_tokens.size(); i++) {
147@@ -3400,7 +3407,7 @@
148
149 // reuse chunks from the cached prompt by shifting their KV cache in the new position
150 if (can_cache_reuse && n_cache_reuse > 0) {
151- GGML_ASSERT(!slot.prompt.tokens.has_mtmd);
152+ GGML_ASSERT(!slot.prompt.tokens.find_next_media_chunk(0).first);
153
154 size_t head_c = n_past; // cache
155 size_t head_p = n_past; // current prompt
156@@ -3701,6 +3708,11 @@
157
158 const auto & spans = slot.task->params.message_spans;
159 const auto last_user_pos = spans.last_user_message_pos();
160+ const auto first_user = std::find_if(spans.spans.begin(), spans.spans.end(), [](const auto & span) {
161+ return span.role == COMMON_CHAT_ROLE_USER;
162+ });
163+ const int64_t first_user_pos = first_user == spans.spans.end() ? -1 : first_user->pos;
164+ const int64_t prefix_pos = std::min<int64_t>(4096, first_user_pos);
165
166 const int32_t n_decision_first = slot.task->type == SERVER_TASK_TYPE_DECISION ? slot.task->decision.pos_first() : -1;
167
168@@ -3743,12 +3755,16 @@
169 }
170 slot.prompt.tokens.push_back(cur_tok);
171
172+ if (do_checkpoint && prefix_cache && slot.prompt.n_tokens() == prefix_pos) {
173+ break;
174+ }
175+
176 // break at the last user message, or at user messages at least min step past the last checkpoint
177 if (do_checkpoint && spans.is_user_start(slot.prompt.n_tokens())) {
178 const auto pos = slot.prompt.n_tokens();
179 const auto & checkpoints = slot.prompt.checkpoints;
180
181- if (pos == last_user_pos || checkpoints.empty() || pos > checkpoints.back().n_tokens + params_base.checkpoint_min_step) {
182+ if (pos == first_user_pos || pos == last_user_pos || checkpoints.empty() || pos > checkpoints.back().n_tokens + params_base.checkpoint_min_step) {
183 break;
184 }
185 }
186@@ -3783,6 +3799,8 @@
187 const bool near_prompt_end = slot.task->n_tokens() < slot.prompt.n_tokens() + n_ubatch;
188
189 const bool is_user_start = spans.is_user_start(n_tokens_start);
190+ const bool is_prefix_boundary = prefix_cache && n_tokens_start > 0 &&
191+ (n_tokens_start == prefix_pos || n_tokens_start == first_user_pos);
192 const bool is_last_user_message = n_tokens_start == last_user_pos;
193
194 // entire prompt has been processed
195@@ -3799,9 +3817,8 @@
196
197 slot.init_sampler();
198 } else {
199- // skip ordinary mid-prompt checkpoints, unless the batch starts a user
200- // message or we are near the end of the prompt
201- if (!is_user_start && !near_prompt_end) {
202+ // Retain system-prefix boundaries even inside a message.
203+ if (!is_user_start && !is_prefix_boundary && !near_prompt_end) {
204 do_checkpoint = false;
205 }
206 }
207@@ -3821,7 +3838,7 @@
208 // no need to create checkpoints that are too close together, unless it's the last user message
209 do_checkpoint = do_checkpoint && (
210 slot.prompt.checkpoints.empty() ||
211- is_last_user_message || near_prompt_end ||
212+ is_last_user_message || is_prefix_boundary || near_prompt_end ||
213 n_tokens_start > slot.prompt.checkpoints.back().n_tokens + params_base.checkpoint_min_step);
214 SLT_DBG(slot, "main/do_checkpoint = %s, pos_min = %d, pos_max = %d\n", do_checkpoint ? "yes" : "no", pos_min, pos_max);
215
216@@ -3829,6 +3846,21 @@
217 // yet processed and therefore it is not part of the checkpoint.
218 if (do_checkpoint) {
219 create_checkpoint(slot, n_tokens_cur, pos_min, pos_max);
220+ if (is_prefix_boundary &&
221+ !slot.prompt.tokens.find_next_media_chunk(0).first) {
222+ server_prompt prefix {slot.prompt.tokens.clone(), {slot.prompt.checkpoints.back()}};
223+ prefix.tokens.keep_first(n_tokens_start);
224+ const size_t size = llama_state_seq_get_size_ext(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_NONE);
225+ const size_t draft_size = ctx_dft ? llama_state_seq_get_size_ext(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_NONE) : 0;
226+ if (auto * state = prefix_cache->alloc(prefix, size, draft_size)) {
227+ llama_state_seq_get_data_ext(ctx_tgt, state->data.main.data(), size, slot.id, LLAMA_STATE_SEQ_FLAGS_NONE);
228+ if (ctx_dft) {
229+ llama_state_seq_get_data_ext(ctx_dft, state->data.drft.data(), draft_size, slot.id, LLAMA_STATE_SEQ_FLAGS_NONE);
230+ }
231+ prefix_cache->update();
232+ SLT_INF(slot, "cached base prefix, n_tokens = %d\n", n_tokens_start);
233+ }
234+ }
235 }
236 }
237
service/local-ai/service.pkl+7-3
...@@ -14,9 +14,9 @@ healthRestartGrace = "30m"...@@ -14,9 +14,9 @@ healthRestartGrace = "30m"
14secrets { ["api_key"] {} }14secrets { ["api_key"] {} }
1515
16container {16container {
17 image = "ghcr.io/ggml-org/llama.cpp@sha256:e8318a7b3988f9ca57b28c03486569e60def08f2b7b060550ebfa046e175e6cb"17 build = "build"
18 cpu = 400018 cpu = 4000
19 memory = 1638419 memory = 18432
20 entrypoint = "/usr/bin/python3"20 entrypoint = "/usr/bin/python3"
21 http {21 http {
22 containerPort = 808022 containerPort = 8080
...@@ -49,6 +49,10 @@ container {...@@ -49,6 +49,10 @@ container {
49 "--cache-type-k"; "q8_0"49 "--cache-type-k"; "q8_0"
50 "--cache-type-v"; "q8_0"50 "--cache-type-v"; "q8_0"
51 "--cache-ram"; "8192"51 "--cache-ram"; "8192"
52 "--spec-type"; "draft-mtp"
53 "--spec-draft-n-max"; "3"
54 "--cache-type-k-draft"; "q8_0"
55 "--cache-type-v-draft"; "q8_0"
52 "--jinja"56 "--jinja"
53 "--timeout"; "28800"57 "--timeout"; "28800"
54 "--sse-ping-interval"; "10"58 "--sse-ping-interval"; "10"
...@@ -57,6 +61,6 @@ container {...@@ -57,6 +61,6 @@ container {
57 "--top-p"; "0.95"61 "--top-p"; "0.95"
58 "--top-k"; "20"62 "--top-k"; "20"
59 "--min-p"; "0"63 "--min-p"; "0"
60 "--chat-template-kwargs"; "{\"enable_thinking\": true, \"reasoning_effort\": \"medium\", \"preserve_thinking\": true}"64 "--chat-template-kwargs"; "{\"enable_thinking\": true, \"reasoning_effort\": \"\(model.default_reasoning_level)\", \"preserve_thinking\": true}"
61 }65 }
62}66}
tools/benchmark-local-ai.py created+87
...@@ -0,0 +1,87 @@
1#!/usr/bin/env python3
2"""Measure llama.cpp decode, retrieval, and prefix reuse without logging prompts."""
3import argparse
4import base64
5import binascii
6import json
7import os
8import struct
9import time
10import urllib.request
11import zlib
12
13
14def request(path, data=None):
15 body = None if data is None else json.dumps(data).encode()
16 headers = {"Authorization": "Bearer " + os.environ["LLAMA_API_KEY"], "Content-Type": "application/json"}
17 started = time.monotonic()
18 with urllib.request.urlopen(urllib.request.Request(args.url + path, data=body, headers=headers), timeout=1800) as response:
19 result = json.load(response)
20 return result, time.monotonic() - started
21
22
23parser = argparse.ArgumentParser(description=__doc__)
24parser.add_argument("--url", default="http://127.0.0.1:8081")
25parser.add_argument("--contexts", type=int, nargs="+", default=[16384, 65536, 122000])
26parser.add_argument("--suite", choices=["decode", "workloads", "cache", "retrieval", "vision", "all"], default="all")
27parser.add_argument("--require-prefix-cache", action="store_true")
28args = parser.parse_args()
29
30def report(label, data, path="/completion"):
31 result, elapsed = request(path, data)
32 print(json.dumps({"case": label, "elapsed_s": round(elapsed, 3), "timings": result.get("timings"), "usage": result.get("usage")}), flush=True)
33 return result
34
35
36filler = "".join(f"Record {i:06d}: amber bridge cedar delta elm forest granite harbor iris jade.\n" for i in range(15000))
37tokens, _ = request("/tokenize", {"content": filler})
38tokens = tokens["tokens"]
39
40if args.suite in {"decode", "all"}:
41 offset = 0
42 for size in args.contexts:
43 prompt = tokens[offset:offset + size]
44 offset += size
45 assert len(prompt) == size, "Requested context exceeds the fixture"
46 data = {"prompt": prompt, "n_predict": 256, "ignore_eos": True, "temperature": 0, "seed": 42, "cache_prompt": True}
47 report(f"decode-{size}-first", data)
48 report(f"decode-{size}-warm", data)
49
50if args.suite in {"cache", "all"}:
51 prefix, _ = request("/detokenize", {"tokens": tokens[:4096]})
52 other, _ = request("/detokenize", {"tokens": tokens[40000:44096]})
53 prefix["content"] += "\nStable system code is prefix-a-731."
54 other["content"] += "\nStable system code is prefix-b-284."
55 for label, system, body in [("prefix-a", prefix["content"], tokens[10000:30000]), ("prefix-b", other["content"], tokens[45000:65000]), ("prefix-a-new", prefix["content"], tokens[70000:90000]), ("prefix-a-repeat", prefix["content"], tokens[70000:90000]), ("prefix-b-again", other["content"], tokens[95000:115000]), ("prefix-a-metadata", prefix["content"].replace("prefix-a-731", "prefix-c-592"), tokens[120000:140000])]:
56 text, _ = request("/detokenize", {"tokens": body})
57 result = report(label, {"model": "qwen3.8-27b", "messages": [{"role": "system", "content": system}, {"role": "user", "content": text["content"] + "\nWhat is the stable system code? Reply with only it."}], "max_tokens": 32, "temperature": 0, "chat_template_kwargs": {"enable_thinking": False}}, "/v1/chat/completions")
58 assert system.rsplit(" ", 1)[-1].rstrip(".") in result["choices"][0]["message"]["content"], "Prefix restore lost the system fact"
59 if args.require_prefix_cache and label in {"prefix-a-new", "prefix-a-metadata"}:
60 assert result["timings"]["cache_n"] >= 4096, "Client switching discarded the base prefix"
61
62if args.suite in {"workloads", "all"}:
63 for task in ["Implement a bounded asynchronous work queue in Python using asyncio. Explain cancellation, backpressure, shutdown, race conditions and error handling. Include tests and a detailed explanation.", "Explain how database isolation levels interact with concurrent read-modify-write operations. Give concrete schedules, anomalies, and remedies for each level, with detailed examples."]:
64 template, _ = request("/apply-template", {"messages": [{"role": "system", "content": "You are a careful software engineer."}, {"role": "user", "content": task}], "chat_template_kwargs": {"enable_thinking": True, "reasoning_effort": "medium"}})
65 report("workload-code" if "asyncio" in task else "workload-reasoning", {"prompt": template["prompt"], "n_predict": 512, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0, "seed": 42, "cache_prompt": True})
66
67if args.suite in {"retrieval", "all"}:
68 for size in args.contexts:
69 facts = ["START password is otter-731.", "MIDDLE password is maple-284.", "END password is comet-965."]
70 segment = tokens[:max(1, (size - 256) // 2)]
71 text, _ = request("/detokenize", {"tokens": segment})
72 messages = [{"role": "system", "content": "Find the three passwords in the records. Reply with only the passwords in START, MIDDLE, END order."}, {"role": "user", "content": facts[0] + "\n" + text["content"] + "\n" + facts[1] + "\n" + text["content"] + "\n" + facts[2]}]
73 template, _ = request("/apply-template", {"messages": messages, "chat_template_kwargs": {"enable_thinking": False}})
74 result = report(f"retrieval-{size}", {"prompt": template["prompt"], "n_predict": 64, "temperature": 0, "cache_prompt": True})
75 positions = [result["content"].find(fact.split()[-1].rstrip(".")) for fact in facts]
76 passed = all(pos >= 0 for pos in positions) and positions == sorted(positions)
77 print(json.dumps({"case": f"retrieval-{size}", "passed": passed}), flush=True)
78 if not passed:
79 raise SystemExit("Retrieval failed")
80
81if args.suite in {"vision", "all"}:
82 png = b"\x89PNG\r\n\x1a\n"
83 pixels = b"".join(b"\0" + b"".join(b"\xff\0\0" if 128 <= x < 384 and 128 <= y < 384 else b"\xff\xff\xff" for x in range(512)) for y in range(512))
84 for kind, data in [(b"IHDR", struct.pack(">IIBBBBB", 512, 512, 8, 2, 0, 0, 0)), (b"IDAT", zlib.compress(pixels)), (b"IEND", b"")]:
85 png += struct.pack(">I", len(data)) + kind + data + struct.pack(">I", binascii.crc32(kind + data))
86 result = report("vision", {"model": "qwen3.8-27b", "messages": [{"role": "user", "content": [{"type": "text", "text": "What color is the square? Reply with one word."}, {"type": "image_url", "image_url": {"url": "data:image/png;base64," + base64.b64encode(png).decode()}}]}], "max_tokens": 32, "temperature": 0, "chat_template_kwargs": {"enable_thinking": False}}, "/v1/chat/completions")
87 assert "red" in result["choices"][0]["message"]["content"].lower(), "Vision failed"
tools/local-ai.md+76-3
...@@ -1,14 +1,27 @@...@@ -1,14 +1,27 @@
1# Snow Globe local AI1# Snow Globe local AI
22
3The first model is Qwen3.8-27B, using Unsloth's UD-Q4_K_M GGUF and F16 vision3The first model is Qwen3.8-27B, using Unsloth's UD-Q4_K_M GGUF and F16 vision
4projector. Both files and the llama.cpp CUDA image are pinned. The model runs4projector. Both files, the base CUDA image, and the llama.cpp source are pinned.
5entirely on the RTX 3090, with one inference slot and a 131,072-token context.5The model runs entirely on the RTX 3090, with one inference slot and a 131,072-token context.
6Codex compacts at 96,000 tokens to leave room for tool results and output.6Codex compacts at 96,000 tokens to leave room for tool results and output.
7The KV cache uses Q8; the single slot leaves more VRAM for vision and runtime7The KV cache uses Q8; the single slot leaves more VRAM for vision and runtime
8working memory. Additional requests queue so simultaneous generation cannot8working memory. Additional requests queue so simultaneous generation cannot
9reduce the active request's decode rate. An 8 GiB host-memory prompt cache9reduce the active request's decode rate. An 8 GiB host-memory prompt cache
10preserves evicted contexts between coding, reviewer, and helper requests.10preserves evicted contexts between coding, reviewer, and helper requests.
11The container has a 16 GiB RAM limit for the runtime and checkpoints.11The patched server also reserves a separate 2 GiB cache for prefixes before
12the first user message, including system instructions and tool definitions.
13A second checkpoint at 4,096 tokens retains the shared base when later workspace
14metadata changes; shorter headers need only their complete checkpoint.
15It captures these from actual requests; there are no copied Codex or Claude
16prompts to update. Prefix states survive conversation-cache eviction and remain
17available after restoration. Only complete token matches restore these states,
18including the recurrent and MTP state. Prefix variants have their own bounded
19LRU eviction; first use, unmatched tokens, and server restarts still need prefill.
20The container has an 18 GiB RAM limit for the runtime and both caches.
21Native MTP drafts up to three tokens, which the target model verifies. The draft
22KV cache also uses Q8. Target weights, target KV precision, context length, and
23reasoning defaults remain unchanged. The build replaces only the server library;
24inference libraries and embedded UI assets come from the pinned base image.
12Medium reasoning is enabled for coding, using Qwen's recommended thinking-mode25Medium reasoning is enabled for coding, using Qwen's recommended thinking-mode
13sampling parameters. This adds26sampling parameters. This adds
14reasoning tokens before answers; the earlier latency baseline disabled thinking.27reasoning tokens before answers; the earlier latency baseline disabled thinking.
...@@ -68,6 +81,22 @@ running classification against the same local endpoint. Both use medium...@@ -68,6 +81,22 @@ running classification against the same local endpoint. Both use medium
68reasoning. Claude's credential store is isolated from ordinary Claude login,81reasoning. Claude's credential store is isolated from ordinary Claude login,
69so using Snow Globe does not log the user's subscription out.82so using Snow Globe does not log the user's subscription out.
7083
84Qwen's raw thinking is streamed as Responses reasoning text, including the
85`content_index` required by stock Codex. To display it in the terminal:
86
87```sh
88snow-codex -c show_raw_agent_reasoning=true -c hide_agent_reasoning=false
89```
90
91Reasoning summaries remain disabled because the model emits raw thinking rather
92than a separate summary. `tools/verify-local-ai.py` checks actual Codex app-server
93reasoning deltas arrive before the answer; stored reasoning in a completed
94response alone does not establish that live display works. The production fix
95was checked with stock Codex 0.161.0: 237 live reasoning deltas arrived before
96the answer, the interactive terminal updated during thinking, both coding
97fixtures passed, and automatic approval authorized its temporary write in
9821.4 seconds.
99
71API and event-stream waits allow eight hours. Claude's byte-stream watchdog100API and event-stream waits allow eight hours. Claude's byte-stream watchdog
72has a 30-minute maximum, kept alive by ten-second server SSE pings. Stock101has a 30-minute maximum, kept alive by ten-second server SSE pings. Stock
73Codex 0.160.0 retains a hardcoded 90-second approval-review deadline that102Codex 0.160.0 retains a hardcoded 90-second approval-review deadline that
...@@ -172,3 +201,47 @@ with synthetic files and credentials, separate from the personal coding profile....@@ -172,3 +201,47 @@ with synthetic files and credentials, separate from the personal coding profile.
172Evaluate coding, tool arguments, prompt injection, unauthorized writes, and201Evaluate coding, tool arguments, prompt injection, unauthorized writes, and
173attempted outbound requests against the base model. Passing those tests is202attempted outbound requests against the base model. Passing those tests is
174evidence about observed behavior, not proof that an unrestricted agent is safe.203evidence about observed behavior, not proof that an unrestricted agent is safe.
204
205## Runtime tuning (2026-10-07)
206
207Measurements on the production RTX 3090, using 256 greedy decode tokens with the
208same target weights, Q8 KV cache, and context size:
209
210| Input context | Stock runtime | MTP, three Q8 draft tokens |
211| --- | ---: | ---: |
212| 16,384 tokens | 39.1 tok/s | 92.0 tok/s |
213| 65,536 tokens | 30.3 tok/s | 73.3 tok/s |
214
215These are warm synthetic decode rates, not a promise for every coding task.
216The n-gram draft trial did not improve throughput. MTP's target verifies every
217accepted draft token; it does not substitute a smaller model's answers.
218Cold prefill remains about 1,200 tok/s at 16K and 990 tok/s at 64K. Caching avoids
219processing unchanged prefixes, but new long inputs still pay that cost.
220
221The production checks exercise stock Codex and Claude Code editing the same
222median fixture, all eight unchanged tests, Codex's automatic approval reviewer,
223vision, and three-fact retrieval in a 121,816-token input. They establish these
224paths remain functional, not a broad model-quality evaluation. The initial
225MTP/prefix-cache check with stock Codex 0.161.0 and Claude Code 2.1.293 took 68.6 and 120.4
226seconds while sharing the slot. All 30 other production containers retained
227their identities; the AI allocation recorded no OOM events, limit hits, or
228restarts during verification.
229
230`tools/benchmark-local-ai.py` measures decode, natural coding/reasoning workloads,
231client switching, changing system metadata, retrieval, and vision. Run inside
232the production container, where `LLAMA_API_KEY` is already inherited, against
233the backend on `http://127.0.0.1:8081`; it emits timings without prompts or keys:
234
235```sh
236python3 /tmp/benchmark-local-ai.py --suite cache --require-prefix-cache
237python3 /tmp/benchmark-local-ai.py --suite decode --contexts 16384 65536
238python3 /tmp/benchmark-local-ai.py --suite retrieval --contexts 122000
239python3 /tmp/benchmark-local-ai.py --suite vision
240```
241
242The prefix test alternates two system headers and long unrelated user histories,
243then changes a trailing system fact. Both restores must reuse at least 4,096
244tokens and answer from the current header. Production retained 4,114 tokens
245across client switches and 4,096 after the system fact changed, with the new
246fact answered correctly. Repeating the conversation retained 24,135 tokens
247and completed in 0.335 seconds, including generation.
tools/verify-local-ai.py+51-1
...@@ -5,6 +5,7 @@ import concurrent.futures...@@ -5,6 +5,7 @@ import concurrent.futures
5from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer5from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
6import json6import json
7import os7import os
8import selectors
8from pathlib import Path9from pathlib import Path
9import subprocess10import subprocess
10import tempfile11import tempfile
...@@ -48,6 +49,54 @@ def run_client(root, provider, prompt, extra_env=None, label=None):...@@ -48,6 +49,54 @@ def run_client(root, provider, prompt, extra_env=None, label=None):
48 return log49 return log
4950
5051
52def verify_reasoning(root, endpoint):
53 env = os.environ.copy()
54 env.update(CODEX_HOME=str(root / "codex-home"), SNOWGLOBE_AI_URL=endpoint)
55 wrapper = Path(os.environ.get("SNOWGLOBE_AI_BIN_DIR", Path.home() / ".local/bin")) / "snow-codex"
56 with (root / "codex-reasoning-stderr.log").open("w") as errors, (root / "codex-reasoning.jsonl").open("wb") as log:
57 process = subprocess.Popen([str(wrapper), "-c", "show_raw_agent_reasoning=true", "-c", "hide_agent_reasoning=false", "app-server"], cwd=root / "codex", env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=errors, bufsize=0)
58 with selectors.DefaultSelector() as selector:
59 selector.register(process.stdout, selectors.EVENT_READ)
60 process.stdin.write(json.dumps({"id": 0, "method": "initialize", "params": {"clientInfo": {"name": "snowglobe_reasoning_test", "version": "1.0"}}}).encode() + b"\n")
61 reasoning = 0
62 answer = ""
63 deadline = time.monotonic() + 240
64 try:
65 while time.monotonic() < deadline:
66 if not selector.select(1):
67 continue
68 line = process.stdout.readline()
69 assert line, "Codex app-server closed before completing the reasoning test"
70 log.write(line)
71 event = json.loads(line)
72 assert "error" not in event, event.get("error")
73 if event.get("id") == 0:
74 process.stdin.write(b'{"method":"initialized","params":{}}\n')
75 process.stdin.write(json.dumps({"id": 1, "method": "thread/start", "params": {"cwd": str(root / "codex")}}).encode() + b"\n")
76 elif event.get("id") == 1:
77 process.stdin.write(json.dumps({"id": 2, "method": "turn/start", "params": {"threadId": event["result"]["thread"]["id"], "input": [{"type": "text", "text": "Do not use tools. Find the smallest positive integer divisible by every integer from 1 through 12. Give the integer and a concise justification."}]}}).encode() + b"\n")
78 method = event.get("method")
79 if method == "item/reasoning/textDelta":
80 reasoning += bool(event["params"]["delta"])
81 elif method == "item/agentMessage/delta":
82 assert reasoning > 0, "Codex received no live reasoning before the answer"
83 answer += event["params"]["delta"]
84 elif method == "turn/completed":
85 assert event["params"]["turn"]["status"] == "completed"
86 assert reasoning > 0 and "27720" in answer.replace(",", ""), "Reasoning fixture failed"
87 print(f"Codex streamed {reasoning} reasoning deltas before its answer", flush=True)
88 break
89 else:
90 raise TimeoutError("Codex reasoning stream timed out")
91 finally:
92 process.stdin.close()
93 try:
94 process.wait(timeout=5)
95 except subprocess.TimeoutExpired:
96 process.terminate()
97 process.wait(timeout=5)
98
99
51def review_delay_test(root, delay, endpoint):100def review_delay_test(root, delay, endpoint):
52 delayed = threading.Event()101 delayed = threading.Event()
53 class Proxy(BaseHTTPRequestHandler):102 class Proxy(BaseHTTPRequestHandler):
...@@ -120,5 +169,6 @@ if __name__ == "__main__":...@@ -120,5 +169,6 @@ if __name__ == "__main__":
120 assert (work / "test_stats.py").read_text() == TESTS, f"{provider} changed the tests"169 assert (work / "test_stats.py").read_text() == TESTS, f"{provider} changed the tests"
121 subprocess.run(["python3", "-m", "unittest", "-v"], cwd=work, check=True)170 subprocess.run(["python3", "-m", "unittest", "-v"], cwd=work, check=True)
122 assert args.approval_delay >= 0, "Delay must be nonnegative."171 assert args.approval_delay >= 0, "Delay must be nonnegative."
172 verify_reasoning(root, args.url)
123 review_delay_test(root, args.approval_delay, args.url)173 review_delay_test(root, args.approval_delay, args.url)
124 print("Real coding and approval checks passed", flush=True)174 print("Real coding, reasoning, and approval checks passed", flush=True)