diff --git a/service/local-ai/build/Dockerfile b/service/local-ai/build/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..6e37e8a46b9672d97de76ee975e87781b435413e --- /dev/null +++ b/service/local-ai/build/Dockerfile @@ -0,0 +1,13 @@ +ARG RUNTIME=ghcr.io/ggml-org/llama.cpp@sha256:e8318a7b3988f9ca57b28c03486569e60def08f2b7b060550ebfa046e175e6cb +FROM ${RUNTIME} AS build +RUN rm -f /etc/apt/sources.list.d/* && printf 'deb https://archive.ubuntu.com/ubuntu noble main universe\ndeb https://archive.ubuntu.com/ubuntu noble-updates main universe\ndeb https://security.ubuntu.com/ubuntu noble-security main universe\n' > /etc/apt/sources.list && apt-get -o Acquire::https::Timeout=30 update && apt-get install -y --no-install-recommends cmake g++ make patch libssl-dev +WORKDIR /src +RUN python3 -c 'import hashlib,urllib.request; data=urllib.request.urlopen("https://api.github.com/repos/ggml-org/llama.cpp/tarball/11fe02151f79c41d0d4af7da708755d73b9c0da6").read(); assert hashlib.sha256(data).hexdigest()=="965db483b5e27c4917bac5bf83e0d2d533099049f8d5234c75e93c969d76443a"; open("source.tar.gz","wb").write(data)' && tar -xzf source.tar.gz --strip-components=1 && rm source.tar.gz +COPY prefix-cache.patch /tmp/prefix-cache.patch +RUN patch -p1 < /tmp/prefix-cache.patch +COPY extract-ui.cpp /tmp/extract-ui.cpp +RUN python3 -c 'import pathlib,subprocess; symbols=subprocess.check_output(["nm","-S","/app/libllama-server-impl.so"],text=True); size=int(next(line.split()[1] for line in symbols.splitlines() if line.split()[-1]=="_ZL8g_assets"),16); header=pathlib.Path("tools/ui/ui.h.in").read_text().replace("#cmakedefine LLAMA_UI_HAS_ASSETS 1","#define LLAMA_UI_HAS_ASSETS 1").replace("@N_ASSETS@",str(size)+"/sizeof(llama_ui_asset)"); pathlib.Path("/tmp/ui.h").write_text(header)' && g++ -std=c++17 -I/tmp /tmp/extract-ui.cpp -L/app -Wl,-rpath,/app -lllama-server-impl -o /tmp/extract-ui && LD_LIBRARY_PATH=/app /tmp/extract-ui && python3 -c 'import gzip,pathlib; [p.write_bytes(gzip.decompress(p.read_bytes())) for p in pathlib.Path("tools/ui/dist").rglob("*") if p.is_file()]' +RUN cmake -S . -B /build -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DGGML_CUDA=OFF -DGGML_CPU=OFF -DGGML_BACKEND_DL=ON -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_UI=OFF -DLLAMA_USE_PREBUILT_UI=OFF -DLLAMA_BUILD_NUMBER=11382 -DLLAMA_BUILD_COMMIT=11fe02151 && cmake --build /build --target llama-ui-assets && ln -s /app/libllama.so* /app/libllama-common.so* /app/libmtmd.so* /app/libggml*.so* /build/bin/ && cmake --build /build --target server-context/fast cpp-httplib/fast llama-common-base/fast llama-ui/fast -j 4 && cmake --build /build --target llama-server-impl/fast -j 4 + +FROM ${RUNTIME} +COPY --from=build /build/bin/libllama-server-impl.so /app/libllama-server-impl.so diff --git a/service/local-ai/build/extract-ui.cpp b/service/local-ai/build/extract-ui.cpp new file mode 100644 index 0000000000000000000000000000000000000000..eeabf6f7c84d9a517dbace08e3f2b9296eb57416 --- /dev/null +++ b/service/local-ai/build/extract-ui.cpp @@ -0,0 +1,18 @@ +#include "ui.h" +#include +#include + +int main() { + if (!llama_ui_use_gzip()) { + return 1; + } + for (const auto & asset : llama_ui_get_assets()) { + const auto path = std::filesystem::path("tools/ui/dist") / asset.name; + std::filesystem::create_directories(path.parent_path()); + std::ofstream output(path, std::ios::binary); + output.write(reinterpret_cast(asset.data), asset.size); + if (!output) { + return 1; + } + } +} diff --git a/service/local-ai/build/prefix-cache.patch b/service/local-ai/build/prefix-cache.patch new file mode 100644 index 0000000000000000000000000000000000000000..c97fce001e32ed0a19cf34610ef0b06f63da356f --- /dev/null +++ b/service/local-ai/build/prefix-cache.patch @@ -0,0 +1,237 @@ +--- a/tools/server/server-task.h ++++ b/tools/server/server-task.h +@@ -650,12 +650,13 @@ + }; + + struct server_prompt_cache { +- server_prompt_cache(int32_t limit_size_mib, size_t limit_tokens) { ++ server_prompt_cache(int32_t limit_size_mib, size_t limit_tokens, bool keep_on_load = false) : keep_on_load(keep_on_load) { + this->limit_size = 1024ull*1024ull*(limit_size_mib < 0 ? 0 : limit_size_mib); + this->limit_tokens = limit_tokens; + } + + std::list states; ++ const bool keep_on_load; + + // in bytes, 0 = no limit + size_t limit_size = 0; +--- a/tools/server/server-task.cpp ++++ b/tools/server/server-task.cpp +@@ -1226,6 +1226,7 @@ + {"event", "response.reasoning_text.delta"}, + {"data", json { + {"type", "response.reasoning_text.delta"}, ++ {"content_index", 0}, + {"delta", diff.reasoning_content_delta}, + {"item_id", oai_resp_reasoning_id}, + }}, +@@ -1724,7 +1725,10 @@ + for (auto it = states.begin(); it != states.end(); ++it) { + const int cur_lcp_len = it->prompt.tokens.get_common_prefix(prompt.tokens); + +- if (cur_lcp_len == (int) prompt.tokens.size()) { ++ if (cur_lcp_len == (int) prompt.tokens.size() && (!keep_on_load || it->prompt.tokens.size() == prompt.tokens.size())) { ++ if (keep_on_load) { ++ states.splice(states.end(), states, it); ++ } + SRV_TRC("%s", " - prompt is already in the cache, skipping\n"); + return nullptr; + } +@@ -1749,7 +1753,7 @@ + for (auto it = states.begin(); it != states.end();) { + const int len = it->prompt.tokens.get_common_prefix(prompt.tokens); + +- if (len == (int) it->prompt.tokens.size()) { ++ if (!keep_on_load && len == (int) it->prompt.tokens.size()) { + SRV_TRC(" - removing obsolete cached prompt with length %d\n", len); + + it = states.erase(it); +@@ -1820,12 +1824,18 @@ + + SRV_TRC(" - prompt with length %7zu, lcp = %7d, f_keep = %.3f, f_sim = %.3f\n", it->prompt.tokens.size(), lcp_cur, f_keep_cur, f_sim_cur); + ++ // Only complete pinned prefixes contain a usable recurrent state. ++ if (keep_on_load && lcp_cur != (int) it->prompt.tokens.size()) { ++ continue; ++ } ++ + // don't trash large prompts + if (f_keep_cur < 0.25f) { + continue; + } + +- if (f_keep_best < f_keep_cur && f_sim_best < f_sim_cur) { ++ if ((keep_on_load || f_keep_best < f_keep_cur) && ++ (f_sim_best < f_sim_cur || (keep_on_load && f_sim_best == f_sim_cur && f_keep_best < f_keep_cur))) { + f_keep_best = f_keep_cur; + f_sim_best = f_sim_cur; + +@@ -1847,8 +1857,10 @@ + return false; + } + +- data.clear(); +- data.shrink_to_fit(); ++ if (!keep_on_load) { ++ data.clear(); ++ data.shrink_to_fit(); ++ } + } + + { +@@ -1865,14 +1877,20 @@ + return false; + } + +- data.clear(); +- data.shrink_to_fit(); ++ if (!keep_on_load) { ++ data.clear(); ++ data.shrink_to_fit(); ++ } + } + } + +- prompt = std::move(it_best->prompt); +- +- states.erase(it_best); ++ if (keep_on_load) { ++ prompt = it_best->prompt.clone(); ++ states.splice(states.end(), states, it_best); ++ } else { ++ prompt = std::move(it_best->prompt); ++ states.erase(it_best); ++ } + } + + return true; +--- a/tools/server/server-context.cpp ++++ b/tools/server/server-context.cpp +@@ -954,6 +954,7 @@ + int n_empty_consecutive = 0; + + std::unique_ptr prompt_cache; ++ std::unique_ptr prefix_cache; + + server_metrics metrics; + +@@ -1424,6 +1425,7 @@ + SRV_TRC("%s", "use `--cache-ram 0` to disable the prompt cache\n"); + + prompt_cache = std::make_unique(params_base.cache_ram_mib, n_ctx); ++ prefix_cache = std::make_unique(2048, n_ctx, true); + } else { + SRV_TRC("%s", "prompt cache is disabled - use `--cache-ram N` to enable it\n"); + } +@@ -1721,6 +1723,11 @@ + + SRV_TRC("prompt cache update took %.2f ms\n", (ggml_time_us() - t_start) / 1000.0); + } ++ } ++ ++ if (ret && prefix_cache && task.type == SERVER_TASK_TYPE_COMPLETION && ++ !ret->prompt_load(*prefix_cache, task.tokens)) { ++ ret->prompt_clear(); + } + + return ret; +@@ -3091,7 +3098,7 @@ + // add generated tokens to cache + // ref: https://github.com/ggml-org/llama.cpp/pull/16818#discussion_r2473269481 + { +- GGML_ASSERT(!slot.prompt.tokens.has_mtmd); ++ GGML_ASSERT(!slot.prompt.tokens.find_next_media_chunk(0).first); + + llama_tokens new_tokens = slot.prompt.tokens.get_tokens(); // copy + for (size_t i = n_keep + n_discard; i < new_tokens.size(); i++) { +@@ -3400,7 +3407,7 @@ + + // reuse chunks from the cached prompt by shifting their KV cache in the new position + if (can_cache_reuse && n_cache_reuse > 0) { +- GGML_ASSERT(!slot.prompt.tokens.has_mtmd); ++ GGML_ASSERT(!slot.prompt.tokens.find_next_media_chunk(0).first); + + size_t head_c = n_past; // cache + size_t head_p = n_past; // current prompt +@@ -3701,6 +3708,11 @@ + + const auto & spans = slot.task->params.message_spans; + const auto last_user_pos = spans.last_user_message_pos(); ++ const auto first_user = std::find_if(spans.spans.begin(), spans.spans.end(), [](const auto & span) { ++ return span.role == COMMON_CHAT_ROLE_USER; ++ }); ++ const int64_t first_user_pos = first_user == spans.spans.end() ? -1 : first_user->pos; ++ const int64_t prefix_pos = std::min(4096, first_user_pos); + + const int32_t n_decision_first = slot.task->type == SERVER_TASK_TYPE_DECISION ? slot.task->decision.pos_first() : -1; + +@@ -3743,12 +3755,16 @@ + } + slot.prompt.tokens.push_back(cur_tok); + ++ if (do_checkpoint && prefix_cache && slot.prompt.n_tokens() == prefix_pos) { ++ break; ++ } ++ + // break at the last user message, or at user messages at least min step past the last checkpoint + if (do_checkpoint && spans.is_user_start(slot.prompt.n_tokens())) { + const auto pos = slot.prompt.n_tokens(); + const auto & checkpoints = slot.prompt.checkpoints; + +- if (pos == last_user_pos || checkpoints.empty() || pos > checkpoints.back().n_tokens + params_base.checkpoint_min_step) { ++ if (pos == first_user_pos || pos == last_user_pos || checkpoints.empty() || pos > checkpoints.back().n_tokens + params_base.checkpoint_min_step) { + break; + } + } +@@ -3783,6 +3799,8 @@ + const bool near_prompt_end = slot.task->n_tokens() < slot.prompt.n_tokens() + n_ubatch; + + const bool is_user_start = spans.is_user_start(n_tokens_start); ++ const bool is_prefix_boundary = prefix_cache && n_tokens_start > 0 && ++ (n_tokens_start == prefix_pos || n_tokens_start == first_user_pos); + const bool is_last_user_message = n_tokens_start == last_user_pos; + + // entire prompt has been processed +@@ -3799,9 +3817,8 @@ + + slot.init_sampler(); + } else { +- // skip ordinary mid-prompt checkpoints, unless the batch starts a user +- // message or we are near the end of the prompt +- if (!is_user_start && !near_prompt_end) { ++ // Retain system-prefix boundaries even inside a message. ++ if (!is_user_start && !is_prefix_boundary && !near_prompt_end) { + do_checkpoint = false; + } + } +@@ -3821,7 +3838,7 @@ + // no need to create checkpoints that are too close together, unless it's the last user message + do_checkpoint = do_checkpoint && ( + slot.prompt.checkpoints.empty() || +- is_last_user_message || near_prompt_end || ++ is_last_user_message || is_prefix_boundary || near_prompt_end || + n_tokens_start > slot.prompt.checkpoints.back().n_tokens + params_base.checkpoint_min_step); + SLT_DBG(slot, "main/do_checkpoint = %s, pos_min = %d, pos_max = %d\n", do_checkpoint ? "yes" : "no", pos_min, pos_max); + +@@ -3829,6 +3846,21 @@ + // yet processed and therefore it is not part of the checkpoint. + if (do_checkpoint) { + create_checkpoint(slot, n_tokens_cur, pos_min, pos_max); ++ if (is_prefix_boundary && ++ !slot.prompt.tokens.find_next_media_chunk(0).first) { ++ server_prompt prefix {slot.prompt.tokens.clone(), {slot.prompt.checkpoints.back()}}; ++ prefix.tokens.keep_first(n_tokens_start); ++ const size_t size = llama_state_seq_get_size_ext(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_NONE); ++ const size_t draft_size = ctx_dft ? llama_state_seq_get_size_ext(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_NONE) : 0; ++ if (auto * state = prefix_cache->alloc(prefix, size, draft_size)) { ++ llama_state_seq_get_data_ext(ctx_tgt, state->data.main.data(), size, slot.id, LLAMA_STATE_SEQ_FLAGS_NONE); ++ if (ctx_dft) { ++ llama_state_seq_get_data_ext(ctx_dft, state->data.drft.data(), draft_size, slot.id, LLAMA_STATE_SEQ_FLAGS_NONE); ++ } ++ prefix_cache->update(); ++ SLT_INF(slot, "cached base prefix, n_tokens = %d\n", n_tokens_start); ++ } ++ } + } + } + diff --git a/service/local-ai/service.pkl b/service/local-ai/service.pkl index 8fbbad05334c3d6ea5a5b07f9058d3a43bab00e1..39be2bedc9773836edd4c0bf632bcea6962b8bea 100644 --- a/service/local-ai/service.pkl +++ b/service/local-ai/service.pkl @@ -14,9 +14,9 @@ healthRestartGrace = "30m" secrets { ["api_key"] {} } container { - image = "ghcr.io/ggml-org/llama.cpp@sha256:e8318a7b3988f9ca57b28c03486569e60def08f2b7b060550ebfa046e175e6cb" + build = "build" cpu = 4000 - memory = 16384 + memory = 18432 entrypoint = "/usr/bin/python3" http { containerPort = 8080 @@ -49,6 +49,10 @@ container { "--cache-type-k"; "q8_0" "--cache-type-v"; "q8_0" "--cache-ram"; "8192" + "--spec-type"; "draft-mtp" + "--spec-draft-n-max"; "3" + "--cache-type-k-draft"; "q8_0" + "--cache-type-v-draft"; "q8_0" "--jinja" "--timeout"; "28800" "--sse-ping-interval"; "10" @@ -57,6 +61,6 @@ container { "--top-p"; "0.95" "--top-k"; "20" "--min-p"; "0" - "--chat-template-kwargs"; "{\"enable_thinking\": true, \"reasoning_effort\": \"medium\", \"preserve_thinking\": true}" + "--chat-template-kwargs"; "{\"enable_thinking\": true, \"reasoning_effort\": \"\(model.default_reasoning_level)\", \"preserve_thinking\": true}" } } diff --git a/tools/benchmark-local-ai.py b/tools/benchmark-local-ai.py new file mode 100644 index 0000000000000000000000000000000000000000..b801a384d8ee1a1a998650f05f105c9fe78585b3 --- /dev/null +++ b/tools/benchmark-local-ai.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Measure llama.cpp decode, retrieval, and prefix reuse without logging prompts.""" +import argparse +import base64 +import binascii +import json +import os +import struct +import time +import urllib.request +import zlib + + +def request(path, data=None): + body = None if data is None else json.dumps(data).encode() + headers = {"Authorization": "Bearer " + os.environ["LLAMA_API_KEY"], "Content-Type": "application/json"} + started = time.monotonic() + with urllib.request.urlopen(urllib.request.Request(args.url + path, data=body, headers=headers), timeout=1800) as response: + result = json.load(response) + return result, time.monotonic() - started + + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument("--url", default="http://127.0.0.1:8081") +parser.add_argument("--contexts", type=int, nargs="+", default=[16384, 65536, 122000]) +parser.add_argument("--suite", choices=["decode", "workloads", "cache", "retrieval", "vision", "all"], default="all") +parser.add_argument("--require-prefix-cache", action="store_true") +args = parser.parse_args() + +def report(label, data, path="/completion"): + result, elapsed = request(path, data) + print(json.dumps({"case": label, "elapsed_s": round(elapsed, 3), "timings": result.get("timings"), "usage": result.get("usage")}), flush=True) + return result + + +filler = "".join(f"Record {i:06d}: amber bridge cedar delta elm forest granite harbor iris jade.\n" for i in range(15000)) +tokens, _ = request("/tokenize", {"content": filler}) +tokens = tokens["tokens"] + +if args.suite in {"decode", "all"}: + offset = 0 + for size in args.contexts: + prompt = tokens[offset:offset + size] + offset += size + assert len(prompt) == size, "Requested context exceeds the fixture" + data = {"prompt": prompt, "n_predict": 256, "ignore_eos": True, "temperature": 0, "seed": 42, "cache_prompt": True} + report(f"decode-{size}-first", data) + report(f"decode-{size}-warm", data) + +if args.suite in {"cache", "all"}: + prefix, _ = request("/detokenize", {"tokens": tokens[:4096]}) + other, _ = request("/detokenize", {"tokens": tokens[40000:44096]}) + prefix["content"] += "\nStable system code is prefix-a-731." + other["content"] += "\nStable system code is prefix-b-284." + for label, system, body in [("prefix-a", prefix["content"], tokens[10000:30000]), ("prefix-b", other["content"], tokens[45000:65000]), ("prefix-a-new", prefix["content"], tokens[70000:90000]), ("prefix-a-repeat", prefix["content"], tokens[70000:90000]), ("prefix-b-again", other["content"], tokens[95000:115000]), ("prefix-a-metadata", prefix["content"].replace("prefix-a-731", "prefix-c-592"), tokens[120000:140000])]: + text, _ = request("/detokenize", {"tokens": body}) + result = report(label, {"model": "qwen3.8-27b", "messages": [{"role": "system", "content": system}, {"role": "user", "content": text["content"] + "\nWhat is the stable system code? Reply with only it."}], "max_tokens": 32, "temperature": 0, "chat_template_kwargs": {"enable_thinking": False}}, "/v1/chat/completions") + assert system.rsplit(" ", 1)[-1].rstrip(".") in result["choices"][0]["message"]["content"], "Prefix restore lost the system fact" + if args.require_prefix_cache and label in {"prefix-a-new", "prefix-a-metadata"}: + assert result["timings"]["cache_n"] >= 4096, "Client switching discarded the base prefix" + +if args.suite in {"workloads", "all"}: + for task in ["Implement a bounded asynchronous work queue in Python using asyncio. Explain cancellation, backpressure, shutdown, race conditions and error handling. Include tests and a detailed explanation.", "Explain how database isolation levels interact with concurrent read-modify-write operations. Give concrete schedules, anomalies, and remedies for each level, with detailed examples."]: + template, _ = request("/apply-template", {"messages": [{"role": "system", "content": "You are a careful software engineer."}, {"role": "user", "content": task}], "chat_template_kwargs": {"enable_thinking": True, "reasoning_effort": "medium"}}) + report("workload-code" if "asyncio" in task else "workload-reasoning", {"prompt": template["prompt"], "n_predict": 512, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0, "seed": 42, "cache_prompt": True}) + +if args.suite in {"retrieval", "all"}: + for size in args.contexts: + facts = ["START password is otter-731.", "MIDDLE password is maple-284.", "END password is comet-965."] + segment = tokens[:max(1, (size - 256) // 2)] + text, _ = request("/detokenize", {"tokens": segment}) + messages = [{"role": "system", "content": "Find the three passwords in the records. Reply with only the passwords in START, MIDDLE, END order."}, {"role": "user", "content": facts[0] + "\n" + text["content"] + "\n" + facts[1] + "\n" + text["content"] + "\n" + facts[2]}] + template, _ = request("/apply-template", {"messages": messages, "chat_template_kwargs": {"enable_thinking": False}}) + result = report(f"retrieval-{size}", {"prompt": template["prompt"], "n_predict": 64, "temperature": 0, "cache_prompt": True}) + positions = [result["content"].find(fact.split()[-1].rstrip(".")) for fact in facts] + passed = all(pos >= 0 for pos in positions) and positions == sorted(positions) + print(json.dumps({"case": f"retrieval-{size}", "passed": passed}), flush=True) + if not passed: + raise SystemExit("Retrieval failed") + +if args.suite in {"vision", "all"}: + png = b"\x89PNG\r\n\x1a\n" + pixels = b"".join(b"\0" + b"".join(b"\xff\0\0" if 128 <= x < 384 and 128 <= y < 384 else b"\xff\xff\xff" for x in range(512)) for y in range(512)) + for kind, data in [(b"IHDR", struct.pack(">IIBBBBB", 512, 512, 8, 2, 0, 0, 0)), (b"IDAT", zlib.compress(pixels)), (b"IEND", b"")]: + png += struct.pack(">I", len(data)) + kind + data + struct.pack(">I", binascii.crc32(kind + data)) + result = report("vision", {"model": "qwen3.8-27b", "messages": [{"role": "user", "content": [{"type": "text", "text": "What color is the square? Reply with one word."}, {"type": "image_url", "image_url": {"url": "data:image/png;base64," + base64.b64encode(png).decode()}}]}], "max_tokens": 32, "temperature": 0, "chat_template_kwargs": {"enable_thinking": False}}, "/v1/chat/completions") + assert "red" in result["choices"][0]["message"]["content"].lower(), "Vision failed" diff --git a/tools/local-ai.md b/tools/local-ai.md index 5bc1f92f0828ab95c9066a68eb341066e7484fa7..c392f7dd203efbd0622a34911ba9a1b86fa06975 100644 --- a/tools/local-ai.md +++ b/tools/local-ai.md @@ -1,14 +1,27 @@ # Snow Globe local AI The first model is Qwen3.8-27B, using Unsloth's UD-Q4_K_M GGUF and F16 vision -projector. Both files and the llama.cpp CUDA image are pinned. The model runs -entirely on the RTX 3090, with one inference slot and a 131,072-token context. +projector. Both files, the base CUDA image, and the llama.cpp source are pinned. +The model runs entirely on the RTX 3090, with one inference slot and a 131,072-token context. Codex compacts at 96,000 tokens to leave room for tool results and output. The KV cache uses Q8; the single slot leaves more VRAM for vision and runtime working memory. Additional requests queue so simultaneous generation cannot reduce the active request's decode rate. An 8 GiB host-memory prompt cache preserves evicted contexts between coding, reviewer, and helper requests. -The container has a 16 GiB RAM limit for the runtime and checkpoints. +The patched server also reserves a separate 2 GiB cache for prefixes before +the first user message, including system instructions and tool definitions. +A second checkpoint at 4,096 tokens retains the shared base when later workspace +metadata changes; shorter headers need only their complete checkpoint. +It captures these from actual requests; there are no copied Codex or Claude +prompts to update. Prefix states survive conversation-cache eviction and remain +available after restoration. Only complete token matches restore these states, +including the recurrent and MTP state. Prefix variants have their own bounded +LRU eviction; first use, unmatched tokens, and server restarts still need prefill. +The container has an 18 GiB RAM limit for the runtime and both caches. +Native MTP drafts up to three tokens, which the target model verifies. The draft +KV cache also uses Q8. Target weights, target KV precision, context length, and +reasoning defaults remain unchanged. The build replaces only the server library; +inference libraries and embedded UI assets come from the pinned base image. Medium reasoning is enabled for coding, using Qwen's recommended thinking-mode sampling parameters. This adds reasoning tokens before answers; the earlier latency baseline disabled thinking. @@ -68,6 +81,22 @@ running classification against the same local endpoint. Both use medium reasoning. Claude's credential store is isolated from ordinary Claude login, so using Snow Globe does not log the user's subscription out. +Qwen's raw thinking is streamed as Responses reasoning text, including the +`content_index` required by stock Codex. To display it in the terminal: + +```sh +snow-codex -c show_raw_agent_reasoning=true -c hide_agent_reasoning=false +``` + +Reasoning summaries remain disabled because the model emits raw thinking rather +than a separate summary. `tools/verify-local-ai.py` checks actual Codex app-server +reasoning deltas arrive before the answer; stored reasoning in a completed +response alone does not establish that live display works. The production fix +was checked with stock Codex 0.161.0: 237 live reasoning deltas arrived before +the answer, the interactive terminal updated during thinking, both coding +fixtures passed, and automatic approval authorized its temporary write in +21.4 seconds. + API and event-stream waits allow eight hours. Claude's byte-stream watchdog has a 30-minute maximum, kept alive by ten-second server SSE pings. Stock Codex 0.160.0 retains a hardcoded 90-second approval-review deadline that @@ -172,3 +201,47 @@ with synthetic files and credentials, separate from the personal coding profile. Evaluate coding, tool arguments, prompt injection, unauthorized writes, and attempted outbound requests against the base model. Passing those tests is evidence about observed behavior, not proof that an unrestricted agent is safe. + +## Runtime tuning (2026-10-07) + +Measurements on the production RTX 3090, using 256 greedy decode tokens with the +same target weights, Q8 KV cache, and context size: + +| Input context | Stock runtime | MTP, three Q8 draft tokens | +| --- | ---: | ---: | +| 16,384 tokens | 39.1 tok/s | 92.0 tok/s | +| 65,536 tokens | 30.3 tok/s | 73.3 tok/s | + +These are warm synthetic decode rates, not a promise for every coding task. +The n-gram draft trial did not improve throughput. MTP's target verifies every +accepted draft token; it does not substitute a smaller model's answers. +Cold prefill remains about 1,200 tok/s at 16K and 990 tok/s at 64K. Caching avoids +processing unchanged prefixes, but new long inputs still pay that cost. + +The production checks exercise stock Codex and Claude Code editing the same +median fixture, all eight unchanged tests, Codex's automatic approval reviewer, +vision, and three-fact retrieval in a 121,816-token input. They establish these +paths remain functional, not a broad model-quality evaluation. The initial +MTP/prefix-cache check with stock Codex 0.161.0 and Claude Code 2.1.293 took 68.6 and 120.4 +seconds while sharing the slot. All 30 other production containers retained +their identities; the AI allocation recorded no OOM events, limit hits, or +restarts during verification. + +`tools/benchmark-local-ai.py` measures decode, natural coding/reasoning workloads, +client switching, changing system metadata, retrieval, and vision. Run inside +the production container, where `LLAMA_API_KEY` is already inherited, against +the backend on `http://127.0.0.1:8081`; it emits timings without prompts or keys: + +```sh +python3 /tmp/benchmark-local-ai.py --suite cache --require-prefix-cache +python3 /tmp/benchmark-local-ai.py --suite decode --contexts 16384 65536 +python3 /tmp/benchmark-local-ai.py --suite retrieval --contexts 122000 +python3 /tmp/benchmark-local-ai.py --suite vision +``` + +The prefix test alternates two system headers and long unrelated user histories, +then changes a trailing system fact. Both restores must reuse at least 4,096 +tokens and answer from the current header. Production retained 4,114 tokens +across client switches and 4,096 after the system fact changed, with the new +fact answered correctly. Repeating the conversation retained 24,135 tokens +and completed in 0.335 seconds, including generation. diff --git a/tools/verify-local-ai.py b/tools/verify-local-ai.py index 9eddbbc5513ea9347f762488f9ebecbe3e9ef0cf..75d7779fb44ae941648e396cb536575ce8110ab2 100644 --- a/tools/verify-local-ai.py +++ b/tools/verify-local-ai.py @@ -5,6 +5,7 @@ import concurrent.futures from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import json import os +import selectors from pathlib import Path import subprocess import tempfile @@ -48,6 +49,54 @@ def run_client(root, provider, prompt, extra_env=None, label=None): return log +def verify_reasoning(root, endpoint): + env = os.environ.copy() + env.update(CODEX_HOME=str(root / "codex-home"), SNOWGLOBE_AI_URL=endpoint) + wrapper = Path(os.environ.get("SNOWGLOBE_AI_BIN_DIR", Path.home() / ".local/bin")) / "snow-codex" + with (root / "codex-reasoning-stderr.log").open("w") as errors, (root / "codex-reasoning.jsonl").open("wb") as log: + process = subprocess.Popen([str(wrapper), "-c", "show_raw_agent_reasoning=true", "-c", "hide_agent_reasoning=false", "app-server"], cwd=root / "codex", env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=errors, bufsize=0) + with selectors.DefaultSelector() as selector: + selector.register(process.stdout, selectors.EVENT_READ) + process.stdin.write(json.dumps({"id": 0, "method": "initialize", "params": {"clientInfo": {"name": "snowglobe_reasoning_test", "version": "1.0"}}}).encode() + b"\n") + reasoning = 0 + answer = "" + deadline = time.monotonic() + 240 + try: + while time.monotonic() < deadline: + if not selector.select(1): + continue + line = process.stdout.readline() + assert line, "Codex app-server closed before completing the reasoning test" + log.write(line) + event = json.loads(line) + assert "error" not in event, event.get("error") + if event.get("id") == 0: + process.stdin.write(b'{"method":"initialized","params":{}}\n') + process.stdin.write(json.dumps({"id": 1, "method": "thread/start", "params": {"cwd": str(root / "codex")}}).encode() + b"\n") + elif event.get("id") == 1: + process.stdin.write(json.dumps({"id": 2, "method": "turn/start", "params": {"threadId": event["result"]["thread"]["id"], "input": [{"type": "text", "text": "Do not use tools. Find the smallest positive integer divisible by every integer from 1 through 12. Give the integer and a concise justification."}]}}).encode() + b"\n") + method = event.get("method") + if method == "item/reasoning/textDelta": + reasoning += bool(event["params"]["delta"]) + elif method == "item/agentMessage/delta": + assert reasoning > 0, "Codex received no live reasoning before the answer" + answer += event["params"]["delta"] + elif method == "turn/completed": + assert event["params"]["turn"]["status"] == "completed" + assert reasoning > 0 and "27720" in answer.replace(",", ""), "Reasoning fixture failed" + print(f"Codex streamed {reasoning} reasoning deltas before its answer", flush=True) + break + else: + raise TimeoutError("Codex reasoning stream timed out") + finally: + process.stdin.close() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.terminate() + process.wait(timeout=5) + + def review_delay_test(root, delay, endpoint): delayed = threading.Event() class Proxy(BaseHTTPRequestHandler): @@ -120,5 +169,6 @@ if __name__ == "__main__": assert (work / "test_stats.py").read_text() == TESTS, f"{provider} changed the tests" subprocess.run(["python3", "-m", "unittest", "-v"], cwd=work, check=True) assert args.approval_delay >= 0, "Delay must be nonnegative." + verify_reasoning(root, args.url) review_delay_test(root, args.approval_delay, args.url) - print("Real coding and approval checks passed", flush=True) + print("Real coding, reasoning, and approval checks passed", flush=True)