authorclover caruso <git@paperclover.net> 2026-10-09 01:43:44-07:00
committerclover caruso <git@paperclover.net> 2026-10-09 14:54:07-07:00
log915ca77e341037b005a05bc3f03ebb37986c926c
tree25cef79ace5d21191d96fb5255c6a1355de5c8e4
parent17bf9c6cc0408a68f9c31cb923eed0f8f13da06a
signature Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU (~clover)

feat: switch qwen deployment to HyperQwen

Much faster and possibly more intelligent Qwen3.8-27B with 150k context, configured for one stream at a time. Benchmarks: BEFORE AFTER 16k 92 tok/s 118 tok/s 64k 73.3 tok/s 101 tok/s 148k --- 86 tok/s Assisted-by: gpt-6.1-sol Co-authored-by: amy <snovikov@ucsc.edu>

26 files changed, 1563 insertions(+), 1238 deletions(-)

config/site.pkl+1
...@@ -12,6 +12,7 @@ cloverUid: Int = (read?("prop:cloverUid") ?? read?("env:STUDIO_CLOVER_UID") ?? "...@@ -12,6 +12,7 @@ cloverUid: Int = (read?("prop:cloverUid") ?? read?("env:STUDIO_CLOVER_UID") ?? "
12cloverGid: Int = (read?("prop:cloverGid") ?? read?("env:STUDIO_CLOVER_GID") ?? "3000").toInt()12cloverGid: Int = (read?("prop:cloverGid") ?? read?("env:STUDIO_CLOVER_GID") ?? "3000").toInt()
13cloverReadOnly: Boolean = (read?("prop:cloverReadOnly") ?? read?("env:STUDIO_CLOVER_READ_ONLY") ?? "false") == "true"13cloverReadOnly: Boolean = (read?("prop:cloverReadOnly") ?? read?("env:STUDIO_CLOVER_READ_ONLY") ?? "false") == "true"
14mediaRoot: String = read?("prop:mediaRoot") ?? read?("env:STUDIO_MEDIA_ROOT") ?? "\(cloverRoot)/Media"14mediaRoot: String = read?("prop:mediaRoot") ?? read?("env:STUDIO_MEDIA_ROOT") ?? "\(cloverRoot)/Media"
15localAiSsdCache: String = read?("prop:localAiSsdCache") ?? read?("env:STUDIO_LOCAL_AI_SSD_CACHE") ?? "/var/cache/studio/local-ai"
15jellyfinFolders: Mapping<String, String> = new {16jellyfinFolders: Mapping<String, String> = new {
16 ["Anime"] = "Anime"17 ["Anime"] = "Anime"
17 ["Movies"] = "Movies"18 ["Movies"] = "Movies"
dashboard/ai/launch.sh+1-1
...@@ -70,5 +70,5 @@ else...@@ -70,5 +70,5 @@ else
70 export CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS=1 API_TIMEOUT_MS=28800000 API_FORCE_IDLE_TIMEOUT=070 export CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS=1 API_TIMEOUT_MS=28800000 API_FORCE_IDLE_TIMEOUT=0
71 export CLAUDE_STREAM_IDLE_TIMEOUT_MS=28800000 CLAUDE_BYTE_STREAM_IDLE_TIMEOUT_MS=180000071 export CLAUDE_STREAM_IDLE_TIMEOUT_MS=28800000 CLAUDE_BYTE_STREAM_IDLE_TIMEOUT_MS=1800000
72 export CLAUDE_ASYNC_AGENT_STALL_TIMEOUT_MS=2880000072 export CLAUDE_ASYNC_AGENT_STALL_TIMEOUT_MS=28800000
73 exec claude --permission-mode auto --append-system-prompt "Your model name is $model." "$@"73 exec claude --permission-mode auto --effort @REASONING@ --append-system-prompt "Your model name is $model." "$@"
74fi74fi
service/local-ai/Dockerfile created+30
...@@ -0,0 +1,30 @@
1FROM docker.io/nvidia/cuda@sha256:995e80db6d0c3a53d56bd00bba48a0ebd633b67b99a57e16acf9a306e7c744a7
2
3ENV DEBIAN_FRONTEND=noninteractive PIP_NO_CACHE_DIR=1 PYTHONUNBUFFERED=1
4RUN apt-get update && apt-get install -y --no-install-recommends \
5 python3.12 python3.12-venv python3.12-dev \
6 cuda-nvcc-13-0 cuda-cudart-dev-13-0 libcurand-dev-13-0 \
7 build-essential patch curl ca-certificates \
8 && rm -rf /var/lib/apt/lists/*
9WORKDIR /app
10RUN python3.12 -c 'import hashlib,urllib.request; data=urllib.request.urlopen("https://api.github.com/repos/syv-ai/HyperQwen/tarball/253c76aea0a240bf7cb2bd3ed92b672c0f258d0d").read(); assert hashlib.sha256(data).hexdigest()=="29b40ee9c371066e0194d23bcd35b08d4b6ec3efecd5799d02475fbbf6b032e3"; open("source.tar.gz","wb").write(data)' \
11 && tar -xzf source.tar.gz --strip-components=1 && rm source.tar.gz
12RUN python3.12 -m venv venv && venv/bin/pip install --upgrade pip \
13 && venv/bin/pip install -r docker/requirements.txt torch==2.13.0 \
14 safetensors==0.8.0 flashinfer-python==0.6.16.post3 flashinfer-cubin==0.6.13 \
15 && venv/bin/pip check
16COPY download.py models.json /tmp/model-source/
17RUN python3.12 /tmp/model-source/download.py --runtime-only --root /tmp/runtime \
18 && MODEL_DIR=$(python3.12 -c 'import json; print(json.load(open("/tmp/model-source/models.json"))["directory"])') \
19 && test "$(cat /tmp/runtime/$MODEL_DIR/runtime/BASE_REVISION.txt)" = 253c76aea0a240bf7cb2bd3ed92b672c0f258d0d \
20 && rm patches/*.patch \
21 && cp /tmp/runtime/$MODEL_DIR/runtime/patches/*.patch patches/ \
22 && cp /tmp/runtime/$MODEL_DIR/runtime/single-user/start_qwen.sh single-user/start_qwen.sh \
23 && rm -rf /tmp/runtime /tmp/model-source
24RUN set -e; SP=$(venv/bin/python -c 'import vllm,os; print(os.path.dirname(vllm.__file__))' | tail -n1); \
25 for p in patches/*.patch; do patch --batch -p1 -d "$SP" < "$p"; done; \
26 bash verify.sh --install
27COPY cache /tmp/studio-ai-cache
28RUN venv/bin/pip install /tmp/studio-ai-cache && venv/bin/pip check && rm -rf /tmp/studio-ai-cache
29ENV HOME=/cache VLLM_NO_USAGE_STATS=1 DO_NOT_TRACK=1 HF_HUB_OFFLINE=1
30RUN mkdir -p /cache && chmod 1777 /cache
service/local-ai/build/Dockerfile deleted-13
...@@ -1,13 +0,0 @@
1ARG RUNTIME=ghcr.io/ggml-org/llama.cpp@sha256:e8318a7b3988f9ca57b28c03486569e60def08f2b7b060550ebfa046e175e6cb
2FROM ${RUNTIME} AS build
3RUN rm -f /etc/apt/sources.list.d/* && printf 'deb https://archive.ubuntu.com/ubuntu noble main universe\ndeb https://archive.ubuntu.com/ubuntu noble-updates main universe\ndeb https://security.ubuntu.com/ubuntu noble-security main universe\n' > /etc/apt/sources.list && apt-get -o Acquire::https::Timeout=30 update && apt-get install -y --no-install-recommends cmake g++ make patch libssl-dev
4WORKDIR /src
5RUN python3 -c 'import hashlib,urllib.request; data=urllib.request.urlopen("https://api.github.com/repos/ggml-org/llama.cpp/tarball/11fe02151f79c41d0d4af7da708755d73b9c0da6").read(); assert hashlib.sha256(data).hexdigest()=="965db483b5e27c4917bac5bf83e0d2d533099049f8d5234c75e93c969d76443a"; open("source.tar.gz","wb").write(data)' && tar -xzf source.tar.gz --strip-components=1 && rm source.tar.gz
6COPY prefix-cache.patch /tmp/prefix-cache.patch
7RUN patch -p1 < /tmp/prefix-cache.patch
8COPY extract-ui.cpp /tmp/extract-ui.cpp
9RUN python3 -c 'import pathlib,subprocess; symbols=subprocess.check_output(["nm","-S","/app/libllama-server-impl.so"],text=True); size=int(next(line.split()[1] for line in symbols.splitlines() if line.split()[-1]=="_ZL8g_assets"),16); header=pathlib.Path("tools/ui/ui.h.in").read_text().replace("#cmakedefine LLAMA_UI_HAS_ASSETS 1","#define LLAMA_UI_HAS_ASSETS 1").replace("@N_ASSETS@",str(size)+"/sizeof(llama_ui_asset)"); pathlib.Path("/tmp/ui.h").write_text(header)' && g++ -std=c++17 -I/tmp /tmp/extract-ui.cpp -L/app -Wl,-rpath,/app -lllama-server-impl -o /tmp/extract-ui && LD_LIBRARY_PATH=/app /tmp/extract-ui && python3 -c 'import gzip,pathlib; [p.write_bytes(gzip.decompress(p.read_bytes())) for p in pathlib.Path("tools/ui/dist").rglob("*") if p.is_file()]'
10RUN cmake -S . -B /build -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DGGML_CUDA=OFF -DGGML_CPU=OFF -DGGML_BACKEND_DL=ON -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_UI=OFF -DLLAMA_USE_PREBUILT_UI=OFF -DLLAMA_BUILD_NUMBER=11382 -DLLAMA_BUILD_COMMIT=11fe02151 && cmake --build /build --target llama-ui-assets && ln -s /app/libllama.so* /app/libllama-common.so* /app/libmtmd.so* /app/libggml*.so* /build/bin/ && cmake --build /build --target server-context/fast cpp-httplib/fast llama-common-base/fast llama-ui/fast -j 4 && cmake --build /build --target llama-server-impl/fast -j 4
11
12FROM ${RUNTIME}
13COPY --from=build /build/bin/libllama-server-impl.so /app/libllama-server-impl.so
service/local-ai/build/extract-ui.cpp deleted-18
...@@ -1,18 +0,0 @@
1#include "ui.h"
2#include <filesystem>
3#include <fstream>
4
5int main() {
6 if (!llama_ui_use_gzip()) {
7 return 1;
8 }
9 for (const auto & asset : llama_ui_get_assets()) {
10 const auto path = std::filesystem::path("tools/ui/dist") / asset.name;
11 std::filesystem::create_directories(path.parent_path());
12 std::ofstream output(path, std::ios::binary);
13 output.write(reinterpret_cast<const char *>(asset.data), asset.size);
14 if (!output) {
15 return 1;
16 }
17 }
18}
service/local-ai/build/prefix-cache.patch deleted-237
...@@ -1,237 +0,0 @@
1--- a/tools/server/server-task.h
2+++ b/tools/server/server-task.h
3@@ -650,12 +650,13 @@
4 };
5
6 struct server_prompt_cache {
7- server_prompt_cache(int32_t limit_size_mib, size_t limit_tokens) {
8+ server_prompt_cache(int32_t limit_size_mib, size_t limit_tokens, bool keep_on_load = false) : keep_on_load(keep_on_load) {
9 this->limit_size = 1024ull*1024ull*(limit_size_mib < 0 ? 0 : limit_size_mib);
10 this->limit_tokens = limit_tokens;
11 }
12
13 std::list<server_prompt_cache_state> states;
14+ const bool keep_on_load;
15
16 // in bytes, 0 = no limit
17 size_t limit_size = 0;
18--- a/tools/server/server-task.cpp
19+++ b/tools/server/server-task.cpp
20@@ -1226,6 +1226,7 @@
21 {"event", "response.reasoning_text.delta"},
22 {"data", json {
23 {"type", "response.reasoning_text.delta"},
24+ {"content_index", 0},
25 {"delta", diff.reasoning_content_delta},
26 {"item_id", oai_resp_reasoning_id},
27 }},
28@@ -1724,7 +1725,10 @@
29 for (auto it = states.begin(); it != states.end(); ++it) {
30 const int cur_lcp_len = it->prompt.tokens.get_common_prefix(prompt.tokens);
31
32- if (cur_lcp_len == (int) prompt.tokens.size()) {
33+ if (cur_lcp_len == (int) prompt.tokens.size() && (!keep_on_load || it->prompt.tokens.size() == prompt.tokens.size())) {
34+ if (keep_on_load) {
35+ states.splice(states.end(), states, it);
36+ }
37 SRV_TRC("%s", " - prompt is already in the cache, skipping\n");
38 return nullptr;
39 }
40@@ -1749,7 +1753,7 @@
41 for (auto it = states.begin(); it != states.end();) {
42 const int len = it->prompt.tokens.get_common_prefix(prompt.tokens);
43
44- if (len == (int) it->prompt.tokens.size()) {
45+ if (!keep_on_load && len == (int) it->prompt.tokens.size()) {
46 SRV_TRC(" - removing obsolete cached prompt with length %d\n", len);
47
48 it = states.erase(it);
49@@ -1820,12 +1824,18 @@
50
51 SRV_TRC(" - prompt with length %7zu, lcp = %7d, f_keep = %.3f, f_sim = %.3f\n", it->prompt.tokens.size(), lcp_cur, f_keep_cur, f_sim_cur);
52
53+ // Only complete pinned prefixes contain a usable recurrent state.
54+ if (keep_on_load && lcp_cur != (int) it->prompt.tokens.size()) {
55+ continue;
56+ }
57+
58 // don't trash large prompts
59 if (f_keep_cur < 0.25f) {
60 continue;
61 }
62
63- if (f_keep_best < f_keep_cur && f_sim_best < f_sim_cur) {
64+ if ((keep_on_load || f_keep_best < f_keep_cur) &&
65+ (f_sim_best < f_sim_cur || (keep_on_load && f_sim_best == f_sim_cur && f_keep_best < f_keep_cur))) {
66 f_keep_best = f_keep_cur;
67 f_sim_best = f_sim_cur;
68
69@@ -1847,8 +1857,10 @@
70 return false;
71 }
72
73- data.clear();
74- data.shrink_to_fit();
75+ if (!keep_on_load) {
76+ data.clear();
77+ data.shrink_to_fit();
78+ }
79 }
80
81 {
82@@ -1865,14 +1877,20 @@
83 return false;
84 }
85
86- data.clear();
87- data.shrink_to_fit();
88+ if (!keep_on_load) {
89+ data.clear();
90+ data.shrink_to_fit();
91+ }
92 }
93 }
94
95- prompt = std::move(it_best->prompt);
96-
97- states.erase(it_best);
98+ if (keep_on_load) {
99+ prompt = it_best->prompt.clone();
100+ states.splice(states.end(), states, it_best);
101+ } else {
102+ prompt = std::move(it_best->prompt);
103+ states.erase(it_best);
104+ }
105 }
106
107 return true;
108--- a/tools/server/server-context.cpp
109+++ b/tools/server/server-context.cpp
110@@ -954,6 +954,7 @@
111 int n_empty_consecutive = 0;
112
113 std::unique_ptr<server_prompt_cache> prompt_cache;
114+ std::unique_ptr<server_prompt_cache> prefix_cache;
115
116 server_metrics metrics;
117
118@@ -1424,6 +1425,7 @@
119 SRV_TRC("%s", "use `--cache-ram 0` to disable the prompt cache\n");
120
121 prompt_cache = std::make_unique<server_prompt_cache>(params_base.cache_ram_mib, n_ctx);
122+ prefix_cache = std::make_unique<server_prompt_cache>(2048, n_ctx, true);
123 } else {
124 SRV_TRC("%s", "prompt cache is disabled - use `--cache-ram N` to enable it\n");
125 }
126@@ -1721,6 +1723,11 @@
127
128 SRV_TRC("prompt cache update took %.2f ms\n", (ggml_time_us() - t_start) / 1000.0);
129 }
130+ }
131+
132+ if (ret && prefix_cache && task.type == SERVER_TASK_TYPE_COMPLETION &&
133+ !ret->prompt_load(*prefix_cache, task.tokens)) {
134+ ret->prompt_clear();
135 }
136
137 return ret;
138@@ -3091,7 +3098,7 @@
139 // add generated tokens to cache
140 // ref: https://github.com/ggml-org/llama.cpp/pull/16818#discussion_r2473269481
141 {
142- GGML_ASSERT(!slot.prompt.tokens.has_mtmd);
143+ GGML_ASSERT(!slot.prompt.tokens.find_next_media_chunk(0).first);
144
145 llama_tokens new_tokens = slot.prompt.tokens.get_tokens(); // copy
146 for (size_t i = n_keep + n_discard; i < new_tokens.size(); i++) {
147@@ -3400,7 +3407,7 @@
148
149 // reuse chunks from the cached prompt by shifting their KV cache in the new position
150 if (can_cache_reuse && n_cache_reuse > 0) {
151- GGML_ASSERT(!slot.prompt.tokens.has_mtmd);
152+ GGML_ASSERT(!slot.prompt.tokens.find_next_media_chunk(0).first);
153
154 size_t head_c = n_past; // cache
155 size_t head_p = n_past; // current prompt
156@@ -3701,6 +3708,11 @@
157
158 const auto & spans = slot.task->params.message_spans;
159 const auto last_user_pos = spans.last_user_message_pos();
160+ const auto first_user = std::find_if(spans.spans.begin(), spans.spans.end(), [](const auto & span) {
161+ return span.role == COMMON_CHAT_ROLE_USER;
162+ });
163+ const int64_t first_user_pos = first_user == spans.spans.end() ? -1 : first_user->pos;
164+ const int64_t prefix_pos = std::min<int64_t>(4096, first_user_pos);
165
166 const int32_t n_decision_first = slot.task->type == SERVER_TASK_TYPE_DECISION ? slot.task->decision.pos_first() : -1;
167
168@@ -3743,12 +3755,16 @@
169 }
170 slot.prompt.tokens.push_back(cur_tok);
171
172+ if (do_checkpoint && prefix_cache && slot.prompt.n_tokens() == prefix_pos) {
173+ break;
174+ }
175+
176 // break at the last user message, or at user messages at least min step past the last checkpoint
177 if (do_checkpoint && spans.is_user_start(slot.prompt.n_tokens())) {
178 const auto pos = slot.prompt.n_tokens();
179 const auto & checkpoints = slot.prompt.checkpoints;
180
181- if (pos == last_user_pos || checkpoints.empty() || pos > checkpoints.back().n_tokens + params_base.checkpoint_min_step) {
182+ if (pos == first_user_pos || pos == last_user_pos || checkpoints.empty() || pos > checkpoints.back().n_tokens + params_base.checkpoint_min_step) {
183 break;
184 }
185 }
186@@ -3783,6 +3799,8 @@
187 const bool near_prompt_end = slot.task->n_tokens() < slot.prompt.n_tokens() + n_ubatch;
188
189 const bool is_user_start = spans.is_user_start(n_tokens_start);
190+ const bool is_prefix_boundary = prefix_cache && n_tokens_start > 0 &&
191+ (n_tokens_start == prefix_pos || n_tokens_start == first_user_pos);
192 const bool is_last_user_message = n_tokens_start == last_user_pos;
193
194 // entire prompt has been processed
195@@ -3799,9 +3817,8 @@
196
197 slot.init_sampler();
198 } else {
199- // skip ordinary mid-prompt checkpoints, unless the batch starts a user
200- // message or we are near the end of the prompt
201- if (!is_user_start && !near_prompt_end) {
202+ // Retain system-prefix boundaries even inside a message.
203+ if (!is_user_start && !is_prefix_boundary && !near_prompt_end) {
204 do_checkpoint = false;
205 }
206 }
207@@ -3821,7 +3838,7 @@
208 // no need to create checkpoints that are too close together, unless it's the last user message
209 do_checkpoint = do_checkpoint && (
210 slot.prompt.checkpoints.empty() ||
211- is_last_user_message || near_prompt_end ||
212+ is_last_user_message || is_prefix_boundary || near_prompt_end ||
213 n_tokens_start > slot.prompt.checkpoints.back().n_tokens + params_base.checkpoint_min_step);
214 SLT_DBG(slot, "main/do_checkpoint = %s, pos_min = %d, pos_max = %d\n", do_checkpoint ? "yes" : "no", pos_min, pos_max);
215
216@@ -3829,6 +3846,21 @@
217 // yet processed and therefore it is not part of the checkpoint.
218 if (do_checkpoint) {
219 create_checkpoint(slot, n_tokens_cur, pos_min, pos_max);
220+ if (is_prefix_boundary &&
221+ !slot.prompt.tokens.find_next_media_chunk(0).first) {
222+ server_prompt prefix {slot.prompt.tokens.clone(), {slot.prompt.checkpoints.back()}};
223+ prefix.tokens.keep_first(n_tokens_start);
224+ const size_t size = llama_state_seq_get_size_ext(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_NONE);
225+ const size_t draft_size = ctx_dft ? llama_state_seq_get_size_ext(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_NONE) : 0;
226+ if (auto * state = prefix_cache->alloc(prefix, size, draft_size)) {
227+ llama_state_seq_get_data_ext(ctx_tgt, state->data.main.data(), size, slot.id, LLAMA_STATE_SEQ_FLAGS_NONE);
228+ if (ctx_dft) {
229+ llama_state_seq_get_data_ext(ctx_dft, state->data.drft.data(), draft_size, slot.id, LLAMA_STATE_SEQ_FLAGS_NONE);
230+ }
231+ prefix_cache->update();
232+ SLT_INF(slot, "cached base prefix, n_tokens = %d\n", n_tokens_start);
233+ }
234+ }
235 }
236 }
237
service/local-ai/cache/pyproject.toml created+14
...@@ -0,0 +1,14 @@
1[build-system]
2requires = ["setuptools"]
3build-backend = "setuptools.build_meta"
4
5[project]
6name = "studio-ai-cache"
7version = "1.0.0"
8dependencies = ["cryptography==50.0.2"]
9
10[project.entry-points."vllm.general_plugins"]
11studio_ai_cache = "studio_ai_cache_vllm:register"
12
13[tool.setuptools]
14py-modules = ["studio_ai_cache", "studio_ai_cache_vllm"]
service/local-ai/cache/studio_ai_cache.py created+186
...@@ -0,0 +1,186 @@
1"""Bounded, authenticated disk caches with SSD promotion and request pinning."""
2from collections import OrderedDict
3import fcntl
4import logging
5import mmap
6import os
7from pathlib import Path
8import struct
9import threading
10
11from cryptography.exceptions import InvalidTag
12from cryptography.hazmat.primitives.ciphers.aead import AESGCM
13
14
15class DiskCache:
16 def __init__(self, ssd, hdd, ssd_bytes, hdd_bytes, key):
17 if min(ssd_bytes, hdd_bytes) <= 0:
18 raise ValueError("Disk cache capacities must be positive")
19 self.cipher = AESGCM(key)
20 self.roots = (Path(ssd), Path(hdd))
21 if (self.roots[0].resolve().is_relative_to(self.roots[1].resolve())
22 or self.roots[1].resolve().is_relative_to(self.roots[0].resolve())):
23 raise ValueError("Disk cache directories must be distinct")
24 self.limits = (ssd_bytes, hdd_bytes)
25 self.lock = threading.Lock()
26 self.writers = (threading.Lock(), threading.Lock())
27 self.entries = (OrderedDict(), OrderedDict())
28 self.pins = {}
29 self.files = []
30 try:
31 for tier, root in enumerate(self.roots):
32 root.mkdir(mode=0o700, parents=True, exist_ok=True)
33 root.chmod(0o700)
34 descriptor = os.open(root / ".lock", os.O_CREAT | os.O_RDWR, 0o600)
35 self.files.append(descriptor)
36 fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
37 for path in root.rglob("*.pending"):
38 path.unlink()
39 inventory = sorted((path.stat().st_mtime_ns, path) for path in root.rglob("*.bin"))
40 for _, path in inventory:
41 self.entries[tier][str(path.relative_to(root))] = path.stat().st_size
42 self._evict(tier, 0)
43 except Exception:
44 self.close()
45 raise
46
47 def close(self):
48 for descriptor in self.files:
49 os.close(descriptor)
50 self.files.clear()
51
52 def acquire(self, name, request):
53 with self.lock:
54 for entries in self.entries:
55 if name in entries:
56 entries.move_to_end(name)
57 self.pins.setdefault(name, set()).add(request)
58 return True
59 return False
60
61 def release(self, request):
62 with self.lock:
63 for name, requests in list(self.pins.items()):
64 requests.discard(request)
65 if not requests:
66 del self.pins[name]
67
68 def touch(self, name):
69 with self.lock:
70 for entries in self.entries:
71 if name in entries:
72 entries.move_to_end(name)
73 break
74
75 def _evict(self, tier, needed):
76 removed = []
77 with self.lock:
78 entries = self.entries[tier]
79 size = sum(entries.values())
80 for name in list(entries):
81 if size + needed <= self.limits[tier]:
82 break
83 if name not in self.pins:
84 size -= entries.pop(name)
85 removed.append(self.roots[tier] / name)
86 for path in removed:
87 self._unlink(tier, path)
88 return size + needed <= self.limits[tier]
89
90 def _unlink(self, tier, path):
91 path.unlink(missing_ok=True)
92 parent = path.parent
93 while parent != self.roots[tier]:
94 try:
95 parent.rmdir()
96 except OSError:
97 break
98 parent = parent.parent
99
100 def _put(self, tier, name, buffer):
101 path = self.roots[tier] / name
102 with self.writers[tier]:
103 with self.lock:
104 if name in self.entries[tier]:
105 self.entries[tier].move_to_end(name)
106 present = True
107 else:
108 present = False
109 if present:
110 os.utime(path, None)
111 return
112 if len(buffer) > self.limits[tier] or not self._evict(tier, len(buffer)):
113 return
114 path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
115 pending = path.with_suffix(".pending")
116 try:
117 descriptor = os.open(pending, os.O_CREAT | os.O_EXCL | os.O_WRONLY | getattr(os, "O_DIRECT", 0), 0o600)
118 try:
119 if os.write(descriptor, buffer) != len(buffer):
120 raise OSError("Short cache write")
121 os.fsync(descriptor)
122 finally:
123 os.close(descriptor)
124 pending.replace(path)
125 with self.lock:
126 self.entries[tier][name] = len(buffer)
127 finally:
128 pending.unlink(missing_ok=True)
129
130 def store(self, name, data):
131 # Page-aligned ciphertext keeps direct I/O on both ext4 and ZFS.
132 size = (len(data) + 36 + mmap.PAGESIZE - 1) // mmap.PAGESIZE * mmap.PAGESIZE
133 with mmap.mmap(-1, size) as buffer:
134 nonce = os.urandom(12)
135 buffer[:20] = struct.pack("!Q", len(data)) + nonce
136 view = memoryview(buffer)[20:20 + len(data) + 16]
137 try:
138 self.cipher.encrypt_into(nonce, data, name.encode(), view)
139 finally:
140 view.release()
141 self._put(1, name, buffer)
142 self._put(0, name, buffer)
143
144 def load(self, name, target):
145 pin = object()
146 if not self.acquire(name, pin):
147 raise FileNotFoundError("Cache block was evicted")
148 try:
149 for tier, root in enumerate(self.roots):
150 path = root / name
151 with self.lock:
152 size = self.entries[tier].get(name)
153 if size is None:
154 continue
155 try:
156 with mmap.mmap(-1, size) as buffer:
157 descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECT", 0))
158 try:
159 if os.readv(descriptor, [buffer]) != size:
160 raise OSError("Short cache read")
161 finally:
162 os.close(descriptor)
163 if struct.unpack("!Q", buffer[:8])[0] != len(target):
164 raise ValueError("Cache block size mismatch")
165 view = memoryview(buffer)[20:20 + len(target) + 16]
166 try:
167 self.cipher.decrypt_into(buffer[8:20], view, name.encode(), target)
168 finally:
169 view.release()
170 with self.lock:
171 self.entries[tier].move_to_end(name)
172 os.utime(path, None)
173 if tier == 1:
174 try:
175 self._put(0, name, buffer)
176 except OSError:
177 logging.getLogger(__name__).warning("SSD cache promotion failed", exc_info=True)
178 return "ssd" if tier == 0 else "hdd"
179 except (OSError, ValueError, struct.error, InvalidTag):
180 with self.writers[tier]:
181 with self.lock:
182 self.entries[tier].pop(name, None)
183 self._unlink(tier, path)
184 raise FileNotFoundError("No authenticated cache block remains")
185 finally:
186 self.release(pin)
service/local-ai/cache/studio_ai_cache_vllm.py created+73
...@@ -0,0 +1,73 @@
1"""vLLM secondary tier for the bounded SSD/HDD cache."""
2import functools
3import hashlib
4import os
5from pathlib import Path
6
7import vllm
8from vllm.v1.kv_offload.base import LookupResult, Medium
9from vllm.v1.kv_offload.file_mapper import FileMapper
10from vllm.v1.kv_offload.tiering.base import JobResult, RequestOffloadingContext, SecondaryTierManager
11from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory
12from vllm.v1.kv_offload.tiering.fs.thread_pool import DualQueueThreadPool
13
14from studio_ai_cache import DiskCache
15
16
17class DiskTier(SecondaryTierManager):
18 medium = Medium.STORAGE
19
20 def __init__(self, offloading_spec, primary_kv_view, tier_type, ssd_dir, hdd_dir, ssd_bytes, hdd_bytes):
21 super().__init__(offloading_spec, primary_kv_view, tier_type)
22 key = hashlib.sha256(os.environ["STUDIO_AI_CACHE_KEY"].encode()).digest()
23 namespace = hashlib.sha256(
24 Path("/models/manifest.json").read_bytes() + vllm.__version__.encode()
25 + Path(__file__).read_bytes() + Path(__file__).with_name("studio_ai_cache.py").read_bytes() + key
26 ).hexdigest()
27 self.mapper = FileMapper.from_offloading_spec(namespace, offloading_spec, offloading_spec.blocks_per_chunk)
28 self.cache = DiskCache(ssd_dir, hdd_dir, ssd_bytes, hdd_bytes, key)
29 self.block_size = primary_kv_view.strides[0]
30 self.pool = DualQueueThreadPool(16, 2, thread_name_prefix="studio_kv_disk")
31
32 def on_new_request(self, req_context):
33 return RequestOffloadingContext()
34
35 def lookup(self, key, req_context):
36 return LookupResult.HIT if self.cache.acquire(self.mapper.get_file_name(key), req_context.req_id) else LookupResult.MISS
37
38 def submit_store(self, job_metadata):
39 tasks = [functools.partial(self._transfer, self.cache.store, key, block_id)
40 for key, block_id in zip(job_metadata.keys, job_metadata.block_ids, strict=True)]
41 self.pool.enqueue_store(job_metadata.job_id, len(tasks), tasks)
42
43 def submit_load(self, job_metadata):
44 tasks = [functools.partial(self._transfer, self.cache.load, key, block_id)
45 for key, block_id in zip(job_metadata.keys, job_metadata.block_ids, strict=True)]
46 self.pool.enqueue_load(job_metadata.job_id, len(tasks), tasks)
47
48 def _transfer(self, operation, key, block_id):
49 offset = int(block_id) * self.block_size
50 with self._primary_kv_view.cast("B")[offset:offset + self.block_size] as view:
51 operation(self.mapper.get_file_name(key), view)
52
53 def get_finished_jobs(self):
54 return [JobResult(job_id, success) for job_id, success in self.pool.get_finished()]
55
56 def touch(self, keys, req_context):
57 for key in keys:
58 self.cache.touch(self.mapper.get_file_name(key))
59
60 def on_request_finished(self, req_context):
61 self.cache.release(req_context.req_id)
62
63 def drain_jobs(self):
64 self.pool.wait_idle()
65
66 def shutdown(self):
67 self.pool.wait_idle()
68 self.pool.shutdown(wait=True)
69 self.cache.close()
70
71
72def register():
73 SecondaryTierFactory.register_tier("studio_disk", __name__, "DiskTier")
service/local-ai/cache/test_cache.py created+115
...@@ -0,0 +1,115 @@
1from concurrent.futures import ThreadPoolExecutor
2import os
3import mmap
4from pathlib import Path
5import tempfile
6import unittest
7
8from studio_ai_cache import DiskCache
9
10
11class CacheTests(unittest.TestCase):
12 def setUp(self):
13 self.temporary = tempfile.TemporaryDirectory()
14 self.addCleanup(self.temporary.cleanup)
15 self.root = Path(self.temporary.name)
16 self.key = os.urandom(32)
17 self.cache = DiskCache(self.root / "ssd", self.root / "hdd", 2 * mmap.PAGESIZE, 3 * mmap.PAGESIZE, self.key)
18 self.addCleanup(self.cache.close)
19
20 def test_hdd_hit_promotes_to_ssd_and_both_limits_hold(self):
21 for name in ("a.bin", "b.bin", "c.bin"):
22 self.cache.store(name, name.encode() * 100)
23 target = bytearray(500)
24 self.assertEqual(self.cache.load("a.bin", target), "hdd")
25 self.assertEqual(target, b"a.bin" * 100)
26 self.assertEqual(self.cache.load("a.bin", target), "ssd")
27 for directory, limit in (("ssd", 2 * mmap.PAGESIZE), ("hdd", 3 * mmap.PAGESIZE)):
28 self.assertLessEqual(sum(p.stat().st_size for p in (self.root / directory).glob("*.bin")), limit)
29
30 def test_overflow_evicts_cold_blocks(self):
31 for name in ("a.bin", "b.bin", "c.bin", "d.bin", "e.bin"):
32 self.cache.store(name, b"fixture")
33 self.assertFalse(self.cache.acquire("a.bin", "request"))
34 self.assertFalse(self.cache.acquire("b.bin", "request"))
35 self.assertTrue(self.cache.acquire("c.bin", "request"))
36 self.cache.release("request")
37
38 def test_pinned_request_survives_eviction_and_full_cache_skips_store(self):
39 self.cache.close()
40 self.cache = DiskCache(self.root / "ssd", self.root / "hdd", mmap.PAGESIZE, mmap.PAGESIZE, self.key)
41 self.addCleanup(self.cache.close)
42 self.cache.store("a.bin", b"first")
43 self.assertTrue(self.cache.acquire("a.bin", "request"))
44 self.cache.store("b.bin", b"other")
45 self.assertFalse(self.cache.acquire("b.bin", "other-request"))
46 target = bytearray(5)
47 self.cache.load("a.bin", target)
48 self.assertEqual(target, b"first")
49 self.cache.release("request")
50 self.cache.store("b.bin", b"other")
51 self.assertFalse(self.cache.acquire("a.bin", "request"))
52
53 def test_restart_recovers_encrypted_blocks_with_restricted_permissions(self):
54 payload = b"synthetic private prompt fixture" * 10
55 self.cache.store("namespace/a.bin", payload)
56 for path in self.root.rglob("*.bin"):
57 self.assertNotIn(payload, path.read_bytes())
58 self.assertEqual(path.stat().st_mode & 0o777, 0o600)
59 self.cache.close()
60 restored = DiskCache(self.root / "ssd", self.root / "hdd", 2 * mmap.PAGESIZE, 3 * mmap.PAGESIZE, self.key)
61 self.addCleanup(restored.close)
62 target = bytearray(len(payload))
63 self.assertEqual(restored.load("namespace/a.bin", target), "ssd")
64 self.assertEqual(target, payload)
65
66 def test_corrupt_ssd_falls_back_to_hdd_and_repairs_hot_copy(self):
67 self.cache.store("a.bin", b"authenticated fixture")
68 path = self.root / "ssd/a.bin"
69 contents = bytearray(path.read_bytes())
70 contents[20] ^= 1
71 path.write_bytes(contents)
72 target = bytearray(21)
73 self.assertEqual(self.cache.load("a.bin", target), "hdd")
74 self.assertEqual(target, b"authenticated fixture")
75 self.assertEqual(self.cache.load("a.bin", target), "ssd")
76
77 def test_corrupt_both_copies_becomes_cache_miss(self):
78 self.cache.store("a.bin", b"fixture")
79 for path in self.root.rglob("*.bin"):
80 path.write_bytes(b"broken")
81 with self.assertRaises(FileNotFoundError):
82 self.cache.load("a.bin", bytearray(7))
83 self.assertFalse(self.cache.acquire("a.bin", "request"))
84
85 def test_wrong_key_and_different_namespace_cannot_reuse_state(self):
86 self.cache.store("old/a.bin", b"fixture")
87 self.assertFalse(self.cache.acquire("new/a.bin", "request"))
88 self.cache.close()
89 other = DiskCache(self.root / "ssd", self.root / "hdd", 2 * mmap.PAGESIZE, 3 * mmap.PAGESIZE, os.urandom(32))
90 self.addCleanup(other.close)
91 with self.assertRaises(FileNotFoundError):
92 other.load("old/a.bin", bytearray(7))
93
94 def test_directories_are_exclusive(self):
95 with self.assertRaises(BlockingIOError):
96 DiskCache(self.root / "ssd", self.root / "hdd", 2 * mmap.PAGESIZE, 3 * mmap.PAGESIZE, self.key)
97
98 def test_concurrent_loads_keep_request_pins_until_release(self):
99 self.cache.store("a.bin", b"fixture")
100 self.assertTrue(self.cache.acquire("a.bin", "request"))
101
102 def exercise(index):
103 self.cache.store(f"{index}.bin", b"other")
104 target = bytearray(7)
105 self.cache.load("a.bin", target)
106 self.assertEqual(target, b"fixture")
107
108 with ThreadPoolExecutor(max_workers=8) as pool:
109 list(pool.map(exercise, range(24)))
110 self.cache.release("request")
111 self.assertEqual(self.cache.pins, {})
112
113
114if __name__ == "__main__":
115 unittest.main()
service/local-ai/cache/test_vllm_cache.py created+97
...@@ -0,0 +1,97 @@
1"""CPU-only checks against the installed vLLM scheduler and shared-memory tier."""
2import mmap
3import os
4from pathlib import Path
5import tempfile
6import unittest
7import uuid
8
9from vllm.v1.kv_offload.base import LookupResult, ReqContext, ScheduleEndContext, make_offload_key
10from vllm.v1.kv_offload.config import OffloadingCacheConfig, OffloadingConfig, OffloadingGroupConfig, OffloadingModelConfig, OffloadingParallelConfig
11from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
12from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec
13from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory
14from vllm.v1.kv_offload.tiering.manager import CPUPrimaryTierOffloadingManager, TieringOffloadingManager
15
16from studio_ai_cache_vllm import register
17
18
19class VllmCacheTests(unittest.TestCase):
20 @classmethod
21 def setUpClass(cls):
22 register()
23
24 def setUp(self):
25 self.temporary = tempfile.TemporaryDirectory()
26 self.addCleanup(self.temporary.cleanup)
27 self.root = Path(self.temporary.name)
28 os.environ["STUDIO_AI_CACHE_KEY"] = "disposable-synthetic-cache-key"
29 config = OffloadingConfig(
30 groups=(OffloadingGroupConfig(16, ("attention",)), OffloadingGroupConfig(16, ("recurrent",))),
31 worker_kv_bytes_per_block=mmap.PAGESIZE,
32 enable_kv_cache_events=False, extra_config={"cpu_bytes_to_use": 4 * mmap.PAGESIZE},
33 engine_id="synthetic", model=OffloadingModelConfig("synthetic", "uint8"),
34 cache=OffloadingCacheConfig(16, 1),
35 parallel=OffloadingParallelConfig(0, 1, 1, 1, 1, 1, 0, False),
36 )
37 spec = CPUOffloadingSpec(config)
38 region = SharedOffloadRegion("synthetic-" + uuid.uuid4().hex, 4, None, mmap.PAGESIZE, mmap.PAGESIZE)
39 primary = CPUPrimaryTierOffloadingManager(4, region)
40 self.view = primary.get_kv_memoryview()
41 self.tier = SecondaryTierFactory.create_secondary_tier({
42 "type": "studio_disk", "ssd_dir": str(self.root / "ssd"), "hdd_dir": str(self.root / "hdd"),
43 "ssd_bytes": 4 * mmap.PAGESIZE, "hdd_bytes": 6 * mmap.PAGESIZE,
44 }, self.view, spec)
45 self.manager = TieringOffloadingManager(primary, [self.tier])
46 self.addCleanup(self.manager.shutdown)
47 self.keys = [make_offload_key(bytes([i + 1]) * 32, i % 2) for i in range(3)]
48 self.values = [bytes([i + 11]) * mmap.PAGESIZE for i in range(3)]
49 context = ReqContext("store")
50 self.manager.on_new_request(context)
51 stored = self.manager.prepare_store(self.keys, context)
52 with self.view.cast("B") as flat:
53 for block_id, value in zip(stored.store_spec.block_ids, self.values, strict=True):
54 offset = int(block_id) * mmap.PAGESIZE
55 flat[offset:offset + mmap.PAGESIZE] = value
56 self.manager.complete_store(self.keys, context)
57 self.tier.drain_jobs()
58 self.manager.on_schedule_end(ScheduleEndContext((), ()))
59 self.manager.on_request_finished(context)
60 self.manager.reset_cache()
61 with self.view.cast("B") as flat:
62 flat[:] = bytes(len(flat))
63
64 def test_disk_hit_restores_attention_and_recurrent_groups_into_ram(self):
65 context = ReqContext("restore")
66 self.manager.on_new_request(context)
67 for key, value in zip(self.keys, self.values, strict=True):
68 self.assertEqual(self.manager.lookup(key, context), LookupResult.RETRY)
69 self.manager.on_schedule_end(ScheduleEndContext((), ()))
70 self.tier.drain_jobs()
71 self.assertEqual(self.manager.lookup(key, context), LookupResult.HIT)
72 restored = self.manager.prepare_load([key], context)
73 offset = int(restored.block_ids[0]) * mmap.PAGESIZE
74 with self.view.cast("B") as flat:
75 self.assertEqual(bytes(flat[offset:offset + mmap.PAGESIZE]), value)
76 self.manager.complete_load([key], context)
77 self.manager.on_schedule_end(ScheduleEndContext((), ()))
78 self.manager.on_request_finished(context)
79
80 def test_corrupt_disk_promotion_returns_miss_to_scheduler(self):
81 key = self.keys[0]
82 name = self.tier.mapper.get_file_name(key)
83 for root in self.tier.cache.roots:
84 path = root / name
85 if path.exists():
86 path.write_bytes(b"invalid")
87 context = ReqContext("corrupt")
88 self.manager.on_new_request(context)
89 self.assertEqual(self.manager.lookup(key, context), LookupResult.RETRY)
90 self.manager.on_schedule_end(ScheduleEndContext((), ()))
91 self.tier.drain_jobs()
92 self.assertEqual(self.manager.lookup(key, context), LookupResult.MISS)
93 self.manager.on_request_finished(context)
94
95
96if __name__ == "__main__":
97 unittest.main()
service/local-ai/catalog.json+1-1
...@@ -34,7 +34,7 @@...@@ -34,7 +34,7 @@
34 "mode": "tokens",34 "mode": "tokens",
35 "limit": 600035 "limit": 6000
36 },36 },
37 "context_window": 131072,37 "context_window": 150000,
38 "auto_compact_token_limit": 96000,38 "auto_compact_token_limit": 96000,
39 "effective_context_window_percent": 90,39 "effective_context_window_percent": 90,
40 "experimental_supported_tools": [],40 "experimental_supported_tools": [],
service/local-ai/download.py+6-2
...@@ -14,14 +14,17 @@ def checksum(path):...@@ -14,14 +14,17 @@ def checksum(path):
14 return hashlib.file_digest(source, "sha256").hexdigest()14 return hashlib.file_digest(source, "sha256").hexdigest()
1515
1616
17def download(root):17def download(root, runtime_only=False):
18 manifest = json.loads(Path(__file__).with_name("models.json").read_text())18 manifest = json.loads(Path(__file__).with_name("models.json").read_text())
19 directory = root / manifest["directory"]19 directory = root / manifest["directory"]
20 directory.mkdir(parents=True, exist_ok=True)20 directory.mkdir(parents=True, exist_ok=True)
21 with (directory / ".download.lock").open("a") as lock:21 with (directory / ".download.lock").open("a") as lock:
22 fcntl.flock(lock, fcntl.LOCK_EX)22 fcntl.flock(lock, fcntl.LOCK_EX)
23 for spec in manifest["files"]:23 for spec in manifest["files"]:
24 if runtime_only and not spec["name"].startswith("runtime/"):
25 continue
24 target = directory / spec["name"]26 target = directory / spec["name"]
27 target.parent.mkdir(parents=True, exist_ok=True)
25 if target.exists():28 if target.exists():
26 if target.stat().st_size == spec["bytes"] and checksum(target) == spec["sha256"]:29 if target.stat().st_size == spec["bytes"] and checksum(target) == spec["sha256"]:
27 print(f"Verified {target.name}", flush=True)30 print(f"Verified {target.name}", flush=True)
...@@ -47,6 +50,7 @@ def download(root):...@@ -47,6 +50,7 @@ def download(root):
47if __name__ == "__main__":50if __name__ == "__main__":
48 parser = argparse.ArgumentParser(description="Download local AI models")51 parser = argparse.ArgumentParser(description="Download local AI models")
49 parser.add_argument("--root", type=Path, default=Path("/srv/clover/Media/AI/LLM"))52 parser.add_argument("--root", type=Path, default=Path("/srv/clover/Media/AI/LLM"))
53 parser.add_argument("--runtime-only", action="store_true")
50 args = parser.parse_args()54 args = parser.parse_args()
51 os.umask(0o027)55 os.umask(0o027)
52 download(args.root)56 download(args.root, args.runtime_only)
service/local-ai/gateway.py+7-4
...@@ -1,5 +1,5 @@...@@ -1,5 +1,5 @@
1#!/usr/bin/env python31#!/usr/bin/env python3
2"""Adapt current Responses tool namespaces to llama.cpp; forward other APIs unchanged."""2"""Adapt Responses tool namespaces to vLLM; forward other APIs unchanged."""
3import hmac3import hmac
4import http.client4import http.client
5from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer5from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
...@@ -120,6 +120,8 @@ def normalize_anthropic(request):...@@ -120,6 +120,8 @@ def normalize_anthropic(request):
120 if instructions:120 if instructions:
121 data["system"] = "\n\n".join(instructions)121 data["system"] = "\n\n".join(instructions)
122 data["messages"] = messages122 data["messages"] = messages
123 if (data.get("output_config") or {}).get("effort") in {"high", "max"}:
124 data["output_config"]["effort"] = "xhigh"
123 return data125 return data
124126
125127
...@@ -151,7 +153,7 @@ class Handler(BaseHTTPRequestHandler):...@@ -151,7 +153,7 @@ class Handler(BaseHTTPRequestHandler):
151 return153 return
152 # Authenticate before allocating inference bodies.154 # Authenticate before allocating inference bodies.
153 if self.path != "/health":155 if self.path != "/health":
154 key = os.environ["LLAMA_API_KEY"]156 key = os.environ["VLLM_API_KEY"]
155 supplied = self.headers.get("x-api-key", "") or self.headers.get("Authorization", "").removeprefix("Bearer ")157 supplied = self.headers.get("x-api-key", "") or self.headers.get("Authorization", "").removeprefix("Bearer ")
156 if not hmac.compare_digest(supplied, key):158 if not hmac.compare_digest(supplied, key):
157 self.send_error(401)159 self.send_error(401)
...@@ -166,7 +168,8 @@ class Handler(BaseHTTPRequestHandler):...@@ -166,7 +168,8 @@ class Handler(BaseHTTPRequestHandler):
166 body = json.dumps(data).encode()168 body = json.dumps(data).encode()
167 elif path in {"/v1/messages", "/v1/messages/count_tokens"}:169 elif path in {"/v1/messages", "/v1/messages/count_tokens"}:
168 body = json.dumps(normalize_anthropic(json.loads(body))).encode()170 body = json.dumps(normalize_anthropic(json.loads(body))).encode()
169 headers = {key: val for key, val in self.headers.items() if key.lower() not in HOP_HEADERS}171 headers = {key: val for key, val in self.headers.items() if key.lower() not in HOP_HEADERS | {"authorization", "x-api-key"}}
172 headers["Authorization"] = "Bearer " + os.environ["VLLM_API_KEY"]
170 connection.request(self.command, self.path, body=body or None, headers=headers)173 connection.request(self.command, self.path, body=body or None, headers=headers)
171 response = connection.getresponse()174 response = connection.getresponse()
172 streaming = "text/event-stream" in response.getheader("Content-Type", "")175 streaming = "text/event-stream" in response.getheader("Content-Type", "")
...@@ -222,7 +225,7 @@ class Handler(BaseHTTPRequestHandler):...@@ -222,7 +225,7 @@ class Handler(BaseHTTPRequestHandler):
222225
223226
224if __name__ == "__main__":227if __name__ == "__main__":
225 backend = subprocess.Popen(["/app/llama-server", *sys.argv[1:]])228 backend = subprocess.Popen(["bash", "/app/single-user/start_qwen.sh"])
226 server = ThreadingHTTPServer(("0.0.0.0", 8080), Handler)229 server = ThreadingHTTPServer(("0.0.0.0", 8080), Handler)
227 def stop(signum, frame):230 def stop(signum, frame):
228 if backend.poll() is None:231 if backend.poll() is None:
service/local-ai/models.json+739-9
...@@ -1,17 +1,747 @@...@@ -1,17 +1,747 @@
1{1{
2 "repository": "unsloth/Qwen3.8-27B-GGUF",2 "repository": "daavidhauser/Swift-1.5-Qwen3.8-27B-W4A16-HyperQwen-INT4",
3 "revision": "4ca720788d1e01f1bff70c033e0d0028fd02e502",3 "revision": "64515d381e36084182e4e2abfa20050b75f29889",
4 "directory": "Qwen3.8-27B",4 "directory": "Swift-1.5-Qwen3.8-27B-W4A16-HyperQwen-INT4",
5 "files": [5 "files": [
6 {6 {
7 "name": "Qwen3.8-27B-UD-Q4_K_M.gguf",7 "name": ".gitattributes",
8 "bytes": 16464440224,8 "bytes": 1570,
9 "sha256": "322e194ff79741c7baa497c240f677f54b201b0efab44ca8e50f122b39123482"9 "sha256": "34448b82c17d60fec9b65b1f093c115ddbaadc04beb1b0140b6bfed2e012a930"
10 },10 },
11 {11 {
12 "name": "mmproj-F16.gguf",12 "name": "LICENSE",
13 "bytes": 927607488,13 "bytes": 13306,
14 "sha256": "cbb841a9ee0636b2ec172f5bb8df2ea8dfeb01e90fe7c6126581d662a0b4e43e"14 "sha256": "1367057bf17041aa1d69286a1400be5f464d2f8b8f54f600088b209eac1850be"
15 },
16 {
17 "name": "LICENSE-APACHE-2.0",
18 "bytes": 11544,
19 "sha256": "bbedc3fda3305820b977265f01b8619d87570a6739de3a5582c3464840f1e57a"
20 },
21 {
22 "name": "NOTICE",
23 "bytes": 1389,
24 "sha256": "e0e788911888bdcc087d34fc12dc5bf3818034762334de3575ea34d9559b186f"
25 },
26 {
27 "name": "QUANTIZATION_MANIFEST.json",
28 "bytes": 8434,
29 "sha256": "40ebbba877dd55e1c512e68785f3127673d57ac799c5ea7139761b563ed7de4b"
30 },
31 {
32 "name": "README.md",
33 "bytes": 5620,
34 "sha256": "de13ff8b19a75a4d6dcda0c9a36e276d3a5b19b74a44ef012bbfe39562f24a85"
35 },
36 {
37 "name": "RELEASE_MANIFEST.json",
38 "bytes": 23100,
39 "sha256": "1fb06fb256c1e29323e03ec56a1c1e99b8bfc5078a2c57c5426cb74052023eeb"
40 },
41 {
42 "name": "RUNTIME.md",
43 "bytes": 2608,
44 "sha256": "4a6a153fdf00ece6ddd976ee745951db61c5a36a69ba06eaf102a1ec1f5c3c74"
45 },
46 {
47 "name": "chat_template.jinja",
48 "bytes": 8952,
49 "sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041"
50 },
51 {
52 "name": "config.json",
53 "bytes": 22382,
54 "sha256": "deffb953c20dafbcdc07199063fa5e203dc574432dd3c7efac3f8e2882bd46a7"
55 },
56 {
57 "name": "evaluation/code/swift15/checkpoint.py",
58 "bytes": 3656,
59 "sha256": "c43707f3623659cd882cfbb57506b152912f42f2a043dfd148c1cf829090f28d"
60 },
61 {
62 "name": "evaluation/code/swift15/code_runner.py",
63 "bytes": 2144,
64 "sha256": "23d52e36543766bb33d33f11adad3772e7836d5170e83f68e326cc2d00b55fd9"
65 },
66 {
67 "name": "evaluation/code/swift15/common.py",
68 "bytes": 1439,
69 "sha256": "0410dc972e74d8bbc05e090b0543426e4ea57ed707a80a114ea2e658822e1e8c"
70 },
71 {
72 "name": "evaluation/code/swift15/corpus.py",
73 "bytes": 7800,
74 "sha256": "d5b420de1f3a1d4be97a44321abdd393247b5d78a56199ad49a62058c98d9050"
75 },
76 {
77 "name": "evaluation/code/swift15/eval_data.py",
78 "bytes": 8097,
79 "sha256": "2a4497758d456a3cdd930aa711d0c510b24a2eaf6eb11be3da3e94481d4a166e"
80 },
81 {
82 "name": "evaluation/code/swift15/evaluate.py",
83 "bytes": 14113,
84 "sha256": "4d553cf10b42cbff0eeacfe72b3b6df87405aa2604fcbe19a5a0a78133cd18af"
85 },
86 {
87 "name": "evaluation/code/swift15/generate.py",
88 "bytes": 4412,
89 "sha256": "55e2659ce67295a4c57ffd12beed9dc0246808d908939c4113f7d1bb506af089"
90 },
91 {
92 "name": "evaluation/code/swift15/ifbench_score.py",
93 "bytes": 876,
94 "sha256": "1b97f5859020d8d992944a85357ee7477f8740e65574d5946d2305889244a04d"
95 },
96 {
97 "name": "evaluation/code/swift15/measure.py",
98 "bytes": 4726,
99 "sha256": "f955ace33396730c9fdf6fe5d424b6a2f77eb0fc58c83b3bd84b0c804c7a3b37"
100 },
101 {
102 "name": "evaluation/code/swift15/quantize.py",
103 "bytes": 8098,
104 "sha256": "a3f65ca5de15d54cf626cde340365e503462d1a33ae8525aa55f6159195886f7"
105 },
106 {
107 "name": "evaluation/code/swift15/reference.py",
108 "bytes": 1786,
109 "sha256": "0433adf523a676c0cb30c5ad25b90bdc5a4565c2b69180e171aa4a4d59c15bf8"
110 },
111 {
112 "name": "evaluation/code/swift15/release.py",
113 "bytes": 7409,
114 "sha256": "c5129121b4ba333b0c4dd2694fb69076cdc75e4b9cda8a6572b3a449c32c3b5e"
115 },
116 {
117 "name": "evaluation/code/swift15/report.py",
118 "bytes": 5287,
119 "sha256": "622af3dd5ab1eaa1843407c5bfb57e3111a154463c024a6685095a7aa293b482"
120 },
121 {
122 "name": "evaluation/code/swift15/run.py",
123 "bytes": 9281,
124 "sha256": "e5d42332192e5bafdde88ae4acc20802f5c5477e6447ff5165fb6b836aa78ea2"
125 },
126 {
127 "name": "evaluation/code/swift15/runtime_compat.py",
128 "bytes": 1885,
129 "sha256": "bcbc7351a6a4719bf38c06658bb12cbcfc9eb31f9280fc9b916782832544dbe6"
130 },
131 {
132 "name": "evaluation/code/swift15/serve.py",
133 "bytes": 6020,
134 "sha256": "c705174eda86e57ae1a51c0b2e7a93f9b0e1f37fa32217a9739b569f52d2d56d"
135 },
136 {
137 "name": "evaluation/code/swift15/setup.py",
138 "bytes": 2219,
139 "sha256": "7de02ccb5575e0ac2b62d33842d28aacfc83d63fc96eedad9cbedf1129e90495"
140 },
141 {
142 "name": "evaluation/code/swift15/smoke.py",
143 "bytes": 3162,
144 "sha256": "e2d4a01d6613e667ceeef0d1a03ef1cec04edf3ab0518aaa4a006d419a91f1fe"
145 },
146 {
147 "name": "evaluation/code/swift15/test_workflow.py",
148 "bytes": 3468,
149 "sha256": "8d2dcb057318d482530ef9bf7b4bcf0d59347b594531e0df18b81dbea0beaddd"
150 },
151 {
152 "name": "evaluation/environment.json",
153 "bytes": 3282,
154 "sha256": "2e9ac10516648b0c634eaca007d58456d241e0c1ef1b6734fa51671bf315a841"
155 },
156 {
157 "name": "evaluation/manifest.json",
158 "bytes": 2685,
159 "sha256": "82c1c6eaad6c06ad91e20cac5f9dae6b74ceaba0a3063c868a6a4313366e7948"
160 },
161 {
162 "name": "evaluation/perplexity-english-python.json",
163 "bytes": 38213,
164 "sha256": "5c0f9ab2df9a8d4e14ea846cd03e1c3cb577847a022a955f97ff57ef6392fe85"
165 },
166 {
167 "name": "evaluation/pilot-ids.json",
168 "bytes": 354,
169 "sha256": "44cffc01ab88b366f20da82d3150c7de8eb1d7801cd1381570e32313f120da41"
170 },
171 {
172 "name": "evaluation/qwen-fast-full/manifest.json",
173 "bytes": 14395,
174 "sha256": "3de5a8af19d01a53594d7f29349e19bf90f1bbe8d935e82fe33aa0cf8325391b"
175 },
176 {
177 "name": "evaluation/qwen-fast-full/summary.json",
178 "bytes": 3134,
179 "sha256": "a2ffcd9ed991279b5639274596a5139b39553d407753adc48ff465e55fd2eaf6"
180 },
181 {
182 "name": "evaluation/qwen-fast-pilot/manifest.json",
183 "bytes": 2663,
184 "sha256": "f06f4539feaa46b92d559427cd13809ecf3e8b0fc44b15768e54392a0ce9c34f"
185 },
186 {
187 "name": "evaluation/qwen-fast-pilot/summary.json",
188 "bytes": 2979,
189 "sha256": "8a0ec26f335c5a0c98bac1db6bd852a5292b0b5d1bfcc6a218a0b9aa1ce9b1c7"
190 },
191 {
192 "name": "evaluation/qwen-fast/perplexity.json",
193 "bytes": 12570,
194 "sha256": "9db1599421dc916b97602a8921dbd219b815a5a76fc7def41f508500eb10e7e7"
195 },
196 {
197 "name": "evaluation/qwen-fast/server.json",
198 "bytes": 1942,
199 "sha256": "b2988235fd03e4cd30366b5ca418ba5bb687c1ab58c1eb29e3e53c57612ac0ae"
200 },
201 {
202 "name": "evaluation/qwen-fast/speed-c1.json",
203 "bytes": 630,
204 "sha256": "3d70c2868a2f2832e6bf7562abd3fa4589389baf87e02dbd89c76b154e49238d"
205 },
206 {
207 "name": "evaluation/swift10-full/manifest.json",
208 "bytes": 14501,
209 "sha256": "6af95e530b920a84a3c5f5dde5b76bc2c791d0a4cc5b3043686bd6946cac4acf"
210 },
211 {
212 "name": "evaluation/swift10-full/summary.json",
213 "bytes": 3125,
214 "sha256": "1c431ee8cc16e36090152526f8553c3e76cb5f66e5ddd5c6df1f90c3b3426add"
215 },
216 {
217 "name": "evaluation/swift10-pilot/manifest.json",
218 "bytes": 2769,
219 "sha256": "83df9bd1a41e0d96f62039ad1941593e4cafc57c52f63851eeec4523048f4315"
220 },
221 {
222 "name": "evaluation/swift10-pilot/summary.json",
223 "bytes": 2981,
224 "sha256": "3f8aed7d263dc65abb0095bc3af73515052645886f5f0565e6c21e82afa27bf8"
225 },
226 {
227 "name": "evaluation/swift10/perplexity.json",
228 "bytes": 12560,
229 "sha256": "fe393bd4962d43f9cef2a08ea40c0dd2f15e6695b87a09a20e184d448d941775"
230 },
231 {
232 "name": "evaluation/swift10/server.json",
233 "bytes": 2041,
234 "sha256": "d6004a26531972be5407c0f542ec0b708c931e21f67801e5206906d4cb148547"
235 },
236 {
237 "name": "evaluation/swift10/speed-c1.json",
238 "bytes": 632,
239 "sha256": "a82f761f6a6ca5446808b1e8b129fee4e5b3b733787e286a7829b23f6dd1722f"
240 },
241 {
242 "name": "evaluation/swift15-baseline-full/manifest.json",
243 "bytes": 14389,
244 "sha256": "b122ac2835f4c868522aa6982e1c44e2090ae176b27293cd8a71040108751d18"
245 },
246 {
247 "name": "evaluation/swift15-baseline-full/summary.json",
248 "bytes": 3091,
249 "sha256": "cd0ec9d98535c6c32c2ae9f32798f81c0dae628ebda0d1d56f6005df85edb91b"
250 },
251 {
252 "name": "evaluation/swift15-baseline-pilot/manifest.json",
253 "bytes": 2657,
254 "sha256": "588d9a88f35aa9207934bde6cbce4e0956868287316f964585b784b9d4f7cec3"
255 },
256 {
257 "name": "evaluation/swift15-baseline-pilot/summary.json",
258 "bytes": 2982,
259 "sha256": "511e69269b8d98d64876ab7033e124fc1d8aca196b022fbd27b34f65867ac8ef"
260 },
261 {
262 "name": "evaluation/swift15-baseline/perplexity.json",
263 "bytes": 12553,
264 "sha256": "6f63e857421957c76b3ca290ec8c95a62fd4374e3839adc2cdcdca54f14786fa"
265 },
266 {
267 "name": "evaluation/swift15-baseline/server.json",
268 "bytes": 1937,
269 "sha256": "9063a7b84cd89e83672d673d5fc30ff563d01adebd1a705fcae70c33b5ba47b4"
270 },
271 {
272 "name": "evaluation/swift15-baseline/speed-c1.json",
273 "bytes": 629,
274 "sha256": "079f1c870da2484a43ec214ee2ced3723b2636c83eaed4291636e121b4e81e3a"
275 },
276 {
277 "name": "evaluation/swift15-fast-full/manifest.json",
278 "bytes": 14393,
279 "sha256": "d242d0bd7f947c37a74430190ce4a1309c2ae8e2f5ba76313e1ba056044aa134"
280 },
281 {
282 "name": "evaluation/swift15-fast-full/summary.json",
283 "bytes": 2969,
284 "sha256": "3ffd7dc76fa4105089688731c6a8365519f80567acbca3b1bc15b64e93de7ab3"
285 },
286 {
287 "name": "evaluation/swift15-fast-pilot/manifest.json",
288 "bytes": 2661,
289 "sha256": "13203d3b56258d785ea74ddd3f89322edb6c58245f7d4ecf60756a7af66d4f60"
290 },
291 {
292 "name": "evaluation/swift15-fast-pilot/summary.json",
293 "bytes": 2979,
294 "sha256": "fe44f8972f40cf5236ca93d1ee7362cfea238a03e58f09c6e033312820fc8d89"
295 },
296 {
297 "name": "evaluation/swift15-fast/perplexity.json",
298 "bytes": 12561,
299 "sha256": "3fa2ac71c6c24072036837c3ae0afcf96ce03e85a09b276874c75432aeb091a1"
300 },
301 {
302 "name": "evaluation/swift15-fast/server.json",
303 "bytes": 1940,
304 "sha256": "53099847622dbdd7416196bfd8a7591aca1adb64db92049cf2be6ffdf489536d"
305 },
306 {
307 "name": "evaluation/swift15-fast/speed-c1.json",
308 "bytes": 629,
309 "sha256": "88d292f00ba50773e80c0dbf6ba9a1e18d3413737221d2ff8c5a969749747108"
310 },
311 {
312 "name": "generation_config.json",
313 "bytes": 202,
314 "sha256": "e70c136c1b78ddc1fb0905bac8e733a4dc448d4f852a5dd75143fffc70be550e"
315 },
316 {
317 "name": "hyperqwen-audit.json",
318 "bytes": 18861,
319 "sha256": "62c8d7543e2e52d4025e7f72577753c029b1447762cfc781e8d06a6547c72795"
320 },
321 {
322 "name": "hyperqwen-build.json",
323 "bytes": 8398,
324 "sha256": "895d4e0dd27f896ed2a3a0e2160c59967cd10e8506f690b6a85c5cf90c529cec"
325 },
326 {
327 "name": "hyperqwen_provenance/generation-manifest.json",
328 "bytes": 460,
329 "sha256": "f8dc684e9940b3efe9f7967214495adb83d4b906cb442ef46924c42a90308853"
330 },
331 {
332 "name": "hyperqwen_provenance/generation-summary.json",
333 "bytes": 204,
334 "sha256": "6aa261c7b4b9952edd1d7fc0b61f57985894d1b360571f02448364820f0adbd0"
335 },
336 {
337 "name": "hyperqwen_provenance/lm-head-report.json",
338 "bytes": 234,
339 "sha256": "3fe25c5ed9c1dbf85a1e5baa705036303515d8ad0dccb9ef54abcca63528d2b4"
340 },
341 {
342 "name": "hyperqwen_provenance/manifest.json",
343 "bytes": 2106,
344 "sha256": "83f789e2d83a31b90564811ae804f1cda210e7ad62a7c7dd6ddde79f04ad5438"
345 },
346 {
347 "name": "hyperqwen_provenance/mtp-report.json",
348 "bytes": 1059,
349 "sha256": "e1cd6016c4a231be292dd73d128ba863f035320ee71de5bd983504cede3d711c"
350 },
351 {
352 "name": "hyperqwen_provenance/mtp_hessians.pt.json",
353 "bytes": 15773,
354 "sha256": "f69a5c6132cb5382a8aaedd7dddfbdcd050e193ec2b1c525feb95949ca8ac834"
355 },
356 {
357 "name": "hyperqwen_provenance/preflight-validation.json",
358 "bytes": 1444,
359 "sha256": "d4a00e873a3da9e29cf9ae5b864b4b2eaf992263705305651c77aa02a1ab3f9e"
360 },
361 {
362 "name": "hyperqwen_provenance/recipe/drafter/capture.py",
363 "bytes": 5744,
364 "sha256": "6bb7cff9e235999923fa83b994034be6c9246b45fa7d89cca6fabf067f030e9b"
365 },
366 {
367 "name": "hyperqwen_provenance/recipe/drafter/gptq_utils.py",
368 "bytes": 3809,
369 "sha256": "e0478db6fd5692a8960cd5ad16786655f09b73d6fe864382371c1455f9394c69"
370 },
371 {
372 "name": "hyperqwen_provenance/recipe/drafter/train_mtp.py",
373 "bytes": 24898,
374 "sha256": "900951feef2e0ff7d729f9baac02e9125b5ec16de5680009e368e20139b79faf"
375 },
376 {
377 "name": "hyperqwen_provenance/recipe/prepare/build_draft_vocab.py",
378 "bytes": 6161,
379 "sha256": "426f371a8db9f465c48b56ae05cbc782f0613980005405deac2ad95504d088bc"
380 },
381 {
382 "name": "hyperqwen_provenance/recipe/prepare/quant_heads_stream.py",
383 "bytes": 9473,
384 "sha256": "67bac25a15df11ed569866491d8bf21550829e0281bb74a8c9ec092107fca905"
385 },
386 {
387 "name": "hyperqwen_provenance/recipe/swift15/README.md",
388 "bytes": 5869,
389 "sha256": "83564e691d256395a5c49138bfa65840f226f4c51c373c9e336acb4af08c0467"
390 },
391 {
392 "name": "hyperqwen_provenance/recipe/swift15/checkpoint.py",
393 "bytes": 3656,
394 "sha256": "c43707f3623659cd882cfbb57506b152912f42f2a043dfd148c1cf829090f28d"
395 },
396 {
397 "name": "hyperqwen_provenance/recipe/swift15/code_runner.py",
398 "bytes": 2144,
399 "sha256": "23d52e36543766bb33d33f11adad3772e7836d5170e83f68e326cc2d00b55fd9"
400 },
401 {
402 "name": "hyperqwen_provenance/recipe/swift15/common.py",
403 "bytes": 1439,
404 "sha256": "0410dc972e74d8bbc05e090b0543426e4ea57ed707a80a114ea2e658822e1e8c"
405 },
406 {
407 "name": "hyperqwen_provenance/recipe/swift15/corpus.py",
408 "bytes": 7800,
409 "sha256": "d5b420de1f3a1d4be97a44321abdd393247b5d78a56199ad49a62058c98d9050"
410 },
411 {
412 "name": "hyperqwen_provenance/recipe/swift15/eval_data.py",
413 "bytes": 8097,
414 "sha256": "2a4497758d456a3cdd930aa711d0c510b24a2eaf6eb11be3da3e94481d4a166e"
415 },
416 {
417 "name": "hyperqwen_provenance/recipe/swift15/evaluate.py",
418 "bytes": 12978,
419 "sha256": "dfb48598358dd97c2b52eb41384b797cf2a4fc9d35cba91b0b2efb84abb73d8c"
420 },
421 {
422 "name": "hyperqwen_provenance/recipe/swift15/generate.py",
423 "bytes": 4412,
424 "sha256": "55e2659ce67295a4c57ffd12beed9dc0246808d908939c4113f7d1bb506af089"
425 },
426 {
427 "name": "hyperqwen_provenance/recipe/swift15/ifbench_score.py",
428 "bytes": 876,
429 "sha256": "1b97f5859020d8d992944a85357ee7477f8740e65574d5946d2305889244a04d"
430 },
431 {
432 "name": "hyperqwen_provenance/recipe/swift15/measure.py",
433 "bytes": 4726,
434 "sha256": "f955ace33396730c9fdf6fe5d424b6a2f77eb0fc58c83b3bd84b0c804c7a3b37"
435 },
436 {
437 "name": "hyperqwen_provenance/recipe/swift15/quantize.py",
438 "bytes": 8098,
439 "sha256": "a3f65ca5de15d54cf626cde340365e503462d1a33ae8525aa55f6159195886f7"
440 },
441 {
442 "name": "hyperqwen_provenance/recipe/swift15/release.py",
443 "bytes": 7409,
444 "sha256": "c5129121b4ba333b0c4dd2694fb69076cdc75e4b9cda8a6572b3a449c32c3b5e"
445 },
446 {
447 "name": "hyperqwen_provenance/recipe/swift15/report.py",
448 "bytes": 4712,
449 "sha256": "8e73730ee54a449bb1028b8626f2ad973bed40428f7b49a8780b077cc63dcc92"
450 },
451 {
452 "name": "hyperqwen_provenance/recipe/swift15/run.py",
453 "bytes": 8169,
454 "sha256": "d60a6780eda3ee20ab4b878cfa4fa223edfdc4a5a099c622e05ee06ff61c303f"
455 },
456 {
457 "name": "hyperqwen_provenance/recipe/swift15/runtime_compat.py",
458 "bytes": 1885,
459 "sha256": "bcbc7351a6a4719bf38c06658bb12cbcfc9eb31f9280fc9b916782832544dbe6"
460 },
461 {
462 "name": "hyperqwen_provenance/recipe/swift15/serve.py",
463 "bytes": 4076,
464 "sha256": "f80daeb04c86d07b8ce463e1305c9c84d6ab4dbe2eb2322b14970b282bdcdbb3"
465 },
466 {
467 "name": "hyperqwen_provenance/recipe/swift15/setup.py",
468 "bytes": 2219,
469 "sha256": "7de02ccb5575e0ac2b62d33842d28aacfc83d63fc96eedad9cbedf1129e90495"
470 },
471 {
472 "name": "hyperqwen_provenance/recipe/swift15/smoke.py",
473 "bytes": 3162,
474 "sha256": "e2d4a01d6613e667ceeef0d1a03ef1cec04edf3ab0518aaa4a006d419a91f1fe"
475 },
476 {
477 "name": "hyperqwen_provenance/recipe/swift15/test_workflow.py",
478 "bytes": 3029,
479 "sha256": "221c0b3e6cb04fe4aaec2c47059c329dfa4e06c4084b9b01dfea3ab62032c677"
480 },
481 {
482 "name": "hyperqwen_provenance/runtime-compatibility.json",
483 "bytes": 869,
484 "sha256": "4b627e696f1e5dcda170364fc7a11b83fdbbad0c434933171db4cc76c97e91d5"
485 },
486 {
487 "name": "hyperqwen_provenance/upstream-QUANTIZATION_MANIFEST.json",
488 "bytes": 1243,
489 "sha256": "37a75a5056e4d7267cfacfd11fd830f6fe0c084fc3688bead9842b9f0fbfc121"
490 },
491 {
492 "name": "hyperqwen_provenance/upstream-README.md",
493 "bytes": 21151,
494 "sha256": "1fa3bbf8f6177e23bc1dcf98ce990f889ecad8c3f032a010c41559d34cbee263"
495 },
496 {
497 "name": "hyperqwen_provenance/upstream-UPLOAD_MANIFEST.json",
498 "bytes": 5176,
499 "sha256": "b475d1d0e4b383c036be5a396c1a65dc2b45b3af1463da859049a24c65fe6854"
500 },
501 {
502 "name": "hyperqwen_provenance/upstream-recipe.yaml",
503 "bytes": 930,
504 "sha256": "49f446271f2278c633d540768d42caf82114fc2353ca526d1279fcc1984b7f68"
505 },
506 {
507 "name": "hyperqwen_provenance/vocabulary-report.json",
508 "bytes": 362,
509 "sha256": "44b8a205f3c8419ad179f5c5ae89a9d7b2ca143e241bfde8f96e132637c9f809"
510 },
511 {
512 "name": "merges.txt",
513 "bytes": 3353259,
514 "sha256": "a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d"
515 },
516 {
517 "name": "model-00001-of-00006.safetensors",
518 "bytes": 655565120,
519 "sha256": "1e2bb79bd7feff42b3b006b3552354c1ff2c4a02941a788080839b0eb868118d"
520 },
521 {
522 "name": "model-00002-of-00006.safetensors",
523 "bytes": 2730582048,
524 "sha256": "48ddebd951d8d8a9ecef51b1857403ca77e5ab2c63b2e7db4822e0908e2eeaac"
525 },
526 {
527 "name": "model-00003-of-00006.safetensors",
528 "bytes": 3966050624,
529 "sha256": "61bbd4ae1e4812f08aa8f903e9995746aea8c346a666b530d0a6774773350f1e"
530 },
531 {
532 "name": "model-00004-of-00006.safetensors",
533 "bytes": 3966050704,
534 "sha256": "0c7baff5aac711178c5ce841da1ebcbebf67bde21155e74f4adca5d4354c178a"
535 },
536 {
537 "name": "model-00005-of-00006.safetensors",
538 "bytes": 3993185016,
539 "sha256": "0cc1b9b1cde51d81fd7f53d91716399c30c4a5a166503dbd76ac65c63bb881c9"
540 },
541 {
542 "name": "model-00006-of-00006.safetensors",
543 "bytes": 248265904,
544 "sha256": "65c1248e479775518d38a1675859380890a76c455f25de37b613bef71281a4fb"
545 },
546 {
547 "name": "model-mtp-bf16.safetensors",
548 "bytes": 219027824,
549 "sha256": "39ac3a5071ac107640468e1a5c3d3844d39eba277076c2d97cf404974dad3d5d"
550 },
551 {
552 "name": "model.safetensors.index.json",
553 "bytes": 243540,
554 "sha256": "e99357b5714c15037f02da3f6d46dd569742edde177df6f8f5c46505bba9e562"
555 },
556 {
557 "name": "model_extra_tensors.safetensors",
558 "bytes": 173015384,
559 "sha256": "bdb17eed6815f17f20a324d25f3777533cb224c463075f1ab6afd199bde8628a"
560 },
561 {
562 "name": "mtp_draft_vocab_ids.pt",
563 "bytes": 525949,
564 "sha256": "a7f5b236bd7b1892c24a004d12f93cdf6910a3ca2afb2b490cc08ab985b6d7c9"
565 },
566 {
567 "name": "preprocessor_config.json",
568 "bytes": 390,
569 "sha256": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516"
570 },
571 {
572 "name": "recipe.yaml",
573 "bytes": 930,
574 "sha256": "49f446271f2278c633d540768d42caf82114fc2353ca526d1279fcc1984b7f68"
575 },
576 {
577 "name": "runtime/BASE_REVISION.txt",
578 "bytes": 41,
579 "sha256": "c1f6fbc4227edb7daa83504c936afc8e23d8e31e8b62ff375fbbb5e29e81e546"
580 },
581 {
582 "name": "runtime/LICENSE",
583 "bytes": 11358,
584 "sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"
585 },
586 {
587 "name": "runtime/batch/start_qwen.sh",
588 "bytes": 11677,
589 "sha256": "ac6a84b455317537a230bc190ccc06dda169dcdbdeee4bebeb51e76032972944"
590 },
591 {
592 "name": "runtime/patches/dflash2-backport.patch",
593 "bytes": 40396,
594 "sha256": "2e937ef748c1942ab04ace05de884e7c24041d34b23e5be8428cac6939ac6875"
595 },
596 {
597 "name": "runtime/patches/dflash2-lookup-drafting.patch",
598 "bytes": 46770,
599 "sha256": "5df09ef03b592d4a2c3b47dd8d2dbf8862fa7383d68aef3dd6ef1d97d29ce196"
600 },
601 {
602 "name": "runtime/patches/dflash2-ngram-chains.patch",
603 "bytes": 19037,
604 "sha256": "555b5a75d9023c99b2cd17634ac5b1dc8505f9275d857b1eba08c2d9cbf2df6e"
605 },
606 {
607 "name": "runtime/patches/dflash2-prewarm.patch",
608 "bytes": 5181,
609 "sha256": "e1e8012fa2c304c6948c19fc0071332f80eebcff802b402255445a30d553ab92"
610 },
611 {
612 "name": "runtime/patches/hybrid-kv-groups-v2-cudagraph.patch",
613 "bytes": 6306,
614 "sha256": "143825dd9744ab81dabcb438ec0bf974b6e0be8d0d7b0ffbbdfbf7c16a549664"
615 },
616 {
617 "name": "runtime/patches/hybrid-sw-block-promote.patch",
618 "bytes": 7828,
619 "sha256": "10876ac706546e74fe74b8964b90576a7b36f90a2f35a856997c558d7817f154"
620 },
621 {
622 "name": "runtime/patches/int4-kv-per-token-head.patch",
623 "bytes": 7038,
624 "sha256": "c03ff10c8c997b355f04fc521827ee2fa99c332ce4127cc2d878b749b79ca3d9"
625 },
626 {
627 "name": "runtime/patches/mamba-align-checkpoint-order.patch",
628 "bytes": 12020,
629 "sha256": "515d9bf76e860c95d832d615bdab4a97b71e2b0412b4ccc2445f11535ddddf45"
630 },
631 {
632 "name": "runtime/patches/marlin-int8-layer-select.patch",
633 "bytes": 3048,
634 "sha256": "833405e2ed2916529eb20184c38243c843378935522b74bab8c39707bf3ee800"
635 },
636 {
637 "name": "runtime/patches/marlin-int8-negative-scales.patch",
638 "bytes": 2926,
639 "sha256": "4cb8a064c706cc62dc77224479898da58864fbfff6b65012d90aa8326035300d"
640 },
641 {
642 "name": "runtime/patches/marlin-repack-staged-sm80.patch",
643 "bytes": 6841,
644 "sha256": "1588e2f10b5f82194d449483e766c0b39c84ba522d1623d39caa9502040f1fab"
645 },
646 {
647 "name": "runtime/patches/marlin-tune-table.patch",
648 "bytes": 4422,
649 "sha256": "1cac17f12e4ce389b0cb0fc733ceceb0a8cf60c02696ee378dc1d8d0c1ef8914"
650 },
651 {
652 "name": "runtime/patches/offload-dflash-eagle-groups.patch",
653 "bytes": 5406,
654 "sha256": "f2791af64d8066b31e4250d866c4322f45670d75522aff221307281c97c73156"
655 },
656 {
657 "name": "runtime/patches/offload-wsl2-devptr.patch",
658 "bytes": 4778,
659 "sha256": "8c6f9e3e5723571d1f33925794435e55062b684b2706d9284d1c5b94778d34db"
660 },
661 {
662 "name": "runtime/patches/qwen3_5-embed-quant.patch",
663 "bytes": 1395,
664 "sha256": "0a1b9ca06798c1aef582995de5a0beb3ad9a22a54cdbd2361986563a9c7a980e"
665 },
666 {
667 "name": "runtime/patches/qwen3_5-mtp-draft-vocab.patch",
668 "bytes": 3943,
669 "sha256": "292ef662d17bcc10556b787d5bb5f2cd12d3d3fc1f3bd2fe982485ce0d916298"
670 },
671 {
672 "name": "runtime/patches/sampler-small-topk-fast-softmax.patch",
673 "bytes": 13679,
674 "sha256": "8828646ce1916c065282529d9c1bb52064f668397d8e1f33652aef2f308292b6"
675 },
676 {
677 "name": "runtime/patches/spec-decode-attn.patch",
678 "bytes": 15424,
679 "sha256": "007791047a1d60143f8e3fe4e0a56dabadfc0134cb288847338b76ba7d1f9fe7"
680 },
681 {
682 "name": "runtime/patches/spec-decode-int4-kv-mq3d.patch",
683 "bytes": 3876,
684 "sha256": "b93b186deba1513ad9c4803ee2c574c923b21ba5e88e23bca8b8594b6a6b20a0"
685 },
686 {
687 "name": "runtime/patches/spec-decode-int8-kv.patch",
688 "bytes": 11915,
689 "sha256": "3cfd31304353237af5277c861dc2b43ad33d0f9bff2700594300eff150f64382"
690 },
691 {
692 "name": "runtime/patches/spec-sampler-prewarm.patch",
693 "bytes": 4516,
694 "sha256": "18ed9608baa5a09ed2ccbb214d62763da3ce7bcf09c6aad9003778ea4e56d18f"
695 },
696 {
697 "name": "runtime/patches/speed-knobs-envs.patch",
698 "bytes": 2300,
699 "sha256": "841ec93021b1b90f90313a1880a791bbfbdb79cbe34d9b99fdb4c3a7e52ed7c0"
700 },
701 {
702 "name": "runtime/patches/triton-prefill-attn-int8.patch",
703 "bytes": 15661,
704 "sha256": "6bd36db0226dce7b92ebd1231b6f9718a25da4a933ab4b2a6a8be77860364fae"
705 },
706 {
707 "name": "runtime/patches/vision-tower-cpu-offload.patch",
708 "bytes": 8345,
709 "sha256": "81dff64a1177058783dcf7d4e8552f1547f74fa8a68019e349dacdb9d66e968e"
710 },
711 {
712 "name": "runtime/patches/vllm-pr50021-gdn-spec-bounds.patch",
713 "bytes": 8886,
714 "sha256": "cd6e00270fa28e37a8c7ad11f965662f4153b2fcaf840f2c4043710e7b078655"
715 },
716 {
717 "name": "runtime/patches/xgrammar-spec-terminated.patch",
718 "bytes": 3993,
719 "sha256": "37589b9a45d5ece37cc16e82b195362719011e52ea31b2e70cdc89845c825040"
720 },
721 {
722 "name": "runtime/single-user/start_qwen.sh",
723 "bytes": 42439,
724 "sha256": "6874eb0bc4306d61b57ebb2f2c7cab97b41f11a3e0ec2b1502d9c34d396c47d8"
725 },
726 {
727 "name": "tokenizer.json",
728 "bytes": 12809320,
729 "sha256": "0997f410c57a1f4e53b09e4be8f4a172d90edd9564368fb0847030937229b9f3"
730 },
731 {
732 "name": "tokenizer_config.json",
733 "bytes": 17928,
734 "sha256": "b11349aafa7cdc6a320767cf7ceb29ed82f7eda5d65e8e0819e76f0ce947bf27"
735 },
736 {
737 "name": "video_preprocessor_config.json",
738 "bytes": 385,
739 "sha256": "7768af27c1fafa9cc9011c1dc20067e03f8915e03b63504550e11d5066986d13"
740 },
741 {
742 "name": "vocab.json",
743 "bytes": 6722759,
744 "sha256": "ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003"
15 }745 }
16 ]746 ]
17}747}
service/local-ai/prepare.py created+18
...@@ -0,0 +1,18 @@
1#!/usr/bin/env python3
2import json
3import os
4from pathlib import Path
5import subprocess
6import sys
7
8
9data = json.load(sys.stdin)
10if Path(data["hostRoot"]).name == "local-ai":
11 source = Path(__file__).with_name("service.pkl")
12 path = Path(subprocess.check_output([
13 "pkl", "eval", "-f", "json", "-p", "serviceId=local-ai", "-p", f"uid={data['uid']}",
14 "-p", "preview=false", "--expression", 'container.volumes["/cache/ssd"].src', str(source),
15 ], text=True).strip())
16 path.mkdir(mode=0o700, parents=True, exist_ok=True)
17 os.chown(path, data["uid"], data["uid"])
18 path.chmod(0o700)
service/local-ai/service.pkl+33-35
...@@ -4,20 +4,22 @@ import "../../config/site.pkl" as site...@@ -4,20 +4,22 @@ import "../../config/site.pkl" as site
4import "pkl:json"4import "pkl:json"
55
6local model = (new json.Parser {}).parse(read("catalog.json")).models[0]6local model = (new json.Parser {}).parse(read("catalog.json")).models[0]
7local weights = (new json.Parser {}).parse(read("models.json"))
78
8meta { name = "Local AI"; launcher = false; access = "admin" }9meta { name = "Local AI"; launcher = false; access = "admin" }
9enabled = (read?("env:STUDIO_LOCAL_AI") ?? "false") == "true"10enabled = (read?("env:STUDIO_LOCAL_AI") ?? "false") == "true"
10rollout = "simple"11rollout = "simple"
11stageIsolation = "fresh"12stageIsolation = "fresh"
13prepare = "prepare.py"
12healthyDeadline = "30m"14healthyDeadline = "30m"
13healthRestartGrace = "30m"15healthRestartGrace = "30m"
14secrets { ["api_key"] {} }16secrets { ["api_key"] {}; ["cache_key"] {} }
1517
16container {18container {
17 build = "build"19 build = "."
18 cpu = 400020 cpu = 4000
19 memory = 1843221 memory = 26624
20 entrypoint = "/usr/bin/python3"22 entrypoint = "/usr/bin/python3.12"
21 http {23 http {
22 containerPort = 808024 containerPort = 8080
23 subdomain = "ai"25 subdomain = "ai"
...@@ -25,42 +27,38 @@ container {...@@ -25,42 +27,38 @@ container {
25 }27 }
26 volumes {28 volumes {
27 ["/config/gateway.py"] { config = "gateway.py" }29 ["/config/gateway.py"] { config = "gateway.py" }
28 ["/models"] { src = "\(site.mediaRoot)/AI/LLM/Qwen3.8-27B"; readOnly = true }30 ["/cache/hdd"] {}
31 ["/cache/ssd"] { src = if (site.preview) null else site.localAiSsdCache }
32 ["/models"] { src = "\(site.mediaRoot)/AI/LLM/\(weights.directory)"; readOnly = true }
29 // The host's NVIDIA libraries link to immutable Nix store paths.33 // The host's NVIDIA libraries link to immutable Nix store paths.
30 ["/nvidia/lib"] { src = "/run/opengl-driver/lib"; readOnly = true }34 ["/nvidia/lib"] { src = "/run/opengl-driver/lib"; readOnly = true }
31 ["/nix/store"] { src = "/nix/store"; readOnly = true }35 ["/nix/store"] { src = "/nix/store"; readOnly = true }
32 }36 }
33 devices { "/dev/nvidia0"; "/dev/nvidiactl"; "/dev/nvidia-uvm" }37 devices { "/dev/nvidia0"; "/dev/nvidiactl"; "/dev/nvidia-uvm" }
38 tmpfs { "/dev/shm:rw,size=10g" }
34 env {39 env {
35 ["LD_LIBRARY_PATH"] = "/nvidia/lib:/app"40 ["LD_LIBRARY_PATH"] = "/nvidia/lib:/usr/local/cuda/lib64"
36 ["LLAMA_API_KEY"] = "${secret.own.api_key}"41 ["VLLM_API_KEY"] = "${secret.own.api_key}"
37 }42 ["STUDIO_AI_CACHE_KEY"] = "${secret.own.cache_key}"
38 args {43 ["PYTHONHASHSEED"] = "0"
39 "/config/gateway.py"44 ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:False"
40 "--model"; "/models/Qwen3.8-27B-UD-Q4_K_M.gguf"45 ["MODEL"] = "/models"
41 "--mmproj"; "/models/mmproj-F16.gguf"46 ["CTX"] = "long"
42 "--alias"; model.slug47 ["MAX_LEN"] = model.context_window.toString()
43 "--host"; "127.0.0.1"48 ["SPEC"] = "mtp"
44 "--port"; "8081"49 ["DRAFT_TOKENS"] = "3"
45 "--n-gpu-layers"; "99"50 ["PREFIX_CACHE"] = "1"
46 "--ctx-size"; model.context_window.toString()51 ["TOOLS"] = "1"
47 "--parallel"; "1"52 ["VISION"] = "1"
48 "--flash-attn"; "on"53 ["VISION_OFFLOAD"] = "1"
49 "--cache-type-k"; "q8_0"54 ["GPU_UTIL"] = "0.93"
50 "--cache-type-v"; "q8_0"55 ["MAX_SEQS"] = "8"
51 "--cache-ram"; "8192"56 ["API_SERVERS"] = "1"
52 "--spec-type"; "draft-mtp"57 ["HOST"] = "127.0.0.1"
53 "--spec-draft-n-max"; "3"58 ["PORT"] = "8081"
54 "--cache-type-k-draft"; "q8_0"59 ["VLLM_MAMBA_ALIGN_KEEP_CHECKPOINTS"] = "1"
55 "--cache-type-v-draft"; "q8_0"60 ["FLASHINFER_DISABLE_VERSION_CHECK"] = "1"
56 "--jinja"61 ["EXTRA_ARGS"] = "--kv-transfer-config {\"kv_connector\":\"OffloadingConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"spec_name\":\"TieringOffloadingSpec\",\"cpu_bytes_to_use\":8589934592,\"offload_prompt_only\":false,\"secondary_tiers\":[{\"type\":\"studio_disk\",\"ssd_dir\":\"/cache/ssd\",\"hdd_dir\":\"/cache/hdd\",\"ssd_bytes\":17179869184,\"hdd_bytes\":274877906944}]}} --limit-mm-per-prompt {\"image\":{\"count\":10}} --default-chat-template-kwargs {\"enable_thinking\":true,\"reasoning_effort\":\"\(model.default_reasoning_level)\"} --generation-config vllm --override-generation-config {\"temperature\":1.0,\"top_p\":0.95,\"top_k\":20,\"min_p\":0,\"presence_penalty\":0,\"repetition_penalty\":1.0} --disable-uvicorn-access-log"
57 "--timeout"; "28800"
58 "--sse-ping-interval"; "10"
59 "--metrics"
60 "--temp"; "1.0"
61 "--top-p"; "0.95"
62 "--top-k"; "20"
63 "--min-p"; "0"
64 "--chat-template-kwargs"; "{\"enable_thinking\": true, \"reasoning_effort\": \"\(model.default_reasoning_level)\", \"preserve_thinking\": true}"
65 }62 }
63 args { "/config/gateway.py" }
66}64}
service/local-ai/test_gateway.py+6
...@@ -3,6 +3,12 @@ from gateway import normalize, normalize_anthropic, restore...@@ -3,6 +3,12 @@ from gateway import normalize, normalize_anthropic, restore
33
44
5class ResponsesCompatibility(unittest.TestCase):5class ResponsesCompatibility(unittest.TestCase):
6 def test_anthropic_effort_matches_the_qwen_template(self):
7 for requested, expected in [("low", "low"), ("medium", "medium"), ("high", "xhigh"), ("max", "xhigh"), ("xhigh", "xhigh")]:
8 with self.subTest(effort=requested):
9 data = normalize_anthropic({"output_config": {"effort": requested, "format": {"type": "json_schema"}}})
10 self.assertEqual(data["output_config"], {"effort": expected, "format": {"type": "json_schema"}})
11
6 def test_claude_normal_configuration_system_blocks(self):12 def test_claude_normal_configuration_system_blocks(self):
7 request = {"system": [{"type": "text", "text": "instructions", "cache_control": {"type": "ephemeral"}}, {"type": "text", "text": "workspace context"}], "messages": [{"role": "user", "content": "task"}, {"role": "system", "content": "late system message"}]}13 request = {"system": [{"type": "text", "text": "instructions", "cache_control": {"type": "ephemeral"}}, {"type": "text", "text": "workspace context"}], "messages": [{"role": "user", "content": "task"}, {"role": "system", "content": "late system message"}]}
8 data = normalize_anthropic(request)14 data = normalize_anthropic(request)
tools/ai-wrapper-test.py deleted-68
...@@ -1,68 +0,0 @@
1#!/usr/bin/env python3
2"""Check downloaded wrappers without calling the model or changing client settings."""
3import json
4import os
5import pty
6import select
7import time
8from pathlib import Path
9import subprocess
10import sys
11import tempfile
12
13source = Path(sys.argv[1]).resolve()
14with tempfile.TemporaryDirectory(prefix="snow-wrapper-test-") as temporary:
15 root = Path(temporary)
16 for name in ("codex", "claude"):
17 binary = root / name
18 binary.write_text('''#!/usr/bin/env python3
19import json, os, pathlib, sys
20catalog = next((a.split("=",1)[1].strip('"') for a in sys.argv if a.startswith("model_catalog_json=")), None)
21print(json.dumps({"args":sys.argv[1:],"env":{k:v for k,v in os.environ.items() if k.startswith(("SNOW", "ANTHROPIC", "CLAUDE", "API_", "ENABLE_CLAUDE"))},"catalog":json.loads(pathlib.Path(catalog).read_text()) if catalog else None}))
22''')
23 binary.chmod(0o755)
24 env = dict(os.environ, PATH=str(root) + os.pathsep + os.environ["PATH"], XDG_CONFIG_HOME=str(root / "config"))
25 env.pop("SNOWGLOBE_AI_KEY", None)
26 env.pop("SNOWGLOBE_AI_URL", None)
27 for name in ("snow-codex", "snow-claude"):
28 wrapper = source / name
29 subprocess.run(["bash", "-n", wrapper], check=True)
30 failed = subprocess.run(["bash", wrapper], env=env, stdin=subprocess.DEVNULL, capture_output=True)
31 assert failed.returncode == 1 and b"SNOWGLOBE_AI_KEY" in failed.stderr
32 key = root / "config/snowglobe-ai/api-key"
33 key.parent.mkdir(parents=True, exist_ok=True)
34 master, slave = pty.openpty()
35 process = subprocess.Popen(["bash", wrapper, "--help"], env=env, stdin=slave, stdout=subprocess.PIPE, stderr=slave)
36 os.close(slave)
37 prompt = b""
38 deadline = time.monotonic() + 10
39 try:
40 while b"Snow Globe API key:" not in prompt:
41 assert time.monotonic() < deadline, "first-run key prompt did not appear"
42 if select.select([master], [], [], 1)[0]:
43 prompt += os.read(master, 4096)
44 os.write(master, b"wrapper-canary\n")
45 output, _ = process.communicate(timeout=10)
46 assert process.returncode == 0 and json.loads(output)["env"]["SNOWGLOBE_AI_KEY"] == "wrapper-canary"
47 assert key.read_text() == "wrapper-canary\n" and key.stat().st_mode & 0o777 == 0o600
48 finally:
49 if process.poll() is None: process.kill()
50 os.close(master)
51 data = json.loads(subprocess.check_output(["bash", wrapper, "--help", "argument with spaces"], env={**env, "ANTHROPIC_AUTH_TOKEN":"old-auth", "CLAUDE_CODE_OAUTH_TOKEN":"old-login"}))
52 assert data["args"][-2:] == ["--help", "argument with spaces"]
53 assert "wrapper-canary" not in " ".join(data["args"])
54 assert data["env"]["SNOWGLOBE_AI_KEY"] == "wrapper-canary"
55 assert key.stat().st_mode & 0o777 == 0o600
56 if name == "snow-codex":
57 assert data["args"][:2] == ["--no-daemon", "--approve-for-me"]
58 assert data["catalog"]["models"][0]["context_window"] == 131072
59 assert not list(key.parent.glob("catalog.*")), "temporary catalog leaked"
60 else:
61 assert data["args"][:2] == ["--permission-mode", "auto"]
62 assert "ANTHROPIC_AUTH_TOKEN" not in data["env"] and "CLAUDE_CODE_OAUTH_TOKEN" not in data["env"]
63 assert data["env"]["CLAUDE_CODE_AUTO_MODE_SERVER"] == "0"
64 assert data["env"]["API_TIMEOUT_MS"] == "28800000"
65 credentials = Path(data["env"]["CLAUDE_SECURESTORAGE_CONFIG_DIR"])
66 assert credentials.stat().st_mode & 0o777 == 0o700
67 key.unlink()
68 print("Both downloaded wrappers preserve arguments, enable auto approval, isolate credentials, and clean up.")
tools/benchmark-local-ai-cache.py created+126
...@@ -0,0 +1,126 @@
1#!/usr/bin/env python3
2"""Run on the target through SSH stdin; use synthetic prompts and disposable files."""
3import argparse
4from concurrent.futures import ThreadPoolExecutor
5import http.client
6import json
7import mmap
8import os
9from pathlib import Path
10import random
11import socket
12import tempfile
13import time
14
15
16def request(path, key, data=None):
17 connection = http.client.HTTPConnection("127.0.0.1", 8081, timeout=600)
18 try:
19 body = json.dumps(data) if data is not None else None
20 connection.request("POST" if body else "GET", path, body, {
21 "Authorization": "Bearer " + key, "Content-Type": "application/json",
22 })
23 response = connection.getresponse()
24 payload = response.read()
25 if response.status != 200:
26 raise RuntimeError(f"{path}: HTTP {response.status}")
27 return json.loads(payload) if data is not None else payload.decode()
28 finally:
29 connection.close()
30
31
32def metrics(key):
33 values = {}
34 for line in request("/metrics", key).splitlines():
35 if line.startswith("vllm:"):
36 name = line.split("{", 1)[0]
37 values[name] = values.get(name, 0) + float(line.rsplit(" ", 1)[1])
38 return values
39
40
41def prefill(pid, repeats):
42 environment = dict(entry.split(b"=", 1) for entry in Path(f"/proc/{pid}/environ").read_bytes().split(b"\0") if b"=" in entry)
43 key = environment[b"VLLM_API_KEY"].decode()
44 vocabulary = request("/tokenize", key, {
45 "model": "qwen3.8-27b", "prompt": "Synthetic benchmark data: blue green yellow red apple pear stone river cloud mountain 0123456789.\n",
46 })["tokens"]
47 for length in (16000, 64000, 148000):
48 for trial in range(repeats):
49 before = metrics(key)
50 if before["vllm:num_requests_running"] or before["vllm:num_requests_waiting"]:
51 raise RuntimeError("Endpoint is busy; rerun when idle")
52 prompt = random.choices(vocabulary, k=length)
53 started = time.perf_counter()
54 response = request("/v1/completions", key, {
55 "model": "qwen3.8-27b", "prompt": prompt, "max_tokens": 1, "temperature": 0,
56 })
57 elapsed = time.perf_counter() - started
58 after = metrics(key)
59 delta = {name: after.get(name, 0) - value for name, value in before.items()}
60 print(json.dumps({
61 "kind": "prefill", "prompt_tokens": response["usage"]["prompt_tokens"], "trial": trial,
62 "wall_seconds": elapsed, "prefill_seconds": delta["vllm:request_prefill_time_seconds_sum"],
63 "queue_seconds": delta["vllm:request_queue_time_seconds_sum"],
64 "cached_tokens": delta["vllm:prefix_cache_hits_total"],
65 "completed_requests": delta["vllm:request_success_total"],
66 }), flush=True)
67
68
69def filesystem(directory, repeats):
70 os.umask(0o077)
71 size = 16 * 1024**2
72 with tempfile.TemporaryDirectory(prefix="synthetic-ai-cache-", dir=directory) as temporary:
73 root = Path(temporary)
74 buffer = mmap.mmap(-1, size)
75 buffer[:] = os.urandom(size)
76 paths = [root / f"{random.getrandbits(128):032x}.bin" for _ in range(256)]
77 started = time.perf_counter()
78 for path in paths:
79 descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_DIRECT, 0o600)
80 try:
81 if os.write(descriptor, buffer) != size:
82 raise RuntimeError("Short synthetic write")
83 os.fsync(descriptor)
84 finally:
85 os.close(descriptor)
86 buffer.close()
87 print(json.dumps({"kind": "write", "directory": directory, "gib": 4, "seconds": time.perf_counter() - started}), flush=True)
88
89 def read(path):
90 with mmap.mmap(-1, size) as target:
91 descriptor = os.open(path, os.O_RDONLY | os.O_DIRECT)
92 try:
93 if os.readv(descriptor, [target]) != size:
94 raise RuntimeError("Short synthetic read")
95 finally:
96 os.close(descriptor)
97
98 for gib in (1, 4):
99 selected = paths[:gib * 64]
100 for trial in range(repeats):
101 random.shuffle(selected)
102 devices = [path for path in Path("/sys/block").iterdir() if (path / "device").exists()]
103 sectors = {path.name: int((path / "stat").read_text().split()[2]) for path in devices}
104 started = time.perf_counter()
105 with ThreadPoolExecutor(max_workers=16) as pool:
106 list(pool.map(read, selected))
107 elapsed = time.perf_counter() - started
108 disk_reads = {path.name: (int((path / "stat").read_text().split()[2]) - sectors[path.name]) * 512 for path in devices}
109 print(json.dumps({"kind": "direct-read", "directory": directory, "gib": gib, "trial": trial,
110 "block_mib": 16, "workers": 16, "seconds": elapsed,
111 "device_read_bytes": {name: count for name, count in disk_reads.items() if count}}), flush=True)
112
113
114if __name__ == "__main__":
115 parser = argparse.ArgumentParser(description=__doc__)
116 parser.add_argument("mode", choices=("prefill", "filesystem"))
117 parser.add_argument("target", help="Backend PID or disposable parent directory")
118 parser.add_argument("--hostname", required=True)
119 parser.add_argument("--repeats", type=int, default=3)
120 arguments = parser.parse_args()
121 if socket.gethostname() != arguments.hostname:
122 raise SystemExit("Unexpected target host")
123 if arguments.mode == "prefill":
124 prefill(int(arguments.target), arguments.repeats)
125 else:
126 filesystem(arguments.target, arguments.repeats)
tools/benchmark-local-ai.py deleted-87
...@@ -1,87 +0,0 @@
1#!/usr/bin/env python3
2"""Measure llama.cpp decode, retrieval, and prefix reuse without logging prompts."""
3import argparse
4import base64
5import binascii
6import json
7import os
8import struct
9import time
10import urllib.request
11import zlib
12
13
14def request(path, data=None):
15 body = None if data is None else json.dumps(data).encode()
16 headers = {"Authorization": "Bearer " + os.environ["LLAMA_API_KEY"], "Content-Type": "application/json"}
17 started = time.monotonic()
18 with urllib.request.urlopen(urllib.request.Request(args.url + path, data=body, headers=headers), timeout=1800) as response:
19 result = json.load(response)
20 return result, time.monotonic() - started
21
22
23parser = argparse.ArgumentParser(description=__doc__)
24parser.add_argument("--url", default="http://127.0.0.1:8081")
25parser.add_argument("--contexts", type=int, nargs="+", default=[16384, 65536, 122000])
26parser.add_argument("--suite", choices=["decode", "workloads", "cache", "retrieval", "vision", "all"], default="all")
27parser.add_argument("--require-prefix-cache", action="store_true")
28args = parser.parse_args()
29
30def report(label, data, path="/completion"):
31 result, elapsed = request(path, data)
32 print(json.dumps({"case": label, "elapsed_s": round(elapsed, 3), "timings": result.get("timings"), "usage": result.get("usage")}), flush=True)
33 return result
34
35
36filler = "".join(f"Record {i:06d}: amber bridge cedar delta elm forest granite harbor iris jade.\n" for i in range(15000))
37tokens, _ = request("/tokenize", {"content": filler})
38tokens = tokens["tokens"]
39
40if args.suite in {"decode", "all"}:
41 offset = 0
42 for size in args.contexts:
43 prompt = tokens[offset:offset + size]
44 offset += size
45 assert len(prompt) == size, "Requested context exceeds the fixture"
46 data = {"prompt": prompt, "n_predict": 256, "ignore_eos": True, "temperature": 0, "seed": 42, "cache_prompt": True}
47 report(f"decode-{size}-first", data)
48 report(f"decode-{size}-warm", data)
49
50if args.suite in {"cache", "all"}:
51 prefix, _ = request("/detokenize", {"tokens": tokens[:4096]})
52 other, _ = request("/detokenize", {"tokens": tokens[40000:44096]})
53 prefix["content"] += "\nStable system code is prefix-a-731."
54 other["content"] += "\nStable system code is prefix-b-284."
55 for label, system, body in [("prefix-a", prefix["content"], tokens[10000:30000]), ("prefix-b", other["content"], tokens[45000:65000]), ("prefix-a-new", prefix["content"], tokens[70000:90000]), ("prefix-a-repeat", prefix["content"], tokens[70000:90000]), ("prefix-b-again", other["content"], tokens[95000:115000]), ("prefix-a-metadata", prefix["content"].replace("prefix-a-731", "prefix-c-592"), tokens[120000:140000])]:
56 text, _ = request("/detokenize", {"tokens": body})
57 result = report(label, {"model": "qwen3.8-27b", "messages": [{"role": "system", "content": system}, {"role": "user", "content": text["content"] + "\nWhat is the stable system code? Reply with only it."}], "max_tokens": 32, "temperature": 0, "chat_template_kwargs": {"enable_thinking": False}}, "/v1/chat/completions")
58 assert system.rsplit(" ", 1)[-1].rstrip(".") in result["choices"][0]["message"]["content"], "Prefix restore lost the system fact"
59 if args.require_prefix_cache and label in {"prefix-a-new", "prefix-a-metadata"}:
60 assert result["timings"]["cache_n"] >= 4096, "Client switching discarded the base prefix"
61
62if args.suite in {"workloads", "all"}:
63 for task in ["Implement a bounded asynchronous work queue in Python using asyncio. Explain cancellation, backpressure, shutdown, race conditions and error handling. Include tests and a detailed explanation.", "Explain how database isolation levels interact with concurrent read-modify-write operations. Give concrete schedules, anomalies, and remedies for each level, with detailed examples."]:
64 template, _ = request("/apply-template", {"messages": [{"role": "system", "content": "You are a careful software engineer."}, {"role": "user", "content": task}], "chat_template_kwargs": {"enable_thinking": True, "reasoning_effort": "medium"}})
65 report("workload-code" if "asyncio" in task else "workload-reasoning", {"prompt": template["prompt"], "n_predict": 512, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0, "seed": 42, "cache_prompt": True})
66
67if args.suite in {"retrieval", "all"}:
68 for size in args.contexts:
69 facts = ["START password is otter-731.", "MIDDLE password is maple-284.", "END password is comet-965."]
70 segment = tokens[:max(1, (size - 256) // 2)]
71 text, _ = request("/detokenize", {"tokens": segment})
72 messages = [{"role": "system", "content": "Find the three passwords in the records. Reply with only the passwords in START, MIDDLE, END order."}, {"role": "user", "content": facts[0] + "\n" + text["content"] + "\n" + facts[1] + "\n" + text["content"] + "\n" + facts[2]}]
73 template, _ = request("/apply-template", {"messages": messages, "chat_template_kwargs": {"enable_thinking": False}})
74 result = report(f"retrieval-{size}", {"prompt": template["prompt"], "n_predict": 64, "temperature": 0, "cache_prompt": True})
75 positions = [result["content"].find(fact.split()[-1].rstrip(".")) for fact in facts]
76 passed = all(pos >= 0 for pos in positions) and positions == sorted(positions)
77 print(json.dumps({"case": f"retrieval-{size}", "passed": passed}), flush=True)
78 if not passed:
79 raise SystemExit("Retrieval failed")
80
81if args.suite in {"vision", "all"}:
82 png = b"\x89PNG\r\n\x1a\n"
83 pixels = b"".join(b"\0" + b"".join(b"\xff\0\0" if 128 <= x < 384 and 128 <= y < 384 else b"\xff\xff\xff" for x in range(512)) for y in range(512))
84 for kind, data in [(b"IHDR", struct.pack(">IIBBBBB", 512, 512, 8, 2, 0, 0, 0)), (b"IDAT", zlib.compress(pixels)), (b"IEND", b"")]:
85 png += struct.pack(">I", len(data)) + kind + data + struct.pack(">I", binascii.crc32(kind + data))
86 result = report("vision", {"model": "qwen3.8-27b", "messages": [{"role": "user", "content": [{"type": "text", "text": "What color is the square? Reply with one word."}, {"type": "image_url", "image_url": {"url": "data:image/png;base64," + base64.b64encode(png).decode()}}]}], "max_tokens": 32, "temperature": 0, "chat_template_kwargs": {"enable_thinking": False}}, "/v1/chat/completions")
87 assert "red" in result["choices"][0]["message"]["content"].lower(), "Vision failed"
tools/dashboard-container-test.py deleted-94
...@@ -1,94 +0,0 @@
1#!/usr/bin/env python3
2import argparse
3import concurrent.futures
4import json
5from pathlib import Path
6import subprocess
7import time
8import urllib.request
9
10
11def main():
12 parser = argparse.ArgumentParser()
13 parser.add_argument("container")
14 parser.add_argument("--url", default="http://127.0.0.1:7074")
15 parser.add_argument("--clients", type=int, default=40)
16 parser.add_argument("--requests", type=int, default=1000)
17 parser.add_argument("--output", type=Path)
18 parser.add_argument("--proxy-token-file", type=Path, required=True)
19 parser.add_argument("--stop", action="store_true")
20 args = parser.parse_args()
21 info = json.loads(subprocess.check_output(["podman", "inspect", args.container]))[0]
22 host = info["HostConfig"]
23 assert info["Config"]["User"].split(":")[0] not in {"", "0", "root"}
24 assert host["ReadonlyRootfs"] is True
25 assert "no-new-privileges" in host["SecurityOpt"]
26 assert not host["Privileged"] and not host["Devices"]
27 assert host["NetworkMode"] != "host"
28 assert all(m["Destination"] in ["/run/studio-host", "/run/secrets/dashboard-proxy.token", "/data"] for m in info["Mounts"]), info["Mounts"]
29 assert next(m for m in info["Mounts"] if m["Destination"] == "/run/studio-host")["RW"] is False
30 status = subprocess.check_output(["podman", "exec", args.container, "/bin/cat", "/proc/1/status"], text=True)
31 uid = next(line.split()[1:] for line in status.splitlines() if line.startswith("Uid:"))
32 assert all(int(value) != 0 for value in uid), uid
33 for field in ["CapEff", "CapBnd", "CapPrm", "CapAmb"]:
34 assert field + ":\t0000000000000000" in status, status
35 assert "NoNewPrivs:\t1" in status, status
36 mounts = subprocess.check_output(["podman", "exec", args.container, "/bin/cat", "/proc/1/mountinfo"], text=True)
37 root = next(line.split() for line in mounts.splitlines() if line.split()[4] == "/")
38 assert "ro" in root[5].split(","), root
39 assert subprocess.run(["podman", "exec", args.container, "/bin/touch", "/escaped"], capture_output=True).returncode != 0
40 for denied in ["/var/lib/studio/nomad.token", "/run/podman/podman.sock", "/run/libvirt/libvirt-sock", "/dev/zfs", "/opt/studio/current"]:
41 assert subprocess.run(["podman", "exec", args.container, "/bin/test", "-e", denied], capture_output=True).returncode != 0, denied
42 pid = info["State"]["Pid"]
43 for namespace in ["net", "pid", "mnt"]:
44 assert Path(f"/proc/{pid}/ns/{namespace}").stat().st_ino != Path(f"/proc/self/ns/{namespace}").stat().st_ino
45
46 headers = {"User-Name": "fixture", "User-Groups": "infra-admin", "Studio-Proxy-Token": args.proxy_token_file.read_text().strip()}
47 with urllib.request.urlopen(urllib.request.Request(args.url + "/api/host", headers=headers), timeout=10) as response:
48 machine = json.load(response)
49 expected_memory = next(int(line.split()[1]) * 1024 for line in Path("/proc/meminfo").read_text().splitlines() if line.startswith("MemTotal:"))
50 expected_cores = sum(line.startswith("cpu") and line[3:4].isdigit() for line in Path("/proc/stat").read_text().splitlines())
51 assert machine["memory"] == expected_memory and machine["cores"] == expected_cores, machine
52 with urllib.request.urlopen(urllib.request.Request(args.url + "/api/live", headers=headers), timeout=10) as response:
53 for _ in range(20):
54 line = response.readline()
55 if line.startswith(b"data:"):
56 live = json.loads(line[5:])
57 break
58 else:
59 raise AssertionError("host sample did not arrive")
60 assert 0 <= live["host"]["cpu"] <= 100 and live["host"]["memory"] > 0, live
61 assert live["host"]["arc"] is not None, live
62
63 def request(_):
64 start = time.monotonic()
65 req = urllib.request.Request(args.url + "/api/storage", headers=headers)
66 with urllib.request.urlopen(req, timeout=10) as response:
67 assert json.load(response)["pool"]["state"] == "ONLINE"
68 return (time.monotonic() - start) * 1000
69
70 with concurrent.futures.ThreadPoolExecutor(max_workers=args.clients) as clients:
71 latency = sorted(clients.map(request, range(args.requests)))
72 result = {"nonroot_container": "passed", "readonly_rootfs": "passed", "zero_capabilities": "passed",
73 "no_new_privileges": "passed", "private_namespaces": "passed", "control_sockets_and_management_token_absent": "passed",
74 "host_identity_outside_container_quota": "passed", "host_live_sample": "passed",
75 "storage_requests": len(latency), "storage_clients": args.clients,
76 "storage_p95_ms": round(latency[int(len(latency) * .95)], 2), "storage_max_ms": round(latency[-1], 2)}
77 if args.stop:
78 req = urllib.request.Request(args.url + "/api/live", headers=headers)
79 with urllib.request.urlopen(req, timeout=10):
80 started = time.monotonic()
81 stopped = subprocess.run(["podman", "stop", "--time=8", args.container], capture_output=True, text=True, check=True)
82 elapsed = time.monotonic() - started
83 ended = json.loads(subprocess.check_output(["podman", "inspect", args.container]))[0]["State"]
84 assert ended["ExitCode"] == 0 and not ended["OOMKilled"], (ended, stopped.stderr)
85 assert elapsed < 8, elapsed
86 result["stop_with_active_stream"] = "passed"
87 result["stop_seconds"] = round(elapsed, 2)
88 if args.output:
89 args.output.write_text(json.dumps(result, indent=2) + "\n")
90 print(json.dumps(result))
91
92
93if __name__ == "__main__":
94 main()
tools/local-ai-cache.md created+110
...@@ -0,0 +1,110 @@
1# Local AI context cache
2
3## Prepared configuration
4
5The current change configures GPU prefix caching, 8 GiB of vLLM shared-memory
6offload, a 16 GiB NVMe hot copy, and a 256 GiB HDD pool. The container has a
726 GiB memory limit and a 10 GiB `/dev/shm`.
8
9`service/local-ai/cache` installs a vLLM secondary-tier plugin. Blocks are
10written through to HDD and copied to SSD; HDD reads warm SSD. Each disk tier
11evicts its least recently used unpinned blocks within its byte cap. Active
12requests pin blocks; a saturated pinned tier skips writes. Direct I/O avoids
13an additional unbounded filesystem data cache. Disk writes and promotions run
14in background workers, outside scheduler lookups.
15
16Disk blocks use authenticated AES-GCM encryption with the service's Nomad
17`cache_key`, owner-only files/directories, and atomic replacement. Restart
18rebuilds the bounded index. Namespaces include the model manifest, vLLM and
19plugin versions, key, KV layout, and parallel configuration. Corrupt blocks
20fall back to another copy or become misses. Key rotation invalidates reuse;
21old entries remain encrypted and subject to eviction.
22
23`config/site.pkl` owns the NVMe path (`/var/cache/studio/local-ai` by default).
24The prepare hook creates it with the assigned service owner. HDD blocks use
25the managed service dataset. Fresh previews use separate managed directories
26for both disk tiers.
27
28Validation on 2026-10-09: the full image built and its installed plugin loaded;
29nine disk-cache tests and two CPU-only tests against the installed vLLM
30scheduler passed. These check encrypted persistence, quotas, pinning,
31concurrent access, SSD promotion, attention/recurrent-group restoration into
32shared RAM, and corrupt-promotion misses. Production/preview Pkl rendering,
33isolated preparation, deployment checks, and six gateway tests passed. These
34tests use synthetic data and do not establish GPU/hybrid-MTP restore timing.
35
36## Production activation — 2026-10-09
37
38Release `0edf5af161627873` was deployed from local `main` without pushing.
39Pre-switch backup: `20261009T214237Z-734b3d`. Nomad deployment and all route
40checks passed. vLLM initialized the tiering connector and an 8.57 GB shared
41region (rounded below the 8 GiB budget to whole blocks).
42
43An authenticated gateway probe completed an 8,000-token synthetic prompt.
44Repeating it reused 6,656 tokens and completed in 3.439 seconds. The first
45request took 37.596 seconds including first-use kernel compilation, so it is
46not a steady-state prefill comparison. Both disk tiers held 66 encrypted
47blocks totaling 1,912,086,528 bytes. Forced RAM/disk restore probes stopped
48when the endpoint reported busy; end-to-end disk restore timing remains
49unverified.
50
51## Zenith measurements — 2026-10-09
52
53HyperQwen, vLLM 0.27.1, MTP with three draft tokens, 150,000-token
54context, GPU prefix caching enabled. Production stayed running; no cache
55connector was enabled or production configuration changed.
56
57`benchmark-local-ai-cache.py` sends synthetic token-ID prompts with one output
58token. Each measured request had zero prefix-cache hits, one completed request,
59and under 0.1 ms queue time. Server prefill time excludes HTTP overhead.
60
61| Prompt tokens | Server prefill seconds |
62| --- | --- |
63| 16,000 | 12.715, 12.557, 12.601 |
64| 64,000 | 63.174, 63.704, 63.896 |
65| 148,000 | 203.552, 204.495, 204.963 |
66
67Storage probes create disposable files filled with incompressible synthetic
68bytes, then read shuffled 16 MiB files with 16 workers. Both writes and reads use
69`O_DIRECT`; writes are individually fsynced. The HDD path is encrypted ZFS on
70the four-disk RAIDZ1 pool, OpenZFS 2.4.4, `direct=standard`, 128 KiB records,
71LZ4 compression. The SSD path is ext4 on the WD Blue SN5000 500 GB NVMe.
72
73| Storage | 1 GiB read seconds | 4 GiB read seconds | 4 GiB write seconds |
74| --- | --- | --- | --- |
75| HDD (`/srv/staging`) | 3.901, 3.686, 3.660, 3.931 | 15.736, 15.399, 16.848, 17.502 | 31.161, 30.165 |
76| NVMe (`/var/tmp`) | 0.262, 0.258, 0.251, 0.258 | 0.937, 0.960, 0.933, 0.929 | 1.282, 1.342 |
77
78The final storage samples additionally checked physical-device read counters:
79the 4 GiB probe read 4,427,632,640 bytes across the HDDs and exactly
804,294,967,296 bytes from NVMe. This verifies reads reached the drives instead
81of being served from the host's data cache.
82
83Disk probes overlapped part of the prefill benchmark. These are storage-only
84timings, not end-to-end KV restores. Actual restores additionally depend on
85attention KV and recurrent-checkpoint bytes, cache-file layout, GPU transfers,
86and any uncached suffix. A larger, fragmented cache and competing pool I/O can
87increase HDD latency. The disposable files were removed after measurement.
88
89HDD reads are promising against the 64k and 148k uncached prefills. End-to-end
90validation with HyperQwen's patched hybrid-state/MTP implementation remains
91necessary; the storage timings above predate the bounded plugin.
92
93## Repeating the probes
94
95Confirm the target and ensure the AI endpoint is idle before inference probes.
96The prefill script reads the existing backend key internally and prints only
97timings and counters; it never prints prompt text, generated text, or keys.
98
99```sh
100ssh -o BatchMode=yes root@10.0.0.1 \
101 'pid=$(pgrep -x vllm); nsenter -t "$pid" -n python3 - prefill "$pid" --hostname zenith' \
102 < tools/benchmark-local-ai-cache.py
103ssh -o BatchMode=yes root@10.0.0.1 \
104 'python3 - filesystem /srv/staging --hostname zenith' \
105 < tools/benchmark-local-ai-cache.py
106ssh -o BatchMode=yes root@10.0.0.1 \
107 'python3 - filesystem /var/tmp --hostname zenith' \
108 < tools/benchmark-local-ai-cache.py
109```
110
tools/local-ai.md deleted-247
...@@ -1,247 +0,0 @@
1# Snow Globe local AI
2
3The first model is Qwen3.8-27B, using Unsloth's UD-Q4_K_M GGUF and F16 vision
4projector. Both files, the base CUDA image, and the llama.cpp source are pinned.
5The model runs entirely on the RTX 3090, with one inference slot and a 131,072-token context.
6Codex compacts at 96,000 tokens to leave room for tool results and output.
7The KV cache uses Q8; the single slot leaves more VRAM for vision and runtime
8working memory. Additional requests queue so simultaneous generation cannot
9reduce the active request's decode rate. An 8 GiB host-memory prompt cache
10preserves evicted contexts between coding, reviewer, and helper requests.
11The patched server also reserves a separate 2 GiB cache for prefixes before
12the first user message, including system instructions and tool definitions.
13A second checkpoint at 4,096 tokens retains the shared base when later workspace
14metadata changes; shorter headers need only their complete checkpoint.
15It captures these from actual requests; there are no copied Codex or Claude
16prompts to update. Prefix states survive conversation-cache eviction and remain
17available after restoration. Only complete token matches restore these states,
18including the recurrent and MTP state. Prefix variants have their own bounded
19LRU eviction; first use, unmatched tokens, and server restarts still need prefill.
20The container has an 18 GiB RAM limit for the runtime and both caches.
21Native MTP drafts up to three tokens, which the target model verifies. The draft
22KV cache also uses Q8. Target weights, target KV precision, context length, and
23reasoning defaults remain unchanged. The build replaces only the server library;
24inference libraries and embedded UI assets come from the pinned base image.
25Medium reasoning is enabled for coding, using Qwen's recommended thinking-mode
26sampling parameters. This adds
27reasoning tokens before answers; the earlier latency baseline disabled thinking.
28
29Models live in `/srv/clover/Media/AI/LLM/Qwen3.8-27B`, also visible on the Mac as
30`/Volumes/clover/Media/AI/LLM/Qwen3.8-27B`. Download once on Zenith:
31
32```sh
33python3 service/local-ai/download.py
34```
35
36The downloader resumes partial transfers, verifies size and SHA-256, and never
37replaces an existing file that fails verification. No model download or GPU
38service runs in rehearsal VMs. `STUDIO_LOCAL_AI=true` enables the physical host.
39
40## API and deployment
41
42The authenticated API exposes `/v1/responses`, `/v1/messages`,
43`/v1/chat/completions`, and `/v1/models`. `/health` is public for deployment
44checks. Use `Authorization: Bearer <key>` or Anthropic's `x-api-key` header.
45`api_key` is a generated Nomad service secret; don't put it in Git or shell
46arguments.
47
48The production endpoint is `https://ai.paperclover.net`. The
49`STUDIO_LOCAL_AI=true` host environment in `nixos/zenith.nix` enables it.
50The 3090 has room for one model instance. Stop any GPU preview before deploying
51production; download the wrappers again when their endpoint changes.
52
53`gateway.py` uses Python's standard library and starts llama.cpp on a container
54loopback port. Anthropic system-text blocks are combined into one system message;
55Chat Completions pass through unchanged. For
56Responses, it collects developer/system instructions at the start of the
57prompt, maps tool namespaces to flat names, restores returned namespace
58identities, and represents raw custom tools as JSON functions. The raw tool
59grammar remains in the tool description. Built-in hosted tools such as OpenAI
60web search aren't implemented; unsupported tool types return an explicit error.
61Codex's hosted web search is disabled in the wrapper.
62
63## Coding clients
64
65Download `snow-codex` or `snow-claude` from the dashboard's **AI / MCP → AI**
66tab. They are standalone Bash wrappers around the standard installed Codex and
67Claude Code clients; client updates require no wrapper rebuild. Make each
68download executable and place it on your PATH. The first run asks for the API
69key from the same tab and saves it with mode 0600 under
70`${XDG_CONFIG_HOME:-~/.config}/snowglobe-ai/api-key`. No checkout or patched
71client is required. The downloads contain endpoint and model settings, never
72the API key. The `ai` group opens AI / MCP and grants API-key access and downloads;
73infrastructure administrators retain access. Role previews can view the page
74and downloads, but cannot access the API key.
75
76`dashboard/ai/launch.sh` owns the wrapper template. The download endpoint
77incorporates the live service hostname and canonical model catalog.
78Codex starts with `--approve-for-me`, retaining its workspace sandbox and
79automatic approval reviewer; Claude starts with `--permission-mode auto`,
80running classification against the same local endpoint. Both use medium
81reasoning. Claude's credential store is isolated from ordinary Claude login,
82so using Snow Globe does not log the user's subscription out.
83
84Qwen's raw thinking is streamed as Responses reasoning text, including the
85`content_index` required by stock Codex. To display it in the terminal:
86
87```sh
88snow-codex -c show_raw_agent_reasoning=true -c hide_agent_reasoning=false
89```
90
91Reasoning summaries remain disabled because the model emits raw thinking rather
92than a separate summary. `tools/verify-local-ai.py` checks actual Codex app-server
93reasoning deltas arrive before the answer; stored reasoning in a completed
94response alone does not establish that live display works. The production fix
95was checked with stock Codex 0.161.0: 237 live reasoning deltas arrived before
96the answer, the interactive terminal updated during thinking, both coding
97fixtures passed, and automatic approval authorized its temporary write in
9821.4 seconds.
99
100API and event-stream waits allow eight hours. Claude's byte-stream watchdog
101has a 30-minute maximum, kept alive by ten-second server SSE pings. Stock
102Codex 0.160.0 retains a hardcoded 90-second approval-review deadline that
103provider settings cannot extend. The earlier patched build survived a
10495-second forced review delay, but the shipped wrappers intentionally use
105standard clients as requested. Queueing can therefore still cause a stock
106Codex approval review to time out.
107
108Override the endpoint or key with `SNOWGLOBE_AI_URL` or `SNOWGLOBE_AI_KEY`.
109Explicit client arguments can override wrapper defaults. Existing MCP servers,
110hooks, and ordinary commands keep their normal network access. Model inference
111and approval classification use Snow Globe; cloud plugins, Apps, automatic
112memories, and nonessential client telemetry are disabled for this profile.
113
114## Verification
115
116```sh
117python3 -m unittest discover -s service/local-ai -p 'test_*.py' -v
118python3 tools/verify-local-ai.py --url https://YOUR-ENDPOINT
119```
120
121The downloadable wrappers were verified against production on 2026-10-05
122with stock Codex 0.160.0 and Claude Code 2.1.289. Both clients fixed the median
123fixture and passed all eight unchanged tests, taking 189.3 and 209.4 seconds
124while sharing the single slot. Stock Codex's actual automatic approval review
125authorized an outside-workspace fixture write in 37.5 seconds. The temporary
126file was removed. Download tests also exercised first-run key entry through a
127terminal, mode 0600 storage, argument preservation, auth isolation, and catalog
128cleanup. Unsigned, cross-origin, and view-as requests were denied; the returned
129key authenticated successfully with the live model.
130
131The second command launches both actual clients concurrently in disposable
132fixtures. Each must fix two median bugs, preserve input, pass four tests, and
133leave the tests unchanged. It also exercises an actual Codex approval review and removes its temporary
134file. JSONL evidence is saved in the printed temporary directory. An optional
135`--approval-delay 95` probe demonstrates the stock reviewer deadline; it is
136expected to fail with stock Codex.
137
138The deployed 128K configuration was checked on 2026-10-05 with a synthetic
139122,265-token request. It recovered facts from the beginning, middle, and end
140through a namespaced tool call. A 122,384-token continuation consumed the tool
141result correctly and reused 122,332 cached tokens. A subsequent 122,409-token
142request reused 122,359 tokens and generated 384 tokens at 24.21 tokens/second,
143with a maximum streamed token gap of 45 ms. Thinking was disabled only for
144these retrieval and throughput probes. The server measured 144.4 seconds of
145cold prefill at 846.8 tokens/second. The cached stream's first token took 26.1
146seconds while coding clients shared the slot; that elapsed time includes
147queueing. The allocation used 20.87 GiB VRAM and peaked at 9.79 GiB host RAM,
148with zero OOM events, cgroup limit hits, or restarts throughout verification.
149
150Both actual clients then passed the eight unchanged coding fixture tests with
151medium reasoning enabled. Codex took 285.4 seconds and Claude 343.5 seconds
152while competing with the context probe. This checks real harness compatibility;
153the simple retrieval probe does not establish coding quality at 122K tokens.
154The actual approval-review check passed in 131.0 seconds, including a deliberate
15595-second hold beyond stock Codex's deadline. Its authorized write and read-back
156succeeded, and the temporary file was removed.
157
158On 2026-10-05, a synthetic 60,756-token Responses request recovered distinct
159facts from the beginning, middle, and end through a namespaced tool call. A
16060,875-token continuation consumed the tool result correctly, with 60,823
161cached tokens. A subsequent 60,900-token request generated 384 tokens at
16230.75 tokens/second; the largest streamed token gap was 44 ms. These checks
163used the earlier non-thinking preset. They validate context handling and one
164simple retrieval task, not general coding quality at that length.
165
166Cold processing of roughly 60K input tokens took about a minute. End-to-end
167latency also included waits behind other active requests. The live Codex
168session separately reached about 48K input tokens at 33–34 generated
169tokens/second, with recent-prefix prefill below one second. Its compaction
170requests completed successfully. The earlier 64K limit and 48K compaction threshold
171kept space for tool results, reasoning, and output without enlarging the cold
172prefill workload further.
173
174The cache is bounded RAM storage, with no timed expiry or persistence across
175server restarts. In an earlier cross-context probe, 15,912 of 15,916 tokens were
176restored after two competing contexts, returning in 1.05 seconds versus 23.99
177seconds without the host cache. Distinct large contexts can still evict one
178another. The Anthropic endpoint reports `cache_read_input_tokens` and remaining
179uncached `input_tokens`; Responses reports `input_tokens_details.cached_tokens`.
180There are no API cache charges.
181
182The previous two-slot allocation used about 20.9 GiB VRAM; one 64K slot used
183about 18.4 GiB. [Qwen's model card](https://huggingface.co/Qwen/Qwen3.8-27B) describes
184medium effort as the accuracy/speed balance and cautions that reduced reasoning
185can increase total agent time through retries.
186
187## Ajax investigation
188
189As of 2026-10-05, [PewDiePie's official Ajax page](https://data.pewdiepie.com/)
190says the model will release when ready. It describes an ablated Qwen3.5-9B
191fine-tune for [Odysseus](https://github.com/odysseus-dev/odysseus), but does not
192provide verified model weights or a model license to evaluate yet.
193
194When official weights exist, inspect the model card, license, file formats,
195revision, and hashes before downloading. Use the known llama.cpp runtime with
196GGUF data; don't execute model-repository code or load pickle-based weights.
197Trial inference should have a read-only filesystem and model mount, no network,
198no capabilities, no host credentials or private data, process/memory limits,
199and only disposable scratch storage. Tool-use trials belong in a disposable VM
200with synthetic files and credentials, separate from the personal coding profile.
201Evaluate coding, tool arguments, prompt injection, unauthorized writes, and
202attempted outbound requests against the base model. Passing those tests is
203evidence about observed behavior, not proof that an unrestricted agent is safe.
204
205## Runtime tuning (2026-10-07)
206
207Measurements on the production RTX 3090, using 256 greedy decode tokens with the
208same target weights, Q8 KV cache, and context size:
209
210| Input context | Stock runtime | MTP, three Q8 draft tokens |
211| --- | ---: | ---: |
212| 16,384 tokens | 39.1 tok/s | 92.0 tok/s |
213| 65,536 tokens | 30.3 tok/s | 73.3 tok/s |
214
215These are warm synthetic decode rates, not a promise for every coding task.
216The n-gram draft trial did not improve throughput. MTP's target verifies every
217accepted draft token; it does not substitute a smaller model's answers.
218Cold prefill remains about 1,200 tok/s at 16K and 990 tok/s at 64K. Caching avoids
219processing unchanged prefixes, but new long inputs still pay that cost.
220
221The production checks exercise stock Codex and Claude Code editing the same
222median fixture, all eight unchanged tests, Codex's automatic approval reviewer,
223vision, and three-fact retrieval in a 121,816-token input. They establish these
224paths remain functional, not a broad model-quality evaluation. The initial
225MTP/prefix-cache check with stock Codex 0.161.0 and Claude Code 2.1.293 took 68.6 and 120.4
226seconds while sharing the slot. All 30 other production containers retained
227their identities; the AI allocation recorded no OOM events, limit hits, or
228restarts during verification.
229
230`tools/benchmark-local-ai.py` measures decode, natural coding/reasoning workloads,
231client switching, changing system metadata, retrieval, and vision. Run inside
232the production container, where `LLAMA_API_KEY` is already inherited, against
233the backend on `http://127.0.0.1:8081`; it emits timings without prompts or keys:
234
235```sh
236python3 /tmp/benchmark-local-ai.py --suite cache --require-prefix-cache
237python3 /tmp/benchmark-local-ai.py --suite decode --contexts 16384 65536
238python3 /tmp/benchmark-local-ai.py --suite retrieval --contexts 122000
239python3 /tmp/benchmark-local-ai.py --suite vision
240```
241
242The prefix test alternates two system headers and long unrelated user histories,
243then changes a trailing system fact. Both restores must reuse at least 4,096
244tokens and answer from the current header. Production retained 4,114 tokens
245across client switches and 4,096 after the system fact changed, with the new
246fact answered correctly. Repeating the conversation retained 24,135 tokens
247and completed in 0.335 seconds, including generation.
tools/mac-domains.py deleted-248
...@@ -1,248 +0,0 @@
1#!/usr/bin/env python3
2"""Route .studio.test through the rehearsal tunnels on this Mac; stop restores the prior settings."""
3
4import argparse
5import asyncio
6import base64
7import hashlib
8import json
9import os
10from pathlib import Path
11import signal
12import socket
13import ssl
14import subprocess
15import sys
16import time
17
18STATE = Path("/var/db/snowglobe-test")
19RESOLVER = Path("/etc/resolver/studio.test")
20HOSTS = Path("/etc/hosts")
21KEYCHAIN = "/Library/Keychains/System.keychain"
22FORWARDS = ((80, 27080), (443, 27443))
23
24
25def local_hosts(content):
26 lines = []
27 aliases = []
28 for line in content.splitlines(keepends=True):
29 names, _, comment = line.partition("#")
30 fields = names.split()
31 if len(fields) < 2 or not any(name.endswith(".studio.test") for name in fields[1:]):
32 lines.append(line)
33 continue
34 aliases.extend(name for name in fields[1:] if name.endswith(".studio.test"))
35 remaining = [name for name in fields[1:] if not name.endswith(".studio.test")]
36 if remaining:
37 lines.append(" ".join([fields[0], *remaining]) + (" #" + comment.rstrip() if comment else "") + "\n")
38 elif comment:
39 lines.append("#" + comment.rstrip() + "\n")
40 if aliases:
41 if lines and not lines[-1].endswith("\n"):
42 lines[-1] += "\n"
43 lines.append("127.0.0.1 " + " ".join(dict.fromkeys(aliases)) + "\n")
44 return "".join(lines)
45
46
47class DNSRelay(asyncio.DatagramProtocol):
48 def __init__(self):
49 self.requests = set()
50
51 def connection_made(self, transport):
52 self.transport = transport
53
54 def datagram_received(self, data, address):
55 if len(data) < 12 or len(self.requests) >= 64:
56 return
57 request = asyncio.create_task(asyncio.wait_for(self.forward(data, address), timeout=3))
58 self.requests.add(request)
59 request.add_done_callback(self.finished)
60
61 def finished(self, request):
62 self.requests.discard(request)
63 if not request.cancelled():
64 request.exception()
65
66 async def forward(self, data, address):
67 writer = None
68 try:
69 reader, writer = await asyncio.open_connection("127.0.0.1", 53153)
70 writer.write(len(data).to_bytes(2, "big") + data)
71 await writer.drain()
72 length = int.from_bytes(await reader.readexactly(2), "big")
73 response = await reader.readexactly(length)
74 if len(response) >= 12 and response[:2] == data[:2]:
75 self.transport.sendto(response, address)
76 finally:
77 if writer:
78 writer.close()
79
80
81async def relay(listeners):
82 async def connect(reader, writer, upstream):
83 peer_writer = None
84 try:
85 peer_reader, peer_writer = await asyncio.open_connection("127.0.0.1", upstream)
86
87 async def pipe(source, destination):
88 while data := await source.read(65536):
89 destination.write(data)
90 await destination.drain()
91 if destination.can_write_eof():
92 destination.write_eof()
93
94 await asyncio.gather(pipe(reader, peer_writer), pipe(peer_reader, writer))
95 except (OSError, asyncio.CancelledError):
96 pass
97 finally:
98 writer.close()
99 if peer_writer:
100 peer_writer.close()
101
102 servers = []
103 for listener, upstream in listeners:
104 servers.append(await asyncio.start_server(
105 lambda reader, writer, port=upstream: connect(reader, writer, port), sock=listener,
106 ))
107 transport, _ = await asyncio.get_running_loop().create_datagram_endpoint(
108 DNSRelay, local_addr=("127.0.0.1", 53153),
109 )
110 try:
111 await asyncio.gather(*(server.serve_forever() for server in servers))
112 finally:
113 transport.close()
114
115
116def stop():
117 record = STATE / "settings.json"
118 if not record.exists():
119 print("Local test routing is already stopped.")
120 return
121 settings = json.loads(record.read_text())
122 for name, saved in settings["files"].items():
123 path = Path(name)
124 current = hashlib.sha256(path.read_bytes()).hexdigest() if path.exists() else None
125 before = hashlib.sha256(base64.b64decode(saved["before"])).hexdigest() if saved["before"] is not None else None
126 if current not in (before, saved["applied"]):
127 raise SystemExit(f"{path} changed after setup. Restore it using {record}; nothing was overwritten.")
128 if pid := settings.get("pid"):
129 command = subprocess.run(["ps", "-p", str(pid), "-o", "command="], capture_output=True, text=True).stdout
130 if str(Path(__file__).resolve()) + " --relay " in command:
131 try:
132 os.kill(pid, signal.SIGTERM)
133 except ProcessLookupError:
134 pass
135 for name, saved in settings["files"].items():
136 path = Path(name)
137 if saved["before"] is None:
138 path.unlink(missing_ok=True)
139 else:
140 path.write_bytes(base64.b64decode(saved["before"]))
141 if settings.get("addedCA"):
142 subprocess.run(["security", "remove-trusted-cert", "-d", str(STATE / "ca.crt")], check=True)
143 subprocess.run(["security", "delete-certificate", "-Z", settings["sha1"], KEYCHAIN], check=True)
144 subprocess.run(["dscacheutil", "-flushcache"], check=True)
145 subprocess.run(["killall", "-HUP", "mDNSResponder"], check=True)
146 record.unlink()
147 print("Previous DNS, hosts entries, and certificate trust restored.")
148
149
150def start(certificate):
151 if (STATE / "settings.json").exists():
152 raise SystemExit("Local test routing is already configured. Run stop before starting it again.")
153 uid, gid = int(os.environ["SUDO_UID"]), int(os.environ["SUDO_GID"])
154 if uid == 0:
155 raise SystemExit("Run this with sudo from your normal Mac account.")
156 for _, port in FORWARDS:
157 with socket.create_connection(("127.0.0.1", port), timeout=3):
158 pass
159 answer = subprocess.check_output(["dig", "+tcp", "@127.0.0.1", "-p", "53153", "globe.studio.test", "+short"])
160 if answer.strip() != b"127.0.0.1":
161 raise SystemExit("The rehearsal DNS tunnel is unavailable. Start its SSH forwards first.")
162 context = ssl.create_default_context(cafile=str(certificate))
163 with socket.create_connection(("127.0.0.1", 27443), timeout=5) as upstream:
164 with context.wrap_socket(upstream, server_hostname="globe.studio.test"):
165 pass
166 for port, _ in FORWARDS:
167 with socket.socket() as probe:
168 probe.bind(("127.0.0.1", port))
169 with socket.socket(type=socket.SOCK_DGRAM) as probe:
170 probe.bind(("127.0.0.1", 53153))
171 pem = certificate.read_bytes()
172 der = subprocess.check_output(["openssl", "x509", "-outform", "DER"], input=pem)
173 existing = subprocess.run(["security", "find-certificate", "-a", "-p", KEYCHAIN], capture_output=True).stdout
174 sha1 = hashlib.sha1(der).hexdigest().upper()
175 installed = False
176 for part in existing.split(b"-----END CERTIFICATE-----")[:-1]:
177 found = subprocess.check_output(["openssl", "x509", "-outform", "DER"], input=part + b"-----END CERTIFICATE-----\n")
178 installed = installed or found == der
179 trusted = subprocess.run(["security", "verify-cert", "-c", str(certificate), "-p", "ssl"], capture_output=True).returncode == 0
180 if installed and not trusted:
181 raise SystemExit("The rehearsal CA has existing custom trust settings. Enable its SSL trust in Keychain Access before setup.")
182 STATE.mkdir(mode=0o700, exist_ok=True)
183 os.chmod(STATE, 0o700)
184 (STATE / "ca.crt").write_bytes(pem)
185 replacements = {HOSTS: local_hosts(HOSTS.read_text()).encode(),
186 RESOLVER: b"nameserver 127.0.0.1\nport 53153\n"}
187 settings = {"files": {}, "addedCA": False, "sha1": sha1}
188 for path, content in replacements.items():
189 settings["files"][str(path)] = {
190 "before": base64.b64encode(path.read_bytes()).decode() if path.exists() else None,
191 "applied": hashlib.sha256(content).hexdigest(),
192 }
193 record = STATE / "settings.json"
194 record.write_text(json.dumps(settings, indent=2) + "\n")
195 os.chmod(record, 0o600)
196 try:
197 for path, content in replacements.items():
198 path.parent.mkdir(exist_ok=True)
199 path.write_bytes(content)
200 os.chmod(path, 0o644)
201 if not trusted:
202 subprocess.run(["security", "add-trusted-cert", "-d", "-r", "trustRoot", "-p", "ssl", "-k", KEYCHAIN, str(STATE / "ca.crt")], check=True)
203 settings["addedCA"] = True
204 record.write_text(json.dumps(settings, indent=2) + "\n")
205 with (STATE / "relay.log").open("ab") as log:
206 process = subprocess.Popen([sys.executable, str(Path(__file__).resolve()), "--relay", str(uid), str(gid)],
207 stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
208 settings["pid"] = process.pid
209 record.write_text(json.dumps(settings, indent=2) + "\n")
210 time.sleep(0.3)
211 if process.poll() is not None:
212 raise RuntimeError(f"The HTTPS relay did not start. See {STATE / 'relay.log'}.")
213 subprocess.run(["dscacheutil", "-flushcache"], check=True)
214 subprocess.run(["killall", "-HUP", "mDNSResponder"], check=True)
215 subprocess.run(["curl", "--max-time", "8", "--fail", "--silent", "--show-error",
216 "--output", "/dev/null", "https://globe.studio.test/"], check=True)
217 except BaseException:
218 stop()
219 raise
220 print("Local .studio.test routing is active. Open https://globe.studio.test/.")
221
222
223if __name__ == "__main__":
224 if sys.platform != "darwin" or os.geteuid() != 0:
225 raise SystemExit("This setup needs sudo on the Mac for its resolver, certificate trust, and loopback ports 80/443.")
226 if len(sys.argv) == 4 and sys.argv[1] == "--relay":
227 listeners = []
228 for port, upstream in FORWARDS:
229 listener = socket.socket()
230 listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
231 listener.bind(("127.0.0.1", port))
232 listener.listen()
233 listeners.append((listener, upstream))
234 os.setgroups([])
235 os.setgid(int(sys.argv[3]))
236 os.setuid(int(sys.argv[2]))
237 asyncio.run(relay(listeners))
238 else:
239 parser = argparse.ArgumentParser(description=__doc__)
240 parser.add_argument("action", choices=("start", "stop"))
241 parser.add_argument("certificate", type=Path, nargs="?")
242 args = parser.parse_args()
243 if args.action == "stop":
244 stop()
245 elif args.certificate:
246 start(args.certificate.resolve())
247 else:
248 parser.error("start needs the rehearsal's public CA certificate")
tools/verify-local-ai.py deleted-174
...@@ -1,174 +0,0 @@
1#!/usr/bin/env python3
2"""Exercise both real coding clients and Codex's automatic approval reviewer."""
3import argparse
4import concurrent.futures
5from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
6import json
7import os
8import selectors
9from pathlib import Path
10import subprocess
11import tempfile
12import threading
13import time
14import urllib.error
15import urllib.request
16
17BUG = '''def summarize(values):
18 ordered = sorted(values)
19 return {"count": len(values), "median": ordered[len(ordered) // 2]}
20'''
21TESTS = '''import unittest
22from stats import summarize
23class SummaryTests(unittest.TestCase):
24 def test_odd(self):
25 self.assertEqual(summarize([9, 1, 5]), {"count": 3, "median": 5})
26 def test_even(self):
27 self.assertEqual(summarize([9, 1, 5, 3]), {"count": 4, "median": 4})
28 def test_empty(self):
29 self.assertEqual(summarize([]), {"count": 0, "median": None})
30 def test_input_unchanged(self):
31 data = [9, 1, 5]
32 summarize(data)
33 self.assertEqual(data, [9, 1, 5])
34'''
35PROMPT = "Fix stats.py so median is correct for even-length lists and empty input returns a null median. Preserve the caller's list. Run python3 -m unittest -v. Do not change tests. Keep the implementation small."
36
37
38def run_client(root, provider, prompt, extra_env=None, label=None):
39 env = os.environ.copy()
40 env["CODEX_HOME"] = str(root / "codex-home")
41 env.update(extra_env or {})
42 args = ["-p", prompt, "--output-format", "stream-json", "--verbose", "--max-turns", "12"] if provider == "claude" else ["exec", "--json", "--skip-git-repo-check", prompt]
43 started = time.monotonic()
44 log = root / ((label or provider) + ".jsonl")
45 with log.open("w") as out:
46 process = subprocess.run([str(Path(os.environ.get("SNOWGLOBE_AI_BIN_DIR", Path.home() / ".local/bin")) / ("snow-" + provider)), *args], cwd=root / provider, env=env, stdin=subprocess.DEVNULL, stdout=out, stderr=subprocess.STDOUT, timeout=8 * 60 * 60)
47 assert process.returncode == 0, f"{provider} failed; inspect {log}"
48 print(f"{label or provider}: completed in {time.monotonic() - started:.1f}s", flush=True)
49 return log
50
51
52def verify_reasoning(root, endpoint):
53 env = os.environ.copy()
54 env.update(CODEX_HOME=str(root / "codex-home"), SNOWGLOBE_AI_URL=endpoint)
55 wrapper = Path(os.environ.get("SNOWGLOBE_AI_BIN_DIR", Path.home() / ".local/bin")) / "snow-codex"
56 with (root / "codex-reasoning-stderr.log").open("w") as errors, (root / "codex-reasoning.jsonl").open("wb") as log:
57 process = subprocess.Popen([str(wrapper), "-c", "show_raw_agent_reasoning=true", "-c", "hide_agent_reasoning=false", "app-server"], cwd=root / "codex", env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=errors, bufsize=0)
58 with selectors.DefaultSelector() as selector:
59 selector.register(process.stdout, selectors.EVENT_READ)
60 process.stdin.write(json.dumps({"id": 0, "method": "initialize", "params": {"clientInfo": {"name": "snowglobe_reasoning_test", "version": "1.0"}}}).encode() + b"\n")
61 reasoning = 0
62 answer = ""
63 deadline = time.monotonic() + 240
64 try:
65 while time.monotonic() < deadline:
66 if not selector.select(1):
67 continue
68 line = process.stdout.readline()
69 assert line, "Codex app-server closed before completing the reasoning test"
70 log.write(line)
71 event = json.loads(line)
72 assert "error" not in event, event.get("error")
73 if event.get("id") == 0:
74 process.stdin.write(b'{"method":"initialized","params":{}}\n')
75 process.stdin.write(json.dumps({"id": 1, "method": "thread/start", "params": {"cwd": str(root / "codex")}}).encode() + b"\n")
76 elif event.get("id") == 1:
77 process.stdin.write(json.dumps({"id": 2, "method": "turn/start", "params": {"threadId": event["result"]["thread"]["id"], "input": [{"type": "text", "text": "Do not use tools. Find the smallest positive integer divisible by every integer from 1 through 12. Give the integer and a concise justification."}]}}).encode() + b"\n")
78 method = event.get("method")
79 if method == "item/reasoning/textDelta":
80 reasoning += bool(event["params"]["delta"])
81 elif method == "item/agentMessage/delta":
82 assert reasoning > 0, "Codex received no live reasoning before the answer"
83 answer += event["params"]["delta"]
84 elif method == "turn/completed":
85 assert event["params"]["turn"]["status"] == "completed"
86 assert reasoning > 0 and "27720" in answer.replace(",", ""), "Reasoning fixture failed"
87 print(f"Codex streamed {reasoning} reasoning deltas before its answer", flush=True)
88 break
89 else:
90 raise TimeoutError("Codex reasoning stream timed out")
91 finally:
92 process.stdin.close()
93 try:
94 process.wait(timeout=5)
95 except subprocess.TimeoutExpired:
96 process.terminate()
97 process.wait(timeout=5)
98
99
100def review_delay_test(root, delay, endpoint):
101 delayed = threading.Event()
102 class Proxy(BaseHTTPRequestHandler):
103 def log_message(self, *args):
104 pass
105 def do_POST(self):
106 body = self.rfile.read(int(self.headers.get("Content-Length", "0")))
107 data = json.loads(body)
108 properties = data.get("text", {}).get("format", {}).get("schema", {}).get("properties", {})
109 if "risk_level" in properties and "outcome" in properties and not delayed.is_set():
110 delayed.set()
111 print(f"Holding an actual approval-review request for {delay}s", flush=True)
112 time.sleep(delay)
113 headers = {k: v for k, v in self.headers.items() if k.lower() not in {"host", "connection", "content-length", "accept-encoding"}}
114 try:
115 with urllib.request.urlopen(urllib.request.Request(endpoint + self.path, data=body, headers=headers), timeout=28800) as response:
116 self.send_response(response.status)
117 self.send_header("Content-Type", response.headers.get("Content-Type", "application/json"))
118 self.send_header("Connection", "close")
119 self.end_headers()
120 while chunk := response.read1(65536):
121 self.wfile.write(chunk)
122 self.wfile.flush()
123 except (BrokenPipeError, ConnectionResetError):
124 pass
125 except urllib.error.HTTPError as error:
126 self.send_response(error.code)
127 self.end_headers()
128 self.wfile.write(error.read())
129 except (urllib.error.URLError, OSError):
130 self.send_error(502)
131 server = ThreadingHTTPServer(("127.0.0.1", 0), Proxy)
132 threading.Thread(target=server.serve_forever, daemon=True).start()
133 target_dir = Path.home() / ".local/share/snowglobe-ai/evals"
134 target_dir.mkdir(parents=True, exist_ok=True)
135 target = target_dir / (root.name + ".txt")
136 prompt = f"Use exec_command to create {target} containing exactly approval works. This path is outside the writable workspace. Request sandbox_permissions require_escalated with a justification: this temporary fixture write is explicitly authorized. Then read the file back and report its contents. Do not bypass a rejected action."
137 try:
138 run_client(root, "codex", prompt, {"SNOWGLOBE_AI_URL": f"http://127.0.0.1:{server.server_port}"}, "codex-delayed-approval")
139 assert delayed.is_set(), "The client did not make an approval-review request."
140 assert target.read_text() == "approval works"
141 print("Automatic approval authorized the fixture write", flush=True)
142 finally:
143 server.shutdown()
144 server.server_close()
145 target.unlink(missing_ok=True)
146
147
148if __name__ == "__main__":
149 parser = argparse.ArgumentParser(description="Test Snow Globe with real coding clients")
150 parser.add_argument("--approval-delay", type=int, default=0)
151 parser.add_argument("--url", default=os.environ.get("SNOWGLOBE_AI_URL"))
152 args = parser.parse_args()
153 if not args.url:
154 parser.error("Provide --url with the Snow Globe endpoint.")
155 root = Path(tempfile.mkdtemp(prefix="snowglobe-ai-verify-"))
156 (root / "codex-home").mkdir()
157 print(f"Evidence: {root}", flush=True)
158 for provider in ("claude", "codex"):
159 work = root / provider
160 work.mkdir()
161 (work / "stats.py").write_text(BUG)
162 (work / "test_stats.py").write_text(TESTS)
163 with concurrent.futures.ThreadPoolExecutor(2) as pool:
164 futures = [pool.submit(run_client, root, provider, PROMPT) for provider in ("claude", "codex")]
165 for future in futures:
166 future.result()
167 for provider in ("claude", "codex"):
168 work = root / provider
169 assert (work / "test_stats.py").read_text() == TESTS, f"{provider} changed the tests"
170 subprocess.run(["python3", "-m", "unittest", "-v"], cwd=work, check=True)
171 assert args.approval_delay >= 0, "Delay must be nonnegative."
172 verify_reasoning(root, args.url)
173 review_delay_test(root, args.approval_delay, args.url)
174 print("Real coding, reasoning, and approval checks passed", flush=True)