Commit 915ca77e341037b005a05bc3f03ebb37986c926c | home-infra

author	clover caruso <git@paperclover.net> 2026-10-09 01:43:44-07:00
committer	clover caruso <git@paperclover.net> 2026-10-09 14:54:07-07:00
log	915ca77e341037b005a05bc3f03ebb37986c926c
tree	25cef79ace5d21191d96fb5255c6a1355de5c8e4
parent	17bf9c6cc0408a68f9c31cb923eed0f8f13da06a
signature	Signed by SSH key SHA256:52mNGHRsVFBDED9IAX5pe+LRWUefqTbxEReunq21QvU (~clover)

feat: switch qwen deployment to HyperQwen

Much faster and possibly more intelligent Qwen3.8-27B with 150k context,
configured for one stream at a time. Benchmarks:

            BEFORE      AFTER
    16k     92 tok/s    118 tok/s
    64k     73.3 tok/s  101 tok/s
    148k    ---         86 tok/s

Assisted-by: gpt-6.1-sol
Co-authored-by: amy <snovikov@ucsc.edu>

26 files changed, 1563 insertions(+), 1238 deletions(-)

config/site.pkl+1

...	...	@@ -12,6 +12,7 @@ cloverUid: Int = (read?("prop:cloverUid") ?? read?("env:STUDIO_CLOVER_UID") ?? "
12	12	cloverGid: Int = (read?("prop:cloverGid") ?? read?("env:STUDIO_CLOVER_GID") ?? "3000").toInt()
13	13	cloverReadOnly: Boolean = (read?("prop:cloverReadOnly") ?? read?("env:STUDIO_CLOVER_READ_ONLY") ?? "false") == "true"
14	14	mediaRoot: String = read?("prop:mediaRoot") ?? read?("env:STUDIO_MEDIA_ROOT") ?? "\(cloverRoot)/Media"
	15	localAiSsdCache: String = read?("prop:localAiSsdCache") ?? read?("env:STUDIO_LOCAL_AI_SSD_CACHE") ?? "/var/cache/studio/local-ai"
15	16	jellyfinFolders: Mapping<String, String> = new {
16	17	["Anime"] = "Anime"
17	18	["Movies"] = "Movies"

dashboard/ai/launch.sh+1-1

...	...	@@ -70,5 +70,5 @@ else
70	70	export CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS=1 API_TIMEOUT_MS=28800000 API_FORCE_IDLE_TIMEOUT=0
71	71	export CLAUDE_STREAM_IDLE_TIMEOUT_MS=28800000 CLAUDE_BYTE_STREAM_IDLE_TIMEOUT_MS=1800000
72	72	export CLAUDE_ASYNC_AGENT_STALL_TIMEOUT_MS=28800000
73		exec claude --permission-mode auto --append-system-prompt "Your model name is $model." "$@"
	73	exec claude --permission-mode auto --effort @REASONING@ --append-system-prompt "Your model name is $model." "$@"
74	74	fi

service/local-ai/Dockerfile created+30

...	...	@@ -0,0 +1,30 @@
	1	FROM docker.io/nvidia/cuda@sha256:995e80db6d0c3a53d56bd00bba48a0ebd633b67b99a57e16acf9a306e7c744a7
	2
	3	ENV DEBIAN_FRONTEND=noninteractive PIP_NO_CACHE_DIR=1 PYTHONUNBUFFERED=1
	4	RUN apt-get update && apt-get install -y --no-install-recommends \
	5	python3.12 python3.12-venv python3.12-dev \
	6	cuda-nvcc-13-0 cuda-cudart-dev-13-0 libcurand-dev-13-0 \
	7	build-essential patch curl ca-certificates \
	8	&& rm -rf /var/lib/apt/lists/*
	9	WORKDIR /app
	10	RUN python3.12 -c 'import hashlib,urllib.request; data=urllib.request.urlopen("https://api.github.com/repos/syv-ai/HyperQwen/tarball/253c76aea0a240bf7cb2bd3ed92b672c0f258d0d").read(); assert hashlib.sha256(data).hexdigest()=="29b40ee9c371066e0194d23bcd35b08d4b6ec3efecd5799d02475fbbf6b032e3"; open("source.tar.gz","wb").write(data)' \
	11	&& tar -xzf source.tar.gz --strip-components=1 && rm source.tar.gz
	12	RUN python3.12 -m venv venv && venv/bin/pip install --upgrade pip \
	13	&& venv/bin/pip install -r docker/requirements.txt torch==2.13.0 \
	14	safetensors==0.8.0 flashinfer-python==0.6.16.post3 flashinfer-cubin==0.6.13 \
	15	&& venv/bin/pip check
	16	COPY download.py models.json /tmp/model-source/
	17	RUN python3.12 /tmp/model-source/download.py --runtime-only --root /tmp/runtime \
	18	&& MODEL_DIR=$(python3.12 -c 'import json; print(json.load(open("/tmp/model-source/models.json"))["directory"])') \
	19	&& test "$(cat /tmp/runtime/$MODEL_DIR/runtime/BASE_REVISION.txt)" = 253c76aea0a240bf7cb2bd3ed92b672c0f258d0d \
	20	&& rm patches/*.patch \
	21	&& cp /tmp/runtime/$MODEL_DIR/runtime/patches/*.patch patches/ \
	22	&& cp /tmp/runtime/$MODEL_DIR/runtime/single-user/start_qwen.sh single-user/start_qwen.sh \
	23	&& rm -rf /tmp/runtime /tmp/model-source
	24	RUN set -e; SP=$(venv/bin/python -c 'import vllm,os; print(os.path.dirname(vllm.__file__))' \| tail -n1); \
	25	for p in patches/*.patch; do patch --batch -p1 -d "$SP" < "$p"; done; \
	26	bash verify.sh --install
	27	COPY cache /tmp/studio-ai-cache
	28	RUN venv/bin/pip install /tmp/studio-ai-cache && venv/bin/pip check && rm -rf /tmp/studio-ai-cache
	29	ENV HOME=/cache VLLM_NO_USAGE_STATS=1 DO_NOT_TRACK=1 HF_HUB_OFFLINE=1
	30	RUN mkdir -p /cache && chmod 1777 /cache

service/local-ai/build/Dockerfile deleted-13

...	...	@@ -1,13 +0,0 @@
1		ARG RUNTIME=ghcr.io/ggml-org/llama.cpp@sha256:e8318a7b3988f9ca57b28c03486569e60def08f2b7b060550ebfa046e175e6cb
2		FROM ${RUNTIME} AS build
3		RUN rm -f /etc/apt/sources.list.d/* && printf 'deb https://archive.ubuntu.com/ubuntu noble main universe\ndeb https://archive.ubuntu.com/ubuntu noble-updates main universe\ndeb https://security.ubuntu.com/ubuntu noble-security main universe\n' > /etc/apt/sources.list && apt-get -o Acquire::https::Timeout=30 update && apt-get install -y --no-install-recommends cmake g++ make patch libssl-dev
4		WORKDIR /src
5		RUN python3 -c 'import hashlib,urllib.request; data=urllib.request.urlopen("https://api.github.com/repos/ggml-org/llama.cpp/tarball/11fe02151f79c41d0d4af7da708755d73b9c0da6").read(); assert hashlib.sha256(data).hexdigest()=="965db483b5e27c4917bac5bf83e0d2d533099049f8d5234c75e93c969d76443a"; open("source.tar.gz","wb").write(data)' && tar -xzf source.tar.gz --strip-components=1 && rm source.tar.gz
6		COPY prefix-cache.patch /tmp/prefix-cache.patch
7		RUN patch -p1 < /tmp/prefix-cache.patch
8		COPY extract-ui.cpp /tmp/extract-ui.cpp
9		RUN python3 -c 'import pathlib,subprocess; symbols=subprocess.check_output(["nm","-S","/app/libllama-server-impl.so"],text=True); size=int(next(line.split()[1] for line in symbols.splitlines() if line.split()[-1]=="_ZL8g_assets"),16); header=pathlib.Path("tools/ui/ui.h.in").read_text().replace("#cmakedefine LLAMA_UI_HAS_ASSETS 1","#define LLAMA_UI_HAS_ASSETS 1").replace("@N_ASSETS@",str(size)+"/sizeof(llama_ui_asset)"); pathlib.Path("/tmp/ui.h").write_text(header)' && g++ -std=c++17 -I/tmp /tmp/extract-ui.cpp -L/app -Wl,-rpath,/app -lllama-server-impl -o /tmp/extract-ui && LD_LIBRARY_PATH=/app /tmp/extract-ui && python3 -c 'import gzip,pathlib; [p.write_bytes(gzip.decompress(p.read_bytes())) for p in pathlib.Path("tools/ui/dist").rglob("*") if p.is_file()]'
10		RUN cmake -S . -B /build -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DGGML_CUDA=OFF -DGGML_CPU=OFF -DGGML_BACKEND_DL=ON -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_UI=OFF -DLLAMA_USE_PREBUILT_UI=OFF -DLLAMA_BUILD_NUMBER=11382 -DLLAMA_BUILD_COMMIT=11fe02151 && cmake --build /build --target llama-ui-assets && ln -s /app/libllama.so* /app/libllama-common.so* /app/libmtmd.so* /app/libggml.so /build/bin/ && cmake --build /build --target server-context/fast cpp-httplib/fast llama-common-base/fast llama-ui/fast -j 4 && cmake --build /build --target llama-server-impl/fast -j 4
11
12		FROM ${RUNTIME}
13		COPY --from=build /build/bin/libllama-server-impl.so /app/libllama-server-impl.so

service/local-ai/build/extract-ui.cpp deleted-18

...	...	@@ -1,18 +0,0 @@
1		#include "ui.h"
2		#include <filesystem>
3		#include <fstream>
4
5		int main() {
6		if (!llama_ui_use_gzip()) {
7		return 1;
8		}
9		for (const auto & asset : llama_ui_get_assets()) {
10		const auto path = std::filesystem::path("tools/ui/dist") / asset.name;
11		std::filesystem::create_directories(path.parent_path());
12		std::ofstream output(path, std::ios::binary);
13		output.write(reinterpret_cast<const char *>(asset.data), asset.size);
14		if (!output) {
15		return 1;
16		}
17		}
18		}

service/local-ai/build/prefix-cache.patch deleted-237

...	...	@@ -1,237 +0,0 @@
1		--- a/tools/server/server-task.h
2		+++ b/tools/server/server-task.h
3		@@ -650,12 +650,13 @@
4		};
5
6		struct server_prompt_cache {
7		- server_prompt_cache(int32_t limit_size_mib, size_t limit_tokens) {
8		+ server_prompt_cache(int32_t limit_size_mib, size_t limit_tokens, bool keep_on_load = false) : keep_on_load(keep_on_load) {
9		this->limit_size = 1024ull1024ull(limit_size_mib < 0 ? 0 : limit_size_mib);
10		this->limit_tokens = limit_tokens;
11		}
12
13		std::list<server_prompt_cache_state> states;
14		+ const bool keep_on_load;
15
16		// in bytes, 0 = no limit
17		size_t limit_size = 0;
18		--- a/tools/server/server-task.cpp
19		+++ b/tools/server/server-task.cpp
20		@@ -1226,6 +1226,7 @@
21		{"event", "response.reasoning_text.delta"},
22		{"data", json {
23		{"type", "response.reasoning_text.delta"},
24		+ {"content_index", 0},
25		{"delta", diff.reasoning_content_delta},
26		{"item_id", oai_resp_reasoning_id},
27		}},
28		@@ -1724,7 +1725,10 @@
29		for (auto it = states.begin(); it != states.end(); ++it) {
30		const int cur_lcp_len = it->prompt.tokens.get_common_prefix(prompt.tokens);
31
32		- if (cur_lcp_len == (int) prompt.tokens.size()) {
33		+ if (cur_lcp_len == (int) prompt.tokens.size() && (!keep_on_load \|\| it->prompt.tokens.size() == prompt.tokens.size())) {
34		+ if (keep_on_load) {
35		+ states.splice(states.end(), states, it);
36		+ }
37		SRV_TRC("%s", " - prompt is already in the cache, skipping\n");
38		return nullptr;
39		}
40		@@ -1749,7 +1753,7 @@
41		for (auto it = states.begin(); it != states.end();) {
42		const int len = it->prompt.tokens.get_common_prefix(prompt.tokens);
43
44		- if (len == (int) it->prompt.tokens.size()) {
45		+ if (!keep_on_load && len == (int) it->prompt.tokens.size()) {
46		SRV_TRC(" - removing obsolete cached prompt with length %d\n", len);
47
48		it = states.erase(it);
49		@@ -1820,12 +1824,18 @@
50
51		SRV_TRC(" - prompt with length %7zu, lcp = %7d, f_keep = %.3f, f_sim = %.3f\n", it->prompt.tokens.size(), lcp_cur, f_keep_cur, f_sim_cur);
52
53		+ // Only complete pinned prefixes contain a usable recurrent state.
54		+ if (keep_on_load && lcp_cur != (int) it->prompt.tokens.size()) {
55		+ continue;
56		+ }
57		+
58		// don't trash large prompts
59		if (f_keep_cur < 0.25f) {
60		continue;
61		}
62
63		- if (f_keep_best < f_keep_cur && f_sim_best < f_sim_cur) {
64		+ if ((keep_on_load \|\| f_keep_best < f_keep_cur) &&
65		+ (f_sim_best < f_sim_cur \|\| (keep_on_load && f_sim_best == f_sim_cur && f_keep_best < f_keep_cur))) {
66		f_keep_best = f_keep_cur;
67		f_sim_best = f_sim_cur;
68
69		@@ -1847,8 +1857,10 @@
70		return false;
71		}
72
73		- data.clear();
74		- data.shrink_to_fit();
75		+ if (!keep_on_load) {
76		+ data.clear();
77		+ data.shrink_to_fit();
78		+ }
79		}
80
81		{
82		@@ -1865,14 +1877,20 @@
83		return false;
84		}
85
86		- data.clear();
87		- data.shrink_to_fit();
88		+ if (!keep_on_load) {
89		+ data.clear();
90		+ data.shrink_to_fit();
91		+ }
92		}
93		}
94
95		- prompt = std::move(it_best->prompt);
96		-
97		- states.erase(it_best);
98		+ if (keep_on_load) {
99		+ prompt = it_best->prompt.clone();
100		+ states.splice(states.end(), states, it_best);
101		+ } else {
102		+ prompt = std::move(it_best->prompt);
103		+ states.erase(it_best);
104		+ }
105		}
106
107		return true;
108		--- a/tools/server/server-context.cpp
109		+++ b/tools/server/server-context.cpp
110		@@ -954,6 +954,7 @@
111		int n_empty_consecutive = 0;
112
113		std::unique_ptr<server_prompt_cache> prompt_cache;
114		+ std::unique_ptr<server_prompt_cache> prefix_cache;
115
116		server_metrics metrics;
117
118		@@ -1424,6 +1425,7 @@
119		SRV_TRC("%s", "use `--cache-ram 0` to disable the prompt cache\n");
120
121		prompt_cache = std::make_unique<server_prompt_cache>(params_base.cache_ram_mib, n_ctx);
122		+ prefix_cache = std::make_unique<server_prompt_cache>(2048, n_ctx, true);
123		} else {
124		SRV_TRC("%s", "prompt cache is disabled - use `--cache-ram N` to enable it\n");
125		}
126		@@ -1721,6 +1723,11 @@
127
128		SRV_TRC("prompt cache update took %.2f ms\n", (ggml_time_us() - t_start) / 1000.0);
129		}
130		+ }
131		+
132		+ if (ret && prefix_cache && task.type == SERVER_TASK_TYPE_COMPLETION &&
133		+ !ret->prompt_load(*prefix_cache, task.tokens)) {
134		+ ret->prompt_clear();
135		}
136
137		return ret;
138		@@ -3091,7 +3098,7 @@
139		// add generated tokens to cache
140		// ref: https://github.com/ggml-org/llama.cpp/pull/16818#discussion_r2473269481
141		{
142		- GGML_ASSERT(!slot.prompt.tokens.has_mtmd);
143		+ GGML_ASSERT(!slot.prompt.tokens.find_next_media_chunk(0).first);
144
145		llama_tokens new_tokens = slot.prompt.tokens.get_tokens(); // copy
146		for (size_t i = n_keep + n_discard; i < new_tokens.size(); i++) {
147		@@ -3400,7 +3407,7 @@
148
149		// reuse chunks from the cached prompt by shifting their KV cache in the new position
150		if (can_cache_reuse && n_cache_reuse > 0) {
151		- GGML_ASSERT(!slot.prompt.tokens.has_mtmd);
152		+ GGML_ASSERT(!slot.prompt.tokens.find_next_media_chunk(0).first);
153
154		size_t head_c = n_past; // cache
155		size_t head_p = n_past; // current prompt
156		@@ -3701,6 +3708,11 @@
157
158		const auto & spans = slot.task->params.message_spans;
159		const auto last_user_pos = spans.last_user_message_pos();
160		+ const auto first_user = std::find_if(spans.spans.begin(), spans.spans.end(), [](const auto & span) {
161		+ return span.role == COMMON_CHAT_ROLE_USER;
162		+ });
163		+ const int64_t first_user_pos = first_user == spans.spans.end() ? -1 : first_user->pos;
164		+ const int64_t prefix_pos = std::min<int64_t>(4096, first_user_pos);
165
166		const int32_t n_decision_first = slot.task->type == SERVER_TASK_TYPE_DECISION ? slot.task->decision.pos_first() : -1;
167
168		@@ -3743,12 +3755,16 @@
169		}
170		slot.prompt.tokens.push_back(cur_tok);
171
172		+ if (do_checkpoint && prefix_cache && slot.prompt.n_tokens() == prefix_pos) {
173		+ break;
174		+ }
175		+
176		// break at the last user message, or at user messages at least min step past the last checkpoint
177		if (do_checkpoint && spans.is_user_start(slot.prompt.n_tokens())) {
178		const auto pos = slot.prompt.n_tokens();
179		const auto & checkpoints = slot.prompt.checkpoints;
180
181		- if (pos == last_user_pos \|\| checkpoints.empty() \|\| pos > checkpoints.back().n_tokens + params_base.checkpoint_min_step) {
182		+ if (pos == first_user_pos \|\| pos == last_user_pos \|\| checkpoints.empty() \|\| pos > checkpoints.back().n_tokens + params_base.checkpoint_min_step) {
183		break;
184		}
185		}
186		@@ -3783,6 +3799,8 @@
187		const bool near_prompt_end = slot.task->n_tokens() < slot.prompt.n_tokens() + n_ubatch;
188
189		const bool is_user_start = spans.is_user_start(n_tokens_start);
190		+ const bool is_prefix_boundary = prefix_cache && n_tokens_start > 0 &&
191		+ (n_tokens_start == prefix_pos \|\| n_tokens_start == first_user_pos);
192		const bool is_last_user_message = n_tokens_start == last_user_pos;
193
194		// entire prompt has been processed
195		@@ -3799,9 +3817,8 @@
196
197		slot.init_sampler();
198		} else {
199		- // skip ordinary mid-prompt checkpoints, unless the batch starts a user
200		- // message or we are near the end of the prompt
201		- if (!is_user_start && !near_prompt_end) {
202		+ // Retain system-prefix boundaries even inside a message.
203		+ if (!is_user_start && !is_prefix_boundary && !near_prompt_end) {
204		do_checkpoint = false;
205		}
206		}
207		@@ -3821,7 +3838,7 @@
208		// no need to create checkpoints that are too close together, unless it's the last user message
209		do_checkpoint = do_checkpoint && (
210		slot.prompt.checkpoints.empty() \|\|
211		- is_last_user_message \|\| near_prompt_end \|\|
212		+ is_last_user_message \|\| is_prefix_boundary \|\| near_prompt_end \|\|
213		n_tokens_start > slot.prompt.checkpoints.back().n_tokens + params_base.checkpoint_min_step);
214		SLT_DBG(slot, "main/do_checkpoint = %s, pos_min = %d, pos_max = %d\n", do_checkpoint ? "yes" : "no", pos_min, pos_max);
215
216		@@ -3829,6 +3846,21 @@
217		// yet processed and therefore it is not part of the checkpoint.
218		if (do_checkpoint) {
219		create_checkpoint(slot, n_tokens_cur, pos_min, pos_max);
220		+ if (is_prefix_boundary &&
221		+ !slot.prompt.tokens.find_next_media_chunk(0).first) {
222		+ server_prompt prefix {slot.prompt.tokens.clone(), {slot.prompt.checkpoints.back()}};
223		+ prefix.tokens.keep_first(n_tokens_start);
224		+ const size_t size = llama_state_seq_get_size_ext(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_NONE);
225		+ const size_t draft_size = ctx_dft ? llama_state_seq_get_size_ext(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_NONE) : 0;
226		+ if (auto * state = prefix_cache->alloc(prefix, size, draft_size)) {
227		+ llama_state_seq_get_data_ext(ctx_tgt, state->data.main.data(), size, slot.id, LLAMA_STATE_SEQ_FLAGS_NONE);
228		+ if (ctx_dft) {
229		+ llama_state_seq_get_data_ext(ctx_dft, state->data.drft.data(), draft_size, slot.id, LLAMA_STATE_SEQ_FLAGS_NONE);
230		+ }
231		+ prefix_cache->update();
232		+ SLT_INF(slot, "cached base prefix, n_tokens = %d\n", n_tokens_start);
233		+ }
234		+ }
235		}
236		}
237

service/local-ai/cache/pyproject.toml created+14

...	...	@@ -0,0 +1,14 @@
	1	[build-system]
	2	requires = ["setuptools"]
	3	build-backend = "setuptools.build_meta"
	4
	5	[project]
	6	name = "studio-ai-cache"
	7	version = "1.0.0"
	8	dependencies = ["cryptography==50.0.2"]
	9
	10	[project.entry-points."vllm.general_plugins"]
	11	studio_ai_cache = "studio_ai_cache_vllm:register"
	12
	13	[tool.setuptools]
	14	py-modules = ["studio_ai_cache", "studio_ai_cache_vllm"]

service/local-ai/cache/studio_ai_cache.py created+186

...	...	@@ -0,0 +1,186 @@
	1	"""Bounded, authenticated disk caches with SSD promotion and request pinning."""
	2	from collections import OrderedDict
	3	import fcntl
	4	import logging
	5	import mmap
	6	import os
	7	from pathlib import Path
	8	import struct
	9	import threading
	10
	11	from cryptography.exceptions import InvalidTag
	12	from cryptography.hazmat.primitives.ciphers.aead import AESGCM
	13
	14
	15	class DiskCache:
	16	def __init__(self, ssd, hdd, ssd_bytes, hdd_bytes, key):
	17	if min(ssd_bytes, hdd_bytes) <= 0:
	18	raise ValueError("Disk cache capacities must be positive")
	19	self.cipher = AESGCM(key)
	20	self.roots = (Path(ssd), Path(hdd))
	21	if (self.roots[0].resolve().is_relative_to(self.roots[1].resolve())
	22	or self.roots[1].resolve().is_relative_to(self.roots[0].resolve())):
	23	raise ValueError("Disk cache directories must be distinct")
	24	self.limits = (ssd_bytes, hdd_bytes)
	25	self.lock = threading.Lock()
	26	self.writers = (threading.Lock(), threading.Lock())
	27	self.entries = (OrderedDict(), OrderedDict())
	28	self.pins = {}
	29	self.files = []
	30	try:
	31	for tier, root in enumerate(self.roots):
	32	root.mkdir(mode=0o700, parents=True, exist_ok=True)
	33	root.chmod(0o700)
	34	descriptor = os.open(root / ".lock", os.O_CREAT \| os.O_RDWR, 0o600)
	35	self.files.append(descriptor)
	36	fcntl.flock(descriptor, fcntl.LOCK_EX \| fcntl.LOCK_NB)
	37	for path in root.rglob("*.pending"):
	38	path.unlink()
	39	inventory = sorted((path.stat().st_mtime_ns, path) for path in root.rglob("*.bin"))
	40	for _, path in inventory:
	41	self.entries[tier][str(path.relative_to(root))] = path.stat().st_size
	42	self._evict(tier, 0)
	43	except Exception:
	44	self.close()
	45	raise
	46
	47	def close(self):
	48	for descriptor in self.files:
	49	os.close(descriptor)
	50	self.files.clear()
	51
	52	def acquire(self, name, request):
	53	with self.lock:
	54	for entries in self.entries:
	55	if name in entries:
	56	entries.move_to_end(name)
	57	self.pins.setdefault(name, set()).add(request)
	58	return True
	59	return False
	60
	61	def release(self, request):
	62	with self.lock:
	63	for name, requests in list(self.pins.items()):
	64	requests.discard(request)
	65	if not requests:
	66	del self.pins[name]
	67
	68	def touch(self, name):
	69	with self.lock:
	70	for entries in self.entries:
	71	if name in entries:
	72	entries.move_to_end(name)
	73	break
	74
	75	def _evict(self, tier, needed):
	76	removed = []
	77	with self.lock:
	78	entries = self.entries[tier]
	79	size = sum(entries.values())
	80	for name in list(entries):
	81	if size + needed <= self.limits[tier]:
	82	break
	83	if name not in self.pins:
	84	size -= entries.pop(name)
	85	removed.append(self.roots[tier] / name)
	86	for path in removed:
	87	self._unlink(tier, path)
	88	return size + needed <= self.limits[tier]
	89
	90	def _unlink(self, tier, path):
	91	path.unlink(missing_ok=True)
	92	parent = path.parent
	93	while parent != self.roots[tier]:
	94	try:
	95	parent.rmdir()
	96	except OSError:
	97	break
	98	parent = parent.parent
	99
	100	def _put(self, tier, name, buffer):
	101	path = self.roots[tier] / name
	102	with self.writers[tier]:
	103	with self.lock:
	104	if name in self.entries[tier]:
	105	self.entries[tier].move_to_end(name)
	106	present = True
	107	else:
	108	present = False
	109	if present:
	110	os.utime(path, None)
	111	return
	112	if len(buffer) > self.limits[tier] or not self._evict(tier, len(buffer)):
	113	return
	114	path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
	115	pending = path.with_suffix(".pending")
	116	try:
	117	descriptor = os.open(pending, os.O_CREAT \| os.O_EXCL \| os.O_WRONLY \| getattr(os, "O_DIRECT", 0), 0o600)
	118	try:
	119	if os.write(descriptor, buffer) != len(buffer):
	120	raise OSError("Short cache write")
	121	os.fsync(descriptor)
	122	finally:
	123	os.close(descriptor)
	124	pending.replace(path)
	125	with self.lock:
	126	self.entries[tier][name] = len(buffer)
	127	finally:
	128	pending.unlink(missing_ok=True)
	129
	130	def store(self, name, data):
	131	# Page-aligned ciphertext keeps direct I/O on both ext4 and ZFS.
	132	size = (len(data) + 36 + mmap.PAGESIZE - 1) // mmap.PAGESIZE * mmap.PAGESIZE
	133	with mmap.mmap(-1, size) as buffer:
	134	nonce = os.urandom(12)
	135	buffer[:20] = struct.pack("!Q", len(data)) + nonce
	136	view = memoryview(buffer)[20:20 + len(data) + 16]
	137	try:
	138	self.cipher.encrypt_into(nonce, data, name.encode(), view)
	139	finally:
	140	view.release()
	141	self._put(1, name, buffer)
	142	self._put(0, name, buffer)
	143
	144	def load(self, name, target):
	145	pin = object()
	146	if not self.acquire(name, pin):
	147	raise FileNotFoundError("Cache block was evicted")
	148	try:
	149	for tier, root in enumerate(self.roots):
	150	path = root / name
	151	with self.lock:
	152	size = self.entries[tier].get(name)
	153	if size is None:
	154	continue
	155	try:
	156	with mmap.mmap(-1, size) as buffer:
	157	descriptor = os.open(path, os.O_RDONLY \| getattr(os, "O_DIRECT", 0))
	158	try:
	159	if os.readv(descriptor, [buffer]) != size:
	160	raise OSError("Short cache read")
	161	finally:
	162	os.close(descriptor)
	163	if struct.unpack("!Q", buffer[:8])[0] != len(target):
	164	raise ValueError("Cache block size mismatch")
	165	view = memoryview(buffer)[20:20 + len(target) + 16]
	166	try:
	167	self.cipher.decrypt_into(buffer[8:20], view, name.encode(), target)
	168	finally:
	169	view.release()
	170	with self.lock:
	171	self.entries[tier].move_to_end(name)
	172	os.utime(path, None)
	173	if tier == 1:
	174	try:
	175	self._put(0, name, buffer)
	176	except OSError:
	177	logging.getLogger(__name__).warning("SSD cache promotion failed", exc_info=True)
	178	return "ssd" if tier == 0 else "hdd"
	179	except (OSError, ValueError, struct.error, InvalidTag):
	180	with self.writers[tier]:
	181	with self.lock:
	182	self.entries[tier].pop(name, None)
	183	self._unlink(tier, path)
	184	raise FileNotFoundError("No authenticated cache block remains")
	185	finally:
	186	self.release(pin)

service/local-ai/cache/studio_ai_cache_vllm.py created+73

...	...	@@ -0,0 +1,73 @@
	1	"""vLLM secondary tier for the bounded SSD/HDD cache."""
	2	import functools
	3	import hashlib
	4	import os
	5	from pathlib import Path
	6
	7	import vllm
	8	from vllm.v1.kv_offload.base import LookupResult, Medium
	9	from vllm.v1.kv_offload.file_mapper import FileMapper
	10	from vllm.v1.kv_offload.tiering.base import JobResult, RequestOffloadingContext, SecondaryTierManager
	11	from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory
	12	from vllm.v1.kv_offload.tiering.fs.thread_pool import DualQueueThreadPool
	13
	14	from studio_ai_cache import DiskCache
	15
	16
	17	class DiskTier(SecondaryTierManager):
	18	medium = Medium.STORAGE
	19
	20	def __init__(self, offloading_spec, primary_kv_view, tier_type, ssd_dir, hdd_dir, ssd_bytes, hdd_bytes):
	21	super().__init__(offloading_spec, primary_kv_view, tier_type)
	22	key = hashlib.sha256(os.environ["STUDIO_AI_CACHE_KEY"].encode()).digest()
	23	namespace = hashlib.sha256(
	24	Path("/models/manifest.json").read_bytes() + vllm.__version__.encode()
	25	+ Path(__file__).read_bytes() + Path(__file__).with_name("studio_ai_cache.py").read_bytes() + key
	26	).hexdigest()
	27	self.mapper = FileMapper.from_offloading_spec(namespace, offloading_spec, offloading_spec.blocks_per_chunk)
	28	self.cache = DiskCache(ssd_dir, hdd_dir, ssd_bytes, hdd_bytes, key)
	29	self.block_size = primary_kv_view.strides[0]
	30	self.pool = DualQueueThreadPool(16, 2, thread_name_prefix="studio_kv_disk")
	31
	32	def on_new_request(self, req_context):
	33	return RequestOffloadingContext()
	34
	35	def lookup(self, key, req_context):
	36	return LookupResult.HIT if self.cache.acquire(self.mapper.get_file_name(key), req_context.req_id) else LookupResult.MISS
	37
	38	def submit_store(self, job_metadata):
	39	tasks = [functools.partial(self._transfer, self.cache.store, key, block_id)
	40	for key, block_id in zip(job_metadata.keys, job_metadata.block_ids, strict=True)]
	41	self.pool.enqueue_store(job_metadata.job_id, len(tasks), tasks)
	42
	43	def submit_load(self, job_metadata):
	44	tasks = [functools.partial(self._transfer, self.cache.load, key, block_id)
	45	for key, block_id in zip(job_metadata.keys, job_metadata.block_ids, strict=True)]
	46	self.pool.enqueue_load(job_metadata.job_id, len(tasks), tasks)
	47
	48	def _transfer(self, operation, key, block_id):
	49	offset = int(block_id) * self.block_size
	50	with self._primary_kv_view.cast("B")[offset:offset + self.block_size] as view:
	51	operation(self.mapper.get_file_name(key), view)
	52
	53	def get_finished_jobs(self):
	54	return [JobResult(job_id, success) for job_id, success in self.pool.get_finished()]
	55
	56	def touch(self, keys, req_context):
	57	for key in keys:
	58	self.cache.touch(self.mapper.get_file_name(key))
	59
	60	def on_request_finished(self, req_context):
	61	self.cache.release(req_context.req_id)
	62
	63	def drain_jobs(self):
	64	self.pool.wait_idle()
	65
	66	def shutdown(self):
	67	self.pool.wait_idle()
	68	self.pool.shutdown(wait=True)
	69	self.cache.close()
	70
	71
	72	def register():
	73	SecondaryTierFactory.register_tier("studio_disk", __name__, "DiskTier")

service/local-ai/cache/test_cache.py created+115

...	...	@@ -0,0 +1,115 @@
	1	from concurrent.futures import ThreadPoolExecutor
	2	import os
	3	import mmap
	4	from pathlib import Path
	5	import tempfile
	6	import unittest
	7
	8	from studio_ai_cache import DiskCache
	9
	10
	11	class CacheTests(unittest.TestCase):
	12	def setUp(self):
	13	self.temporary = tempfile.TemporaryDirectory()
	14	self.addCleanup(self.temporary.cleanup)
	15	self.root = Path(self.temporary.name)
	16	self.key = os.urandom(32)
	17	self.cache = DiskCache(self.root / "ssd", self.root / "hdd", 2 * mmap.PAGESIZE, 3 * mmap.PAGESIZE, self.key)
	18	self.addCleanup(self.cache.close)
	19
	20	def test_hdd_hit_promotes_to_ssd_and_both_limits_hold(self):
	21	for name in ("a.bin", "b.bin", "c.bin"):
	22	self.cache.store(name, name.encode() * 100)
	23	target = bytearray(500)
	24	self.assertEqual(self.cache.load("a.bin", target), "hdd")
	25	self.assertEqual(target, b"a.bin" * 100)
	26	self.assertEqual(self.cache.load("a.bin", target), "ssd")
	27	for directory, limit in (("ssd", 2 * mmap.PAGESIZE), ("hdd", 3 * mmap.PAGESIZE)):
	28	self.assertLessEqual(sum(p.stat().st_size for p in (self.root / directory).glob("*.bin")), limit)
	29
	30	def test_overflow_evicts_cold_blocks(self):
	31	for name in ("a.bin", "b.bin", "c.bin", "d.bin", "e.bin"):
	32	self.cache.store(name, b"fixture")
	33	self.assertFalse(self.cache.acquire("a.bin", "request"))
	34	self.assertFalse(self.cache.acquire("b.bin", "request"))
	35	self.assertTrue(self.cache.acquire("c.bin", "request"))
	36	self.cache.release("request")
	37
	38	def test_pinned_request_survives_eviction_and_full_cache_skips_store(self):
	39	self.cache.close()
	40	self.cache = DiskCache(self.root / "ssd", self.root / "hdd", mmap.PAGESIZE, mmap.PAGESIZE, self.key)
	41	self.addCleanup(self.cache.close)
	42	self.cache.store("a.bin", b"first")
	43	self.assertTrue(self.cache.acquire("a.bin", "request"))
	44	self.cache.store("b.bin", b"other")
	45	self.assertFalse(self.cache.acquire("b.bin", "other-request"))
	46	target = bytearray(5)
	47	self.cache.load("a.bin", target)
	48	self.assertEqual(target, b"first")
	49	self.cache.release("request")
	50	self.cache.store("b.bin", b"other")
	51	self.assertFalse(self.cache.acquire("a.bin", "request"))
	52
	53	def test_restart_recovers_encrypted_blocks_with_restricted_permissions(self):
	54	payload = b"synthetic private prompt fixture" * 10
	55	self.cache.store("namespace/a.bin", payload)
	56	for path in self.root.rglob("*.bin"):
	57	self.assertNotIn(payload, path.read_bytes())
	58	self.assertEqual(path.stat().st_mode & 0o777, 0o600)
	59	self.cache.close()
	60	restored = DiskCache(self.root / "ssd", self.root / "hdd", 2 * mmap.PAGESIZE, 3 * mmap.PAGESIZE, self.key)
	61	self.addCleanup(restored.close)
	62	target = bytearray(len(payload))
	63	self.assertEqual(restored.load("namespace/a.bin", target), "ssd")
	64	self.assertEqual(target, payload)
	65
	66	def test_corrupt_ssd_falls_back_to_hdd_and_repairs_hot_copy(self):
	67	self.cache.store("a.bin", b"authenticated fixture")
	68	path = self.root / "ssd/a.bin"
	69	contents = bytearray(path.read_bytes())
	70	contents[20] ^= 1
	71	path.write_bytes(contents)
	72	target = bytearray(21)
	73	self.assertEqual(self.cache.load("a.bin", target), "hdd")
	74	self.assertEqual(target, b"authenticated fixture")
	75	self.assertEqual(self.cache.load("a.bin", target), "ssd")
	76
	77	def test_corrupt_both_copies_becomes_cache_miss(self):
	78	self.cache.store("a.bin", b"fixture")
	79	for path in self.root.rglob("*.bin"):
	80	path.write_bytes(b"broken")
	81	with self.assertRaises(FileNotFoundError):
	82	self.cache.load("a.bin", bytearray(7))
	83	self.assertFalse(self.cache.acquire("a.bin", "request"))
	84
	85	def test_wrong_key_and_different_namespace_cannot_reuse_state(self):
	86	self.cache.store("old/a.bin", b"fixture")
	87	self.assertFalse(self.cache.acquire("new/a.bin", "request"))
	88	self.cache.close()
	89	other = DiskCache(self.root / "ssd", self.root / "hdd", 2 * mmap.PAGESIZE, 3 * mmap.PAGESIZE, os.urandom(32))
	90	self.addCleanup(other.close)
	91	with self.assertRaises(FileNotFoundError):
	92	other.load("old/a.bin", bytearray(7))
	93
	94	def test_directories_are_exclusive(self):
	95	with self.assertRaises(BlockingIOError):
	96	DiskCache(self.root / "ssd", self.root / "hdd", 2 * mmap.PAGESIZE, 3 * mmap.PAGESIZE, self.key)
	97
	98	def test_concurrent_loads_keep_request_pins_until_release(self):
	99	self.cache.store("a.bin", b"fixture")
	100	self.assertTrue(self.cache.acquire("a.bin", "request"))
	101
	102	def exercise(index):
	103	self.cache.store(f"{index}.bin", b"other")
	104	target = bytearray(7)
	105	self.cache.load("a.bin", target)
	106	self.assertEqual(target, b"fixture")
	107
	108	with ThreadPoolExecutor(max_workers=8) as pool:
	109	list(pool.map(exercise, range(24)))
	110	self.cache.release("request")
	111	self.assertEqual(self.cache.pins, {})
	112
	113
	114	if __name__ == "__main__":
	115	unittest.main()

service/local-ai/cache/test_vllm_cache.py created+97

...	...	@@ -0,0 +1,97 @@
	1	"""CPU-only checks against the installed vLLM scheduler and shared-memory tier."""
	2	import mmap
	3	import os
	4	from pathlib import Path
	5	import tempfile
	6	import unittest
	7	import uuid
	8
	9	from vllm.v1.kv_offload.base import LookupResult, ReqContext, ScheduleEndContext, make_offload_key
	10	from vllm.v1.kv_offload.config import OffloadingCacheConfig, OffloadingConfig, OffloadingGroupConfig, OffloadingModelConfig, OffloadingParallelConfig
	11	from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
	12	from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec
	13	from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory
	14	from vllm.v1.kv_offload.tiering.manager import CPUPrimaryTierOffloadingManager, TieringOffloadingManager
	15
	16	from studio_ai_cache_vllm import register
	17
	18
	19	class VllmCacheTests(unittest.TestCase):
	20	@classmethod
	21	def setUpClass(cls):
	22	register()
	23
	24	def setUp(self):
	25	self.temporary = tempfile.TemporaryDirectory()
	26	self.addCleanup(self.temporary.cleanup)
	27	self.root = Path(self.temporary.name)
	28	os.environ["STUDIO_AI_CACHE_KEY"] = "disposable-synthetic-cache-key"
	29	config = OffloadingConfig(
	30	groups=(OffloadingGroupConfig(16, ("attention",)), OffloadingGroupConfig(16, ("recurrent",))),
	31	worker_kv_bytes_per_block=mmap.PAGESIZE,
	32	enable_kv_cache_events=False, extra_config={"cpu_bytes_to_use": 4 * mmap.PAGESIZE},
	33	engine_id="synthetic", model=OffloadingModelConfig("synthetic", "uint8"),
	34	cache=OffloadingCacheConfig(16, 1),
	35	parallel=OffloadingParallelConfig(0, 1, 1, 1, 1, 1, 0, False),
	36	)
	37	spec = CPUOffloadingSpec(config)
	38	region = SharedOffloadRegion("synthetic-" + uuid.uuid4().hex, 4, None, mmap.PAGESIZE, mmap.PAGESIZE)
	39	primary = CPUPrimaryTierOffloadingManager(4, region)
	40	self.view = primary.get_kv_memoryview()
	41	self.tier = SecondaryTierFactory.create_secondary_tier({
	42	"type": "studio_disk", "ssd_dir": str(self.root / "ssd"), "hdd_dir": str(self.root / "hdd"),
	43	"ssd_bytes": 4 * mmap.PAGESIZE, "hdd_bytes": 6 * mmap.PAGESIZE,
	44	}, self.view, spec)
	45	self.manager = TieringOffloadingManager(primary, [self.tier])
	46	self.addCleanup(self.manager.shutdown)
	47	self.keys = [make_offload_key(bytes([i + 1]) * 32, i % 2) for i in range(3)]
	48	self.values = [bytes([i + 11]) * mmap.PAGESIZE for i in range(3)]
	49	context = ReqContext("store")
	50	self.manager.on_new_request(context)
	51	stored = self.manager.prepare_store(self.keys, context)
	52	with self.view.cast("B") as flat:
	53	for block_id, value in zip(stored.store_spec.block_ids, self.values, strict=True):
	54	offset = int(block_id) * mmap.PAGESIZE
	55	flat[offset:offset + mmap.PAGESIZE] = value
	56	self.manager.complete_store(self.keys, context)
	57	self.tier.drain_jobs()
	58	self.manager.on_schedule_end(ScheduleEndContext((), ()))
	59	self.manager.on_request_finished(context)
	60	self.manager.reset_cache()
	61	with self.view.cast("B") as flat:
	62	flat[:] = bytes(len(flat))
	63
	64	def test_disk_hit_restores_attention_and_recurrent_groups_into_ram(self):
	65	context = ReqContext("restore")
	66	self.manager.on_new_request(context)
	67	for key, value in zip(self.keys, self.values, strict=True):
	68	self.assertEqual(self.manager.lookup(key, context), LookupResult.RETRY)
	69	self.manager.on_schedule_end(ScheduleEndContext((), ()))
	70	self.tier.drain_jobs()
	71	self.assertEqual(self.manager.lookup(key, context), LookupResult.HIT)
	72	restored = self.manager.prepare_load([key], context)
	73	offset = int(restored.block_ids[0]) * mmap.PAGESIZE
	74	with self.view.cast("B") as flat:
	75	self.assertEqual(bytes(flat[offset:offset + mmap.PAGESIZE]), value)
	76	self.manager.complete_load([key], context)
	77	self.manager.on_schedule_end(ScheduleEndContext((), ()))
	78	self.manager.on_request_finished(context)
	79
	80	def test_corrupt_disk_promotion_returns_miss_to_scheduler(self):
	81	key = self.keys[0]
	82	name = self.tier.mapper.get_file_name(key)
	83	for root in self.tier.cache.roots:
	84	path = root / name
	85	if path.exists():
	86	path.write_bytes(b"invalid")
	87	context = ReqContext("corrupt")
	88	self.manager.on_new_request(context)
	89	self.assertEqual(self.manager.lookup(key, context), LookupResult.RETRY)
	90	self.manager.on_schedule_end(ScheduleEndContext((), ()))
	91	self.tier.drain_jobs()
	92	self.assertEqual(self.manager.lookup(key, context), LookupResult.MISS)
	93	self.manager.on_request_finished(context)
	94
	95
	96	if __name__ == "__main__":
	97	unittest.main()

service/local-ai/catalog.json+1-1

...	...	@@ -34,7 +34,7 @@
34	34	"mode": "tokens",
35	35	"limit": 6000
36	36	},
37		"context_window": 131072,
	37	"context_window": 150000,
38	38	"auto_compact_token_limit": 96000,
39	39	"effective_context_window_percent": 90,
40	40	"experimental_supported_tools": [],

service/local-ai/download.py+6-2

...	...	@@ -14,14 +14,17 @@ def checksum(path):
14	14	return hashlib.file_digest(source, "sha256").hexdigest()
15	15
16	16
17		def download(root):
	17	def download(root, runtime_only=False):
18	18	manifest = json.loads(Path(__file__).with_name("models.json").read_text())
19	19	directory = root / manifest["directory"]
20	20	directory.mkdir(parents=True, exist_ok=True)
21	21	with (directory / ".download.lock").open("a") as lock:
22	22	fcntl.flock(lock, fcntl.LOCK_EX)
23	23	for spec in manifest["files"]:
	24	if runtime_only and not spec["name"].startswith("runtime/"):
	25	continue
24	26	target = directory / spec["name"]
	27	target.parent.mkdir(parents=True, exist_ok=True)
25	28	if target.exists():
26	29	if target.stat().st_size == spec["bytes"] and checksum(target) == spec["sha256"]:
27	30	print(f"Verified {target.name}", flush=True)
...	...	@@ -47,6 +50,7 @@ def download(root):
47	50	if __name__ == "__main__":
48	51	parser = argparse.ArgumentParser(description="Download local AI models")
49	52	parser.add_argument("--root", type=Path, default=Path("/srv/clover/Media/AI/LLM"))
	53	parser.add_argument("--runtime-only", action="store_true")
50	54	args = parser.parse_args()
51	55	os.umask(0o027)
52		download(args.root)
	56	download(args.root, args.runtime_only)

service/local-ai/gateway.py+7-4

...	...	@@ -1,5 +1,5 @@
1	1	#!/usr/bin/env python3
2		"""Adapt current Responses tool namespaces to llama.cpp; forward other APIs unchanged."""
	2	"""Adapt Responses tool namespaces to vLLM; forward other APIs unchanged."""
3	3	import hmac
4	4	import http.client
5	5	from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
...	...	@@ -120,6 +120,8 @@ def normalize_anthropic(request):
120	120	if instructions:
121	121	data["system"] = "\n\n".join(instructions)
122	122	data["messages"] = messages
	123	if (data.get("output_config") or {}).get("effort") in {"high", "max"}:
	124	data["output_config"]["effort"] = "xhigh"
123	125	return data
124	126
125	127
...	...	@@ -151,7 +153,7 @@ class Handler(BaseHTTPRequestHandler):
151	153	return
152	154	# Authenticate before allocating inference bodies.
153	155	if self.path != "/health":
154		key = os.environ["LLAMA_API_KEY"]
	156	key = os.environ["VLLM_API_KEY"]
155	157	supplied = self.headers.get("x-api-key", "") or self.headers.get("Authorization", "").removeprefix("Bearer ")
156	158	if not hmac.compare_digest(supplied, key):
157	159	self.send_error(401)
...	...	@@ -166,7 +168,8 @@ class Handler(BaseHTTPRequestHandler):
166	168	body = json.dumps(data).encode()
167	169	elif path in {"/v1/messages", "/v1/messages/count_tokens"}:
168	170	body = json.dumps(normalize_anthropic(json.loads(body))).encode()
169		headers = {key: val for key, val in self.headers.items() if key.lower() not in HOP_HEADERS}
	171	headers = {key: val for key, val in self.headers.items() if key.lower() not in HOP_HEADERS \| {"authorization", "x-api-key"}}
	172	headers["Authorization"] = "Bearer " + os.environ["VLLM_API_KEY"]
170	173	connection.request(self.command, self.path, body=body or None, headers=headers)
171	174	response = connection.getresponse()
172	175	streaming = "text/event-stream" in response.getheader("Content-Type", "")
...	...	@@ -222,7 +225,7 @@ class Handler(BaseHTTPRequestHandler):
222	225
223	226
224	227	if __name__ == "__main__":
225		backend = subprocess.Popen(["/app/llama-server", *sys.argv[1:]])
	228	backend = subprocess.Popen(["bash", "/app/single-user/start_qwen.sh"])
226	229	server = ThreadingHTTPServer(("0.0.0.0", 8080), Handler)
227	230	def stop(signum, frame):
228	231	if backend.poll() is None:

service/local-ai/models.json+739-9

...	...	@@ -1,17 +1,747 @@
1	1	{
2		"repository": "unsloth/Qwen3.8-27B-GGUF",
3		"revision": "4ca720788d1e01f1bff70c033e0d0028fd02e502",
4		"directory": "Qwen3.8-27B",
	2	"repository": "daavidhauser/Swift-1.5-Qwen3.8-27B-W4A16-HyperQwen-INT4",
	3	"revision": "64515d381e36084182e4e2abfa20050b75f29889",
	4	"directory": "Swift-1.5-Qwen3.8-27B-W4A16-HyperQwen-INT4",
5	5	"files": [
6	6	{
7		"name": "Qwen3.8-27B-UD-Q4_K_M.gguf",
8		"bytes": 16464440224,
9		"sha256": "322e194ff79741c7baa497c240f677f54b201b0efab44ca8e50f122b39123482"
	7	"name": ".gitattributes",
	8	"bytes": 1570,
	9	"sha256": "34448b82c17d60fec9b65b1f093c115ddbaadc04beb1b0140b6bfed2e012a930"
10	10	},
11	11	{
12		"name": "mmproj-F16.gguf",
13		"bytes": 927607488,
14		"sha256": "cbb841a9ee0636b2ec172f5bb8df2ea8dfeb01e90fe7c6126581d662a0b4e43e"
	12	"name": "LICENSE",
	13	"bytes": 13306,
	14	"sha256": "1367057bf17041aa1d69286a1400be5f464d2f8b8f54f600088b209eac1850be"
	15	},
	16	{
	17	"name": "LICENSE-APACHE-2.0",
	18	"bytes": 11544,
	19	"sha256": "bbedc3fda3305820b977265f01b8619d87570a6739de3a5582c3464840f1e57a"
	20	},
	21	{
	22	"name": "NOTICE",
	23	"bytes": 1389,
	24	"sha256": "e0e788911888bdcc087d34fc12dc5bf3818034762334de3575ea34d9559b186f"
	25	},
	26	{
	27	"name": "QUANTIZATION_MANIFEST.json",
	28	"bytes": 8434,
	29	"sha256": "40ebbba877dd55e1c512e68785f3127673d57ac799c5ea7139761b563ed7de4b"
	30	},
	31	{
	32	"name": "README.md",
	33	"bytes": 5620,
	34	"sha256": "de13ff8b19a75a4d6dcda0c9a36e276d3a5b19b74a44ef012bbfe39562f24a85"
	35	},
	36	{
	37	"name": "RELEASE_MANIFEST.json",
	38	"bytes": 23100,
	39	"sha256": "1fb06fb256c1e29323e03ec56a1c1e99b8bfc5078a2c57c5426cb74052023eeb"
	40	},
	41	{
	42	"name": "RUNTIME.md",
	43	"bytes": 2608,
	44	"sha256": "4a6a153fdf00ece6ddd976ee745951db61c5a36a69ba06eaf102a1ec1f5c3c74"
	45	},
	46	{
	47	"name": "chat_template.jinja",
	48	"bytes": 8952,
	49	"sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041"
	50	},
	51	{
	52	"name": "config.json",
	53	"bytes": 22382,
	54	"sha256": "deffb953c20dafbcdc07199063fa5e203dc574432dd3c7efac3f8e2882bd46a7"
	55	},
	56	{
	57	"name": "evaluation/code/swift15/checkpoint.py",
	58	"bytes": 3656,
	59	"sha256": "c43707f3623659cd882cfbb57506b152912f42f2a043dfd148c1cf829090f28d"
	60	},
	61	{
	62	"name": "evaluation/code/swift15/code_runner.py",
	63	"bytes": 2144,
	64	"sha256": "23d52e36543766bb33d33f11adad3772e7836d5170e83f68e326cc2d00b55fd9"
	65	},
	66	{
	67	"name": "evaluation/code/swift15/common.py",
	68	"bytes": 1439,
	69	"sha256": "0410dc972e74d8bbc05e090b0543426e4ea57ed707a80a114ea2e658822e1e8c"
	70	},
	71	{
	72	"name": "evaluation/code/swift15/corpus.py",
	73	"bytes": 7800,
	74	"sha256": "d5b420de1f3a1d4be97a44321abdd393247b5d78a56199ad49a62058c98d9050"
	75	},
	76	{
	77	"name": "evaluation/code/swift15/eval_data.py",
	78	"bytes": 8097,
	79	"sha256": "2a4497758d456a3cdd930aa711d0c510b24a2eaf6eb11be3da3e94481d4a166e"
	80	},
	81	{
	82	"name": "evaluation/code/swift15/evaluate.py",
	83	"bytes": 14113,
	84	"sha256": "4d553cf10b42cbff0eeacfe72b3b6df87405aa2604fcbe19a5a0a78133cd18af"
	85	},
	86	{
	87	"name": "evaluation/code/swift15/generate.py",
	88	"bytes": 4412,
	89	"sha256": "55e2659ce67295a4c57ffd12beed9dc0246808d908939c4113f7d1bb506af089"
	90	},
	91	{
	92	"name": "evaluation/code/swift15/ifbench_score.py",
	93	"bytes": 876,
	94	"sha256": "1b97f5859020d8d992944a85357ee7477f8740e65574d5946d2305889244a04d"
	95	},
	96	{
	97	"name": "evaluation/code/swift15/measure.py",
	98	"bytes": 4726,
	99	"sha256": "f955ace33396730c9fdf6fe5d424b6a2f77eb0fc58c83b3bd84b0c804c7a3b37"
	100	},
	101	{
	102	"name": "evaluation/code/swift15/quantize.py",
	103	"bytes": 8098,
	104	"sha256": "a3f65ca5de15d54cf626cde340365e503462d1a33ae8525aa55f6159195886f7"
	105	},
	106	{
	107	"name": "evaluation/code/swift15/reference.py",
	108	"bytes": 1786,
	109	"sha256": "0433adf523a676c0cb30c5ad25b90bdc5a4565c2b69180e171aa4a4d59c15bf8"
	110	},
	111	{
	112	"name": "evaluation/code/swift15/release.py",
	113	"bytes": 7409,
	114	"sha256": "c5129121b4ba333b0c4dd2694fb69076cdc75e4b9cda8a6572b3a449c32c3b5e"
	115	},
	116	{
	117	"name": "evaluation/code/swift15/report.py",
	118	"bytes": 5287,
	119	"sha256": "622af3dd5ab1eaa1843407c5bfb57e3111a154463c024a6685095a7aa293b482"
	120	},
	121	{
	122	"name": "evaluation/code/swift15/run.py",
	123	"bytes": 9281,
	124	"sha256": "e5d42332192e5bafdde88ae4acc20802f5c5477e6447ff5165fb6b836aa78ea2"
	125	},
	126	{
	127	"name": "evaluation/code/swift15/runtime_compat.py",
	128	"bytes": 1885,
	129	"sha256": "bcbc7351a6a4719bf38c06658bb12cbcfc9eb31f9280fc9b916782832544dbe6"
	130	},
	131	{
	132	"name": "evaluation/code/swift15/serve.py",
	133	"bytes": 6020,
	134	"sha256": "c705174eda86e57ae1a51c0b2e7a93f9b0e1f37fa32217a9739b569f52d2d56d"
	135	},
	136	{
	137	"name": "evaluation/code/swift15/setup.py",
	138	"bytes": 2219,
	139	"sha256": "7de02ccb5575e0ac2b62d33842d28aacfc83d63fc96eedad9cbedf1129e90495"
	140	},
	141	{
	142	"name": "evaluation/code/swift15/smoke.py",
	143	"bytes": 3162,
	144	"sha256": "e2d4a01d6613e667ceeef0d1a03ef1cec04edf3ab0518aaa4a006d419a91f1fe"
	145	},
	146	{
	147	"name": "evaluation/code/swift15/test_workflow.py",
	148	"bytes": 3468,
	149	"sha256": "8d2dcb057318d482530ef9bf7b4bcf0d59347b594531e0df18b81dbea0beaddd"
	150	},
	151	{
	152	"name": "evaluation/environment.json",
	153	"bytes": 3282,
	154	"sha256": "2e9ac10516648b0c634eaca007d58456d241e0c1ef1b6734fa51671bf315a841"
	155	},
	156	{
	157	"name": "evaluation/manifest.json",
	158	"bytes": 2685,
	159	"sha256": "82c1c6eaad6c06ad91e20cac5f9dae6b74ceaba0a3063c868a6a4313366e7948"
	160	},
	161	{
	162	"name": "evaluation/perplexity-english-python.json",
	163	"bytes": 38213,
	164	"sha256": "5c0f9ab2df9a8d4e14ea846cd03e1c3cb577847a022a955f97ff57ef6392fe85"
	165	},
	166	{
	167	"name": "evaluation/pilot-ids.json",
	168	"bytes": 354,
	169	"sha256": "44cffc01ab88b366f20da82d3150c7de8eb1d7801cd1381570e32313f120da41"
	170	},
	171	{
	172	"name": "evaluation/qwen-fast-full/manifest.json",
	173	"bytes": 14395,
	174	"sha256": "3de5a8af19d01a53594d7f29349e19bf90f1bbe8d935e82fe33aa0cf8325391b"
	175	},
	176	{
	177	"name": "evaluation/qwen-fast-full/summary.json",
	178	"bytes": 3134,
	179	"sha256": "a2ffcd9ed991279b5639274596a5139b39553d407753adc48ff465e55fd2eaf6"
	180	},
	181	{
	182	"name": "evaluation/qwen-fast-pilot/manifest.json",
	183	"bytes": 2663,
	184	"sha256": "f06f4539feaa46b92d559427cd13809ecf3e8b0fc44b15768e54392a0ce9c34f"
	185	},
	186	{
	187	"name": "evaluation/qwen-fast-pilot/summary.json",
	188	"bytes": 2979,
	189	"sha256": "8a0ec26f335c5a0c98bac1db6bd852a5292b0b5d1bfcc6a218a0b9aa1ce9b1c7"
	190	},
	191	{
	192	"name": "evaluation/qwen-fast/perplexity.json",
	193	"bytes": 12570,
	194	"sha256": "9db1599421dc916b97602a8921dbd219b815a5a76fc7def41f508500eb10e7e7"
	195	},
	196	{
	197	"name": "evaluation/qwen-fast/server.json",
	198	"bytes": 1942,
	199	"sha256": "b2988235fd03e4cd30366b5ca418ba5bb687c1ab58c1eb29e3e53c57612ac0ae"
	200	},
	201	{
	202	"name": "evaluation/qwen-fast/speed-c1.json",
	203	"bytes": 630,
	204	"sha256": "3d70c2868a2f2832e6bf7562abd3fa4589389baf87e02dbd89c76b154e49238d"
	205	},
	206	{
	207	"name": "evaluation/swift10-full/manifest.json",
	208	"bytes": 14501,
	209	"sha256": "6af95e530b920a84a3c5f5dde5b76bc2c791d0a4cc5b3043686bd6946cac4acf"
	210	},
	211	{
	212	"name": "evaluation/swift10-full/summary.json",
	213	"bytes": 3125,
	214	"sha256": "1c431ee8cc16e36090152526f8553c3e76cb5f66e5ddd5c6df1f90c3b3426add"
	215	},
	216	{
	217	"name": "evaluation/swift10-pilot/manifest.json",
	218	"bytes": 2769,
	219	"sha256": "83df9bd1a41e0d96f62039ad1941593e4cafc57c52f63851eeec4523048f4315"
	220	},
	221	{
	222	"name": "evaluation/swift10-pilot/summary.json",
	223	"bytes": 2981,
	224	"sha256": "3f8aed7d263dc65abb0095bc3af73515052645886f5f0565e6c21e82afa27bf8"
	225	},
	226	{
	227	"name": "evaluation/swift10/perplexity.json",
	228	"bytes": 12560,
	229	"sha256": "fe393bd4962d43f9cef2a08ea40c0dd2f15e6695b87a09a20e184d448d941775"
	230	},
	231	{
	232	"name": "evaluation/swift10/server.json",
	233	"bytes": 2041,
	234	"sha256": "d6004a26531972be5407c0f542ec0b708c931e21f67801e5206906d4cb148547"
	235	},
	236	{
	237	"name": "evaluation/swift10/speed-c1.json",
	238	"bytes": 632,
	239	"sha256": "a82f761f6a6ca5446808b1e8b129fee4e5b3b733787e286a7829b23f6dd1722f"
	240	},
	241	{
	242	"name": "evaluation/swift15-baseline-full/manifest.json",
	243	"bytes": 14389,
	244	"sha256": "b122ac2835f4c868522aa6982e1c44e2090ae176b27293cd8a71040108751d18"
	245	},
	246	{
	247	"name": "evaluation/swift15-baseline-full/summary.json",
	248	"bytes": 3091,
	249	"sha256": "cd0ec9d98535c6c32c2ae9f32798f81c0dae628ebda0d1d56f6005df85edb91b"
	250	},
	251	{
	252	"name": "evaluation/swift15-baseline-pilot/manifest.json",
	253	"bytes": 2657,
	254	"sha256": "588d9a88f35aa9207934bde6cbce4e0956868287316f964585b784b9d4f7cec3"
	255	},
	256	{
	257	"name": "evaluation/swift15-baseline-pilot/summary.json",
	258	"bytes": 2982,
	259	"sha256": "511e69269b8d98d64876ab7033e124fc1d8aca196b022fbd27b34f65867ac8ef"
	260	},
	261	{
	262	"name": "evaluation/swift15-baseline/perplexity.json",
	263	"bytes": 12553,
	264	"sha256": "6f63e857421957c76b3ca290ec8c95a62fd4374e3839adc2cdcdca54f14786fa"
	265	},
	266	{
	267	"name": "evaluation/swift15-baseline/server.json",
	268	"bytes": 1937,
	269	"sha256": "9063a7b84cd89e83672d673d5fc30ff563d01adebd1a705fcae70c33b5ba47b4"
	270	},
	271	{
	272	"name": "evaluation/swift15-baseline/speed-c1.json",
	273	"bytes": 629,
	274	"sha256": "079f1c870da2484a43ec214ee2ced3723b2636c83eaed4291636e121b4e81e3a"
	275	},
	276	{
	277	"name": "evaluation/swift15-fast-full/manifest.json",
	278	"bytes": 14393,
	279	"sha256": "d242d0bd7f947c37a74430190ce4a1309c2ae8e2f5ba76313e1ba056044aa134"
	280	},
	281	{
	282	"name": "evaluation/swift15-fast-full/summary.json",
	283	"bytes": 2969,
	284	"sha256": "3ffd7dc76fa4105089688731c6a8365519f80567acbca3b1bc15b64e93de7ab3"
	285	},
	286	{
	287	"name": "evaluation/swift15-fast-pilot/manifest.json",
	288	"bytes": 2661,
	289	"sha256": "13203d3b56258d785ea74ddd3f89322edb6c58245f7d4ecf60756a7af66d4f60"
	290	},
	291	{
	292	"name": "evaluation/swift15-fast-pilot/summary.json",
	293	"bytes": 2979,
	294	"sha256": "fe44f8972f40cf5236ca93d1ee7362cfea238a03e58f09c6e033312820fc8d89"
	295	},
	296	{
	297	"name": "evaluation/swift15-fast/perplexity.json",
	298	"bytes": 12561,
	299	"sha256": "3fa2ac71c6c24072036837c3ae0afcf96ce03e85a09b276874c75432aeb091a1"
	300	},
	301	{
	302	"name": "evaluation/swift15-fast/server.json",
	303	"bytes": 1940,
	304	"sha256": "53099847622dbdd7416196bfd8a7591aca1adb64db92049cf2be6ffdf489536d"
	305	},
	306	{
	307	"name": "evaluation/swift15-fast/speed-c1.json",
	308	"bytes": 629,
	309	"sha256": "88d292f00ba50773e80c0dbf6ba9a1e18d3413737221d2ff8c5a969749747108"
	310	},
	311	{
	312	"name": "generation_config.json",
	313	"bytes": 202,
	314	"sha256": "e70c136c1b78ddc1fb0905bac8e733a4dc448d4f852a5dd75143fffc70be550e"
	315	},
	316	{
	317	"name": "hyperqwen-audit.json",
	318	"bytes": 18861,
	319	"sha256": "62c8d7543e2e52d4025e7f72577753c029b1447762cfc781e8d06a6547c72795"
	320	},
	321	{
	322	"name": "hyperqwen-build.json",
	323	"bytes": 8398,
	324	"sha256": "895d4e0dd27f896ed2a3a0e2160c59967cd10e8506f690b6a85c5cf90c529cec"
	325	},
	326	{
	327	"name": "hyperqwen_provenance/generation-manifest.json",
	328	"bytes": 460,
	329	"sha256": "f8dc684e9940b3efe9f7967214495adb83d4b906cb442ef46924c42a90308853"
	330	},
	331	{
	332	"name": "hyperqwen_provenance/generation-summary.json",
	333	"bytes": 204,
	334	"sha256": "6aa261c7b4b9952edd1d7fc0b61f57985894d1b360571f02448364820f0adbd0"
	335	},
	336	{
	337	"name": "hyperqwen_provenance/lm-head-report.json",
	338	"bytes": 234,
	339	"sha256": "3fe25c5ed9c1dbf85a1e5baa705036303515d8ad0dccb9ef54abcca63528d2b4"
	340	},
	341	{
	342	"name": "hyperqwen_provenance/manifest.json",
	343	"bytes": 2106,
	344	"sha256": "83f789e2d83a31b90564811ae804f1cda210e7ad62a7c7dd6ddde79f04ad5438"
	345	},
	346	{
	347	"name": "hyperqwen_provenance/mtp-report.json",
	348	"bytes": 1059,
	349	"sha256": "e1cd6016c4a231be292dd73d128ba863f035320ee71de5bd983504cede3d711c"
	350	},
	351	{
	352	"name": "hyperqwen_provenance/mtp_hessians.pt.json",
	353	"bytes": 15773,
	354	"sha256": "f69a5c6132cb5382a8aaedd7dddfbdcd050e193ec2b1c525feb95949ca8ac834"
	355	},
	356	{
	357	"name": "hyperqwen_provenance/preflight-validation.json",
	358	"bytes": 1444,
	359	"sha256": "d4a00e873a3da9e29cf9ae5b864b4b2eaf992263705305651c77aa02a1ab3f9e"
	360	},
	361	{
	362	"name": "hyperqwen_provenance/recipe/drafter/capture.py",
	363	"bytes": 5744,
	364	"sha256": "6bb7cff9e235999923fa83b994034be6c9246b45fa7d89cca6fabf067f030e9b"
	365	},
	366	{
	367	"name": "hyperqwen_provenance/recipe/drafter/gptq_utils.py",
	368	"bytes": 3809,
	369	"sha256": "e0478db6fd5692a8960cd5ad16786655f09b73d6fe864382371c1455f9394c69"
	370	},
	371	{
	372	"name": "hyperqwen_provenance/recipe/drafter/train_mtp.py",
	373	"bytes": 24898,
	374	"sha256": "900951feef2e0ff7d729f9baac02e9125b5ec16de5680009e368e20139b79faf"
	375	},
	376	{
	377	"name": "hyperqwen_provenance/recipe/prepare/build_draft_vocab.py",
	378	"bytes": 6161,
	379	"sha256": "426f371a8db9f465c48b56ae05cbc782f0613980005405deac2ad95504d088bc"
	380	},
	381	{
	382	"name": "hyperqwen_provenance/recipe/prepare/quant_heads_stream.py",
	383	"bytes": 9473,
	384	"sha256": "67bac25a15df11ed569866491d8bf21550829e0281bb74a8c9ec092107fca905"
	385	},
	386	{
	387	"name": "hyperqwen_provenance/recipe/swift15/README.md",
	388	"bytes": 5869,
	389	"sha256": "83564e691d256395a5c49138bfa65840f226f4c51c373c9e336acb4af08c0467"
	390	},
	391	{
	392	"name": "hyperqwen_provenance/recipe/swift15/checkpoint.py",
	393	"bytes": 3656,
	394	"sha256": "c43707f3623659cd882cfbb57506b152912f42f2a043dfd148c1cf829090f28d"
	395	},
	396	{
	397	"name": "hyperqwen_provenance/recipe/swift15/code_runner.py",
	398	"bytes": 2144,
	399	"sha256": "23d52e36543766bb33d33f11adad3772e7836d5170e83f68e326cc2d00b55fd9"
	400	},
	401	{
	402	"name": "hyperqwen_provenance/recipe/swift15/common.py",
	403	"bytes": 1439,
	404	"sha256": "0410dc972e74d8bbc05e090b0543426e4ea57ed707a80a114ea2e658822e1e8c"
	405	},
	406	{
	407	"name": "hyperqwen_provenance/recipe/swift15/corpus.py",
	408	"bytes": 7800,
	409	"sha256": "d5b420de1f3a1d4be97a44321abdd393247b5d78a56199ad49a62058c98d9050"
	410	},
	411	{
	412	"name": "hyperqwen_provenance/recipe/swift15/eval_data.py",
	413	"bytes": 8097,
	414	"sha256": "2a4497758d456a3cdd930aa711d0c510b24a2eaf6eb11be3da3e94481d4a166e"
	415	},
	416	{
	417	"name": "hyperqwen_provenance/recipe/swift15/evaluate.py",
	418	"bytes": 12978,
	419	"sha256": "dfb48598358dd97c2b52eb41384b797cf2a4fc9d35cba91b0b2efb84abb73d8c"
	420	},
	421	{
	422	"name": "hyperqwen_provenance/recipe/swift15/generate.py",
	423	"bytes": 4412,
	424	"sha256": "55e2659ce67295a4c57ffd12beed9dc0246808d908939c4113f7d1bb506af089"
	425	},
	426	{
	427	"name": "hyperqwen_provenance/recipe/swift15/ifbench_score.py",
	428	"bytes": 876,
	429	"sha256": "1b97f5859020d8d992944a85357ee7477f8740e65574d5946d2305889244a04d"
	430	},
	431	{
	432	"name": "hyperqwen_provenance/recipe/swift15/measure.py",
	433	"bytes": 4726,
	434	"sha256": "f955ace33396730c9fdf6fe5d424b6a2f77eb0fc58c83b3bd84b0c804c7a3b37"
	435	},
	436	{
	437	"name": "hyperqwen_provenance/recipe/swift15/quantize.py",
	438	"bytes": 8098,
	439	"sha256": "a3f65ca5de15d54cf626cde340365e503462d1a33ae8525aa55f6159195886f7"
	440	},
	441	{
	442	"name": "hyperqwen_provenance/recipe/swift15/release.py",
	443	"bytes": 7409,
	444	"sha256": "c5129121b4ba333b0c4dd2694fb69076cdc75e4b9cda8a6572b3a449c32c3b5e"
	445	},
	446	{
	447	"name": "hyperqwen_provenance/recipe/swift15/report.py",
	448	"bytes": 4712,
	449	"sha256": "8e73730ee54a449bb1028b8626f2ad973bed40428f7b49a8780b077cc63dcc92"
	450	},
	451	{
	452	"name": "hyperqwen_provenance/recipe/swift15/run.py",
	453	"bytes": 8169,
	454	"sha256": "d60a6780eda3ee20ab4b878cfa4fa223edfdc4a5a099c622e05ee06ff61c303f"
	455	},
	456	{
	457	"name": "hyperqwen_provenance/recipe/swift15/runtime_compat.py",
	458	"bytes": 1885,
	459	"sha256": "bcbc7351a6a4719bf38c06658bb12cbcfc9eb31f9280fc9b916782832544dbe6"
	460	},
	461	{
	462	"name": "hyperqwen_provenance/recipe/swift15/serve.py",
	463	"bytes": 4076,
	464	"sha256": "f80daeb04c86d07b8ce463e1305c9c84d6ab4dbe2eb2322b14970b282bdcdbb3"
	465	},
	466	{
	467	"name": "hyperqwen_provenance/recipe/swift15/setup.py",
	468	"bytes": 2219,
	469	"sha256": "7de02ccb5575e0ac2b62d33842d28aacfc83d63fc96eedad9cbedf1129e90495"
	470	},
	471	{
	472	"name": "hyperqwen_provenance/recipe/swift15/smoke.py",
	473	"bytes": 3162,
	474	"sha256": "e2d4a01d6613e667ceeef0d1a03ef1cec04edf3ab0518aaa4a006d419a91f1fe"
	475	},
	476	{
	477	"name": "hyperqwen_provenance/recipe/swift15/test_workflow.py",
	478	"bytes": 3029,
	479	"sha256": "221c0b3e6cb04fe4aaec2c47059c329dfa4e06c4084b9b01dfea3ab62032c677"
	480	},
	481	{
	482	"name": "hyperqwen_provenance/runtime-compatibility.json",
	483	"bytes": 869,
	484	"sha256": "4b627e696f1e5dcda170364fc7a11b83fdbbad0c434933171db4cc76c97e91d5"
	485	},
	486	{
	487	"name": "hyperqwen_provenance/upstream-QUANTIZATION_MANIFEST.json",
	488	"bytes": 1243,
	489	"sha256": "37a75a5056e4d7267cfacfd11fd830f6fe0c084fc3688bead9842b9f0fbfc121"
	490	},
	491	{
	492	"name": "hyperqwen_provenance/upstream-README.md",
	493	"bytes": 21151,
	494	"sha256": "1fa3bbf8f6177e23bc1dcf98ce990f889ecad8c3f032a010c41559d34cbee263"
	495	},
	496	{
	497	"name": "hyperqwen_provenance/upstream-UPLOAD_MANIFEST.json",
	498	"bytes": 5176,
	499	"sha256": "b475d1d0e4b383c036be5a396c1a65dc2b45b3af1463da859049a24c65fe6854"
	500	},
	501	{
	502	"name": "hyperqwen_provenance/upstream-recipe.yaml",
	503	"bytes": 930,
	504	"sha256": "49f446271f2278c633d540768d42caf82114fc2353ca526d1279fcc1984b7f68"
	505	},
	506	{
	507	"name": "hyperqwen_provenance/vocabulary-report.json",
	508	"bytes": 362,
	509	"sha256": "44b8a205f3c8419ad179f5c5ae89a9d7b2ca143e241bfde8f96e132637c9f809"
	510	},
	511	{
	512	"name": "merges.txt",
	513	"bytes": 3353259,
	514	"sha256": "a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d"
	515	},
	516	{
	517	"name": "model-00001-of-00006.safetensors",
	518	"bytes": 655565120,
	519	"sha256": "1e2bb79bd7feff42b3b006b3552354c1ff2c4a02941a788080839b0eb868118d"
	520	},
	521	{
	522	"name": "model-00002-of-00006.safetensors",
	523	"bytes": 2730582048,
	524	"sha256": "48ddebd951d8d8a9ecef51b1857403ca77e5ab2c63b2e7db4822e0908e2eeaac"
	525	},
	526	{
	527	"name": "model-00003-of-00006.safetensors",
	528	"bytes": 3966050624,
	529	"sha256": "61bbd4ae1e4812f08aa8f903e9995746aea8c346a666b530d0a6774773350f1e"
	530	},
	531	{
	532	"name": "model-00004-of-00006.safetensors",
	533	"bytes": 3966050704,
	534	"sha256": "0c7baff5aac711178c5ce841da1ebcbebf67bde21155e74f4adca5d4354c178a"
	535	},
	536	{
	537	"name": "model-00005-of-00006.safetensors",
	538	"bytes": 3993185016,
	539	"sha256": "0cc1b9b1cde51d81fd7f53d91716399c30c4a5a166503dbd76ac65c63bb881c9"
	540	},
	541	{
	542	"name": "model-00006-of-00006.safetensors",
	543	"bytes": 248265904,
	544	"sha256": "65c1248e479775518d38a1675859380890a76c455f25de37b613bef71281a4fb"
	545	},
	546	{
	547	"name": "model-mtp-bf16.safetensors",
	548	"bytes": 219027824,
	549	"sha256": "39ac3a5071ac107640468e1a5c3d3844d39eba277076c2d97cf404974dad3d5d"
	550	},
	551	{
	552	"name": "model.safetensors.index.json",
	553	"bytes": 243540,
	554	"sha256": "e99357b5714c15037f02da3f6d46dd569742edde177df6f8f5c46505bba9e562"
	555	},
	556	{
	557	"name": "model_extra_tensors.safetensors",
	558	"bytes": 173015384,
	559	"sha256": "bdb17eed6815f17f20a324d25f3777533cb224c463075f1ab6afd199bde8628a"
	560	},
	561	{
	562	"name": "mtp_draft_vocab_ids.pt",
	563	"bytes": 525949,
	564	"sha256": "a7f5b236bd7b1892c24a004d12f93cdf6910a3ca2afb2b490cc08ab985b6d7c9"
	565	},
	566	{
	567	"name": "preprocessor_config.json",
	568	"bytes": 390,
	569	"sha256": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516"
	570	},
	571	{
	572	"name": "recipe.yaml",
	573	"bytes": 930,
	574	"sha256": "49f446271f2278c633d540768d42caf82114fc2353ca526d1279fcc1984b7f68"
	575	},
	576	{
	577	"name": "runtime/BASE_REVISION.txt",
	578	"bytes": 41,
	579	"sha256": "c1f6fbc4227edb7daa83504c936afc8e23d8e31e8b62ff375fbbb5e29e81e546"
	580	},
	581	{
	582	"name": "runtime/LICENSE",
	583	"bytes": 11358,
	584	"sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"
	585	},
	586	{
	587	"name": "runtime/batch/start_qwen.sh",
	588	"bytes": 11677,
	589	"sha256": "ac6a84b455317537a230bc190ccc06dda169dcdbdeee4bebeb51e76032972944"
	590	},
	591	{
	592	"name": "runtime/patches/dflash2-backport.patch",
	593	"bytes": 40396,
	594	"sha256": "2e937ef748c1942ab04ace05de884e7c24041d34b23e5be8428cac6939ac6875"
	595	},
	596	{
	597	"name": "runtime/patches/dflash2-lookup-drafting.patch",
	598	"bytes": 46770,
	599	"sha256": "5df09ef03b592d4a2c3b47dd8d2dbf8862fa7383d68aef3dd6ef1d97d29ce196"
	600	},
	601	{
	602	"name": "runtime/patches/dflash2-ngram-chains.patch",
	603	"bytes": 19037,
	604	"sha256": "555b5a75d9023c99b2cd17634ac5b1dc8505f9275d857b1eba08c2d9cbf2df6e"
	605	},
	606	{
	607	"name": "runtime/patches/dflash2-prewarm.patch",
	608	"bytes": 5181,
	609	"sha256": "e1e8012fa2c304c6948c19fc0071332f80eebcff802b402255445a30d553ab92"
	610	},
	611	{
	612	"name": "runtime/patches/hybrid-kv-groups-v2-cudagraph.patch",
	613	"bytes": 6306,
	614	"sha256": "143825dd9744ab81dabcb438ec0bf974b6e0be8d0d7b0ffbbdfbf7c16a549664"
	615	},
	616	{
	617	"name": "runtime/patches/hybrid-sw-block-promote.patch",
	618	"bytes": 7828,
	619	"sha256": "10876ac706546e74fe74b8964b90576a7b36f90a2f35a856997c558d7817f154"
	620	},
	621	{
	622	"name": "runtime/patches/int4-kv-per-token-head.patch",
	623	"bytes": 7038,
	624	"sha256": "c03ff10c8c997b355f04fc521827ee2fa99c332ce4127cc2d878b749b79ca3d9"
	625	},
	626	{
	627	"name": "runtime/patches/mamba-align-checkpoint-order.patch",
	628	"bytes": 12020,
	629	"sha256": "515d9bf76e860c95d832d615bdab4a97b71e2b0412b4ccc2445f11535ddddf45"
	630	},
	631	{
	632	"name": "runtime/patches/marlin-int8-layer-select.patch",
	633	"bytes": 3048,
	634	"sha256": "833405e2ed2916529eb20184c38243c843378935522b74bab8c39707bf3ee800"
	635	},
	636	{
	637	"name": "runtime/patches/marlin-int8-negative-scales.patch",
	638	"bytes": 2926,
	639	"sha256": "4cb8a064c706cc62dc77224479898da58864fbfff6b65012d90aa8326035300d"
	640	},
	641	{
	642	"name": "runtime/patches/marlin-repack-staged-sm80.patch",
	643	"bytes": 6841,
	644	"sha256": "1588e2f10b5f82194d449483e766c0b39c84ba522d1623d39caa9502040f1fab"
	645	},
	646	{
	647	"name": "runtime/patches/marlin-tune-table.patch",
	648	"bytes": 4422,
	649	"sha256": "1cac17f12e4ce389b0cb0fc733ceceb0a8cf60c02696ee378dc1d8d0c1ef8914"
	650	},
	651	{
	652	"name": "runtime/patches/offload-dflash-eagle-groups.patch",
	653	"bytes": 5406,
	654	"sha256": "f2791af64d8066b31e4250d866c4322f45670d75522aff221307281c97c73156"
	655	},
	656	{
	657	"name": "runtime/patches/offload-wsl2-devptr.patch",
	658	"bytes": 4778,
	659	"sha256": "8c6f9e3e5723571d1f33925794435e55062b684b2706d9284d1c5b94778d34db"
	660	},
	661	{
	662	"name": "runtime/patches/qwen3_5-embed-quant.patch",
	663	"bytes": 1395,
	664	"sha256": "0a1b9ca06798c1aef582995de5a0beb3ad9a22a54cdbd2361986563a9c7a980e"
	665	},
	666	{
	667	"name": "runtime/patches/qwen3_5-mtp-draft-vocab.patch",
	668	"bytes": 3943,
	669	"sha256": "292ef662d17bcc10556b787d5bb5f2cd12d3d3fc1f3bd2fe982485ce0d916298"
	670	},
	671	{
	672	"name": "runtime/patches/sampler-small-topk-fast-softmax.patch",
	673	"bytes": 13679,
	674	"sha256": "8828646ce1916c065282529d9c1bb52064f668397d8e1f33652aef2f308292b6"
	675	},
	676	{
	677	"name": "runtime/patches/spec-decode-attn.patch",
	678	"bytes": 15424,
	679	"sha256": "007791047a1d60143f8e3fe4e0a56dabadfc0134cb288847338b76ba7d1f9fe7"
	680	},
	681	{
	682	"name": "runtime/patches/spec-decode-int4-kv-mq3d.patch",
	683	"bytes": 3876,
	684	"sha256": "b93b186deba1513ad9c4803ee2c574c923b21ba5e88e23bca8b8594b6a6b20a0"
	685	},
	686	{
	687	"name": "runtime/patches/spec-decode-int8-kv.patch",
	688	"bytes": 11915,
	689	"sha256": "3cfd31304353237af5277c861dc2b43ad33d0f9bff2700594300eff150f64382"
	690	},
	691	{
	692	"name": "runtime/patches/spec-sampler-prewarm.patch",
	693	"bytes": 4516,
	694	"sha256": "18ed9608baa5a09ed2ccbb214d62763da3ce7bcf09c6aad9003778ea4e56d18f"
	695	},
	696	{
	697	"name": "runtime/patches/speed-knobs-envs.patch",
	698	"bytes": 2300,
	699	"sha256": "841ec93021b1b90f90313a1880a791bbfbdb79cbe34d9b99fdb4c3a7e52ed7c0"
	700	},
	701	{
	702	"name": "runtime/patches/triton-prefill-attn-int8.patch",
	703	"bytes": 15661,
	704	"sha256": "6bd36db0226dce7b92ebd1231b6f9718a25da4a933ab4b2a6a8be77860364fae"
	705	},
	706	{
	707	"name": "runtime/patches/vision-tower-cpu-offload.patch",
	708	"bytes": 8345,
	709	"sha256": "81dff64a1177058783dcf7d4e8552f1547f74fa8a68019e349dacdb9d66e968e"
	710	},
	711	{
	712	"name": "runtime/patches/vllm-pr50021-gdn-spec-bounds.patch",
	713	"bytes": 8886,
	714	"sha256": "cd6e00270fa28e37a8c7ad11f965662f4153b2fcaf840f2c4043710e7b078655"
	715	},
	716	{
	717	"name": "runtime/patches/xgrammar-spec-terminated.patch",
	718	"bytes": 3993,
	719	"sha256": "37589b9a45d5ece37cc16e82b195362719011e52ea31b2e70cdc89845c825040"
	720	},
	721	{
	722	"name": "runtime/single-user/start_qwen.sh",
	723	"bytes": 42439,
	724	"sha256": "6874eb0bc4306d61b57ebb2f2c7cab97b41f11a3e0ec2b1502d9c34d396c47d8"
	725	},
	726	{
	727	"name": "tokenizer.json",
	728	"bytes": 12809320,
	729	"sha256": "0997f410c57a1f4e53b09e4be8f4a172d90edd9564368fb0847030937229b9f3"
	730	},
	731	{
	732	"name": "tokenizer_config.json",
	733	"bytes": 17928,
	734	"sha256": "b11349aafa7cdc6a320767cf7ceb29ed82f7eda5d65e8e0819e76f0ce947bf27"
	735	},
	736	{
	737	"name": "video_preprocessor_config.json",
	738	"bytes": 385,
	739	"sha256": "7768af27c1fafa9cc9011c1dc20067e03f8915e03b63504550e11d5066986d13"
	740	},
	741	{
	742	"name": "vocab.json",
	743	"bytes": 6722759,
	744	"sha256": "ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003"
15	745	}
16	746	]
17	747	}

service/local-ai/prepare.py created+18

...	...	@@ -0,0 +1,18 @@
	1	#!/usr/bin/env python3
	2	import json
	3	import os
	4	from pathlib import Path
	5	import subprocess
	6	import sys
	7
	8
	9	data = json.load(sys.stdin)
	10	if Path(data["hostRoot"]).name == "local-ai":
	11	source = Path(__file__).with_name("service.pkl")
	12	path = Path(subprocess.check_output([
	13	"pkl", "eval", "-f", "json", "-p", "serviceId=local-ai", "-p", f"uid={data['uid']}",
	14	"-p", "preview=false", "--expression", 'container.volumes["/cache/ssd"].src', str(source),
	15	], text=True).strip())
	16	path.mkdir(mode=0o700, parents=True, exist_ok=True)
	17	os.chown(path, data["uid"], data["uid"])
	18	path.chmod(0o700)

service/local-ai/service.pkl+33-35

...	...	@@ -4,20 +4,22 @@ import "../../config/site.pkl" as site
4	4	import "pkl:json"
5	5
6	6	local model = (new json.Parser {}).parse(read("catalog.json")).models[0]
	7	local weights = (new json.Parser {}).parse(read("models.json"))
7	8
8	9	meta { name = "Local AI"; launcher = false; access = "admin" }
9	10	enabled = (read?("env:STUDIO_LOCAL_AI") ?? "false") == "true"
10	11	rollout = "simple"
11	12	stageIsolation = "fresh"
	13	prepare = "prepare.py"
12	14	healthyDeadline = "30m"
13	15	healthRestartGrace = "30m"
14		secrets { ["api_key"] {} }
	16	secrets { ["api_key"] {}; ["cache_key"] {} }
15	17
16	18	container {
17		build = "build"
	19	build = "."
18	20	cpu = 4000
19		memory = 18432
20		entrypoint = "/usr/bin/python3"
	21	memory = 26624
	22	entrypoint = "/usr/bin/python3.12"
21	23	http {
22	24	containerPort = 8080
23	25	subdomain = "ai"
...	...	@@ -25,42 +27,38 @@ container {
25	27	}
26	28	volumes {
27	29	["/config/gateway.py"] { config = "gateway.py" }
28		["/models"] { src = "\(site.mediaRoot)/AI/LLM/Qwen3.8-27B"; readOnly = true }
	30	["/cache/hdd"] {}
	31	["/cache/ssd"] { src = if (site.preview) null else site.localAiSsdCache }
	32	["/models"] { src = "\(site.mediaRoot)/AI/LLM/\(weights.directory)"; readOnly = true }
29	33	// The host's NVIDIA libraries link to immutable Nix store paths.
30	34	["/nvidia/lib"] { src = "/run/opengl-driver/lib"; readOnly = true }
31	35	["/nix/store"] { src = "/nix/store"; readOnly = true }
32	36	}
33	37	devices { "/dev/nvidia0"; "/dev/nvidiactl"; "/dev/nvidia-uvm" }
	38	tmpfs { "/dev/shm:rw,size=10g" }
34	39	env {
35		["LD_LIBRARY_PATH"] = "/nvidia/lib:/app"
36		["LLAMA_API_KEY"] = "${secret.own.api_key}"
37		}
38		args {
39		"/config/gateway.py"
40		"--model"; "/models/Qwen3.8-27B-UD-Q4_K_M.gguf"
41		"--mmproj"; "/models/mmproj-F16.gguf"
42		"--alias"; model.slug
43		"--host"; "127.0.0.1"
44		"--port"; "8081"
45		"--n-gpu-layers"; "99"
46		"--ctx-size"; model.context_window.toString()
47		"--parallel"; "1"
48		"--flash-attn"; "on"
49		"--cache-type-k"; "q8_0"
50		"--cache-type-v"; "q8_0"
51		"--cache-ram"; "8192"
52		"--spec-type"; "draft-mtp"
53		"--spec-draft-n-max"; "3"
54		"--cache-type-k-draft"; "q8_0"
55		"--cache-type-v-draft"; "q8_0"
56		"--jinja"
57		"--timeout"; "28800"
58		"--sse-ping-interval"; "10"
59		"--metrics"
60		"--temp"; "1.0"
61		"--top-p"; "0.95"
62		"--top-k"; "20"
63		"--min-p"; "0"
64		"--chat-template-kwargs"; "{\"enable_thinking\": true, \"reasoning_effort\": \"\(model.default_reasoning_level)\", \"preserve_thinking\": true}"
	40	["LD_LIBRARY_PATH"] = "/nvidia/lib:/usr/local/cuda/lib64"
	41	["VLLM_API_KEY"] = "${secret.own.api_key}"
	42	["STUDIO_AI_CACHE_KEY"] = "${secret.own.cache_key}"
	43	["PYTHONHASHSEED"] = "0"
	44	["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:False"
	45	["MODEL"] = "/models"
	46	["CTX"] = "long"
	47	["MAX_LEN"] = model.context_window.toString()
	48	["SPEC"] = "mtp"
	49	["DRAFT_TOKENS"] = "3"
	50	["PREFIX_CACHE"] = "1"
	51	["TOOLS"] = "1"
	52	["VISION"] = "1"
	53	["VISION_OFFLOAD"] = "1"
	54	["GPU_UTIL"] = "0.93"
	55	["MAX_SEQS"] = "8"
	56	["API_SERVERS"] = "1"
	57	["HOST"] = "127.0.0.1"
	58	["PORT"] = "8081"
	59	["VLLM_MAMBA_ALIGN_KEEP_CHECKPOINTS"] = "1"
	60	["FLASHINFER_DISABLE_VERSION_CHECK"] = "1"
	61	["EXTRA_ARGS"] = "--kv-transfer-config {\"kv_connector\":\"OffloadingConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"spec_name\":\"TieringOffloadingSpec\",\"cpu_bytes_to_use\":8589934592,\"offload_prompt_only\":false,\"secondary_tiers\":[{\"type\":\"studio_disk\",\"ssd_dir\":\"/cache/ssd\",\"hdd_dir\":\"/cache/hdd\",\"ssd_bytes\":17179869184,\"hdd_bytes\":274877906944}]}} --limit-mm-per-prompt {\"image\":{\"count\":10}} --default-chat-template-kwargs {\"enable_thinking\":true,\"reasoning_effort\":\"\(model.default_reasoning_level)\"} --generation-config vllm --override-generation-config {\"temperature\":1.0,\"top_p\":0.95,\"top_k\":20,\"min_p\":0,\"presence_penalty\":0,\"repetition_penalty\":1.0} --disable-uvicorn-access-log"
65	62	}
	63	args { "/config/gateway.py" }
66	64	}

service/local-ai/test_gateway.py+6

...	...	@@ -3,6 +3,12 @@ from gateway import normalize, normalize_anthropic, restore
3	3
4	4
5	5	class ResponsesCompatibility(unittest.TestCase):
	6	def test_anthropic_effort_matches_the_qwen_template(self):
	7	for requested, expected in [("low", "low"), ("medium", "medium"), ("high", "xhigh"), ("max", "xhigh"), ("xhigh", "xhigh")]:
	8	with self.subTest(effort=requested):
	9	data = normalize_anthropic({"output_config": {"effort": requested, "format": {"type": "json_schema"}}})
	10	self.assertEqual(data["output_config"], {"effort": expected, "format": {"type": "json_schema"}})
	11
6	12	def test_claude_normal_configuration_system_blocks(self):
7	13	request = {"system": [{"type": "text", "text": "instructions", "cache_control": {"type": "ephemeral"}}, {"type": "text", "text": "workspace context"}], "messages": [{"role": "user", "content": "task"}, {"role": "system", "content": "late system message"}]}
8	14	data = normalize_anthropic(request)

tools/ai-wrapper-test.py deleted-68

...	...	@@ -1,68 +0,0 @@
1		#!/usr/bin/env python3
2		"""Check downloaded wrappers without calling the model or changing client settings."""
3		import json
4		import os
5		import pty
6		import select
7		import time
8		from pathlib import Path
9		import subprocess
10		import sys
11		import tempfile
12
13		source = Path(sys.argv[1]).resolve()
14		with tempfile.TemporaryDirectory(prefix="snow-wrapper-test-") as temporary:
15		root = Path(temporary)
16		for name in ("codex", "claude"):
17		binary = root / name
18		binary.write_text('''#!/usr/bin/env python3
19		import json, os, pathlib, sys
20		catalog = next((a.split("=",1)[1].strip('"') for a in sys.argv if a.startswith("model_catalog_json=")), None)
21		print(json.dumps({"args":sys.argv[1:],"env":{k:v for k,v in os.environ.items() if k.startswith(("SNOW", "ANTHROPIC", "CLAUDE", "API_", "ENABLE_CLAUDE"))},"catalog":json.loads(pathlib.Path(catalog).read_text()) if catalog else None}))
22		''')
23		binary.chmod(0o755)
24		env = dict(os.environ, PATH=str(root) + os.pathsep + os.environ["PATH"], XDG_CONFIG_HOME=str(root / "config"))
25		env.pop("SNOWGLOBE_AI_KEY", None)
26		env.pop("SNOWGLOBE_AI_URL", None)
27		for name in ("snow-codex", "snow-claude"):
28		wrapper = source / name
29		subprocess.run(["bash", "-n", wrapper], check=True)
30		failed = subprocess.run(["bash", wrapper], env=env, stdin=subprocess.DEVNULL, capture_output=True)
31		assert failed.returncode == 1 and b"SNOWGLOBE_AI_KEY" in failed.stderr
32		key = root / "config/snowglobe-ai/api-key"
33		key.parent.mkdir(parents=True, exist_ok=True)
34		master, slave = pty.openpty()
35		process = subprocess.Popen(["bash", wrapper, "--help"], env=env, stdin=slave, stdout=subprocess.PIPE, stderr=slave)
36		os.close(slave)
37		prompt = b""
38		deadline = time.monotonic() + 10
39		try:
40		while b"Snow Globe API key:" not in prompt:
41		assert time.monotonic() < deadline, "first-run key prompt did not appear"
42		if select.select([master], [], [], 1)[0]:
43		prompt += os.read(master, 4096)
44		os.write(master, b"wrapper-canary\n")
45		output, _ = process.communicate(timeout=10)
46		assert process.returncode == 0 and json.loads(output)["env"]["SNOWGLOBE_AI_KEY"] == "wrapper-canary"
47		assert key.read_text() == "wrapper-canary\n" and key.stat().st_mode & 0o777 == 0o600
48		finally:
49		if process.poll() is None: process.kill()
50		os.close(master)
51		data = json.loads(subprocess.check_output(["bash", wrapper, "--help", "argument with spaces"], env={**env, "ANTHROPIC_AUTH_TOKEN":"old-auth", "CLAUDE_CODE_OAUTH_TOKEN":"old-login"}))
52		assert data["args"][-2:] == ["--help", "argument with spaces"]
53		assert "wrapper-canary" not in " ".join(data["args"])
54		assert data["env"]["SNOWGLOBE_AI_KEY"] == "wrapper-canary"
55		assert key.stat().st_mode & 0o777 == 0o600
56		if name == "snow-codex":
57		assert data["args"][:2] == ["--no-daemon", "--approve-for-me"]
58		assert data["catalog"]["models"][0]["context_window"] == 131072
59		assert not list(key.parent.glob("catalog.*")), "temporary catalog leaked"
60		else:
61		assert data["args"][:2] == ["--permission-mode", "auto"]
62		assert "ANTHROPIC_AUTH_TOKEN" not in data["env"] and "CLAUDE_CODE_OAUTH_TOKEN" not in data["env"]
63		assert data["env"]["CLAUDE_CODE_AUTO_MODE_SERVER"] == "0"
64		assert data["env"]["API_TIMEOUT_MS"] == "28800000"
65		credentials = Path(data["env"]["CLAUDE_SECURESTORAGE_CONFIG_DIR"])
66		assert credentials.stat().st_mode & 0o777 == 0o700
67		key.unlink()
68		print("Both downloaded wrappers preserve arguments, enable auto approval, isolate credentials, and clean up.")

tools/benchmark-local-ai-cache.py created+126

...	...	@@ -0,0 +1,126 @@
	1	#!/usr/bin/env python3
	2	"""Run on the target through SSH stdin; use synthetic prompts and disposable files."""
	3	import argparse
	4	from concurrent.futures import ThreadPoolExecutor
	5	import http.client
	6	import json
	7	import mmap
	8	import os
	9	from pathlib import Path
	10	import random
	11	import socket
	12	import tempfile
	13	import time
	14
	15
	16	def request(path, key, data=None):
	17	connection = http.client.HTTPConnection("127.0.0.1", 8081, timeout=600)
	18	try:
	19	body = json.dumps(data) if data is not None else None
	20	connection.request("POST" if body else "GET", path, body, {
	21	"Authorization": "Bearer " + key, "Content-Type": "application/json",
	22	})
	23	response = connection.getresponse()
	24	payload = response.read()
	25	if response.status != 200:
	26	raise RuntimeError(f"{path}: HTTP {response.status}")
	27	return json.loads(payload) if data is not None else payload.decode()
	28	finally:
	29	connection.close()
	30
	31
	32	def metrics(key):
	33	values = {}
	34	for line in request("/metrics", key).splitlines():
	35	if line.startswith("vllm:"):
	36	name = line.split("{", 1)[0]
	37	values[name] = values.get(name, 0) + float(line.rsplit(" ", 1)[1])
	38	return values
	39
	40
	41	def prefill(pid, repeats):
	42	environment = dict(entry.split(b"=", 1) for entry in Path(f"/proc/{pid}/environ").read_bytes().split(b"\0") if b"=" in entry)
	43	key = environment[b"VLLM_API_KEY"].decode()
	44	vocabulary = request("/tokenize", key, {
	45	"model": "qwen3.8-27b", "prompt": "Synthetic benchmark data: blue green yellow red apple pear stone river cloud mountain 0123456789.\n",
	46	})["tokens"]
	47	for length in (16000, 64000, 148000):
	48	for trial in range(repeats):
	49	before = metrics(key)
	50	if before["vllm:num_requests_running"] or before["vllm:num_requests_waiting"]:
	51	raise RuntimeError("Endpoint is busy; rerun when idle")
	52	prompt = random.choices(vocabulary, k=length)
	53	started = time.perf_counter()
	54	response = request("/v1/completions", key, {
	55	"model": "qwen3.8-27b", "prompt": prompt, "max_tokens": 1, "temperature": 0,
	56	})
	57	elapsed = time.perf_counter() - started
	58	after = metrics(key)
	59	delta = {name: after.get(name, 0) - value for name, value in before.items()}
	60	print(json.dumps({
	61	"kind": "prefill", "prompt_tokens": response["usage"]["prompt_tokens"], "trial": trial,
	62	"wall_seconds": elapsed, "prefill_seconds": delta["vllm:request_prefill_time_seconds_sum"],
	63	"queue_seconds": delta["vllm:request_queue_time_seconds_sum"],
	64	"cached_tokens": delta["vllm:prefix_cache_hits_total"],
	65	"completed_requests": delta["vllm:request_success_total"],
	66	}), flush=True)
	67
	68
	69	def filesystem(directory, repeats):
	70	os.umask(0o077)
	71	size = 16 * 1024**2
	72	with tempfile.TemporaryDirectory(prefix="synthetic-ai-cache-", dir=directory) as temporary:
	73	root = Path(temporary)
	74	buffer = mmap.mmap(-1, size)
	75	buffer[:] = os.urandom(size)
	76	paths = [root / f"{random.getrandbits(128):032x}.bin" for _ in range(256)]
	77	started = time.perf_counter()
	78	for path in paths:
	79	descriptor = os.open(path, os.O_WRONLY \| os.O_CREAT \| os.O_EXCL \| os.O_DIRECT, 0o600)
	80	try:
	81	if os.write(descriptor, buffer) != size:
	82	raise RuntimeError("Short synthetic write")
	83	os.fsync(descriptor)
	84	finally:
	85	os.close(descriptor)
	86	buffer.close()
	87	print(json.dumps({"kind": "write", "directory": directory, "gib": 4, "seconds": time.perf_counter() - started}), flush=True)
	88
	89	def read(path):
	90	with mmap.mmap(-1, size) as target:
	91	descriptor = os.open(path, os.O_RDONLY \| os.O_DIRECT)
	92	try:
	93	if os.readv(descriptor, [target]) != size:
	94	raise RuntimeError("Short synthetic read")
	95	finally:
	96	os.close(descriptor)
	97
	98	for gib in (1, 4):
	99	selected = paths[:gib * 64]
	100	for trial in range(repeats):
	101	random.shuffle(selected)
	102	devices = [path for path in Path("/sys/block").iterdir() if (path / "device").exists()]
	103	sectors = {path.name: int((path / "stat").read_text().split()[2]) for path in devices}
	104	started = time.perf_counter()
	105	with ThreadPoolExecutor(max_workers=16) as pool:
	106	list(pool.map(read, selected))
	107	elapsed = time.perf_counter() - started
	108	disk_reads = {path.name: (int((path / "stat").read_text().split()[2]) - sectors[path.name]) * 512 for path in devices}
	109	print(json.dumps({"kind": "direct-read", "directory": directory, "gib": gib, "trial": trial,
	110	"block_mib": 16, "workers": 16, "seconds": elapsed,
	111	"device_read_bytes": {name: count for name, count in disk_reads.items() if count}}), flush=True)
	112
	113
	114	if __name__ == "__main__":
	115	parser = argparse.ArgumentParser(description=__doc__)
	116	parser.add_argument("mode", choices=("prefill", "filesystem"))
	117	parser.add_argument("target", help="Backend PID or disposable parent directory")
	118	parser.add_argument("--hostname", required=True)
	119	parser.add_argument("--repeats", type=int, default=3)
	120	arguments = parser.parse_args()
	121	if socket.gethostname() != arguments.hostname:
	122	raise SystemExit("Unexpected target host")
	123	if arguments.mode == "prefill":
	124	prefill(int(arguments.target), arguments.repeats)
	125	else:
	126	filesystem(arguments.target, arguments.repeats)

tools/benchmark-local-ai.py deleted-87

...	...	@@ -1,87 +0,0 @@
1		#!/usr/bin/env python3
2		"""Measure llama.cpp decode, retrieval, and prefix reuse without logging prompts."""
3		import argparse
4		import base64
5		import binascii
6		import json
7		import os
8		import struct
9		import time
10		import urllib.request
11		import zlib
12
13
14		def request(path, data=None):
15		body = None if data is None else json.dumps(data).encode()
16		headers = {"Authorization": "Bearer " + os.environ["LLAMA_API_KEY"], "Content-Type": "application/json"}
17		started = time.monotonic()
18		with urllib.request.urlopen(urllib.request.Request(args.url + path, data=body, headers=headers), timeout=1800) as response:
19		result = json.load(response)
20		return result, time.monotonic() - started
21
22
23		parser = argparse.ArgumentParser(description=__doc__)
24		parser.add_argument("--url", default="http://127.0.0.1:8081")
25		parser.add_argument("--contexts", type=int, nargs="+", default=[16384, 65536, 122000])
26		parser.add_argument("--suite", choices=["decode", "workloads", "cache", "retrieval", "vision", "all"], default="all")
27		parser.add_argument("--require-prefix-cache", action="store_true")
28		args = parser.parse_args()
29
30		def report(label, data, path="/completion"):
31		result, elapsed = request(path, data)
32		print(json.dumps({"case": label, "elapsed_s": round(elapsed, 3), "timings": result.get("timings"), "usage": result.get("usage")}), flush=True)
33		return result
34
35
36		filler = "".join(f"Record {i:06d}: amber bridge cedar delta elm forest granite harbor iris jade.\n" for i in range(15000))
37		tokens, _ = request("/tokenize", {"content": filler})
38		tokens = tokens["tokens"]
39
40		if args.suite in {"decode", "all"}:
41		offset = 0
42		for size in args.contexts:
43		prompt = tokens[offset:offset + size]
44		offset += size
45		assert len(prompt) == size, "Requested context exceeds the fixture"
46		data = {"prompt": prompt, "n_predict": 256, "ignore_eos": True, "temperature": 0, "seed": 42, "cache_prompt": True}
47		report(f"decode-{size}-first", data)
48		report(f"decode-{size}-warm", data)
49
50		if args.suite in {"cache", "all"}:
51		prefix, _ = request("/detokenize", {"tokens": tokens[:4096]})
52		other, _ = request("/detokenize", {"tokens": tokens[40000:44096]})
53		prefix["content"] += "\nStable system code is prefix-a-731."
54		other["content"] += "\nStable system code is prefix-b-284."
55		for label, system, body in [("prefix-a", prefix["content"], tokens[10000:30000]), ("prefix-b", other["content"], tokens[45000:65000]), ("prefix-a-new", prefix["content"], tokens[70000:90000]), ("prefix-a-repeat", prefix["content"], tokens[70000:90000]), ("prefix-b-again", other["content"], tokens[95000:115000]), ("prefix-a-metadata", prefix["content"].replace("prefix-a-731", "prefix-c-592"), tokens[120000:140000])]:
56		text, _ = request("/detokenize", {"tokens": body})
57		result = report(label, {"model": "qwen3.8-27b", "messages": [{"role": "system", "content": system}, {"role": "user", "content": text["content"] + "\nWhat is the stable system code? Reply with only it."}], "max_tokens": 32, "temperature": 0, "chat_template_kwargs": {"enable_thinking": False}}, "/v1/chat/completions")
58		assert system.rsplit(" ", 1)[-1].rstrip(".") in result["choices"][0]["message"]["content"], "Prefix restore lost the system fact"
59		if args.require_prefix_cache and label in {"prefix-a-new", "prefix-a-metadata"}:
60		assert result["timings"]["cache_n"] >= 4096, "Client switching discarded the base prefix"
61
62		if args.suite in {"workloads", "all"}:
63		for task in ["Implement a bounded asynchronous work queue in Python using asyncio. Explain cancellation, backpressure, shutdown, race conditions and error handling. Include tests and a detailed explanation.", "Explain how database isolation levels interact with concurrent read-modify-write operations. Give concrete schedules, anomalies, and remedies for each level, with detailed examples."]:
64		template, _ = request("/apply-template", {"messages": [{"role": "system", "content": "You are a careful software engineer."}, {"role": "user", "content": task}], "chat_template_kwargs": {"enable_thinking": True, "reasoning_effort": "medium"}})
65		report("workload-code" if "asyncio" in task else "workload-reasoning", {"prompt": template["prompt"], "n_predict": 512, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0, "seed": 42, "cache_prompt": True})
66
67		if args.suite in {"retrieval", "all"}:
68		for size in args.contexts:
69		facts = ["START password is otter-731.", "MIDDLE password is maple-284.", "END password is comet-965."]
70		segment = tokens[:max(1, (size - 256) // 2)]
71		text, _ = request("/detokenize", {"tokens": segment})
72		messages = [{"role": "system", "content": "Find the three passwords in the records. Reply with only the passwords in START, MIDDLE, END order."}, {"role": "user", "content": facts[0] + "\n" + text["content"] + "\n" + facts[1] + "\n" + text["content"] + "\n" + facts[2]}]
73		template, _ = request("/apply-template", {"messages": messages, "chat_template_kwargs": {"enable_thinking": False}})
74		result = report(f"retrieval-{size}", {"prompt": template["prompt"], "n_predict": 64, "temperature": 0, "cache_prompt": True})
75		positions = [result["content"].find(fact.split()[-1].rstrip(".")) for fact in facts]
76		passed = all(pos >= 0 for pos in positions) and positions == sorted(positions)
77		print(json.dumps({"case": f"retrieval-{size}", "passed": passed}), flush=True)
78		if not passed:
79		raise SystemExit("Retrieval failed")
80
81		if args.suite in {"vision", "all"}:
82		png = b"\x89PNG\r\n\x1a\n"
83		pixels = b"".join(b"\0" + b"".join(b"\xff\0\0" if 128 <= x < 384 and 128 <= y < 384 else b"\xff\xff\xff" for x in range(512)) for y in range(512))
84		for kind, data in [(b"IHDR", struct.pack(">IIBBBBB", 512, 512, 8, 2, 0, 0, 0)), (b"IDAT", zlib.compress(pixels)), (b"IEND", b"")]:
85		png += struct.pack(">I", len(data)) + kind + data + struct.pack(">I", binascii.crc32(kind + data))
86		result = report("vision", {"model": "qwen3.8-27b", "messages": [{"role": "user", "content": [{"type": "text", "text": "What color is the square? Reply with one word."}, {"type": "image_url", "image_url": {"url": "data:image/png;base64," + base64.b64encode(png).decode()}}]}], "max_tokens": 32, "temperature": 0, "chat_template_kwargs": {"enable_thinking": False}}, "/v1/chat/completions")
87		assert "red" in result["choices"][0]["message"]["content"].lower(), "Vision failed"

tools/dashboard-container-test.py deleted-94

...	...	@@ -1,94 +0,0 @@
1		#!/usr/bin/env python3
2		import argparse
3		import concurrent.futures
4		import json
5		from pathlib import Path
6		import subprocess
7		import time
8		import urllib.request
9
10
11		def main():
12		parser = argparse.ArgumentParser()
13		parser.add_argument("container")
14		parser.add_argument("--url", default="http://127.0.0.1:7074")
15		parser.add_argument("--clients", type=int, default=40)
16		parser.add_argument("--requests", type=int, default=1000)
17		parser.add_argument("--output", type=Path)
18		parser.add_argument("--proxy-token-file", type=Path, required=True)
19		parser.add_argument("--stop", action="store_true")
20		args = parser.parse_args()
21		info = json.loads(subprocess.check_output(["podman", "inspect", args.container]))[0]
22		host = info["HostConfig"]
23		assert info["Config"]["User"].split(":")[0] not in {"", "0", "root"}
24		assert host["ReadonlyRootfs"] is True
25		assert "no-new-privileges" in host["SecurityOpt"]
26		assert not host["Privileged"] and not host["Devices"]
27		assert host["NetworkMode"] != "host"
28		assert all(m["Destination"] in ["/run/studio-host", "/run/secrets/dashboard-proxy.token", "/data"] for m in info["Mounts"]), info["Mounts"]
29		assert next(m for m in info["Mounts"] if m["Destination"] == "/run/studio-host")["RW"] is False
30		status = subprocess.check_output(["podman", "exec", args.container, "/bin/cat", "/proc/1/status"], text=True)
31		uid = next(line.split()[1:] for line in status.splitlines() if line.startswith("Uid:"))
32		assert all(int(value) != 0 for value in uid), uid
33		for field in ["CapEff", "CapBnd", "CapPrm", "CapAmb"]:
34		assert field + ":\t0000000000000000" in status, status
35		assert "NoNewPrivs:\t1" in status, status
36		mounts = subprocess.check_output(["podman", "exec", args.container, "/bin/cat", "/proc/1/mountinfo"], text=True)
37		root = next(line.split() for line in mounts.splitlines() if line.split()[4] == "/")
38		assert "ro" in root[5].split(","), root
39		assert subprocess.run(["podman", "exec", args.container, "/bin/touch", "/escaped"], capture_output=True).returncode != 0
40		for denied in ["/var/lib/studio/nomad.token", "/run/podman/podman.sock", "/run/libvirt/libvirt-sock", "/dev/zfs", "/opt/studio/current"]:
41		assert subprocess.run(["podman", "exec", args.container, "/bin/test", "-e", denied], capture_output=True).returncode != 0, denied
42		pid = info["State"]["Pid"]
43		for namespace in ["net", "pid", "mnt"]:
44		assert Path(f"/proc/{pid}/ns/{namespace}").stat().st_ino != Path(f"/proc/self/ns/{namespace}").stat().st_ino
45
46		headers = {"User-Name": "fixture", "User-Groups": "infra-admin", "Studio-Proxy-Token": args.proxy_token_file.read_text().strip()}
47		with urllib.request.urlopen(urllib.request.Request(args.url + "/api/host", headers=headers), timeout=10) as response:
48		machine = json.load(response)
49		expected_memory = next(int(line.split()[1]) * 1024 for line in Path("/proc/meminfo").read_text().splitlines() if line.startswith("MemTotal:"))
50		expected_cores = sum(line.startswith("cpu") and line[3:4].isdigit() for line in Path("/proc/stat").read_text().splitlines())
51		assert machine["memory"] == expected_memory and machine["cores"] == expected_cores, machine
52		with urllib.request.urlopen(urllib.request.Request(args.url + "/api/live", headers=headers), timeout=10) as response:
53		for _ in range(20):
54		line = response.readline()
55		if line.startswith(b"data:"):
56		live = json.loads(line[5:])
57		break
58		else:
59		raise AssertionError("host sample did not arrive")
60		assert 0 <= live["host"]["cpu"] <= 100 and live["host"]["memory"] > 0, live
61		assert live["host"]["arc"] is not None, live
62
63		def request(_):
64		start = time.monotonic()
65		req = urllib.request.Request(args.url + "/api/storage", headers=headers)
66		with urllib.request.urlopen(req, timeout=10) as response:
67		assert json.load(response)["pool"]["state"] == "ONLINE"
68		return (time.monotonic() - start) * 1000
69
70		with concurrent.futures.ThreadPoolExecutor(max_workers=args.clients) as clients:
71		latency = sorted(clients.map(request, range(args.requests)))
72		result = {"nonroot_container": "passed", "readonly_rootfs": "passed", "zero_capabilities": "passed",
73		"no_new_privileges": "passed", "private_namespaces": "passed", "control_sockets_and_management_token_absent": "passed",
74		"host_identity_outside_container_quota": "passed", "host_live_sample": "passed",
75		"storage_requests": len(latency), "storage_clients": args.clients,
76		"storage_p95_ms": round(latency[int(len(latency) * .95)], 2), "storage_max_ms": round(latency[-1], 2)}
77		if args.stop:
78		req = urllib.request.Request(args.url + "/api/live", headers=headers)
79		with urllib.request.urlopen(req, timeout=10):
80		started = time.monotonic()
81		stopped = subprocess.run(["podman", "stop", "--time=8", args.container], capture_output=True, text=True, check=True)
82		elapsed = time.monotonic() - started
83		ended = json.loads(subprocess.check_output(["podman", "inspect", args.container]))[0]["State"]
84		assert ended["ExitCode"] == 0 and not ended["OOMKilled"], (ended, stopped.stderr)
85		assert elapsed < 8, elapsed
86		result["stop_with_active_stream"] = "passed"
87		result["stop_seconds"] = round(elapsed, 2)
88		if args.output:
89		args.output.write_text(json.dumps(result, indent=2) + "\n")
90		print(json.dumps(result))
91
92
93		if __name__ == "__main__":
94		main()

tools/local-ai-cache.md created+110

...	...	@@ -0,0 +1,110 @@
	1	# Local AI context cache
	2
	3	## Prepared configuration
	4
	5	The current change configures GPU prefix caching, 8 GiB of vLLM shared-memory
	6	offload, a 16 GiB NVMe hot copy, and a 256 GiB HDD pool. The container has a
	7	26 GiB memory limit and a 10 GiB `/dev/shm`.
	8
	9	`service/local-ai/cache` installs a vLLM secondary-tier plugin. Blocks are
	10	written through to HDD and copied to SSD; HDD reads warm SSD. Each disk tier
	11	evicts its least recently used unpinned blocks within its byte cap. Active
	12	requests pin blocks; a saturated pinned tier skips writes. Direct I/O avoids
	13	an additional unbounded filesystem data cache. Disk writes and promotions run
	14	in background workers, outside scheduler lookups.
	15
	16	Disk blocks use authenticated AES-GCM encryption with the service's Nomad
	17	`cache_key`, owner-only files/directories, and atomic replacement. Restart
	18	rebuilds the bounded index. Namespaces include the model manifest, vLLM and
	19	plugin versions, key, KV layout, and parallel configuration. Corrupt blocks
	20	fall back to another copy or become misses. Key rotation invalidates reuse;
	21	old entries remain encrypted and subject to eviction.
	22
	23	`config/site.pkl` owns the NVMe path (`/var/cache/studio/local-ai` by default).
	24	The prepare hook creates it with the assigned service owner. HDD blocks use
	25	the managed service dataset. Fresh previews use separate managed directories
	26	for both disk tiers.
	27
	28	Validation on 2026-10-09: the full image built and its installed plugin loaded;
	29	nine disk-cache tests and two CPU-only tests against the installed vLLM
	30	scheduler passed. These check encrypted persistence, quotas, pinning,
	31	concurrent access, SSD promotion, attention/recurrent-group restoration into
	32	shared RAM, and corrupt-promotion misses. Production/preview Pkl rendering,
	33	isolated preparation, deployment checks, and six gateway tests passed. These
	34	tests use synthetic data and do not establish GPU/hybrid-MTP restore timing.
	35
	36	## Production activation — 2026-10-09
	37
	38	Release `0edf5af161627873` was deployed from local `main` without pushing.
	39	Pre-switch backup: `20261009T214237Z-734b3d`. Nomad deployment and all route
	40	checks passed. vLLM initialized the tiering connector and an 8.57 GB shared
	41	region (rounded below the 8 GiB budget to whole blocks).
	42
	43	An authenticated gateway probe completed an 8,000-token synthetic prompt.
	44	Repeating it reused 6,656 tokens and completed in 3.439 seconds. The first
	45	request took 37.596 seconds including first-use kernel compilation, so it is
	46	not a steady-state prefill comparison. Both disk tiers held 66 encrypted
	47	blocks totaling 1,912,086,528 bytes. Forced RAM/disk restore probes stopped
	48	when the endpoint reported busy; end-to-end disk restore timing remains
	49	unverified.
	50
	51	## Zenith measurements — 2026-10-09
	52
	53	HyperQwen, vLLM 0.27.1, MTP with three draft tokens, 150,000-token
	54	context, GPU prefix caching enabled. Production stayed running; no cache
	55	connector was enabled or production configuration changed.
	56
	57	`benchmark-local-ai-cache.py` sends synthetic token-ID prompts with one output
	58	token. Each measured request had zero prefix-cache hits, one completed request,
	59	and under 0.1 ms queue time. Server prefill time excludes HTTP overhead.
	60
	61	\| Prompt tokens \| Server prefill seconds \|
	62	\| --- \| --- \|
	63	\| 16,000 \| 12.715, 12.557, 12.601 \|
	64	\| 64,000 \| 63.174, 63.704, 63.896 \|
	65	\| 148,000 \| 203.552, 204.495, 204.963 \|
	66
	67	Storage probes create disposable files filled with incompressible synthetic
	68	bytes, then read shuffled 16 MiB files with 16 workers. Both writes and reads use
	69	`O_DIRECT`; writes are individually fsynced. The HDD path is encrypted ZFS on
	70	the four-disk RAIDZ1 pool, OpenZFS 2.4.4, `direct=standard`, 128 KiB records,
	71	LZ4 compression. The SSD path is ext4 on the WD Blue SN5000 500 GB NVMe.
	72
	73	\| Storage \| 1 GiB read seconds \| 4 GiB read seconds \| 4 GiB write seconds \|
	74	\| --- \| --- \| --- \| --- \|
	75	\| HDD (`/srv/staging`) \| 3.901, 3.686, 3.660, 3.931 \| 15.736, 15.399, 16.848, 17.502 \| 31.161, 30.165 \|
	76	\| NVMe (`/var/tmp`) \| 0.262, 0.258, 0.251, 0.258 \| 0.937, 0.960, 0.933, 0.929 \| 1.282, 1.342 \|
	77
	78	The final storage samples additionally checked physical-device read counters:
	79	the 4 GiB probe read 4,427,632,640 bytes across the HDDs and exactly
	80	4,294,967,296 bytes from NVMe. This verifies reads reached the drives instead
	81	of being served from the host's data cache.
	82
	83	Disk probes overlapped part of the prefill benchmark. These are storage-only
	84	timings, not end-to-end KV restores. Actual restores additionally depend on
	85	attention KV and recurrent-checkpoint bytes, cache-file layout, GPU transfers,
	86	and any uncached suffix. A larger, fragmented cache and competing pool I/O can
	87	increase HDD latency. The disposable files were removed after measurement.
	88
	89	HDD reads are promising against the 64k and 148k uncached prefills. End-to-end
	90	validation with HyperQwen's patched hybrid-state/MTP implementation remains
	91	necessary; the storage timings above predate the bounded plugin.
	92
	93	## Repeating the probes
	94
	95	Confirm the target and ensure the AI endpoint is idle before inference probes.
	96	The prefill script reads the existing backend key internally and prints only
	97	timings and counters; it never prints prompt text, generated text, or keys.
	98
	99	```sh
	100	ssh -o BatchMode=yes root@10.0.0.1 \
	101	'pid=$(pgrep -x vllm); nsenter -t "$pid" -n python3 - prefill "$pid" --hostname zenith' \
	102	< tools/benchmark-local-ai-cache.py
	103	ssh -o BatchMode=yes root@10.0.0.1 \
	104	'python3 - filesystem /srv/staging --hostname zenith' \
	105	< tools/benchmark-local-ai-cache.py
	106	ssh -o BatchMode=yes root@10.0.0.1 \
	107	'python3 - filesystem /var/tmp --hostname zenith' \
	108	< tools/benchmark-local-ai-cache.py
	109	```
	110

tools/local-ai.md deleted-247

...	...	@@ -1,247 +0,0 @@
1		# Snow Globe local AI
2
3		The first model is Qwen3.8-27B, using Unsloth's UD-Q4_K_M GGUF and F16 vision
4		projector. Both files, the base CUDA image, and the llama.cpp source are pinned.
5		The model runs entirely on the RTX 3090, with one inference slot and a 131,072-token context.
6		Codex compacts at 96,000 tokens to leave room for tool results and output.
7		The KV cache uses Q8; the single slot leaves more VRAM for vision and runtime
8		working memory. Additional requests queue so simultaneous generation cannot
9		reduce the active request's decode rate. An 8 GiB host-memory prompt cache
10		preserves evicted contexts between coding, reviewer, and helper requests.
11		The patched server also reserves a separate 2 GiB cache for prefixes before
12		the first user message, including system instructions and tool definitions.
13		A second checkpoint at 4,096 tokens retains the shared base when later workspace
14		metadata changes; shorter headers need only their complete checkpoint.
15		It captures these from actual requests; there are no copied Codex or Claude
16		prompts to update. Prefix states survive conversation-cache eviction and remain
17		available after restoration. Only complete token matches restore these states,
18		including the recurrent and MTP state. Prefix variants have their own bounded
19		LRU eviction; first use, unmatched tokens, and server restarts still need prefill.
20		The container has an 18 GiB RAM limit for the runtime and both caches.
21		Native MTP drafts up to three tokens, which the target model verifies. The draft
22		KV cache also uses Q8. Target weights, target KV precision, context length, and
23		reasoning defaults remain unchanged. The build replaces only the server library;
24		inference libraries and embedded UI assets come from the pinned base image.
25		Medium reasoning is enabled for coding, using Qwen's recommended thinking-mode
26		sampling parameters. This adds
27		reasoning tokens before answers; the earlier latency baseline disabled thinking.
28
29		Models live in `/srv/clover/Media/AI/LLM/Qwen3.8-27B`, also visible on the Mac as
30		`/Volumes/clover/Media/AI/LLM/Qwen3.8-27B`. Download once on Zenith:
31
32		```sh
33		python3 service/local-ai/download.py
34		```
35
36		The downloader resumes partial transfers, verifies size and SHA-256, and never
37		replaces an existing file that fails verification. No model download or GPU
38		service runs in rehearsal VMs. `STUDIO_LOCAL_AI=true` enables the physical host.
39
40		## API and deployment
41
42		The authenticated API exposes `/v1/responses`, `/v1/messages`,
43		`/v1/chat/completions`, and `/v1/models`. `/health` is public for deployment
44		checks. Use `Authorization: Bearer <key>` or Anthropic's `x-api-key` header.
45		`api_key` is a generated Nomad service secret; don't put it in Git or shell
46		arguments.
47
48		The production endpoint is `https://ai.paperclover.net`. The
49		`STUDIO_LOCAL_AI=true` host environment in `nixos/zenith.nix` enables it.
50		The 3090 has room for one model instance. Stop any GPU preview before deploying
51		production; download the wrappers again when their endpoint changes.
52
53		`gateway.py` uses Python's standard library and starts llama.cpp on a container
54		loopback port. Anthropic system-text blocks are combined into one system message;
55		Chat Completions pass through unchanged. For
56		Responses, it collects developer/system instructions at the start of the
57		prompt, maps tool namespaces to flat names, restores returned namespace
58		identities, and represents raw custom tools as JSON functions. The raw tool
59		grammar remains in the tool description. Built-in hosted tools such as OpenAI
60		web search aren't implemented; unsupported tool types return an explicit error.
61		Codex's hosted web search is disabled in the wrapper.
62
63		## Coding clients
64
65		Download `snow-codex` or `snow-claude` from the dashboard's AI / MCP → AI
66		tab. They are standalone Bash wrappers around the standard installed Codex and
67		Claude Code clients; client updates require no wrapper rebuild. Make each
68		download executable and place it on your PATH. The first run asks for the API
69		key from the same tab and saves it with mode 0600 under
70		`${XDG_CONFIG_HOME:-~/.config}/snowglobe-ai/api-key`. No checkout or patched
71		client is required. The downloads contain endpoint and model settings, never
72		the API key. The `ai` group opens AI / MCP and grants API-key access and downloads;
73		infrastructure administrators retain access. Role previews can view the page
74		and downloads, but cannot access the API key.
75
76		`dashboard/ai/launch.sh` owns the wrapper template. The download endpoint
77		incorporates the live service hostname and canonical model catalog.
78		Codex starts with `--approve-for-me`, retaining its workspace sandbox and
79		automatic approval reviewer; Claude starts with `--permission-mode auto`,
80		running classification against the same local endpoint. Both use medium
81		reasoning. Claude's credential store is isolated from ordinary Claude login,
82		so using Snow Globe does not log the user's subscription out.
83
84		Qwen's raw thinking is streamed as Responses reasoning text, including the
85		`content_index` required by stock Codex. To display it in the terminal:
86
87		```sh
88		snow-codex -c show_raw_agent_reasoning=true -c hide_agent_reasoning=false
89		```
90
91		Reasoning summaries remain disabled because the model emits raw thinking rather
92		than a separate summary. `tools/verify-local-ai.py` checks actual Codex app-server
93		reasoning deltas arrive before the answer; stored reasoning in a completed
94		response alone does not establish that live display works. The production fix
95		was checked with stock Codex 0.161.0: 237 live reasoning deltas arrived before
96		the answer, the interactive terminal updated during thinking, both coding
97		fixtures passed, and automatic approval authorized its temporary write in
98		21.4 seconds.
99
100		API and event-stream waits allow eight hours. Claude's byte-stream watchdog
101		has a 30-minute maximum, kept alive by ten-second server SSE pings. Stock
102		Codex 0.160.0 retains a hardcoded 90-second approval-review deadline that
103		provider settings cannot extend. The earlier patched build survived a
104		95-second forced review delay, but the shipped wrappers intentionally use
105		standard clients as requested. Queueing can therefore still cause a stock
106		Codex approval review to time out.
107
108		Override the endpoint or key with `SNOWGLOBE_AI_URL` or `SNOWGLOBE_AI_KEY`.
109		Explicit client arguments can override wrapper defaults. Existing MCP servers,
110		hooks, and ordinary commands keep their normal network access. Model inference
111		and approval classification use Snow Globe; cloud plugins, Apps, automatic
112		memories, and nonessential client telemetry are disabled for this profile.
113
114		## Verification
115
116		```sh
117		python3 -m unittest discover -s service/local-ai -p 'test_*.py' -v
118		python3 tools/verify-local-ai.py --url https://YOUR-ENDPOINT
119		```
120
121		The downloadable wrappers were verified against production on 2026-10-05
122		with stock Codex 0.160.0 and Claude Code 2.1.289. Both clients fixed the median
123		fixture and passed all eight unchanged tests, taking 189.3 and 209.4 seconds
124		while sharing the single slot. Stock Codex's actual automatic approval review
125		authorized an outside-workspace fixture write in 37.5 seconds. The temporary
126		file was removed. Download tests also exercised first-run key entry through a
127		terminal, mode 0600 storage, argument preservation, auth isolation, and catalog
128		cleanup. Unsigned, cross-origin, and view-as requests were denied; the returned
129		key authenticated successfully with the live model.
130
131		The second command launches both actual clients concurrently in disposable
132		fixtures. Each must fix two median bugs, preserve input, pass four tests, and
133		leave the tests unchanged. It also exercises an actual Codex approval review and removes its temporary
134		file. JSONL evidence is saved in the printed temporary directory. An optional
135		`--approval-delay 95` probe demonstrates the stock reviewer deadline; it is
136		expected to fail with stock Codex.
137
138		The deployed 128K configuration was checked on 2026-10-05 with a synthetic
139		122,265-token request. It recovered facts from the beginning, middle, and end
140		through a namespaced tool call. A 122,384-token continuation consumed the tool
141		result correctly and reused 122,332 cached tokens. A subsequent 122,409-token
142		request reused 122,359 tokens and generated 384 tokens at 24.21 tokens/second,
143		with a maximum streamed token gap of 45 ms. Thinking was disabled only for
144		these retrieval and throughput probes. The server measured 144.4 seconds of
145		cold prefill at 846.8 tokens/second. The cached stream's first token took 26.1
146		seconds while coding clients shared the slot; that elapsed time includes
147		queueing. The allocation used 20.87 GiB VRAM and peaked at 9.79 GiB host RAM,
148		with zero OOM events, cgroup limit hits, or restarts throughout verification.
149
150		Both actual clients then passed the eight unchanged coding fixture tests with
151		medium reasoning enabled. Codex took 285.4 seconds and Claude 343.5 seconds
152		while competing with the context probe. This checks real harness compatibility;
153		the simple retrieval probe does not establish coding quality at 122K tokens.
154		The actual approval-review check passed in 131.0 seconds, including a deliberate
155		95-second hold beyond stock Codex's deadline. Its authorized write and read-back
156		succeeded, and the temporary file was removed.
157
158		On 2026-10-05, a synthetic 60,756-token Responses request recovered distinct
159		facts from the beginning, middle, and end through a namespaced tool call. A
160		60,875-token continuation consumed the tool result correctly, with 60,823
161		cached tokens. A subsequent 60,900-token request generated 384 tokens at
162		30.75 tokens/second; the largest streamed token gap was 44 ms. These checks
163		used the earlier non-thinking preset. They validate context handling and one
164		simple retrieval task, not general coding quality at that length.
165
166		Cold processing of roughly 60K input tokens took about a minute. End-to-end
167		latency also included waits behind other active requests. The live Codex
168		session separately reached about 48K input tokens at 33–34 generated
169		tokens/second, with recent-prefix prefill below one second. Its compaction
170		requests completed successfully. The earlier 64K limit and 48K compaction threshold
171		kept space for tool results, reasoning, and output without enlarging the cold
172		prefill workload further.
173
174		The cache is bounded RAM storage, with no timed expiry or persistence across
175		server restarts. In an earlier cross-context probe, 15,912 of 15,916 tokens were
176		restored after two competing contexts, returning in 1.05 seconds versus 23.99
177		seconds without the host cache. Distinct large contexts can still evict one
178		another. The Anthropic endpoint reports `cache_read_input_tokens` and remaining
179		uncached `input_tokens`; Responses reports `input_tokens_details.cached_tokens`.
180		There are no API cache charges.
181
182		The previous two-slot allocation used about 20.9 GiB VRAM; one 64K slot used
183		about 18.4 GiB. [Qwen's model card](https://huggingface.co/Qwen/Qwen3.8-27B) describes
184		medium effort as the accuracy/speed balance and cautions that reduced reasoning
185		can increase total agent time through retries.
186
187		## Ajax investigation
188
189		As of 2026-10-05, [PewDiePie's official Ajax page](https://data.pewdiepie.com/)
190		says the model will release when ready. It describes an ablated Qwen3.5-9B
191		fine-tune for [Odysseus](https://github.com/odysseus-dev/odysseus), but does not
192		provide verified model weights or a model license to evaluate yet.
193
194		When official weights exist, inspect the model card, license, file formats,
195		revision, and hashes before downloading. Use the known llama.cpp runtime with
196		GGUF data; don't execute model-repository code or load pickle-based weights.
197		Trial inference should have a read-only filesystem and model mount, no network,
198		no capabilities, no host credentials or private data, process/memory limits,
199		and only disposable scratch storage. Tool-use trials belong in a disposable VM
200		with synthetic files and credentials, separate from the personal coding profile.
201		Evaluate coding, tool arguments, prompt injection, unauthorized writes, and
202		attempted outbound requests against the base model. Passing those tests is
203		evidence about observed behavior, not proof that an unrestricted agent is safe.
204
205		## Runtime tuning (2026-10-07)
206
207		Measurements on the production RTX 3090, using 256 greedy decode tokens with the
208		same target weights, Q8 KV cache, and context size:
209
210		\| Input context \| Stock runtime \| MTP, three Q8 draft tokens \|
211		\| --- \| ---: \| ---: \|
212		\| 16,384 tokens \| 39.1 tok/s \| 92.0 tok/s \|
213		\| 65,536 tokens \| 30.3 tok/s \| 73.3 tok/s \|
214
215		These are warm synthetic decode rates, not a promise for every coding task.
216		The n-gram draft trial did not improve throughput. MTP's target verifies every
217		accepted draft token; it does not substitute a smaller model's answers.
218		Cold prefill remains about 1,200 tok/s at 16K and 990 tok/s at 64K. Caching avoids
219		processing unchanged prefixes, but new long inputs still pay that cost.
220
221		The production checks exercise stock Codex and Claude Code editing the same
222		median fixture, all eight unchanged tests, Codex's automatic approval reviewer,
223		vision, and three-fact retrieval in a 121,816-token input. They establish these
224		paths remain functional, not a broad model-quality evaluation. The initial
225		MTP/prefix-cache check with stock Codex 0.161.0 and Claude Code 2.1.293 took 68.6 and 120.4
226		seconds while sharing the slot. All 30 other production containers retained
227		their identities; the AI allocation recorded no OOM events, limit hits, or
228		restarts during verification.
229
230		`tools/benchmark-local-ai.py` measures decode, natural coding/reasoning workloads,
231		client switching, changing system metadata, retrieval, and vision. Run inside
232		the production container, where `LLAMA_API_KEY` is already inherited, against
233		the backend on `http://127.0.0.1:8081`; it emits timings without prompts or keys:
234
235		```sh
236		python3 /tmp/benchmark-local-ai.py --suite cache --require-prefix-cache
237		python3 /tmp/benchmark-local-ai.py --suite decode --contexts 16384 65536
238		python3 /tmp/benchmark-local-ai.py --suite retrieval --contexts 122000
239		python3 /tmp/benchmark-local-ai.py --suite vision
240		```
241
242		The prefix test alternates two system headers and long unrelated user histories,
243		then changes a trailing system fact. Both restores must reuse at least 4,096
244		tokens and answer from the current header. Production retained 4,114 tokens
245		across client switches and 4,096 after the system fact changed, with the new
246		fact answered correctly. Repeating the conversation retained 24,135 tokens
247		and completed in 0.335 seconds, including generation.

tools/mac-domains.py deleted-248

...	...	@@ -1,248 +0,0 @@
1		#!/usr/bin/env python3
2		"""Route .studio.test through the rehearsal tunnels on this Mac; stop restores the prior settings."""
3
4		import argparse
5		import asyncio
6		import base64
7		import hashlib
8		import json
9		import os
10		from pathlib import Path
11		import signal
12		import socket
13		import ssl
14		import subprocess
15		import sys
16		import time
17
18		STATE = Path("/var/db/snowglobe-test")
19		RESOLVER = Path("/etc/resolver/studio.test")
20		HOSTS = Path("/etc/hosts")
21		KEYCHAIN = "/Library/Keychains/System.keychain"
22		FORWARDS = ((80, 27080), (443, 27443))
23
24
25		def local_hosts(content):
26		lines = []
27		aliases = []
28		for line in content.splitlines(keepends=True):
29		names, _, comment = line.partition("#")
30		fields = names.split()
31		if len(fields) < 2 or not any(name.endswith(".studio.test") for name in fields[1:]):
32		lines.append(line)
33		continue
34		aliases.extend(name for name in fields[1:] if name.endswith(".studio.test"))
35		remaining = [name for name in fields[1:] if not name.endswith(".studio.test")]
36		if remaining:
37		lines.append(" ".join([fields[0], *remaining]) + (" #" + comment.rstrip() if comment else "") + "\n")
38		elif comment:
39		lines.append("#" + comment.rstrip() + "\n")
40		if aliases:
41		if lines and not lines[-1].endswith("\n"):
42		lines[-1] += "\n"
43		lines.append("127.0.0.1 " + " ".join(dict.fromkeys(aliases)) + "\n")
44		return "".join(lines)
45
46
47		class DNSRelay(asyncio.DatagramProtocol):
48		def __init__(self):
49		self.requests = set()
50
51		def connection_made(self, transport):
52		self.transport = transport
53
54		def datagram_received(self, data, address):
55		if len(data) < 12 or len(self.requests) >= 64:
56		return
57		request = asyncio.create_task(asyncio.wait_for(self.forward(data, address), timeout=3))
58		self.requests.add(request)
59		request.add_done_callback(self.finished)
60
61		def finished(self, request):
62		self.requests.discard(request)
63		if not request.cancelled():
64		request.exception()
65
66		async def forward(self, data, address):
67		writer = None
68		try:
69		reader, writer = await asyncio.open_connection("127.0.0.1", 53153)
70		writer.write(len(data).to_bytes(2, "big") + data)
71		await writer.drain()
72		length = int.from_bytes(await reader.readexactly(2), "big")
73		response = await reader.readexactly(length)
74		if len(response) >= 12 and response[:2] == data[:2]:
75		self.transport.sendto(response, address)
76		finally:
77		if writer:
78		writer.close()
79
80
81		async def relay(listeners):
82		async def connect(reader, writer, upstream):
83		peer_writer = None
84		try:
85		peer_reader, peer_writer = await asyncio.open_connection("127.0.0.1", upstream)
86
87		async def pipe(source, destination):
88		while data := await source.read(65536):
89		destination.write(data)
90		await destination.drain()
91		if destination.can_write_eof():
92		destination.write_eof()
93
94		await asyncio.gather(pipe(reader, peer_writer), pipe(peer_reader, writer))
95		except (OSError, asyncio.CancelledError):
96		pass
97		finally:
98		writer.close()
99		if peer_writer:
100		peer_writer.close()
101
102		servers = []
103		for listener, upstream in listeners:
104		servers.append(await asyncio.start_server(
105		lambda reader, writer, port=upstream: connect(reader, writer, port), sock=listener,
106		))
107		transport, _ = await asyncio.get_running_loop().create_datagram_endpoint(
108		DNSRelay, local_addr=("127.0.0.1", 53153),
109		)
110		try:
111		await asyncio.gather(*(server.serve_forever() for server in servers))
112		finally:
113		transport.close()
114
115
116		def stop():
117		record = STATE / "settings.json"
118		if not record.exists():
119		print("Local test routing is already stopped.")
120		return
121		settings = json.loads(record.read_text())
122		for name, saved in settings["files"].items():
123		path = Path(name)
124		current = hashlib.sha256(path.read_bytes()).hexdigest() if path.exists() else None
125		before = hashlib.sha256(base64.b64decode(saved["before"])).hexdigest() if saved["before"] is not None else None
126		if current not in (before, saved["applied"]):
127		raise SystemExit(f"{path} changed after setup. Restore it using {record}; nothing was overwritten.")
128		if pid := settings.get("pid"):
129		command = subprocess.run(["ps", "-p", str(pid), "-o", "command="], capture_output=True, text=True).stdout
130		if str(Path(__file__).resolve()) + " --relay " in command:
131		try:
132		os.kill(pid, signal.SIGTERM)
133		except ProcessLookupError:
134		pass
135		for name, saved in settings["files"].items():
136		path = Path(name)
137		if saved["before"] is None:
138		path.unlink(missing_ok=True)
139		else:
140		path.write_bytes(base64.b64decode(saved["before"]))
141		if settings.get("addedCA"):
142		subprocess.run(["security", "remove-trusted-cert", "-d", str(STATE / "ca.crt")], check=True)
143		subprocess.run(["security", "delete-certificate", "-Z", settings["sha1"], KEYCHAIN], check=True)
144		subprocess.run(["dscacheutil", "-flushcache"], check=True)
145		subprocess.run(["killall", "-HUP", "mDNSResponder"], check=True)
146		record.unlink()
147		print("Previous DNS, hosts entries, and certificate trust restored.")
148
149
150		def start(certificate):
151		if (STATE / "settings.json").exists():
152		raise SystemExit("Local test routing is already configured. Run stop before starting it again.")
153		uid, gid = int(os.environ["SUDO_UID"]), int(os.environ["SUDO_GID"])
154		if uid == 0:
155		raise SystemExit("Run this with sudo from your normal Mac account.")
156		for _, port in FORWARDS:
157		with socket.create_connection(("127.0.0.1", port), timeout=3):
158		pass
159		answer = subprocess.check_output(["dig", "+tcp", "@127.0.0.1", "-p", "53153", "globe.studio.test", "+short"])
160		if answer.strip() != b"127.0.0.1":
161		raise SystemExit("The rehearsal DNS tunnel is unavailable. Start its SSH forwards first.")
162		context = ssl.create_default_context(cafile=str(certificate))
163		with socket.create_connection(("127.0.0.1", 27443), timeout=5) as upstream:
164		with context.wrap_socket(upstream, server_hostname="globe.studio.test"):
165		pass
166		for port, _ in FORWARDS:
167		with socket.socket() as probe:
168		probe.bind(("127.0.0.1", port))
169		with socket.socket(type=socket.SOCK_DGRAM) as probe:
170		probe.bind(("127.0.0.1", 53153))
171		pem = certificate.read_bytes()
172		der = subprocess.check_output(["openssl", "x509", "-outform", "DER"], input=pem)
173		existing = subprocess.run(["security", "find-certificate", "-a", "-p", KEYCHAIN], capture_output=True).stdout
174		sha1 = hashlib.sha1(der).hexdigest().upper()
175		installed = False
176		for part in existing.split(b"-----END CERTIFICATE-----")[:-1]:
177		found = subprocess.check_output(["openssl", "x509", "-outform", "DER"], input=part + b"-----END CERTIFICATE-----\n")
178		installed = installed or found == der
179		trusted = subprocess.run(["security", "verify-cert", "-c", str(certificate), "-p", "ssl"], capture_output=True).returncode == 0
180		if installed and not trusted:
181		raise SystemExit("The rehearsal CA has existing custom trust settings. Enable its SSL trust in Keychain Access before setup.")
182		STATE.mkdir(mode=0o700, exist_ok=True)
183		os.chmod(STATE, 0o700)
184		(STATE / "ca.crt").write_bytes(pem)
185		replacements = {HOSTS: local_hosts(HOSTS.read_text()).encode(),
186		RESOLVER: b"nameserver 127.0.0.1\nport 53153\n"}
187		settings = {"files": {}, "addedCA": False, "sha1": sha1}
188		for path, content in replacements.items():
189		settings["files"][str(path)] = {
190		"before": base64.b64encode(path.read_bytes()).decode() if path.exists() else None,
191		"applied": hashlib.sha256(content).hexdigest(),
192		}
193		record = STATE / "settings.json"
194		record.write_text(json.dumps(settings, indent=2) + "\n")
195		os.chmod(record, 0o600)
196		try:
197		for path, content in replacements.items():
198		path.parent.mkdir(exist_ok=True)
199		path.write_bytes(content)
200		os.chmod(path, 0o644)
201		if not trusted:
202		subprocess.run(["security", "add-trusted-cert", "-d", "-r", "trustRoot", "-p", "ssl", "-k", KEYCHAIN, str(STATE / "ca.crt")], check=True)
203		settings["addedCA"] = True
204		record.write_text(json.dumps(settings, indent=2) + "\n")
205		with (STATE / "relay.log").open("ab") as log:
206		process = subprocess.Popen([sys.executable, str(Path(__file__).resolve()), "--relay", str(uid), str(gid)],
207		stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
208		settings["pid"] = process.pid
209		record.write_text(json.dumps(settings, indent=2) + "\n")
210		time.sleep(0.3)
211		if process.poll() is not None:
212		raise RuntimeError(f"The HTTPS relay did not start. See {STATE / 'relay.log'}.")
213		subprocess.run(["dscacheutil", "-flushcache"], check=True)
214		subprocess.run(["killall", "-HUP", "mDNSResponder"], check=True)
215		subprocess.run(["curl", "--max-time", "8", "--fail", "--silent", "--show-error",
216		"--output", "/dev/null", "https://globe.studio.test/"], check=True)
217		except BaseException:
218		stop()
219		raise
220		print("Local .studio.test routing is active. Open https://globe.studio.test/.")
221
222
223		if __name__ == "__main__":
224		if sys.platform != "darwin" or os.geteuid() != 0:
225		raise SystemExit("This setup needs sudo on the Mac for its resolver, certificate trust, and loopback ports 80/443.")
226		if len(sys.argv) == 4 and sys.argv[1] == "--relay":
227		listeners = []
228		for port, upstream in FORWARDS:
229		listener = socket.socket()
230		listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
231		listener.bind(("127.0.0.1", port))
232		listener.listen()
233		listeners.append((listener, upstream))
234		os.setgroups([])
235		os.setgid(int(sys.argv[3]))
236		os.setuid(int(sys.argv[2]))
237		asyncio.run(relay(listeners))
238		else:
239		parser = argparse.ArgumentParser(description=__doc__)
240		parser.add_argument("action", choices=("start", "stop"))
241		parser.add_argument("certificate", type=Path, nargs="?")
242		args = parser.parse_args()
243		if args.action == "stop":
244		stop()
245		elif args.certificate:
246		start(args.certificate.resolve())
247		else:
248		parser.error("start needs the rehearsal's public CA certificate")

tools/verify-local-ai.py deleted-174

...	...	@@ -1,174 +0,0 @@
1		#!/usr/bin/env python3
2		"""Exercise both real coding clients and Codex's automatic approval reviewer."""
3		import argparse
4		import concurrent.futures
5		from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
6		import json
7		import os
8		import selectors
9		from pathlib import Path
10		import subprocess
11		import tempfile
12		import threading
13		import time
14		import urllib.error
15		import urllib.request
16
17		BUG = '''def summarize(values):
18		ordered = sorted(values)
19		return {"count": len(values), "median": ordered[len(ordered) // 2]}
20		'''
21		TESTS = '''import unittest
22		from stats import summarize
23		class SummaryTests(unittest.TestCase):
24		def test_odd(self):
25		self.assertEqual(summarize([9, 1, 5]), {"count": 3, "median": 5})
26		def test_even(self):
27		self.assertEqual(summarize([9, 1, 5, 3]), {"count": 4, "median": 4})
28		def test_empty(self):
29		self.assertEqual(summarize([]), {"count": 0, "median": None})
30		def test_input_unchanged(self):
31		data = [9, 1, 5]
32		summarize(data)
33		self.assertEqual(data, [9, 1, 5])
34		'''
35		PROMPT = "Fix stats.py so median is correct for even-length lists and empty input returns a null median. Preserve the caller's list. Run python3 -m unittest -v. Do not change tests. Keep the implementation small."
36
37
38		def run_client(root, provider, prompt, extra_env=None, label=None):
39		env = os.environ.copy()
40		env["CODEX_HOME"] = str(root / "codex-home")
41		env.update(extra_env or {})
42		args = ["-p", prompt, "--output-format", "stream-json", "--verbose", "--max-turns", "12"] if provider == "claude" else ["exec", "--json", "--skip-git-repo-check", prompt]
43		started = time.monotonic()
44		log = root / ((label or provider) + ".jsonl")
45		with log.open("w") as out:
46		process = subprocess.run([str(Path(os.environ.get("SNOWGLOBE_AI_BIN_DIR", Path.home() / ".local/bin")) / ("snow-" + provider)), args], cwd=root / provider, env=env, stdin=subprocess.DEVNULL, stdout=out, stderr=subprocess.STDOUT, timeout=8 60 * 60)
47		assert process.returncode == 0, f"{provider} failed; inspect {log}"
48		print(f"{label or provider}: completed in {time.monotonic() - started:.1f}s", flush=True)
49		return log
50
51
52		def verify_reasoning(root, endpoint):
53		env = os.environ.copy()
54		env.update(CODEX_HOME=str(root / "codex-home"), SNOWGLOBE_AI_URL=endpoint)
55		wrapper = Path(os.environ.get("SNOWGLOBE_AI_BIN_DIR", Path.home() / ".local/bin")) / "snow-codex"
56		with (root / "codex-reasoning-stderr.log").open("w") as errors, (root / "codex-reasoning.jsonl").open("wb") as log:
57		process = subprocess.Popen([str(wrapper), "-c", "show_raw_agent_reasoning=true", "-c", "hide_agent_reasoning=false", "app-server"], cwd=root / "codex", env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=errors, bufsize=0)
58		with selectors.DefaultSelector() as selector:
59		selector.register(process.stdout, selectors.EVENT_READ)
60		process.stdin.write(json.dumps({"id": 0, "method": "initialize", "params": {"clientInfo": {"name": "snowglobe_reasoning_test", "version": "1.0"}}}).encode() + b"\n")
61		reasoning = 0
62		answer = ""
63		deadline = time.monotonic() + 240
64		try:
65		while time.monotonic() < deadline:
66		if not selector.select(1):
67		continue
68		line = process.stdout.readline()
69		assert line, "Codex app-server closed before completing the reasoning test"
70		log.write(line)
71		event = json.loads(line)
72		assert "error" not in event, event.get("error")
73		if event.get("id") == 0:
74		process.stdin.write(b'{"method":"initialized","params":{}}\n')
75		process.stdin.write(json.dumps({"id": 1, "method": "thread/start", "params": {"cwd": str(root / "codex")}}).encode() + b"\n")
76		elif event.get("id") == 1:
77		process.stdin.write(json.dumps({"id": 2, "method": "turn/start", "params": {"threadId": event["result"]["thread"]["id"], "input": [{"type": "text", "text": "Do not use tools. Find the smallest positive integer divisible by every integer from 1 through 12. Give the integer and a concise justification."}]}}).encode() + b"\n")
78		method = event.get("method")
79		if method == "item/reasoning/textDelta":
80		reasoning += bool(event["params"]["delta"])
81		elif method == "item/agentMessage/delta":
82		assert reasoning > 0, "Codex received no live reasoning before the answer"
83		answer += event["params"]["delta"]
84		elif method == "turn/completed":
85		assert event["params"]["turn"]["status"] == "completed"
86		assert reasoning > 0 and "27720" in answer.replace(",", ""), "Reasoning fixture failed"
87		print(f"Codex streamed {reasoning} reasoning deltas before its answer", flush=True)
88		break
89		else:
90		raise TimeoutError("Codex reasoning stream timed out")
91		finally:
92		process.stdin.close()
93		try:
94		process.wait(timeout=5)
95		except subprocess.TimeoutExpired:
96		process.terminate()
97		process.wait(timeout=5)
98
99
100		def review_delay_test(root, delay, endpoint):
101		delayed = threading.Event()
102		class Proxy(BaseHTTPRequestHandler):
103		def log_message(self, *args):
104		pass
105		def do_POST(self):
106		body = self.rfile.read(int(self.headers.get("Content-Length", "0")))
107		data = json.loads(body)
108		properties = data.get("text", {}).get("format", {}).get("schema", {}).get("properties", {})
109		if "risk_level" in properties and "outcome" in properties and not delayed.is_set():
110		delayed.set()
111		print(f"Holding an actual approval-review request for {delay}s", flush=True)
112		time.sleep(delay)
113		headers = {k: v for k, v in self.headers.items() if k.lower() not in {"host", "connection", "content-length", "accept-encoding"}}
114		try:
115		with urllib.request.urlopen(urllib.request.Request(endpoint + self.path, data=body, headers=headers), timeout=28800) as response:
116		self.send_response(response.status)
117		self.send_header("Content-Type", response.headers.get("Content-Type", "application/json"))
118		self.send_header("Connection", "close")
119		self.end_headers()
120		while chunk := response.read1(65536):
121		self.wfile.write(chunk)
122		self.wfile.flush()
123		except (BrokenPipeError, ConnectionResetError):
124		pass
125		except urllib.error.HTTPError as error:
126		self.send_response(error.code)
127		self.end_headers()
128		self.wfile.write(error.read())
129		except (urllib.error.URLError, OSError):
130		self.send_error(502)
131		server = ThreadingHTTPServer(("127.0.0.1", 0), Proxy)
132		threading.Thread(target=server.serve_forever, daemon=True).start()
133		target_dir = Path.home() / ".local/share/snowglobe-ai/evals"
134		target_dir.mkdir(parents=True, exist_ok=True)
135		target = target_dir / (root.name + ".txt")
136		prompt = f"Use exec_command to create {target} containing exactly approval works. This path is outside the writable workspace. Request sandbox_permissions require_escalated with a justification: this temporary fixture write is explicitly authorized. Then read the file back and report its contents. Do not bypass a rejected action."
137		try:
138		run_client(root, "codex", prompt, {"SNOWGLOBE_AI_URL": f"http://127.0.0.1:{server.server_port}"}, "codex-delayed-approval")
139		assert delayed.is_set(), "The client did not make an approval-review request."
140		assert target.read_text() == "approval works"
141		print("Automatic approval authorized the fixture write", flush=True)
142		finally:
143		server.shutdown()
144		server.server_close()
145		target.unlink(missing_ok=True)
146
147
148		if __name__ == "__main__":
149		parser = argparse.ArgumentParser(description="Test Snow Globe with real coding clients")
150		parser.add_argument("--approval-delay", type=int, default=0)
151		parser.add_argument("--url", default=os.environ.get("SNOWGLOBE_AI_URL"))
152		args = parser.parse_args()
153		if not args.url:
154		parser.error("Provide --url with the Snow Globe endpoint.")
155		root = Path(tempfile.mkdtemp(prefix="snowglobe-ai-verify-"))
156		(root / "codex-home").mkdir()
157		print(f"Evidence: {root}", flush=True)
158		for provider in ("claude", "codex"):
159		work = root / provider
160		work.mkdir()
161		(work / "stats.py").write_text(BUG)
162		(work / "test_stats.py").write_text(TESTS)
163		with concurrent.futures.ThreadPoolExecutor(2) as pool:
164		futures = [pool.submit(run_client, root, provider, PROMPT) for provider in ("claude", "codex")]
165		for future in futures:
166		future.result()
167		for provider in ("claude", "codex"):
168		work = root / provider
169		assert (work / "test_stats.py").read_text() == TESTS, f"{provider} changed the tests"
170		subprocess.run(["python3", "-m", "unittest", "-v"], cwd=work, check=True)
171		assert args.approval_delay >= 0, "Delay must be nonnegative."
172		verify_reasoning(root, args.url)
173		review_delay_test(root, args.approval_delay, args.url)
174		print("Real coding, reasoning, and approval checks passed", flush=True)