diff --git a/config/site.pkl b/config/site.pkl index 04bcaf29ad745bcda027ad3b42915fb9c6f24551..f69227748696e0419051e11b1da376a091f61067 100644 --- a/config/site.pkl +++ b/config/site.pkl @@ -12,6 +12,7 @@ cloverUid: Int = (read?("prop:cloverUid") ?? read?("env:STUDIO_CLOVER_UID") ?? " cloverGid: Int = (read?("prop:cloverGid") ?? read?("env:STUDIO_CLOVER_GID") ?? "3000").toInt() cloverReadOnly: Boolean = (read?("prop:cloverReadOnly") ?? read?("env:STUDIO_CLOVER_READ_ONLY") ?? "false") == "true" mediaRoot: String = read?("prop:mediaRoot") ?? read?("env:STUDIO_MEDIA_ROOT") ?? "\(cloverRoot)/Media" +localAiSsdCache: String = read?("prop:localAiSsdCache") ?? read?("env:STUDIO_LOCAL_AI_SSD_CACHE") ?? "/var/cache/studio/local-ai" jellyfinFolders: Mapping = new { ["Anime"] = "Anime" ["Movies"] = "Movies" diff --git a/dashboard/ai/launch.sh b/dashboard/ai/launch.sh index 2b9927905b06dc9c892138bd17db8e5d518daedd..c209202272068e535b75784bc23c3dbc9ba0fe0b 100644 --- a/dashboard/ai/launch.sh +++ b/dashboard/ai/launch.sh @@ -70,5 +70,5 @@ else export CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS=1 API_TIMEOUT_MS=28800000 API_FORCE_IDLE_TIMEOUT=0 export CLAUDE_STREAM_IDLE_TIMEOUT_MS=28800000 CLAUDE_BYTE_STREAM_IDLE_TIMEOUT_MS=1800000 export CLAUDE_ASYNC_AGENT_STALL_TIMEOUT_MS=28800000 - exec claude --permission-mode auto --append-system-prompt "Your model name is $model." "$@" + exec claude --permission-mode auto --effort @REASONING@ --append-system-prompt "Your model name is $model." "$@" fi diff --git a/service/local-ai/Dockerfile b/service/local-ai/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..656d2b304ed88330ea951fbb2b8f01df49c21d49 --- /dev/null +++ b/service/local-ai/Dockerfile @@ -0,0 +1,30 @@ +FROM docker.io/nvidia/cuda@sha256:995e80db6d0c3a53d56bd00bba48a0ebd633b67b99a57e16acf9a306e7c744a7 + +ENV DEBIAN_FRONTEND=noninteractive PIP_NO_CACHE_DIR=1 PYTHONUNBUFFERED=1 +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3.12 python3.12-venv python3.12-dev \ + cuda-nvcc-13-0 cuda-cudart-dev-13-0 libcurand-dev-13-0 \ + build-essential patch curl ca-certificates \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /app +RUN python3.12 -c 'import hashlib,urllib.request; data=urllib.request.urlopen("https://api.github.com/repos/syv-ai/HyperQwen/tarball/253c76aea0a240bf7cb2bd3ed92b672c0f258d0d").read(); assert hashlib.sha256(data).hexdigest()=="29b40ee9c371066e0194d23bcd35b08d4b6ec3efecd5799d02475fbbf6b032e3"; open("source.tar.gz","wb").write(data)' \ + && tar -xzf source.tar.gz --strip-components=1 && rm source.tar.gz +RUN python3.12 -m venv venv && venv/bin/pip install --upgrade pip \ + && venv/bin/pip install -r docker/requirements.txt torch==2.13.0 \ + safetensors==0.8.0 flashinfer-python==0.6.16.post3 flashinfer-cubin==0.6.13 \ + && venv/bin/pip check +COPY download.py models.json /tmp/model-source/ +RUN python3.12 /tmp/model-source/download.py --runtime-only --root /tmp/runtime \ + && MODEL_DIR=$(python3.12 -c 'import json; print(json.load(open("/tmp/model-source/models.json"))["directory"])') \ + && test "$(cat /tmp/runtime/$MODEL_DIR/runtime/BASE_REVISION.txt)" = 253c76aea0a240bf7cb2bd3ed92b672c0f258d0d \ + && rm patches/*.patch \ + && cp /tmp/runtime/$MODEL_DIR/runtime/patches/*.patch patches/ \ + && cp /tmp/runtime/$MODEL_DIR/runtime/single-user/start_qwen.sh single-user/start_qwen.sh \ + && rm -rf /tmp/runtime /tmp/model-source +RUN set -e; SP=$(venv/bin/python -c 'import vllm,os; print(os.path.dirname(vllm.__file__))' | tail -n1); \ + for p in patches/*.patch; do patch --batch -p1 -d "$SP" < "$p"; done; \ + bash verify.sh --install +COPY cache /tmp/studio-ai-cache +RUN venv/bin/pip install /tmp/studio-ai-cache && venv/bin/pip check && rm -rf /tmp/studio-ai-cache +ENV HOME=/cache VLLM_NO_USAGE_STATS=1 DO_NOT_TRACK=1 HF_HUB_OFFLINE=1 +RUN mkdir -p /cache && chmod 1777 /cache diff --git a/service/local-ai/build/Dockerfile b/service/local-ai/build/Dockerfile deleted file mode 100644 index 6e37e8a46b9672d97de76ee975e87781b435413e..0000000000000000000000000000000000000000 --- a/service/local-ai/build/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -ARG RUNTIME=ghcr.io/ggml-org/llama.cpp@sha256:e8318a7b3988f9ca57b28c03486569e60def08f2b7b060550ebfa046e175e6cb -FROM ${RUNTIME} AS build -RUN rm -f /etc/apt/sources.list.d/* && printf 'deb https://archive.ubuntu.com/ubuntu noble main universe\ndeb https://archive.ubuntu.com/ubuntu noble-updates main universe\ndeb https://security.ubuntu.com/ubuntu noble-security main universe\n' > /etc/apt/sources.list && apt-get -o Acquire::https::Timeout=30 update && apt-get install -y --no-install-recommends cmake g++ make patch libssl-dev -WORKDIR /src -RUN python3 -c 'import hashlib,urllib.request; data=urllib.request.urlopen("https://api.github.com/repos/ggml-org/llama.cpp/tarball/11fe02151f79c41d0d4af7da708755d73b9c0da6").read(); assert hashlib.sha256(data).hexdigest()=="965db483b5e27c4917bac5bf83e0d2d533099049f8d5234c75e93c969d76443a"; open("source.tar.gz","wb").write(data)' && tar -xzf source.tar.gz --strip-components=1 && rm source.tar.gz -COPY prefix-cache.patch /tmp/prefix-cache.patch -RUN patch -p1 < /tmp/prefix-cache.patch -COPY extract-ui.cpp /tmp/extract-ui.cpp -RUN python3 -c 'import pathlib,subprocess; symbols=subprocess.check_output(["nm","-S","/app/libllama-server-impl.so"],text=True); size=int(next(line.split()[1] for line in symbols.splitlines() if line.split()[-1]=="_ZL8g_assets"),16); header=pathlib.Path("tools/ui/ui.h.in").read_text().replace("#cmakedefine LLAMA_UI_HAS_ASSETS 1","#define LLAMA_UI_HAS_ASSETS 1").replace("@N_ASSETS@",str(size)+"/sizeof(llama_ui_asset)"); pathlib.Path("/tmp/ui.h").write_text(header)' && g++ -std=c++17 -I/tmp /tmp/extract-ui.cpp -L/app -Wl,-rpath,/app -lllama-server-impl -o /tmp/extract-ui && LD_LIBRARY_PATH=/app /tmp/extract-ui && python3 -c 'import gzip,pathlib; [p.write_bytes(gzip.decompress(p.read_bytes())) for p in pathlib.Path("tools/ui/dist").rglob("*") if p.is_file()]' -RUN cmake -S . -B /build -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DGGML_CUDA=OFF -DGGML_CPU=OFF -DGGML_BACKEND_DL=ON -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_UI=OFF -DLLAMA_USE_PREBUILT_UI=OFF -DLLAMA_BUILD_NUMBER=11382 -DLLAMA_BUILD_COMMIT=11fe02151 && cmake --build /build --target llama-ui-assets && ln -s /app/libllama.so* /app/libllama-common.so* /app/libmtmd.so* /app/libggml*.so* /build/bin/ && cmake --build /build --target server-context/fast cpp-httplib/fast llama-common-base/fast llama-ui/fast -j 4 && cmake --build /build --target llama-server-impl/fast -j 4 - -FROM ${RUNTIME} -COPY --from=build /build/bin/libllama-server-impl.so /app/libllama-server-impl.so diff --git a/service/local-ai/build/extract-ui.cpp b/service/local-ai/build/extract-ui.cpp deleted file mode 100644 index eeabf6f7c84d9a517dbace08e3f2b9296eb57416..0000000000000000000000000000000000000000 --- a/service/local-ai/build/extract-ui.cpp +++ /dev/null @@ -1,18 +0,0 @@ -#include "ui.h" -#include -#include - -int main() { - if (!llama_ui_use_gzip()) { - return 1; - } - for (const auto & asset : llama_ui_get_assets()) { - const auto path = std::filesystem::path("tools/ui/dist") / asset.name; - std::filesystem::create_directories(path.parent_path()); - std::ofstream output(path, std::ios::binary); - output.write(reinterpret_cast(asset.data), asset.size); - if (!output) { - return 1; - } - } -} diff --git a/service/local-ai/build/prefix-cache.patch b/service/local-ai/build/prefix-cache.patch deleted file mode 100644 index c97fce001e32ed0a19cf34610ef0b06f63da356f..0000000000000000000000000000000000000000 --- a/service/local-ai/build/prefix-cache.patch +++ /dev/null @@ -1,237 +0,0 @@ ---- a/tools/server/server-task.h -+++ b/tools/server/server-task.h -@@ -650,12 +650,13 @@ - }; - - struct server_prompt_cache { -- server_prompt_cache(int32_t limit_size_mib, size_t limit_tokens) { -+ server_prompt_cache(int32_t limit_size_mib, size_t limit_tokens, bool keep_on_load = false) : keep_on_load(keep_on_load) { - this->limit_size = 1024ull*1024ull*(limit_size_mib < 0 ? 0 : limit_size_mib); - this->limit_tokens = limit_tokens; - } - - std::list states; -+ const bool keep_on_load; - - // in bytes, 0 = no limit - size_t limit_size = 0; ---- a/tools/server/server-task.cpp -+++ b/tools/server/server-task.cpp -@@ -1226,6 +1226,7 @@ - {"event", "response.reasoning_text.delta"}, - {"data", json { - {"type", "response.reasoning_text.delta"}, -+ {"content_index", 0}, - {"delta", diff.reasoning_content_delta}, - {"item_id", oai_resp_reasoning_id}, - }}, -@@ -1724,7 +1725,10 @@ - for (auto it = states.begin(); it != states.end(); ++it) { - const int cur_lcp_len = it->prompt.tokens.get_common_prefix(prompt.tokens); - -- if (cur_lcp_len == (int) prompt.tokens.size()) { -+ if (cur_lcp_len == (int) prompt.tokens.size() && (!keep_on_load || it->prompt.tokens.size() == prompt.tokens.size())) { -+ if (keep_on_load) { -+ states.splice(states.end(), states, it); -+ } - SRV_TRC("%s", " - prompt is already in the cache, skipping\n"); - return nullptr; - } -@@ -1749,7 +1753,7 @@ - for (auto it = states.begin(); it != states.end();) { - const int len = it->prompt.tokens.get_common_prefix(prompt.tokens); - -- if (len == (int) it->prompt.tokens.size()) { -+ if (!keep_on_load && len == (int) it->prompt.tokens.size()) { - SRV_TRC(" - removing obsolete cached prompt with length %d\n", len); - - it = states.erase(it); -@@ -1820,12 +1824,18 @@ - - SRV_TRC(" - prompt with length %7zu, lcp = %7d, f_keep = %.3f, f_sim = %.3f\n", it->prompt.tokens.size(), lcp_cur, f_keep_cur, f_sim_cur); - -+ // Only complete pinned prefixes contain a usable recurrent state. -+ if (keep_on_load && lcp_cur != (int) it->prompt.tokens.size()) { -+ continue; -+ } -+ - // don't trash large prompts - if (f_keep_cur < 0.25f) { - continue; - } - -- if (f_keep_best < f_keep_cur && f_sim_best < f_sim_cur) { -+ if ((keep_on_load || f_keep_best < f_keep_cur) && -+ (f_sim_best < f_sim_cur || (keep_on_load && f_sim_best == f_sim_cur && f_keep_best < f_keep_cur))) { - f_keep_best = f_keep_cur; - f_sim_best = f_sim_cur; - -@@ -1847,8 +1857,10 @@ - return false; - } - -- data.clear(); -- data.shrink_to_fit(); -+ if (!keep_on_load) { -+ data.clear(); -+ data.shrink_to_fit(); -+ } - } - - { -@@ -1865,14 +1877,20 @@ - return false; - } - -- data.clear(); -- data.shrink_to_fit(); -+ if (!keep_on_load) { -+ data.clear(); -+ data.shrink_to_fit(); -+ } - } - } - -- prompt = std::move(it_best->prompt); -- -- states.erase(it_best); -+ if (keep_on_load) { -+ prompt = it_best->prompt.clone(); -+ states.splice(states.end(), states, it_best); -+ } else { -+ prompt = std::move(it_best->prompt); -+ states.erase(it_best); -+ } - } - - return true; ---- a/tools/server/server-context.cpp -+++ b/tools/server/server-context.cpp -@@ -954,6 +954,7 @@ - int n_empty_consecutive = 0; - - std::unique_ptr prompt_cache; -+ std::unique_ptr prefix_cache; - - server_metrics metrics; - -@@ -1424,6 +1425,7 @@ - SRV_TRC("%s", "use `--cache-ram 0` to disable the prompt cache\n"); - - prompt_cache = std::make_unique(params_base.cache_ram_mib, n_ctx); -+ prefix_cache = std::make_unique(2048, n_ctx, true); - } else { - SRV_TRC("%s", "prompt cache is disabled - use `--cache-ram N` to enable it\n"); - } -@@ -1721,6 +1723,11 @@ - - SRV_TRC("prompt cache update took %.2f ms\n", (ggml_time_us() - t_start) / 1000.0); - } -+ } -+ -+ if (ret && prefix_cache && task.type == SERVER_TASK_TYPE_COMPLETION && -+ !ret->prompt_load(*prefix_cache, task.tokens)) { -+ ret->prompt_clear(); - } - - return ret; -@@ -3091,7 +3098,7 @@ - // add generated tokens to cache - // ref: https://github.com/ggml-org/llama.cpp/pull/16818#discussion_r2473269481 - { -- GGML_ASSERT(!slot.prompt.tokens.has_mtmd); -+ GGML_ASSERT(!slot.prompt.tokens.find_next_media_chunk(0).first); - - llama_tokens new_tokens = slot.prompt.tokens.get_tokens(); // copy - for (size_t i = n_keep + n_discard; i < new_tokens.size(); i++) { -@@ -3400,7 +3407,7 @@ - - // reuse chunks from the cached prompt by shifting their KV cache in the new position - if (can_cache_reuse && n_cache_reuse > 0) { -- GGML_ASSERT(!slot.prompt.tokens.has_mtmd); -+ GGML_ASSERT(!slot.prompt.tokens.find_next_media_chunk(0).first); - - size_t head_c = n_past; // cache - size_t head_p = n_past; // current prompt -@@ -3701,6 +3708,11 @@ - - const auto & spans = slot.task->params.message_spans; - const auto last_user_pos = spans.last_user_message_pos(); -+ const auto first_user = std::find_if(spans.spans.begin(), spans.spans.end(), [](const auto & span) { -+ return span.role == COMMON_CHAT_ROLE_USER; -+ }); -+ const int64_t first_user_pos = first_user == spans.spans.end() ? -1 : first_user->pos; -+ const int64_t prefix_pos = std::min(4096, first_user_pos); - - const int32_t n_decision_first = slot.task->type == SERVER_TASK_TYPE_DECISION ? slot.task->decision.pos_first() : -1; - -@@ -3743,12 +3755,16 @@ - } - slot.prompt.tokens.push_back(cur_tok); - -+ if (do_checkpoint && prefix_cache && slot.prompt.n_tokens() == prefix_pos) { -+ break; -+ } -+ - // break at the last user message, or at user messages at least min step past the last checkpoint - if (do_checkpoint && spans.is_user_start(slot.prompt.n_tokens())) { - const auto pos = slot.prompt.n_tokens(); - const auto & checkpoints = slot.prompt.checkpoints; - -- if (pos == last_user_pos || checkpoints.empty() || pos > checkpoints.back().n_tokens + params_base.checkpoint_min_step) { -+ if (pos == first_user_pos || pos == last_user_pos || checkpoints.empty() || pos > checkpoints.back().n_tokens + params_base.checkpoint_min_step) { - break; - } - } -@@ -3783,6 +3799,8 @@ - const bool near_prompt_end = slot.task->n_tokens() < slot.prompt.n_tokens() + n_ubatch; - - const bool is_user_start = spans.is_user_start(n_tokens_start); -+ const bool is_prefix_boundary = prefix_cache && n_tokens_start > 0 && -+ (n_tokens_start == prefix_pos || n_tokens_start == first_user_pos); - const bool is_last_user_message = n_tokens_start == last_user_pos; - - // entire prompt has been processed -@@ -3799,9 +3817,8 @@ - - slot.init_sampler(); - } else { -- // skip ordinary mid-prompt checkpoints, unless the batch starts a user -- // message or we are near the end of the prompt -- if (!is_user_start && !near_prompt_end) { -+ // Retain system-prefix boundaries even inside a message. -+ if (!is_user_start && !is_prefix_boundary && !near_prompt_end) { - do_checkpoint = false; - } - } -@@ -3821,7 +3838,7 @@ - // no need to create checkpoints that are too close together, unless it's the last user message - do_checkpoint = do_checkpoint && ( - slot.prompt.checkpoints.empty() || -- is_last_user_message || near_prompt_end || -+ is_last_user_message || is_prefix_boundary || near_prompt_end || - n_tokens_start > slot.prompt.checkpoints.back().n_tokens + params_base.checkpoint_min_step); - SLT_DBG(slot, "main/do_checkpoint = %s, pos_min = %d, pos_max = %d\n", do_checkpoint ? "yes" : "no", pos_min, pos_max); - -@@ -3829,6 +3846,21 @@ - // yet processed and therefore it is not part of the checkpoint. - if (do_checkpoint) { - create_checkpoint(slot, n_tokens_cur, pos_min, pos_max); -+ if (is_prefix_boundary && -+ !slot.prompt.tokens.find_next_media_chunk(0).first) { -+ server_prompt prefix {slot.prompt.tokens.clone(), {slot.prompt.checkpoints.back()}}; -+ prefix.tokens.keep_first(n_tokens_start); -+ const size_t size = llama_state_seq_get_size_ext(ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_NONE); -+ const size_t draft_size = ctx_dft ? llama_state_seq_get_size_ext(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_NONE) : 0; -+ if (auto * state = prefix_cache->alloc(prefix, size, draft_size)) { -+ llama_state_seq_get_data_ext(ctx_tgt, state->data.main.data(), size, slot.id, LLAMA_STATE_SEQ_FLAGS_NONE); -+ if (ctx_dft) { -+ llama_state_seq_get_data_ext(ctx_dft, state->data.drft.data(), draft_size, slot.id, LLAMA_STATE_SEQ_FLAGS_NONE); -+ } -+ prefix_cache->update(); -+ SLT_INF(slot, "cached base prefix, n_tokens = %d\n", n_tokens_start); -+ } -+ } - } - } - diff --git a/service/local-ai/cache/pyproject.toml b/service/local-ai/cache/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..d64c48a71321ad844c83253aa562085d9e94555f --- /dev/null +++ b/service/local-ai/cache/pyproject.toml @@ -0,0 +1,14 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "studio-ai-cache" +version = "1.0.0" +dependencies = ["cryptography==50.0.2"] + +[project.entry-points."vllm.general_plugins"] +studio_ai_cache = "studio_ai_cache_vllm:register" + +[tool.setuptools] +py-modules = ["studio_ai_cache", "studio_ai_cache_vllm"] diff --git a/service/local-ai/cache/studio_ai_cache.py b/service/local-ai/cache/studio_ai_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..f390b59b32e022a7e7f554bacdfe1fdfdcc55089 --- /dev/null +++ b/service/local-ai/cache/studio_ai_cache.py @@ -0,0 +1,186 @@ +"""Bounded, authenticated disk caches with SSD promotion and request pinning.""" +from collections import OrderedDict +import fcntl +import logging +import mmap +import os +from pathlib import Path +import struct +import threading + +from cryptography.exceptions import InvalidTag +from cryptography.hazmat.primitives.ciphers.aead import AESGCM + + +class DiskCache: + def __init__(self, ssd, hdd, ssd_bytes, hdd_bytes, key): + if min(ssd_bytes, hdd_bytes) <= 0: + raise ValueError("Disk cache capacities must be positive") + self.cipher = AESGCM(key) + self.roots = (Path(ssd), Path(hdd)) + if (self.roots[0].resolve().is_relative_to(self.roots[1].resolve()) + or self.roots[1].resolve().is_relative_to(self.roots[0].resolve())): + raise ValueError("Disk cache directories must be distinct") + self.limits = (ssd_bytes, hdd_bytes) + self.lock = threading.Lock() + self.writers = (threading.Lock(), threading.Lock()) + self.entries = (OrderedDict(), OrderedDict()) + self.pins = {} + self.files = [] + try: + for tier, root in enumerate(self.roots): + root.mkdir(mode=0o700, parents=True, exist_ok=True) + root.chmod(0o700) + descriptor = os.open(root / ".lock", os.O_CREAT | os.O_RDWR, 0o600) + self.files.append(descriptor) + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + for path in root.rglob("*.pending"): + path.unlink() + inventory = sorted((path.stat().st_mtime_ns, path) for path in root.rglob("*.bin")) + for _, path in inventory: + self.entries[tier][str(path.relative_to(root))] = path.stat().st_size + self._evict(tier, 0) + except Exception: + self.close() + raise + + def close(self): + for descriptor in self.files: + os.close(descriptor) + self.files.clear() + + def acquire(self, name, request): + with self.lock: + for entries in self.entries: + if name in entries: + entries.move_to_end(name) + self.pins.setdefault(name, set()).add(request) + return True + return False + + def release(self, request): + with self.lock: + for name, requests in list(self.pins.items()): + requests.discard(request) + if not requests: + del self.pins[name] + + def touch(self, name): + with self.lock: + for entries in self.entries: + if name in entries: + entries.move_to_end(name) + break + + def _evict(self, tier, needed): + removed = [] + with self.lock: + entries = self.entries[tier] + size = sum(entries.values()) + for name in list(entries): + if size + needed <= self.limits[tier]: + break + if name not in self.pins: + size -= entries.pop(name) + removed.append(self.roots[tier] / name) + for path in removed: + self._unlink(tier, path) + return size + needed <= self.limits[tier] + + def _unlink(self, tier, path): + path.unlink(missing_ok=True) + parent = path.parent + while parent != self.roots[tier]: + try: + parent.rmdir() + except OSError: + break + parent = parent.parent + + def _put(self, tier, name, buffer): + path = self.roots[tier] / name + with self.writers[tier]: + with self.lock: + if name in self.entries[tier]: + self.entries[tier].move_to_end(name) + present = True + else: + present = False + if present: + os.utime(path, None) + return + if len(buffer) > self.limits[tier] or not self._evict(tier, len(buffer)): + return + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + pending = path.with_suffix(".pending") + try: + descriptor = os.open(pending, os.O_CREAT | os.O_EXCL | os.O_WRONLY | getattr(os, "O_DIRECT", 0), 0o600) + try: + if os.write(descriptor, buffer) != len(buffer): + raise OSError("Short cache write") + os.fsync(descriptor) + finally: + os.close(descriptor) + pending.replace(path) + with self.lock: + self.entries[tier][name] = len(buffer) + finally: + pending.unlink(missing_ok=True) + + def store(self, name, data): + # Page-aligned ciphertext keeps direct I/O on both ext4 and ZFS. + size = (len(data) + 36 + mmap.PAGESIZE - 1) // mmap.PAGESIZE * mmap.PAGESIZE + with mmap.mmap(-1, size) as buffer: + nonce = os.urandom(12) + buffer[:20] = struct.pack("!Q", len(data)) + nonce + view = memoryview(buffer)[20:20 + len(data) + 16] + try: + self.cipher.encrypt_into(nonce, data, name.encode(), view) + finally: + view.release() + self._put(1, name, buffer) + self._put(0, name, buffer) + + def load(self, name, target): + pin = object() + if not self.acquire(name, pin): + raise FileNotFoundError("Cache block was evicted") + try: + for tier, root in enumerate(self.roots): + path = root / name + with self.lock: + size = self.entries[tier].get(name) + if size is None: + continue + try: + with mmap.mmap(-1, size) as buffer: + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECT", 0)) + try: + if os.readv(descriptor, [buffer]) != size: + raise OSError("Short cache read") + finally: + os.close(descriptor) + if struct.unpack("!Q", buffer[:8])[0] != len(target): + raise ValueError("Cache block size mismatch") + view = memoryview(buffer)[20:20 + len(target) + 16] + try: + self.cipher.decrypt_into(buffer[8:20], view, name.encode(), target) + finally: + view.release() + with self.lock: + self.entries[tier].move_to_end(name) + os.utime(path, None) + if tier == 1: + try: + self._put(0, name, buffer) + except OSError: + logging.getLogger(__name__).warning("SSD cache promotion failed", exc_info=True) + return "ssd" if tier == 0 else "hdd" + except (OSError, ValueError, struct.error, InvalidTag): + with self.writers[tier]: + with self.lock: + self.entries[tier].pop(name, None) + self._unlink(tier, path) + raise FileNotFoundError("No authenticated cache block remains") + finally: + self.release(pin) diff --git a/service/local-ai/cache/studio_ai_cache_vllm.py b/service/local-ai/cache/studio_ai_cache_vllm.py new file mode 100644 index 0000000000000000000000000000000000000000..afd2164d6fc9c3deda769ad4e3ca0774f44888ae --- /dev/null +++ b/service/local-ai/cache/studio_ai_cache_vllm.py @@ -0,0 +1,73 @@ +"""vLLM secondary tier for the bounded SSD/HDD cache.""" +import functools +import hashlib +import os +from pathlib import Path + +import vllm +from vllm.v1.kv_offload.base import LookupResult, Medium +from vllm.v1.kv_offload.file_mapper import FileMapper +from vllm.v1.kv_offload.tiering.base import JobResult, RequestOffloadingContext, SecondaryTierManager +from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory +from vllm.v1.kv_offload.tiering.fs.thread_pool import DualQueueThreadPool + +from studio_ai_cache import DiskCache + + +class DiskTier(SecondaryTierManager): + medium = Medium.STORAGE + + def __init__(self, offloading_spec, primary_kv_view, tier_type, ssd_dir, hdd_dir, ssd_bytes, hdd_bytes): + super().__init__(offloading_spec, primary_kv_view, tier_type) + key = hashlib.sha256(os.environ["STUDIO_AI_CACHE_KEY"].encode()).digest() + namespace = hashlib.sha256( + Path("/models/manifest.json").read_bytes() + vllm.__version__.encode() + + Path(__file__).read_bytes() + Path(__file__).with_name("studio_ai_cache.py").read_bytes() + key + ).hexdigest() + self.mapper = FileMapper.from_offloading_spec(namespace, offloading_spec, offloading_spec.blocks_per_chunk) + self.cache = DiskCache(ssd_dir, hdd_dir, ssd_bytes, hdd_bytes, key) + self.block_size = primary_kv_view.strides[0] + self.pool = DualQueueThreadPool(16, 2, thread_name_prefix="studio_kv_disk") + + def on_new_request(self, req_context): + return RequestOffloadingContext() + + def lookup(self, key, req_context): + return LookupResult.HIT if self.cache.acquire(self.mapper.get_file_name(key), req_context.req_id) else LookupResult.MISS + + def submit_store(self, job_metadata): + tasks = [functools.partial(self._transfer, self.cache.store, key, block_id) + for key, block_id in zip(job_metadata.keys, job_metadata.block_ids, strict=True)] + self.pool.enqueue_store(job_metadata.job_id, len(tasks), tasks) + + def submit_load(self, job_metadata): + tasks = [functools.partial(self._transfer, self.cache.load, key, block_id) + for key, block_id in zip(job_metadata.keys, job_metadata.block_ids, strict=True)] + self.pool.enqueue_load(job_metadata.job_id, len(tasks), tasks) + + def _transfer(self, operation, key, block_id): + offset = int(block_id) * self.block_size + with self._primary_kv_view.cast("B")[offset:offset + self.block_size] as view: + operation(self.mapper.get_file_name(key), view) + + def get_finished_jobs(self): + return [JobResult(job_id, success) for job_id, success in self.pool.get_finished()] + + def touch(self, keys, req_context): + for key in keys: + self.cache.touch(self.mapper.get_file_name(key)) + + def on_request_finished(self, req_context): + self.cache.release(req_context.req_id) + + def drain_jobs(self): + self.pool.wait_idle() + + def shutdown(self): + self.pool.wait_idle() + self.pool.shutdown(wait=True) + self.cache.close() + + +def register(): + SecondaryTierFactory.register_tier("studio_disk", __name__, "DiskTier") diff --git a/service/local-ai/cache/test_cache.py b/service/local-ai/cache/test_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..381b9399dbbc1cfbcff7251d834e1ec4fd9053dc --- /dev/null +++ b/service/local-ai/cache/test_cache.py @@ -0,0 +1,115 @@ +from concurrent.futures import ThreadPoolExecutor +import os +import mmap +from pathlib import Path +import tempfile +import unittest + +from studio_ai_cache import DiskCache + + +class CacheTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.key = os.urandom(32) + self.cache = DiskCache(self.root / "ssd", self.root / "hdd", 2 * mmap.PAGESIZE, 3 * mmap.PAGESIZE, self.key) + self.addCleanup(self.cache.close) + + def test_hdd_hit_promotes_to_ssd_and_both_limits_hold(self): + for name in ("a.bin", "b.bin", "c.bin"): + self.cache.store(name, name.encode() * 100) + target = bytearray(500) + self.assertEqual(self.cache.load("a.bin", target), "hdd") + self.assertEqual(target, b"a.bin" * 100) + self.assertEqual(self.cache.load("a.bin", target), "ssd") + for directory, limit in (("ssd", 2 * mmap.PAGESIZE), ("hdd", 3 * mmap.PAGESIZE)): + self.assertLessEqual(sum(p.stat().st_size for p in (self.root / directory).glob("*.bin")), limit) + + def test_overflow_evicts_cold_blocks(self): + for name in ("a.bin", "b.bin", "c.bin", "d.bin", "e.bin"): + self.cache.store(name, b"fixture") + self.assertFalse(self.cache.acquire("a.bin", "request")) + self.assertFalse(self.cache.acquire("b.bin", "request")) + self.assertTrue(self.cache.acquire("c.bin", "request")) + self.cache.release("request") + + def test_pinned_request_survives_eviction_and_full_cache_skips_store(self): + self.cache.close() + self.cache = DiskCache(self.root / "ssd", self.root / "hdd", mmap.PAGESIZE, mmap.PAGESIZE, self.key) + self.addCleanup(self.cache.close) + self.cache.store("a.bin", b"first") + self.assertTrue(self.cache.acquire("a.bin", "request")) + self.cache.store("b.bin", b"other") + self.assertFalse(self.cache.acquire("b.bin", "other-request")) + target = bytearray(5) + self.cache.load("a.bin", target) + self.assertEqual(target, b"first") + self.cache.release("request") + self.cache.store("b.bin", b"other") + self.assertFalse(self.cache.acquire("a.bin", "request")) + + def test_restart_recovers_encrypted_blocks_with_restricted_permissions(self): + payload = b"synthetic private prompt fixture" * 10 + self.cache.store("namespace/a.bin", payload) + for path in self.root.rglob("*.bin"): + self.assertNotIn(payload, path.read_bytes()) + self.assertEqual(path.stat().st_mode & 0o777, 0o600) + self.cache.close() + restored = DiskCache(self.root / "ssd", self.root / "hdd", 2 * mmap.PAGESIZE, 3 * mmap.PAGESIZE, self.key) + self.addCleanup(restored.close) + target = bytearray(len(payload)) + self.assertEqual(restored.load("namespace/a.bin", target), "ssd") + self.assertEqual(target, payload) + + def test_corrupt_ssd_falls_back_to_hdd_and_repairs_hot_copy(self): + self.cache.store("a.bin", b"authenticated fixture") + path = self.root / "ssd/a.bin" + contents = bytearray(path.read_bytes()) + contents[20] ^= 1 + path.write_bytes(contents) + target = bytearray(21) + self.assertEqual(self.cache.load("a.bin", target), "hdd") + self.assertEqual(target, b"authenticated fixture") + self.assertEqual(self.cache.load("a.bin", target), "ssd") + + def test_corrupt_both_copies_becomes_cache_miss(self): + self.cache.store("a.bin", b"fixture") + for path in self.root.rglob("*.bin"): + path.write_bytes(b"broken") + with self.assertRaises(FileNotFoundError): + self.cache.load("a.bin", bytearray(7)) + self.assertFalse(self.cache.acquire("a.bin", "request")) + + def test_wrong_key_and_different_namespace_cannot_reuse_state(self): + self.cache.store("old/a.bin", b"fixture") + self.assertFalse(self.cache.acquire("new/a.bin", "request")) + self.cache.close() + other = DiskCache(self.root / "ssd", self.root / "hdd", 2 * mmap.PAGESIZE, 3 * mmap.PAGESIZE, os.urandom(32)) + self.addCleanup(other.close) + with self.assertRaises(FileNotFoundError): + other.load("old/a.bin", bytearray(7)) + + def test_directories_are_exclusive(self): + with self.assertRaises(BlockingIOError): + DiskCache(self.root / "ssd", self.root / "hdd", 2 * mmap.PAGESIZE, 3 * mmap.PAGESIZE, self.key) + + def test_concurrent_loads_keep_request_pins_until_release(self): + self.cache.store("a.bin", b"fixture") + self.assertTrue(self.cache.acquire("a.bin", "request")) + + def exercise(index): + self.cache.store(f"{index}.bin", b"other") + target = bytearray(7) + self.cache.load("a.bin", target) + self.assertEqual(target, b"fixture") + + with ThreadPoolExecutor(max_workers=8) as pool: + list(pool.map(exercise, range(24))) + self.cache.release("request") + self.assertEqual(self.cache.pins, {}) + + +if __name__ == "__main__": + unittest.main() diff --git a/service/local-ai/cache/test_vllm_cache.py b/service/local-ai/cache/test_vllm_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..ad3537a1faa379e2b9b2068db3b320388f0482d0 --- /dev/null +++ b/service/local-ai/cache/test_vllm_cache.py @@ -0,0 +1,97 @@ +"""CPU-only checks against the installed vLLM scheduler and shared-memory tier.""" +import mmap +import os +from pathlib import Path +import tempfile +import unittest +import uuid + +from vllm.v1.kv_offload.base import LookupResult, ReqContext, ScheduleEndContext, make_offload_key +from vllm.v1.kv_offload.config import OffloadingCacheConfig, OffloadingConfig, OffloadingGroupConfig, OffloadingModelConfig, OffloadingParallelConfig +from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion +from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec +from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory +from vllm.v1.kv_offload.tiering.manager import CPUPrimaryTierOffloadingManager, TieringOffloadingManager + +from studio_ai_cache_vllm import register + + +class VllmCacheTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + register() + + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + os.environ["STUDIO_AI_CACHE_KEY"] = "disposable-synthetic-cache-key" + config = OffloadingConfig( + groups=(OffloadingGroupConfig(16, ("attention",)), OffloadingGroupConfig(16, ("recurrent",))), + worker_kv_bytes_per_block=mmap.PAGESIZE, + enable_kv_cache_events=False, extra_config={"cpu_bytes_to_use": 4 * mmap.PAGESIZE}, + engine_id="synthetic", model=OffloadingModelConfig("synthetic", "uint8"), + cache=OffloadingCacheConfig(16, 1), + parallel=OffloadingParallelConfig(0, 1, 1, 1, 1, 1, 0, False), + ) + spec = CPUOffloadingSpec(config) + region = SharedOffloadRegion("synthetic-" + uuid.uuid4().hex, 4, None, mmap.PAGESIZE, mmap.PAGESIZE) + primary = CPUPrimaryTierOffloadingManager(4, region) + self.view = primary.get_kv_memoryview() + self.tier = SecondaryTierFactory.create_secondary_tier({ + "type": "studio_disk", "ssd_dir": str(self.root / "ssd"), "hdd_dir": str(self.root / "hdd"), + "ssd_bytes": 4 * mmap.PAGESIZE, "hdd_bytes": 6 * mmap.PAGESIZE, + }, self.view, spec) + self.manager = TieringOffloadingManager(primary, [self.tier]) + self.addCleanup(self.manager.shutdown) + self.keys = [make_offload_key(bytes([i + 1]) * 32, i % 2) for i in range(3)] + self.values = [bytes([i + 11]) * mmap.PAGESIZE for i in range(3)] + context = ReqContext("store") + self.manager.on_new_request(context) + stored = self.manager.prepare_store(self.keys, context) + with self.view.cast("B") as flat: + for block_id, value in zip(stored.store_spec.block_ids, self.values, strict=True): + offset = int(block_id) * mmap.PAGESIZE + flat[offset:offset + mmap.PAGESIZE] = value + self.manager.complete_store(self.keys, context) + self.tier.drain_jobs() + self.manager.on_schedule_end(ScheduleEndContext((), ())) + self.manager.on_request_finished(context) + self.manager.reset_cache() + with self.view.cast("B") as flat: + flat[:] = bytes(len(flat)) + + def test_disk_hit_restores_attention_and_recurrent_groups_into_ram(self): + context = ReqContext("restore") + self.manager.on_new_request(context) + for key, value in zip(self.keys, self.values, strict=True): + self.assertEqual(self.manager.lookup(key, context), LookupResult.RETRY) + self.manager.on_schedule_end(ScheduleEndContext((), ())) + self.tier.drain_jobs() + self.assertEqual(self.manager.lookup(key, context), LookupResult.HIT) + restored = self.manager.prepare_load([key], context) + offset = int(restored.block_ids[0]) * mmap.PAGESIZE + with self.view.cast("B") as flat: + self.assertEqual(bytes(flat[offset:offset + mmap.PAGESIZE]), value) + self.manager.complete_load([key], context) + self.manager.on_schedule_end(ScheduleEndContext((), ())) + self.manager.on_request_finished(context) + + def test_corrupt_disk_promotion_returns_miss_to_scheduler(self): + key = self.keys[0] + name = self.tier.mapper.get_file_name(key) + for root in self.tier.cache.roots: + path = root / name + if path.exists(): + path.write_bytes(b"invalid") + context = ReqContext("corrupt") + self.manager.on_new_request(context) + self.assertEqual(self.manager.lookup(key, context), LookupResult.RETRY) + self.manager.on_schedule_end(ScheduleEndContext((), ())) + self.tier.drain_jobs() + self.assertEqual(self.manager.lookup(key, context), LookupResult.MISS) + self.manager.on_request_finished(context) + + +if __name__ == "__main__": + unittest.main() diff --git a/service/local-ai/catalog.json b/service/local-ai/catalog.json index 251bcf287e5665f3fa94e57ed56132e8f372bdc0..4f75832c9e4e3395a7ebf6da2a1f5c8296f00b10 100644 --- a/service/local-ai/catalog.json +++ b/service/local-ai/catalog.json @@ -34,7 +34,7 @@ "mode": "tokens", "limit": 6000 }, - "context_window": 131072, + "context_window": 150000, "auto_compact_token_limit": 96000, "effective_context_window_percent": 90, "experimental_supported_tools": [], diff --git a/service/local-ai/download.py b/service/local-ai/download.py index ea003b217f6af12b10101eb9902c3d443bd12606..726d18558e6c47d39ff7fb4d348f2e9728031755 100644 --- a/service/local-ai/download.py +++ b/service/local-ai/download.py @@ -14,14 +14,17 @@ def checksum(path): return hashlib.file_digest(source, "sha256").hexdigest() -def download(root): +def download(root, runtime_only=False): manifest = json.loads(Path(__file__).with_name("models.json").read_text()) directory = root / manifest["directory"] directory.mkdir(parents=True, exist_ok=True) with (directory / ".download.lock").open("a") as lock: fcntl.flock(lock, fcntl.LOCK_EX) for spec in manifest["files"]: + if runtime_only and not spec["name"].startswith("runtime/"): + continue target = directory / spec["name"] + target.parent.mkdir(parents=True, exist_ok=True) if target.exists(): if target.stat().st_size == spec["bytes"] and checksum(target) == spec["sha256"]: print(f"Verified {target.name}", flush=True) @@ -47,6 +50,7 @@ def download(root): if __name__ == "__main__": parser = argparse.ArgumentParser(description="Download local AI models") parser.add_argument("--root", type=Path, default=Path("/srv/clover/Media/AI/LLM")) + parser.add_argument("--runtime-only", action="store_true") args = parser.parse_args() os.umask(0o027) - download(args.root) + download(args.root, args.runtime_only) diff --git a/service/local-ai/gateway.py b/service/local-ai/gateway.py index 1be60cf1afb52ed6dd82a5d4d1b8b0821ed436f6..e13d52f8337b2b6888b9c1e84934d70dbeb74d75 100644 --- a/service/local-ai/gateway.py +++ b/service/local-ai/gateway.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Adapt current Responses tool namespaces to llama.cpp; forward other APIs unchanged.""" +"""Adapt Responses tool namespaces to vLLM; forward other APIs unchanged.""" import hmac import http.client from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -120,6 +120,8 @@ def normalize_anthropic(request): if instructions: data["system"] = "\n\n".join(instructions) data["messages"] = messages + if (data.get("output_config") or {}).get("effort") in {"high", "max"}: + data["output_config"]["effort"] = "xhigh" return data @@ -151,7 +153,7 @@ class Handler(BaseHTTPRequestHandler): return # Authenticate before allocating inference bodies. if self.path != "/health": - key = os.environ["LLAMA_API_KEY"] + key = os.environ["VLLM_API_KEY"] supplied = self.headers.get("x-api-key", "") or self.headers.get("Authorization", "").removeprefix("Bearer ") if not hmac.compare_digest(supplied, key): self.send_error(401) @@ -166,7 +168,8 @@ class Handler(BaseHTTPRequestHandler): body = json.dumps(data).encode() elif path in {"/v1/messages", "/v1/messages/count_tokens"}: body = json.dumps(normalize_anthropic(json.loads(body))).encode() - headers = {key: val for key, val in self.headers.items() if key.lower() not in HOP_HEADERS} + headers = {key: val for key, val in self.headers.items() if key.lower() not in HOP_HEADERS | {"authorization", "x-api-key"}} + headers["Authorization"] = "Bearer " + os.environ["VLLM_API_KEY"] connection.request(self.command, self.path, body=body or None, headers=headers) response = connection.getresponse() streaming = "text/event-stream" in response.getheader("Content-Type", "") @@ -222,7 +225,7 @@ class Handler(BaseHTTPRequestHandler): if __name__ == "__main__": - backend = subprocess.Popen(["/app/llama-server", *sys.argv[1:]]) + backend = subprocess.Popen(["bash", "/app/single-user/start_qwen.sh"]) server = ThreadingHTTPServer(("0.0.0.0", 8080), Handler) def stop(signum, frame): if backend.poll() is None: diff --git a/service/local-ai/models.json b/service/local-ai/models.json index e6c7c1283dfc4097232d4cd501db5e32477e2aa3..493d59f7f6d1f5092846f516284e96657fa80fd0 100644 --- a/service/local-ai/models.json +++ b/service/local-ai/models.json @@ -1,17 +1,747 @@ { - "repository": "unsloth/Qwen3.8-27B-GGUF", - "revision": "4ca720788d1e01f1bff70c033e0d0028fd02e502", - "directory": "Qwen3.8-27B", + "repository": "daavidhauser/Swift-1.5-Qwen3.8-27B-W4A16-HyperQwen-INT4", + "revision": "64515d381e36084182e4e2abfa20050b75f29889", + "directory": "Swift-1.5-Qwen3.8-27B-W4A16-HyperQwen-INT4", "files": [ { - "name": "Qwen3.8-27B-UD-Q4_K_M.gguf", - "bytes": 16464440224, - "sha256": "322e194ff79741c7baa497c240f677f54b201b0efab44ca8e50f122b39123482" + "name": ".gitattributes", + "bytes": 1570, + "sha256": "34448b82c17d60fec9b65b1f093c115ddbaadc04beb1b0140b6bfed2e012a930" }, { - "name": "mmproj-F16.gguf", - "bytes": 927607488, - "sha256": "cbb841a9ee0636b2ec172f5bb8df2ea8dfeb01e90fe7c6126581d662a0b4e43e" + "name": "LICENSE", + "bytes": 13306, + "sha256": "1367057bf17041aa1d69286a1400be5f464d2f8b8f54f600088b209eac1850be" + }, + { + "name": "LICENSE-APACHE-2.0", + "bytes": 11544, + "sha256": "bbedc3fda3305820b977265f01b8619d87570a6739de3a5582c3464840f1e57a" + }, + { + "name": "NOTICE", + "bytes": 1389, + "sha256": "e0e788911888bdcc087d34fc12dc5bf3818034762334de3575ea34d9559b186f" + }, + { + "name": "QUANTIZATION_MANIFEST.json", + "bytes": 8434, + "sha256": "40ebbba877dd55e1c512e68785f3127673d57ac799c5ea7139761b563ed7de4b" + }, + { + "name": "README.md", + "bytes": 5620, + "sha256": "de13ff8b19a75a4d6dcda0c9a36e276d3a5b19b74a44ef012bbfe39562f24a85" + }, + { + "name": "RELEASE_MANIFEST.json", + "bytes": 23100, + "sha256": "1fb06fb256c1e29323e03ec56a1c1e99b8bfc5078a2c57c5426cb74052023eeb" + }, + { + "name": "RUNTIME.md", + "bytes": 2608, + "sha256": "4a6a153fdf00ece6ddd976ee745951db61c5a36a69ba06eaf102a1ec1f5c3c74" + }, + { + "name": "chat_template.jinja", + "bytes": 8952, + "sha256": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041" + }, + { + "name": "config.json", + "bytes": 22382, + "sha256": "deffb953c20dafbcdc07199063fa5e203dc574432dd3c7efac3f8e2882bd46a7" + }, + { + "name": "evaluation/code/swift15/checkpoint.py", + "bytes": 3656, + "sha256": "c43707f3623659cd882cfbb57506b152912f42f2a043dfd148c1cf829090f28d" + }, + { + "name": "evaluation/code/swift15/code_runner.py", + "bytes": 2144, + "sha256": "23d52e36543766bb33d33f11adad3772e7836d5170e83f68e326cc2d00b55fd9" + }, + { + "name": "evaluation/code/swift15/common.py", + "bytes": 1439, + "sha256": "0410dc972e74d8bbc05e090b0543426e4ea57ed707a80a114ea2e658822e1e8c" + }, + { + "name": "evaluation/code/swift15/corpus.py", + "bytes": 7800, + "sha256": "d5b420de1f3a1d4be97a44321abdd393247b5d78a56199ad49a62058c98d9050" + }, + { + "name": "evaluation/code/swift15/eval_data.py", + "bytes": 8097, + "sha256": "2a4497758d456a3cdd930aa711d0c510b24a2eaf6eb11be3da3e94481d4a166e" + }, + { + "name": "evaluation/code/swift15/evaluate.py", + "bytes": 14113, + "sha256": "4d553cf10b42cbff0eeacfe72b3b6df87405aa2604fcbe19a5a0a78133cd18af" + }, + { + "name": "evaluation/code/swift15/generate.py", + "bytes": 4412, + "sha256": "55e2659ce67295a4c57ffd12beed9dc0246808d908939c4113f7d1bb506af089" + }, + { + "name": "evaluation/code/swift15/ifbench_score.py", + "bytes": 876, + "sha256": "1b97f5859020d8d992944a85357ee7477f8740e65574d5946d2305889244a04d" + }, + { + "name": "evaluation/code/swift15/measure.py", + "bytes": 4726, + "sha256": "f955ace33396730c9fdf6fe5d424b6a2f77eb0fc58c83b3bd84b0c804c7a3b37" + }, + { + "name": "evaluation/code/swift15/quantize.py", + "bytes": 8098, + "sha256": "a3f65ca5de15d54cf626cde340365e503462d1a33ae8525aa55f6159195886f7" + }, + { + "name": "evaluation/code/swift15/reference.py", + "bytes": 1786, + "sha256": "0433adf523a676c0cb30c5ad25b90bdc5a4565c2b69180e171aa4a4d59c15bf8" + }, + { + "name": "evaluation/code/swift15/release.py", + "bytes": 7409, + "sha256": "c5129121b4ba333b0c4dd2694fb69076cdc75e4b9cda8a6572b3a449c32c3b5e" + }, + { + "name": "evaluation/code/swift15/report.py", + "bytes": 5287, + "sha256": "622af3dd5ab1eaa1843407c5bfb57e3111a154463c024a6685095a7aa293b482" + }, + { + "name": "evaluation/code/swift15/run.py", + "bytes": 9281, + "sha256": "e5d42332192e5bafdde88ae4acc20802f5c5477e6447ff5165fb6b836aa78ea2" + }, + { + "name": "evaluation/code/swift15/runtime_compat.py", + "bytes": 1885, + "sha256": "bcbc7351a6a4719bf38c06658bb12cbcfc9eb31f9280fc9b916782832544dbe6" + }, + { + "name": "evaluation/code/swift15/serve.py", + "bytes": 6020, + "sha256": "c705174eda86e57ae1a51c0b2e7a93f9b0e1f37fa32217a9739b569f52d2d56d" + }, + { + "name": "evaluation/code/swift15/setup.py", + "bytes": 2219, + "sha256": "7de02ccb5575e0ac2b62d33842d28aacfc83d63fc96eedad9cbedf1129e90495" + }, + { + "name": "evaluation/code/swift15/smoke.py", + "bytes": 3162, + "sha256": "e2d4a01d6613e667ceeef0d1a03ef1cec04edf3ab0518aaa4a006d419a91f1fe" + }, + { + "name": "evaluation/code/swift15/test_workflow.py", + "bytes": 3468, + "sha256": "8d2dcb057318d482530ef9bf7b4bcf0d59347b594531e0df18b81dbea0beaddd" + }, + { + "name": "evaluation/environment.json", + "bytes": 3282, + "sha256": "2e9ac10516648b0c634eaca007d58456d241e0c1ef1b6734fa51671bf315a841" + }, + { + "name": "evaluation/manifest.json", + "bytes": 2685, + "sha256": "82c1c6eaad6c06ad91e20cac5f9dae6b74ceaba0a3063c868a6a4313366e7948" + }, + { + "name": "evaluation/perplexity-english-python.json", + "bytes": 38213, + "sha256": "5c0f9ab2df9a8d4e14ea846cd03e1c3cb577847a022a955f97ff57ef6392fe85" + }, + { + "name": "evaluation/pilot-ids.json", + "bytes": 354, + "sha256": "44cffc01ab88b366f20da82d3150c7de8eb1d7801cd1381570e32313f120da41" + }, + { + "name": "evaluation/qwen-fast-full/manifest.json", + "bytes": 14395, + "sha256": "3de5a8af19d01a53594d7f29349e19bf90f1bbe8d935e82fe33aa0cf8325391b" + }, + { + "name": "evaluation/qwen-fast-full/summary.json", + "bytes": 3134, + "sha256": "a2ffcd9ed991279b5639274596a5139b39553d407753adc48ff465e55fd2eaf6" + }, + { + "name": "evaluation/qwen-fast-pilot/manifest.json", + "bytes": 2663, + "sha256": "f06f4539feaa46b92d559427cd13809ecf3e8b0fc44b15768e54392a0ce9c34f" + }, + { + "name": "evaluation/qwen-fast-pilot/summary.json", + "bytes": 2979, + "sha256": "8a0ec26f335c5a0c98bac1db6bd852a5292b0b5d1bfcc6a218a0b9aa1ce9b1c7" + }, + { + "name": "evaluation/qwen-fast/perplexity.json", + "bytes": 12570, + "sha256": "9db1599421dc916b97602a8921dbd219b815a5a76fc7def41f508500eb10e7e7" + }, + { + "name": "evaluation/qwen-fast/server.json", + "bytes": 1942, + "sha256": "b2988235fd03e4cd30366b5ca418ba5bb687c1ab58c1eb29e3e53c57612ac0ae" + }, + { + "name": "evaluation/qwen-fast/speed-c1.json", + "bytes": 630, + "sha256": "3d70c2868a2f2832e6bf7562abd3fa4589389baf87e02dbd89c76b154e49238d" + }, + { + "name": "evaluation/swift10-full/manifest.json", + "bytes": 14501, + "sha256": "6af95e530b920a84a3c5f5dde5b76bc2c791d0a4cc5b3043686bd6946cac4acf" + }, + { + "name": "evaluation/swift10-full/summary.json", + "bytes": 3125, + "sha256": "1c431ee8cc16e36090152526f8553c3e76cb5f66e5ddd5c6df1f90c3b3426add" + }, + { + "name": "evaluation/swift10-pilot/manifest.json", + "bytes": 2769, + "sha256": "83df9bd1a41e0d96f62039ad1941593e4cafc57c52f63851eeec4523048f4315" + }, + { + "name": "evaluation/swift10-pilot/summary.json", + "bytes": 2981, + "sha256": "3f8aed7d263dc65abb0095bc3af73515052645886f5f0565e6c21e82afa27bf8" + }, + { + "name": "evaluation/swift10/perplexity.json", + "bytes": 12560, + "sha256": "fe393bd4962d43f9cef2a08ea40c0dd2f15e6695b87a09a20e184d448d941775" + }, + { + "name": "evaluation/swift10/server.json", + "bytes": 2041, + "sha256": "d6004a26531972be5407c0f542ec0b708c931e21f67801e5206906d4cb148547" + }, + { + "name": "evaluation/swift10/speed-c1.json", + "bytes": 632, + "sha256": "a82f761f6a6ca5446808b1e8b129fee4e5b3b733787e286a7829b23f6dd1722f" + }, + { + "name": "evaluation/swift15-baseline-full/manifest.json", + "bytes": 14389, + "sha256": "b122ac2835f4c868522aa6982e1c44e2090ae176b27293cd8a71040108751d18" + }, + { + "name": "evaluation/swift15-baseline-full/summary.json", + "bytes": 3091, + "sha256": "cd0ec9d98535c6c32c2ae9f32798f81c0dae628ebda0d1d56f6005df85edb91b" + }, + { + "name": "evaluation/swift15-baseline-pilot/manifest.json", + "bytes": 2657, + "sha256": "588d9a88f35aa9207934bde6cbce4e0956868287316f964585b784b9d4f7cec3" + }, + { + "name": "evaluation/swift15-baseline-pilot/summary.json", + "bytes": 2982, + "sha256": "511e69269b8d98d64876ab7033e124fc1d8aca196b022fbd27b34f65867ac8ef" + }, + { + "name": "evaluation/swift15-baseline/perplexity.json", + "bytes": 12553, + "sha256": "6f63e857421957c76b3ca290ec8c95a62fd4374e3839adc2cdcdca54f14786fa" + }, + { + "name": "evaluation/swift15-baseline/server.json", + "bytes": 1937, + "sha256": "9063a7b84cd89e83672d673d5fc30ff563d01adebd1a705fcae70c33b5ba47b4" + }, + { + "name": "evaluation/swift15-baseline/speed-c1.json", + "bytes": 629, + "sha256": "079f1c870da2484a43ec214ee2ced3723b2636c83eaed4291636e121b4e81e3a" + }, + { + "name": "evaluation/swift15-fast-full/manifest.json", + "bytes": 14393, + "sha256": "d242d0bd7f947c37a74430190ce4a1309c2ae8e2f5ba76313e1ba056044aa134" + }, + { + "name": "evaluation/swift15-fast-full/summary.json", + "bytes": 2969, + "sha256": "3ffd7dc76fa4105089688731c6a8365519f80567acbca3b1bc15b64e93de7ab3" + }, + { + "name": "evaluation/swift15-fast-pilot/manifest.json", + "bytes": 2661, + "sha256": "13203d3b56258d785ea74ddd3f89322edb6c58245f7d4ecf60756a7af66d4f60" + }, + { + "name": "evaluation/swift15-fast-pilot/summary.json", + "bytes": 2979, + "sha256": "fe44f8972f40cf5236ca93d1ee7362cfea238a03e58f09c6e033312820fc8d89" + }, + { + "name": "evaluation/swift15-fast/perplexity.json", + "bytes": 12561, + "sha256": "3fa2ac71c6c24072036837c3ae0afcf96ce03e85a09b276874c75432aeb091a1" + }, + { + "name": "evaluation/swift15-fast/server.json", + "bytes": 1940, + "sha256": "53099847622dbdd7416196bfd8a7591aca1adb64db92049cf2be6ffdf489536d" + }, + { + "name": "evaluation/swift15-fast/speed-c1.json", + "bytes": 629, + "sha256": "88d292f00ba50773e80c0dbf6ba9a1e18d3413737221d2ff8c5a969749747108" + }, + { + "name": "generation_config.json", + "bytes": 202, + "sha256": "e70c136c1b78ddc1fb0905bac8e733a4dc448d4f852a5dd75143fffc70be550e" + }, + { + "name": "hyperqwen-audit.json", + "bytes": 18861, + "sha256": "62c8d7543e2e52d4025e7f72577753c029b1447762cfc781e8d06a6547c72795" + }, + { + "name": "hyperqwen-build.json", + "bytes": 8398, + "sha256": "895d4e0dd27f896ed2a3a0e2160c59967cd10e8506f690b6a85c5cf90c529cec" + }, + { + "name": "hyperqwen_provenance/generation-manifest.json", + "bytes": 460, + "sha256": "f8dc684e9940b3efe9f7967214495adb83d4b906cb442ef46924c42a90308853" + }, + { + "name": "hyperqwen_provenance/generation-summary.json", + "bytes": 204, + "sha256": "6aa261c7b4b9952edd1d7fc0b61f57985894d1b360571f02448364820f0adbd0" + }, + { + "name": "hyperqwen_provenance/lm-head-report.json", + "bytes": 234, + "sha256": "3fe25c5ed9c1dbf85a1e5baa705036303515d8ad0dccb9ef54abcca63528d2b4" + }, + { + "name": "hyperqwen_provenance/manifest.json", + "bytes": 2106, + "sha256": "83f789e2d83a31b90564811ae804f1cda210e7ad62a7c7dd6ddde79f04ad5438" + }, + { + "name": "hyperqwen_provenance/mtp-report.json", + "bytes": 1059, + "sha256": "e1cd6016c4a231be292dd73d128ba863f035320ee71de5bd983504cede3d711c" + }, + { + "name": "hyperqwen_provenance/mtp_hessians.pt.json", + "bytes": 15773, + "sha256": "f69a5c6132cb5382a8aaedd7dddfbdcd050e193ec2b1c525feb95949ca8ac834" + }, + { + "name": "hyperqwen_provenance/preflight-validation.json", + "bytes": 1444, + "sha256": "d4a00e873a3da9e29cf9ae5b864b4b2eaf992263705305651c77aa02a1ab3f9e" + }, + { + "name": "hyperqwen_provenance/recipe/drafter/capture.py", + "bytes": 5744, + "sha256": "6bb7cff9e235999923fa83b994034be6c9246b45fa7d89cca6fabf067f030e9b" + }, + { + "name": "hyperqwen_provenance/recipe/drafter/gptq_utils.py", + "bytes": 3809, + "sha256": "e0478db6fd5692a8960cd5ad16786655f09b73d6fe864382371c1455f9394c69" + }, + { + "name": "hyperqwen_provenance/recipe/drafter/train_mtp.py", + "bytes": 24898, + "sha256": "900951feef2e0ff7d729f9baac02e9125b5ec16de5680009e368e20139b79faf" + }, + { + "name": "hyperqwen_provenance/recipe/prepare/build_draft_vocab.py", + "bytes": 6161, + "sha256": "426f371a8db9f465c48b56ae05cbc782f0613980005405deac2ad95504d088bc" + }, + { + "name": "hyperqwen_provenance/recipe/prepare/quant_heads_stream.py", + "bytes": 9473, + "sha256": "67bac25a15df11ed569866491d8bf21550829e0281bb74a8c9ec092107fca905" + }, + { + "name": "hyperqwen_provenance/recipe/swift15/README.md", + "bytes": 5869, + "sha256": "83564e691d256395a5c49138bfa65840f226f4c51c373c9e336acb4af08c0467" + }, + { + "name": "hyperqwen_provenance/recipe/swift15/checkpoint.py", + "bytes": 3656, + "sha256": "c43707f3623659cd882cfbb57506b152912f42f2a043dfd148c1cf829090f28d" + }, + { + "name": "hyperqwen_provenance/recipe/swift15/code_runner.py", + "bytes": 2144, + "sha256": "23d52e36543766bb33d33f11adad3772e7836d5170e83f68e326cc2d00b55fd9" + }, + { + "name": "hyperqwen_provenance/recipe/swift15/common.py", + "bytes": 1439, + "sha256": "0410dc972e74d8bbc05e090b0543426e4ea57ed707a80a114ea2e658822e1e8c" + }, + { + "name": "hyperqwen_provenance/recipe/swift15/corpus.py", + "bytes": 7800, + "sha256": "d5b420de1f3a1d4be97a44321abdd393247b5d78a56199ad49a62058c98d9050" + }, + { + "name": "hyperqwen_provenance/recipe/swift15/eval_data.py", + "bytes": 8097, + "sha256": "2a4497758d456a3cdd930aa711d0c510b24a2eaf6eb11be3da3e94481d4a166e" + }, + { + "name": "hyperqwen_provenance/recipe/swift15/evaluate.py", + "bytes": 12978, + "sha256": "dfb48598358dd97c2b52eb41384b797cf2a4fc9d35cba91b0b2efb84abb73d8c" + }, + { + "name": "hyperqwen_provenance/recipe/swift15/generate.py", + "bytes": 4412, + "sha256": "55e2659ce67295a4c57ffd12beed9dc0246808d908939c4113f7d1bb506af089" + }, + { + "name": "hyperqwen_provenance/recipe/swift15/ifbench_score.py", + "bytes": 876, + "sha256": "1b97f5859020d8d992944a85357ee7477f8740e65574d5946d2305889244a04d" + }, + { + "name": "hyperqwen_provenance/recipe/swift15/measure.py", + "bytes": 4726, + "sha256": "f955ace33396730c9fdf6fe5d424b6a2f77eb0fc58c83b3bd84b0c804c7a3b37" + }, + { + "name": "hyperqwen_provenance/recipe/swift15/quantize.py", + "bytes": 8098, + "sha256": "a3f65ca5de15d54cf626cde340365e503462d1a33ae8525aa55f6159195886f7" + }, + { + "name": "hyperqwen_provenance/recipe/swift15/release.py", + "bytes": 7409, + "sha256": "c5129121b4ba333b0c4dd2694fb69076cdc75e4b9cda8a6572b3a449c32c3b5e" + }, + { + "name": "hyperqwen_provenance/recipe/swift15/report.py", + "bytes": 4712, + "sha256": "8e73730ee54a449bb1028b8626f2ad973bed40428f7b49a8780b077cc63dcc92" + }, + { + "name": "hyperqwen_provenance/recipe/swift15/run.py", + "bytes": 8169, + "sha256": "d60a6780eda3ee20ab4b878cfa4fa223edfdc4a5a099c622e05ee06ff61c303f" + }, + { + "name": "hyperqwen_provenance/recipe/swift15/runtime_compat.py", + "bytes": 1885, + "sha256": "bcbc7351a6a4719bf38c06658bb12cbcfc9eb31f9280fc9b916782832544dbe6" + }, + { + "name": "hyperqwen_provenance/recipe/swift15/serve.py", + "bytes": 4076, + "sha256": "f80daeb04c86d07b8ce463e1305c9c84d6ab4dbe2eb2322b14970b282bdcdbb3" + }, + { + "name": "hyperqwen_provenance/recipe/swift15/setup.py", + "bytes": 2219, + "sha256": "7de02ccb5575e0ac2b62d33842d28aacfc83d63fc96eedad9cbedf1129e90495" + }, + { + "name": "hyperqwen_provenance/recipe/swift15/smoke.py", + "bytes": 3162, + "sha256": "e2d4a01d6613e667ceeef0d1a03ef1cec04edf3ab0518aaa4a006d419a91f1fe" + }, + { + "name": "hyperqwen_provenance/recipe/swift15/test_workflow.py", + "bytes": 3029, + "sha256": "221c0b3e6cb04fe4aaec2c47059c329dfa4e06c4084b9b01dfea3ab62032c677" + }, + { + "name": "hyperqwen_provenance/runtime-compatibility.json", + "bytes": 869, + "sha256": "4b627e696f1e5dcda170364fc7a11b83fdbbad0c434933171db4cc76c97e91d5" + }, + { + "name": "hyperqwen_provenance/upstream-QUANTIZATION_MANIFEST.json", + "bytes": 1243, + "sha256": "37a75a5056e4d7267cfacfd11fd830f6fe0c084fc3688bead9842b9f0fbfc121" + }, + { + "name": "hyperqwen_provenance/upstream-README.md", + "bytes": 21151, + "sha256": "1fa3bbf8f6177e23bc1dcf98ce990f889ecad8c3f032a010c41559d34cbee263" + }, + { + "name": "hyperqwen_provenance/upstream-UPLOAD_MANIFEST.json", + "bytes": 5176, + "sha256": "b475d1d0e4b383c036be5a396c1a65dc2b45b3af1463da859049a24c65fe6854" + }, + { + "name": "hyperqwen_provenance/upstream-recipe.yaml", + "bytes": 930, + "sha256": "49f446271f2278c633d540768d42caf82114fc2353ca526d1279fcc1984b7f68" + }, + { + "name": "hyperqwen_provenance/vocabulary-report.json", + "bytes": 362, + "sha256": "44b8a205f3c8419ad179f5c5ae89a9d7b2ca143e241bfde8f96e132637c9f809" + }, + { + "name": "merges.txt", + "bytes": 3353259, + "sha256": "a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d" + }, + { + "name": "model-00001-of-00006.safetensors", + "bytes": 655565120, + "sha256": "1e2bb79bd7feff42b3b006b3552354c1ff2c4a02941a788080839b0eb868118d" + }, + { + "name": "model-00002-of-00006.safetensors", + "bytes": 2730582048, + "sha256": "48ddebd951d8d8a9ecef51b1857403ca77e5ab2c63b2e7db4822e0908e2eeaac" + }, + { + "name": "model-00003-of-00006.safetensors", + "bytes": 3966050624, + "sha256": "61bbd4ae1e4812f08aa8f903e9995746aea8c346a666b530d0a6774773350f1e" + }, + { + "name": "model-00004-of-00006.safetensors", + "bytes": 3966050704, + "sha256": "0c7baff5aac711178c5ce841da1ebcbebf67bde21155e74f4adca5d4354c178a" + }, + { + "name": "model-00005-of-00006.safetensors", + "bytes": 3993185016, + "sha256": "0cc1b9b1cde51d81fd7f53d91716399c30c4a5a166503dbd76ac65c63bb881c9" + }, + { + "name": "model-00006-of-00006.safetensors", + "bytes": 248265904, + "sha256": "65c1248e479775518d38a1675859380890a76c455f25de37b613bef71281a4fb" + }, + { + "name": "model-mtp-bf16.safetensors", + "bytes": 219027824, + "sha256": "39ac3a5071ac107640468e1a5c3d3844d39eba277076c2d97cf404974dad3d5d" + }, + { + "name": "model.safetensors.index.json", + "bytes": 243540, + "sha256": "e99357b5714c15037f02da3f6d46dd569742edde177df6f8f5c46505bba9e562" + }, + { + "name": "model_extra_tensors.safetensors", + "bytes": 173015384, + "sha256": "bdb17eed6815f17f20a324d25f3777533cb224c463075f1ab6afd199bde8628a" + }, + { + "name": "mtp_draft_vocab_ids.pt", + "bytes": 525949, + "sha256": "a7f5b236bd7b1892c24a004d12f93cdf6910a3ca2afb2b490cc08ab985b6d7c9" + }, + { + "name": "preprocessor_config.json", + "bytes": 390, + "sha256": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516" + }, + { + "name": "recipe.yaml", + "bytes": 930, + "sha256": "49f446271f2278c633d540768d42caf82114fc2353ca526d1279fcc1984b7f68" + }, + { + "name": "runtime/BASE_REVISION.txt", + "bytes": 41, + "sha256": "c1f6fbc4227edb7daa83504c936afc8e23d8e31e8b62ff375fbbb5e29e81e546" + }, + { + "name": "runtime/LICENSE", + "bytes": 11358, + "sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30" + }, + { + "name": "runtime/batch/start_qwen.sh", + "bytes": 11677, + "sha256": "ac6a84b455317537a230bc190ccc06dda169dcdbdeee4bebeb51e76032972944" + }, + { + "name": "runtime/patches/dflash2-backport.patch", + "bytes": 40396, + "sha256": "2e937ef748c1942ab04ace05de884e7c24041d34b23e5be8428cac6939ac6875" + }, + { + "name": "runtime/patches/dflash2-lookup-drafting.patch", + "bytes": 46770, + "sha256": "5df09ef03b592d4a2c3b47dd8d2dbf8862fa7383d68aef3dd6ef1d97d29ce196" + }, + { + "name": "runtime/patches/dflash2-ngram-chains.patch", + "bytes": 19037, + "sha256": "555b5a75d9023c99b2cd17634ac5b1dc8505f9275d857b1eba08c2d9cbf2df6e" + }, + { + "name": "runtime/patches/dflash2-prewarm.patch", + "bytes": 5181, + "sha256": "e1e8012fa2c304c6948c19fc0071332f80eebcff802b402255445a30d553ab92" + }, + { + "name": "runtime/patches/hybrid-kv-groups-v2-cudagraph.patch", + "bytes": 6306, + "sha256": "143825dd9744ab81dabcb438ec0bf974b6e0be8d0d7b0ffbbdfbf7c16a549664" + }, + { + "name": "runtime/patches/hybrid-sw-block-promote.patch", + "bytes": 7828, + "sha256": "10876ac706546e74fe74b8964b90576a7b36f90a2f35a856997c558d7817f154" + }, + { + "name": "runtime/patches/int4-kv-per-token-head.patch", + "bytes": 7038, + "sha256": "c03ff10c8c997b355f04fc521827ee2fa99c332ce4127cc2d878b749b79ca3d9" + }, + { + "name": "runtime/patches/mamba-align-checkpoint-order.patch", + "bytes": 12020, + "sha256": "515d9bf76e860c95d832d615bdab4a97b71e2b0412b4ccc2445f11535ddddf45" + }, + { + "name": "runtime/patches/marlin-int8-layer-select.patch", + "bytes": 3048, + "sha256": "833405e2ed2916529eb20184c38243c843378935522b74bab8c39707bf3ee800" + }, + { + "name": "runtime/patches/marlin-int8-negative-scales.patch", + "bytes": 2926, + "sha256": "4cb8a064c706cc62dc77224479898da58864fbfff6b65012d90aa8326035300d" + }, + { + "name": "runtime/patches/marlin-repack-staged-sm80.patch", + "bytes": 6841, + "sha256": "1588e2f10b5f82194d449483e766c0b39c84ba522d1623d39caa9502040f1fab" + }, + { + "name": "runtime/patches/marlin-tune-table.patch", + "bytes": 4422, + "sha256": "1cac17f12e4ce389b0cb0fc733ceceb0a8cf60c02696ee378dc1d8d0c1ef8914" + }, + { + "name": "runtime/patches/offload-dflash-eagle-groups.patch", + "bytes": 5406, + "sha256": "f2791af64d8066b31e4250d866c4322f45670d75522aff221307281c97c73156" + }, + { + "name": "runtime/patches/offload-wsl2-devptr.patch", + "bytes": 4778, + "sha256": "8c6f9e3e5723571d1f33925794435e55062b684b2706d9284d1c5b94778d34db" + }, + { + "name": "runtime/patches/qwen3_5-embed-quant.patch", + "bytes": 1395, + "sha256": "0a1b9ca06798c1aef582995de5a0beb3ad9a22a54cdbd2361986563a9c7a980e" + }, + { + "name": "runtime/patches/qwen3_5-mtp-draft-vocab.patch", + "bytes": 3943, + "sha256": "292ef662d17bcc10556b787d5bb5f2cd12d3d3fc1f3bd2fe982485ce0d916298" + }, + { + "name": "runtime/patches/sampler-small-topk-fast-softmax.patch", + "bytes": 13679, + "sha256": "8828646ce1916c065282529d9c1bb52064f668397d8e1f33652aef2f308292b6" + }, + { + "name": "runtime/patches/spec-decode-attn.patch", + "bytes": 15424, + "sha256": "007791047a1d60143f8e3fe4e0a56dabadfc0134cb288847338b76ba7d1f9fe7" + }, + { + "name": "runtime/patches/spec-decode-int4-kv-mq3d.patch", + "bytes": 3876, + "sha256": "b93b186deba1513ad9c4803ee2c574c923b21ba5e88e23bca8b8594b6a6b20a0" + }, + { + "name": "runtime/patches/spec-decode-int8-kv.patch", + "bytes": 11915, + "sha256": "3cfd31304353237af5277c861dc2b43ad33d0f9bff2700594300eff150f64382" + }, + { + "name": "runtime/patches/spec-sampler-prewarm.patch", + "bytes": 4516, + "sha256": "18ed9608baa5a09ed2ccbb214d62763da3ce7bcf09c6aad9003778ea4e56d18f" + }, + { + "name": "runtime/patches/speed-knobs-envs.patch", + "bytes": 2300, + "sha256": "841ec93021b1b90f90313a1880a791bbfbdb79cbe34d9b99fdb4c3a7e52ed7c0" + }, + { + "name": "runtime/patches/triton-prefill-attn-int8.patch", + "bytes": 15661, + "sha256": "6bd36db0226dce7b92ebd1231b6f9718a25da4a933ab4b2a6a8be77860364fae" + }, + { + "name": "runtime/patches/vision-tower-cpu-offload.patch", + "bytes": 8345, + "sha256": "81dff64a1177058783dcf7d4e8552f1547f74fa8a68019e349dacdb9d66e968e" + }, + { + "name": "runtime/patches/vllm-pr50021-gdn-spec-bounds.patch", + "bytes": 8886, + "sha256": "cd6e00270fa28e37a8c7ad11f965662f4153b2fcaf840f2c4043710e7b078655" + }, + { + "name": "runtime/patches/xgrammar-spec-terminated.patch", + "bytes": 3993, + "sha256": "37589b9a45d5ece37cc16e82b195362719011e52ea31b2e70cdc89845c825040" + }, + { + "name": "runtime/single-user/start_qwen.sh", + "bytes": 42439, + "sha256": "6874eb0bc4306d61b57ebb2f2c7cab97b41f11a3e0ec2b1502d9c34d396c47d8" + }, + { + "name": "tokenizer.json", + "bytes": 12809320, + "sha256": "0997f410c57a1f4e53b09e4be8f4a172d90edd9564368fb0847030937229b9f3" + }, + { + "name": "tokenizer_config.json", + "bytes": 17928, + "sha256": "b11349aafa7cdc6a320767cf7ceb29ed82f7eda5d65e8e0819e76f0ce947bf27" + }, + { + "name": "video_preprocessor_config.json", + "bytes": 385, + "sha256": "7768af27c1fafa9cc9011c1dc20067e03f8915e03b63504550e11d5066986d13" + }, + { + "name": "vocab.json", + "bytes": 6722759, + "sha256": "ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003" } ] } diff --git a/service/local-ai/prepare.py b/service/local-ai/prepare.py new file mode 100644 index 0000000000000000000000000000000000000000..2467dcabf1a9dde4cca7f89023112d458ed73851 --- /dev/null +++ b/service/local-ai/prepare.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +import json +import os +from pathlib import Path +import subprocess +import sys + + +data = json.load(sys.stdin) +if Path(data["hostRoot"]).name == "local-ai": + source = Path(__file__).with_name("service.pkl") + path = Path(subprocess.check_output([ + "pkl", "eval", "-f", "json", "-p", "serviceId=local-ai", "-p", f"uid={data['uid']}", + "-p", "preview=false", "--expression", 'container.volumes["/cache/ssd"].src', str(source), + ], text=True).strip()) + path.mkdir(mode=0o700, parents=True, exist_ok=True) + os.chown(path, data["uid"], data["uid"]) + path.chmod(0o700) diff --git a/service/local-ai/service.pkl b/service/local-ai/service.pkl index 39be2bedc9773836edd4c0bf632bcea6962b8bea..2f9f8baeff9922926057a9a5245c8e01135f7203 100644 --- a/service/local-ai/service.pkl +++ b/service/local-ai/service.pkl @@ -4,20 +4,22 @@ import "../../config/site.pkl" as site import "pkl:json" local model = (new json.Parser {}).parse(read("catalog.json")).models[0] +local weights = (new json.Parser {}).parse(read("models.json")) meta { name = "Local AI"; launcher = false; access = "admin" } enabled = (read?("env:STUDIO_LOCAL_AI") ?? "false") == "true" rollout = "simple" stageIsolation = "fresh" +prepare = "prepare.py" healthyDeadline = "30m" healthRestartGrace = "30m" -secrets { ["api_key"] {} } +secrets { ["api_key"] {}; ["cache_key"] {} } container { - build = "build" + build = "." cpu = 4000 - memory = 18432 - entrypoint = "/usr/bin/python3" + memory = 26624 + entrypoint = "/usr/bin/python3.12" http { containerPort = 8080 subdomain = "ai" @@ -25,42 +27,38 @@ container { } volumes { ["/config/gateway.py"] { config = "gateway.py" } - ["/models"] { src = "\(site.mediaRoot)/AI/LLM/Qwen3.8-27B"; readOnly = true } + ["/cache/hdd"] {} + ["/cache/ssd"] { src = if (site.preview) null else site.localAiSsdCache } + ["/models"] { src = "\(site.mediaRoot)/AI/LLM/\(weights.directory)"; readOnly = true } // The host's NVIDIA libraries link to immutable Nix store paths. ["/nvidia/lib"] { src = "/run/opengl-driver/lib"; readOnly = true } ["/nix/store"] { src = "/nix/store"; readOnly = true } } devices { "/dev/nvidia0"; "/dev/nvidiactl"; "/dev/nvidia-uvm" } + tmpfs { "/dev/shm:rw,size=10g" } env { - ["LD_LIBRARY_PATH"] = "/nvidia/lib:/app" - ["LLAMA_API_KEY"] = "${secret.own.api_key}" - } - args { - "/config/gateway.py" - "--model"; "/models/Qwen3.8-27B-UD-Q4_K_M.gguf" - "--mmproj"; "/models/mmproj-F16.gguf" - "--alias"; model.slug - "--host"; "127.0.0.1" - "--port"; "8081" - "--n-gpu-layers"; "99" - "--ctx-size"; model.context_window.toString() - "--parallel"; "1" - "--flash-attn"; "on" - "--cache-type-k"; "q8_0" - "--cache-type-v"; "q8_0" - "--cache-ram"; "8192" - "--spec-type"; "draft-mtp" - "--spec-draft-n-max"; "3" - "--cache-type-k-draft"; "q8_0" - "--cache-type-v-draft"; "q8_0" - "--jinja" - "--timeout"; "28800" - "--sse-ping-interval"; "10" - "--metrics" - "--temp"; "1.0" - "--top-p"; "0.95" - "--top-k"; "20" - "--min-p"; "0" - "--chat-template-kwargs"; "{\"enable_thinking\": true, \"reasoning_effort\": \"\(model.default_reasoning_level)\", \"preserve_thinking\": true}" + ["LD_LIBRARY_PATH"] = "/nvidia/lib:/usr/local/cuda/lib64" + ["VLLM_API_KEY"] = "${secret.own.api_key}" + ["STUDIO_AI_CACHE_KEY"] = "${secret.own.cache_key}" + ["PYTHONHASHSEED"] = "0" + ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:False" + ["MODEL"] = "/models" + ["CTX"] = "long" + ["MAX_LEN"] = model.context_window.toString() + ["SPEC"] = "mtp" + ["DRAFT_TOKENS"] = "3" + ["PREFIX_CACHE"] = "1" + ["TOOLS"] = "1" + ["VISION"] = "1" + ["VISION_OFFLOAD"] = "1" + ["GPU_UTIL"] = "0.93" + ["MAX_SEQS"] = "8" + ["API_SERVERS"] = "1" + ["HOST"] = "127.0.0.1" + ["PORT"] = "8081" + ["VLLM_MAMBA_ALIGN_KEEP_CHECKPOINTS"] = "1" + ["FLASHINFER_DISABLE_VERSION_CHECK"] = "1" + ["EXTRA_ARGS"] = "--kv-transfer-config {\"kv_connector\":\"OffloadingConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"spec_name\":\"TieringOffloadingSpec\",\"cpu_bytes_to_use\":8589934592,\"offload_prompt_only\":false,\"secondary_tiers\":[{\"type\":\"studio_disk\",\"ssd_dir\":\"/cache/ssd\",\"hdd_dir\":\"/cache/hdd\",\"ssd_bytes\":17179869184,\"hdd_bytes\":274877906944}]}} --limit-mm-per-prompt {\"image\":{\"count\":10}} --default-chat-template-kwargs {\"enable_thinking\":true,\"reasoning_effort\":\"\(model.default_reasoning_level)\"} --generation-config vllm --override-generation-config {\"temperature\":1.0,\"top_p\":0.95,\"top_k\":20,\"min_p\":0,\"presence_penalty\":0,\"repetition_penalty\":1.0} --disable-uvicorn-access-log" } + args { "/config/gateway.py" } } diff --git a/service/local-ai/test_gateway.py b/service/local-ai/test_gateway.py index 55aeee230ea3c3ad91552561517cbe3de67c8147..11b81a0aba7ab3269c60abde754cfd785f8daea1 100644 --- a/service/local-ai/test_gateway.py +++ b/service/local-ai/test_gateway.py @@ -3,6 +3,12 @@ from gateway import normalize, normalize_anthropic, restore class ResponsesCompatibility(unittest.TestCase): + def test_anthropic_effort_matches_the_qwen_template(self): + for requested, expected in [("low", "low"), ("medium", "medium"), ("high", "xhigh"), ("max", "xhigh"), ("xhigh", "xhigh")]: + with self.subTest(effort=requested): + data = normalize_anthropic({"output_config": {"effort": requested, "format": {"type": "json_schema"}}}) + self.assertEqual(data["output_config"], {"effort": expected, "format": {"type": "json_schema"}}) + def test_claude_normal_configuration_system_blocks(self): request = {"system": [{"type": "text", "text": "instructions", "cache_control": {"type": "ephemeral"}}, {"type": "text", "text": "workspace context"}], "messages": [{"role": "user", "content": "task"}, {"role": "system", "content": "late system message"}]} data = normalize_anthropic(request) diff --git a/tools/ai-wrapper-test.py b/tools/ai-wrapper-test.py deleted file mode 100644 index 4b7427b3e7a24fa48716b71cfe9b5ef3188a4d62..0000000000000000000000000000000000000000 --- a/tools/ai-wrapper-test.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python3 -"""Check downloaded wrappers without calling the model or changing client settings.""" -import json -import os -import pty -import select -import time -from pathlib import Path -import subprocess -import sys -import tempfile - -source = Path(sys.argv[1]).resolve() -with tempfile.TemporaryDirectory(prefix="snow-wrapper-test-") as temporary: - root = Path(temporary) - for name in ("codex", "claude"): - binary = root / name - binary.write_text('''#!/usr/bin/env python3 -import json, os, pathlib, sys -catalog = next((a.split("=",1)[1].strip('"') for a in sys.argv if a.startswith("model_catalog_json=")), None) -print(json.dumps({"args":sys.argv[1:],"env":{k:v for k,v in os.environ.items() if k.startswith(("SNOW", "ANTHROPIC", "CLAUDE", "API_", "ENABLE_CLAUDE"))},"catalog":json.loads(pathlib.Path(catalog).read_text()) if catalog else None})) -''') - binary.chmod(0o755) - env = dict(os.environ, PATH=str(root) + os.pathsep + os.environ["PATH"], XDG_CONFIG_HOME=str(root / "config")) - env.pop("SNOWGLOBE_AI_KEY", None) - env.pop("SNOWGLOBE_AI_URL", None) - for name in ("snow-codex", "snow-claude"): - wrapper = source / name - subprocess.run(["bash", "-n", wrapper], check=True) - failed = subprocess.run(["bash", wrapper], env=env, stdin=subprocess.DEVNULL, capture_output=True) - assert failed.returncode == 1 and b"SNOWGLOBE_AI_KEY" in failed.stderr - key = root / "config/snowglobe-ai/api-key" - key.parent.mkdir(parents=True, exist_ok=True) - master, slave = pty.openpty() - process = subprocess.Popen(["bash", wrapper, "--help"], env=env, stdin=slave, stdout=subprocess.PIPE, stderr=slave) - os.close(slave) - prompt = b"" - deadline = time.monotonic() + 10 - try: - while b"Snow Globe API key:" not in prompt: - assert time.monotonic() < deadline, "first-run key prompt did not appear" - if select.select([master], [], [], 1)[0]: - prompt += os.read(master, 4096) - os.write(master, b"wrapper-canary\n") - output, _ = process.communicate(timeout=10) - assert process.returncode == 0 and json.loads(output)["env"]["SNOWGLOBE_AI_KEY"] == "wrapper-canary" - assert key.read_text() == "wrapper-canary\n" and key.stat().st_mode & 0o777 == 0o600 - finally: - if process.poll() is None: process.kill() - os.close(master) - data = json.loads(subprocess.check_output(["bash", wrapper, "--help", "argument with spaces"], env={**env, "ANTHROPIC_AUTH_TOKEN":"old-auth", "CLAUDE_CODE_OAUTH_TOKEN":"old-login"})) - assert data["args"][-2:] == ["--help", "argument with spaces"] - assert "wrapper-canary" not in " ".join(data["args"]) - assert data["env"]["SNOWGLOBE_AI_KEY"] == "wrapper-canary" - assert key.stat().st_mode & 0o777 == 0o600 - if name == "snow-codex": - assert data["args"][:2] == ["--no-daemon", "--approve-for-me"] - assert data["catalog"]["models"][0]["context_window"] == 131072 - assert not list(key.parent.glob("catalog.*")), "temporary catalog leaked" - else: - assert data["args"][:2] == ["--permission-mode", "auto"] - assert "ANTHROPIC_AUTH_TOKEN" not in data["env"] and "CLAUDE_CODE_OAUTH_TOKEN" not in data["env"] - assert data["env"]["CLAUDE_CODE_AUTO_MODE_SERVER"] == "0" - assert data["env"]["API_TIMEOUT_MS"] == "28800000" - credentials = Path(data["env"]["CLAUDE_SECURESTORAGE_CONFIG_DIR"]) - assert credentials.stat().st_mode & 0o777 == 0o700 - key.unlink() - print("Both downloaded wrappers preserve arguments, enable auto approval, isolate credentials, and clean up.") diff --git a/tools/benchmark-local-ai-cache.py b/tools/benchmark-local-ai-cache.py new file mode 100644 index 0000000000000000000000000000000000000000..f7896e12da204a08c81f957b0fffe8924ae2f0cc --- /dev/null +++ b/tools/benchmark-local-ai-cache.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Run on the target through SSH stdin; use synthetic prompts and disposable files.""" +import argparse +from concurrent.futures import ThreadPoolExecutor +import http.client +import json +import mmap +import os +from pathlib import Path +import random +import socket +import tempfile +import time + + +def request(path, key, data=None): + connection = http.client.HTTPConnection("127.0.0.1", 8081, timeout=600) + try: + body = json.dumps(data) if data is not None else None + connection.request("POST" if body else "GET", path, body, { + "Authorization": "Bearer " + key, "Content-Type": "application/json", + }) + response = connection.getresponse() + payload = response.read() + if response.status != 200: + raise RuntimeError(f"{path}: HTTP {response.status}") + return json.loads(payload) if data is not None else payload.decode() + finally: + connection.close() + + +def metrics(key): + values = {} + for line in request("/metrics", key).splitlines(): + if line.startswith("vllm:"): + name = line.split("{", 1)[0] + values[name] = values.get(name, 0) + float(line.rsplit(" ", 1)[1]) + return values + + +def prefill(pid, repeats): + environment = dict(entry.split(b"=", 1) for entry in Path(f"/proc/{pid}/environ").read_bytes().split(b"\0") if b"=" in entry) + key = environment[b"VLLM_API_KEY"].decode() + vocabulary = request("/tokenize", key, { + "model": "qwen3.8-27b", "prompt": "Synthetic benchmark data: blue green yellow red apple pear stone river cloud mountain 0123456789.\n", + })["tokens"] + for length in (16000, 64000, 148000): + for trial in range(repeats): + before = metrics(key) + if before["vllm:num_requests_running"] or before["vllm:num_requests_waiting"]: + raise RuntimeError("Endpoint is busy; rerun when idle") + prompt = random.choices(vocabulary, k=length) + started = time.perf_counter() + response = request("/v1/completions", key, { + "model": "qwen3.8-27b", "prompt": prompt, "max_tokens": 1, "temperature": 0, + }) + elapsed = time.perf_counter() - started + after = metrics(key) + delta = {name: after.get(name, 0) - value for name, value in before.items()} + print(json.dumps({ + "kind": "prefill", "prompt_tokens": response["usage"]["prompt_tokens"], "trial": trial, + "wall_seconds": elapsed, "prefill_seconds": delta["vllm:request_prefill_time_seconds_sum"], + "queue_seconds": delta["vllm:request_queue_time_seconds_sum"], + "cached_tokens": delta["vllm:prefix_cache_hits_total"], + "completed_requests": delta["vllm:request_success_total"], + }), flush=True) + + +def filesystem(directory, repeats): + os.umask(0o077) + size = 16 * 1024**2 + with tempfile.TemporaryDirectory(prefix="synthetic-ai-cache-", dir=directory) as temporary: + root = Path(temporary) + buffer = mmap.mmap(-1, size) + buffer[:] = os.urandom(size) + paths = [root / f"{random.getrandbits(128):032x}.bin" for _ in range(256)] + started = time.perf_counter() + for path in paths: + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_DIRECT, 0o600) + try: + if os.write(descriptor, buffer) != size: + raise RuntimeError("Short synthetic write") + os.fsync(descriptor) + finally: + os.close(descriptor) + buffer.close() + print(json.dumps({"kind": "write", "directory": directory, "gib": 4, "seconds": time.perf_counter() - started}), flush=True) + + def read(path): + with mmap.mmap(-1, size) as target: + descriptor = os.open(path, os.O_RDONLY | os.O_DIRECT) + try: + if os.readv(descriptor, [target]) != size: + raise RuntimeError("Short synthetic read") + finally: + os.close(descriptor) + + for gib in (1, 4): + selected = paths[:gib * 64] + for trial in range(repeats): + random.shuffle(selected) + devices = [path for path in Path("/sys/block").iterdir() if (path / "device").exists()] + sectors = {path.name: int((path / "stat").read_text().split()[2]) for path in devices} + started = time.perf_counter() + with ThreadPoolExecutor(max_workers=16) as pool: + list(pool.map(read, selected)) + elapsed = time.perf_counter() - started + disk_reads = {path.name: (int((path / "stat").read_text().split()[2]) - sectors[path.name]) * 512 for path in devices} + print(json.dumps({"kind": "direct-read", "directory": directory, "gib": gib, "trial": trial, + "block_mib": 16, "workers": 16, "seconds": elapsed, + "device_read_bytes": {name: count for name, count in disk_reads.items() if count}}), flush=True) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("mode", choices=("prefill", "filesystem")) + parser.add_argument("target", help="Backend PID or disposable parent directory") + parser.add_argument("--hostname", required=True) + parser.add_argument("--repeats", type=int, default=3) + arguments = parser.parse_args() + if socket.gethostname() != arguments.hostname: + raise SystemExit("Unexpected target host") + if arguments.mode == "prefill": + prefill(int(arguments.target), arguments.repeats) + else: + filesystem(arguments.target, arguments.repeats) diff --git a/tools/benchmark-local-ai.py b/tools/benchmark-local-ai.py deleted file mode 100644 index b801a384d8ee1a1a998650f05f105c9fe78585b3..0000000000000000000000000000000000000000 --- a/tools/benchmark-local-ai.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python3 -"""Measure llama.cpp decode, retrieval, and prefix reuse without logging prompts.""" -import argparse -import base64 -import binascii -import json -import os -import struct -import time -import urllib.request -import zlib - - -def request(path, data=None): - body = None if data is None else json.dumps(data).encode() - headers = {"Authorization": "Bearer " + os.environ["LLAMA_API_KEY"], "Content-Type": "application/json"} - started = time.monotonic() - with urllib.request.urlopen(urllib.request.Request(args.url + path, data=body, headers=headers), timeout=1800) as response: - result = json.load(response) - return result, time.monotonic() - started - - -parser = argparse.ArgumentParser(description=__doc__) -parser.add_argument("--url", default="http://127.0.0.1:8081") -parser.add_argument("--contexts", type=int, nargs="+", default=[16384, 65536, 122000]) -parser.add_argument("--suite", choices=["decode", "workloads", "cache", "retrieval", "vision", "all"], default="all") -parser.add_argument("--require-prefix-cache", action="store_true") -args = parser.parse_args() - -def report(label, data, path="/completion"): - result, elapsed = request(path, data) - print(json.dumps({"case": label, "elapsed_s": round(elapsed, 3), "timings": result.get("timings"), "usage": result.get("usage")}), flush=True) - return result - - -filler = "".join(f"Record {i:06d}: amber bridge cedar delta elm forest granite harbor iris jade.\n" for i in range(15000)) -tokens, _ = request("/tokenize", {"content": filler}) -tokens = tokens["tokens"] - -if args.suite in {"decode", "all"}: - offset = 0 - for size in args.contexts: - prompt = tokens[offset:offset + size] - offset += size - assert len(prompt) == size, "Requested context exceeds the fixture" - data = {"prompt": prompt, "n_predict": 256, "ignore_eos": True, "temperature": 0, "seed": 42, "cache_prompt": True} - report(f"decode-{size}-first", data) - report(f"decode-{size}-warm", data) - -if args.suite in {"cache", "all"}: - prefix, _ = request("/detokenize", {"tokens": tokens[:4096]}) - other, _ = request("/detokenize", {"tokens": tokens[40000:44096]}) - prefix["content"] += "\nStable system code is prefix-a-731." - other["content"] += "\nStable system code is prefix-b-284." - for label, system, body in [("prefix-a", prefix["content"], tokens[10000:30000]), ("prefix-b", other["content"], tokens[45000:65000]), ("prefix-a-new", prefix["content"], tokens[70000:90000]), ("prefix-a-repeat", prefix["content"], tokens[70000:90000]), ("prefix-b-again", other["content"], tokens[95000:115000]), ("prefix-a-metadata", prefix["content"].replace("prefix-a-731", "prefix-c-592"), tokens[120000:140000])]: - text, _ = request("/detokenize", {"tokens": body}) - result = report(label, {"model": "qwen3.8-27b", "messages": [{"role": "system", "content": system}, {"role": "user", "content": text["content"] + "\nWhat is the stable system code? Reply with only it."}], "max_tokens": 32, "temperature": 0, "chat_template_kwargs": {"enable_thinking": False}}, "/v1/chat/completions") - assert system.rsplit(" ", 1)[-1].rstrip(".") in result["choices"][0]["message"]["content"], "Prefix restore lost the system fact" - if args.require_prefix_cache and label in {"prefix-a-new", "prefix-a-metadata"}: - assert result["timings"]["cache_n"] >= 4096, "Client switching discarded the base prefix" - -if args.suite in {"workloads", "all"}: - for task in ["Implement a bounded asynchronous work queue in Python using asyncio. Explain cancellation, backpressure, shutdown, race conditions and error handling. Include tests and a detailed explanation.", "Explain how database isolation levels interact with concurrent read-modify-write operations. Give concrete schedules, anomalies, and remedies for each level, with detailed examples."]: - template, _ = request("/apply-template", {"messages": [{"role": "system", "content": "You are a careful software engineer."}, {"role": "user", "content": task}], "chat_template_kwargs": {"enable_thinking": True, "reasoning_effort": "medium"}}) - report("workload-code" if "asyncio" in task else "workload-reasoning", {"prompt": template["prompt"], "n_predict": 512, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0, "seed": 42, "cache_prompt": True}) - -if args.suite in {"retrieval", "all"}: - for size in args.contexts: - facts = ["START password is otter-731.", "MIDDLE password is maple-284.", "END password is comet-965."] - segment = tokens[:max(1, (size - 256) // 2)] - text, _ = request("/detokenize", {"tokens": segment}) - messages = [{"role": "system", "content": "Find the three passwords in the records. Reply with only the passwords in START, MIDDLE, END order."}, {"role": "user", "content": facts[0] + "\n" + text["content"] + "\n" + facts[1] + "\n" + text["content"] + "\n" + facts[2]}] - template, _ = request("/apply-template", {"messages": messages, "chat_template_kwargs": {"enable_thinking": False}}) - result = report(f"retrieval-{size}", {"prompt": template["prompt"], "n_predict": 64, "temperature": 0, "cache_prompt": True}) - positions = [result["content"].find(fact.split()[-1].rstrip(".")) for fact in facts] - passed = all(pos >= 0 for pos in positions) and positions == sorted(positions) - print(json.dumps({"case": f"retrieval-{size}", "passed": passed}), flush=True) - if not passed: - raise SystemExit("Retrieval failed") - -if args.suite in {"vision", "all"}: - png = b"\x89PNG\r\n\x1a\n" - pixels = b"".join(b"\0" + b"".join(b"\xff\0\0" if 128 <= x < 384 and 128 <= y < 384 else b"\xff\xff\xff" for x in range(512)) for y in range(512)) - for kind, data in [(b"IHDR", struct.pack(">IIBBBBB", 512, 512, 8, 2, 0, 0, 0)), (b"IDAT", zlib.compress(pixels)), (b"IEND", b"")]: - png += struct.pack(">I", len(data)) + kind + data + struct.pack(">I", binascii.crc32(kind + data)) - result = report("vision", {"model": "qwen3.8-27b", "messages": [{"role": "user", "content": [{"type": "text", "text": "What color is the square? Reply with one word."}, {"type": "image_url", "image_url": {"url": "data:image/png;base64," + base64.b64encode(png).decode()}}]}], "max_tokens": 32, "temperature": 0, "chat_template_kwargs": {"enable_thinking": False}}, "/v1/chat/completions") - assert "red" in result["choices"][0]["message"]["content"].lower(), "Vision failed" diff --git a/tools/dashboard-container-test.py b/tools/dashboard-container-test.py deleted file mode 100644 index 0301a46353b7b9232ed94f2ce815edf768b3e94e..0000000000000000000000000000000000000000 --- a/tools/dashboard-container-test.py +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import concurrent.futures -import json -from pathlib import Path -import subprocess -import time -import urllib.request - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("container") - parser.add_argument("--url", default="http://127.0.0.1:7074") - parser.add_argument("--clients", type=int, default=40) - parser.add_argument("--requests", type=int, default=1000) - parser.add_argument("--output", type=Path) - parser.add_argument("--proxy-token-file", type=Path, required=True) - parser.add_argument("--stop", action="store_true") - args = parser.parse_args() - info = json.loads(subprocess.check_output(["podman", "inspect", args.container]))[0] - host = info["HostConfig"] - assert info["Config"]["User"].split(":")[0] not in {"", "0", "root"} - assert host["ReadonlyRootfs"] is True - assert "no-new-privileges" in host["SecurityOpt"] - assert not host["Privileged"] and not host["Devices"] - assert host["NetworkMode"] != "host" - assert all(m["Destination"] in ["/run/studio-host", "/run/secrets/dashboard-proxy.token", "/data"] for m in info["Mounts"]), info["Mounts"] - assert next(m for m in info["Mounts"] if m["Destination"] == "/run/studio-host")["RW"] is False - status = subprocess.check_output(["podman", "exec", args.container, "/bin/cat", "/proc/1/status"], text=True) - uid = next(line.split()[1:] for line in status.splitlines() if line.startswith("Uid:")) - assert all(int(value) != 0 for value in uid), uid - for field in ["CapEff", "CapBnd", "CapPrm", "CapAmb"]: - assert field + ":\t0000000000000000" in status, status - assert "NoNewPrivs:\t1" in status, status - mounts = subprocess.check_output(["podman", "exec", args.container, "/bin/cat", "/proc/1/mountinfo"], text=True) - root = next(line.split() for line in mounts.splitlines() if line.split()[4] == "/") - assert "ro" in root[5].split(","), root - assert subprocess.run(["podman", "exec", args.container, "/bin/touch", "/escaped"], capture_output=True).returncode != 0 - for denied in ["/var/lib/studio/nomad.token", "/run/podman/podman.sock", "/run/libvirt/libvirt-sock", "/dev/zfs", "/opt/studio/current"]: - assert subprocess.run(["podman", "exec", args.container, "/bin/test", "-e", denied], capture_output=True).returncode != 0, denied - pid = info["State"]["Pid"] - for namespace in ["net", "pid", "mnt"]: - assert Path(f"/proc/{pid}/ns/{namespace}").stat().st_ino != Path(f"/proc/self/ns/{namespace}").stat().st_ino - - headers = {"User-Name": "fixture", "User-Groups": "infra-admin", "Studio-Proxy-Token": args.proxy_token_file.read_text().strip()} - with urllib.request.urlopen(urllib.request.Request(args.url + "/api/host", headers=headers), timeout=10) as response: - machine = json.load(response) - expected_memory = next(int(line.split()[1]) * 1024 for line in Path("/proc/meminfo").read_text().splitlines() if line.startswith("MemTotal:")) - expected_cores = sum(line.startswith("cpu") and line[3:4].isdigit() for line in Path("/proc/stat").read_text().splitlines()) - assert machine["memory"] == expected_memory and machine["cores"] == expected_cores, machine - with urllib.request.urlopen(urllib.request.Request(args.url + "/api/live", headers=headers), timeout=10) as response: - for _ in range(20): - line = response.readline() - if line.startswith(b"data:"): - live = json.loads(line[5:]) - break - else: - raise AssertionError("host sample did not arrive") - assert 0 <= live["host"]["cpu"] <= 100 and live["host"]["memory"] > 0, live - assert live["host"]["arc"] is not None, live - - def request(_): - start = time.monotonic() - req = urllib.request.Request(args.url + "/api/storage", headers=headers) - with urllib.request.urlopen(req, timeout=10) as response: - assert json.load(response)["pool"]["state"] == "ONLINE" - return (time.monotonic() - start) * 1000 - - with concurrent.futures.ThreadPoolExecutor(max_workers=args.clients) as clients: - latency = sorted(clients.map(request, range(args.requests))) - result = {"nonroot_container": "passed", "readonly_rootfs": "passed", "zero_capabilities": "passed", - "no_new_privileges": "passed", "private_namespaces": "passed", "control_sockets_and_management_token_absent": "passed", - "host_identity_outside_container_quota": "passed", "host_live_sample": "passed", - "storage_requests": len(latency), "storage_clients": args.clients, - "storage_p95_ms": round(latency[int(len(latency) * .95)], 2), "storage_max_ms": round(latency[-1], 2)} - if args.stop: - req = urllib.request.Request(args.url + "/api/live", headers=headers) - with urllib.request.urlopen(req, timeout=10): - started = time.monotonic() - stopped = subprocess.run(["podman", "stop", "--time=8", args.container], capture_output=True, text=True, check=True) - elapsed = time.monotonic() - started - ended = json.loads(subprocess.check_output(["podman", "inspect", args.container]))[0]["State"] - assert ended["ExitCode"] == 0 and not ended["OOMKilled"], (ended, stopped.stderr) - assert elapsed < 8, elapsed - result["stop_with_active_stream"] = "passed" - result["stop_seconds"] = round(elapsed, 2) - if args.output: - args.output.write_text(json.dumps(result, indent=2) + "\n") - print(json.dumps(result)) - - -if __name__ == "__main__": - main() diff --git a/tools/local-ai-cache.md b/tools/local-ai-cache.md new file mode 100644 index 0000000000000000000000000000000000000000..78362dcd5318ef7d9daea77589dc6b6c45908bde --- /dev/null +++ b/tools/local-ai-cache.md @@ -0,0 +1,110 @@ +# Local AI context cache + +## Prepared configuration + +The current change configures GPU prefix caching, 8 GiB of vLLM shared-memory +offload, a 16 GiB NVMe hot copy, and a 256 GiB HDD pool. The container has a +26 GiB memory limit and a 10 GiB `/dev/shm`. + +`service/local-ai/cache` installs a vLLM secondary-tier plugin. Blocks are +written through to HDD and copied to SSD; HDD reads warm SSD. Each disk tier +evicts its least recently used unpinned blocks within its byte cap. Active +requests pin blocks; a saturated pinned tier skips writes. Direct I/O avoids +an additional unbounded filesystem data cache. Disk writes and promotions run +in background workers, outside scheduler lookups. + +Disk blocks use authenticated AES-GCM encryption with the service's Nomad +`cache_key`, owner-only files/directories, and atomic replacement. Restart +rebuilds the bounded index. Namespaces include the model manifest, vLLM and +plugin versions, key, KV layout, and parallel configuration. Corrupt blocks +fall back to another copy or become misses. Key rotation invalidates reuse; +old entries remain encrypted and subject to eviction. + +`config/site.pkl` owns the NVMe path (`/var/cache/studio/local-ai` by default). +The prepare hook creates it with the assigned service owner. HDD blocks use +the managed service dataset. Fresh previews use separate managed directories +for both disk tiers. + +Validation on 2026-10-09: the full image built and its installed plugin loaded; +nine disk-cache tests and two CPU-only tests against the installed vLLM +scheduler passed. These check encrypted persistence, quotas, pinning, +concurrent access, SSD promotion, attention/recurrent-group restoration into +shared RAM, and corrupt-promotion misses. Production/preview Pkl rendering, +isolated preparation, deployment checks, and six gateway tests passed. These +tests use synthetic data and do not establish GPU/hybrid-MTP restore timing. + +## Production activation — 2026-10-09 + +Release `0edf5af161627873` was deployed from local `main` without pushing. +Pre-switch backup: `20261009T214237Z-734b3d`. Nomad deployment and all route +checks passed. vLLM initialized the tiering connector and an 8.57 GB shared +region (rounded below the 8 GiB budget to whole blocks). + +An authenticated gateway probe completed an 8,000-token synthetic prompt. +Repeating it reused 6,656 tokens and completed in 3.439 seconds. The first +request took 37.596 seconds including first-use kernel compilation, so it is +not a steady-state prefill comparison. Both disk tiers held 66 encrypted +blocks totaling 1,912,086,528 bytes. Forced RAM/disk restore probes stopped +when the endpoint reported busy; end-to-end disk restore timing remains +unverified. + +## Zenith measurements — 2026-10-09 + +HyperQwen, vLLM 0.27.1, MTP with three draft tokens, 150,000-token +context, GPU prefix caching enabled. Production stayed running; no cache +connector was enabled or production configuration changed. + +`benchmark-local-ai-cache.py` sends synthetic token-ID prompts with one output +token. Each measured request had zero prefix-cache hits, one completed request, +and under 0.1 ms queue time. Server prefill time excludes HTTP overhead. + +| Prompt tokens | Server prefill seconds | +| --- | --- | +| 16,000 | 12.715, 12.557, 12.601 | +| 64,000 | 63.174, 63.704, 63.896 | +| 148,000 | 203.552, 204.495, 204.963 | + +Storage probes create disposable files filled with incompressible synthetic +bytes, then read shuffled 16 MiB files with 16 workers. Both writes and reads use +`O_DIRECT`; writes are individually fsynced. The HDD path is encrypted ZFS on +the four-disk RAIDZ1 pool, OpenZFS 2.4.4, `direct=standard`, 128 KiB records, +LZ4 compression. The SSD path is ext4 on the WD Blue SN5000 500 GB NVMe. + +| Storage | 1 GiB read seconds | 4 GiB read seconds | 4 GiB write seconds | +| --- | --- | --- | --- | +| HDD (`/srv/staging`) | 3.901, 3.686, 3.660, 3.931 | 15.736, 15.399, 16.848, 17.502 | 31.161, 30.165 | +| NVMe (`/var/tmp`) | 0.262, 0.258, 0.251, 0.258 | 0.937, 0.960, 0.933, 0.929 | 1.282, 1.342 | + +The final storage samples additionally checked physical-device read counters: +the 4 GiB probe read 4,427,632,640 bytes across the HDDs and exactly +4,294,967,296 bytes from NVMe. This verifies reads reached the drives instead +of being served from the host's data cache. + +Disk probes overlapped part of the prefill benchmark. These are storage-only +timings, not end-to-end KV restores. Actual restores additionally depend on +attention KV and recurrent-checkpoint bytes, cache-file layout, GPU transfers, +and any uncached suffix. A larger, fragmented cache and competing pool I/O can +increase HDD latency. The disposable files were removed after measurement. + +HDD reads are promising against the 64k and 148k uncached prefills. End-to-end +validation with HyperQwen's patched hybrid-state/MTP implementation remains +necessary; the storage timings above predate the bounded plugin. + +## Repeating the probes + +Confirm the target and ensure the AI endpoint is idle before inference probes. +The prefill script reads the existing backend key internally and prints only +timings and counters; it never prints prompt text, generated text, or keys. + +```sh +ssh -o BatchMode=yes root@10.0.0.1 \ + 'pid=$(pgrep -x vllm); nsenter -t "$pid" -n python3 - prefill "$pid" --hostname zenith' \ + < tools/benchmark-local-ai-cache.py +ssh -o BatchMode=yes root@10.0.0.1 \ + 'python3 - filesystem /srv/staging --hostname zenith' \ + < tools/benchmark-local-ai-cache.py +ssh -o BatchMode=yes root@10.0.0.1 \ + 'python3 - filesystem /var/tmp --hostname zenith' \ + < tools/benchmark-local-ai-cache.py +``` + diff --git a/tools/local-ai.md b/tools/local-ai.md deleted file mode 100644 index c392f7dd203efbd0622a34911ba9a1b86fa06975..0000000000000000000000000000000000000000 --- a/tools/local-ai.md +++ /dev/null @@ -1,247 +0,0 @@ -# Snow Globe local AI - -The first model is Qwen3.8-27B, using Unsloth's UD-Q4_K_M GGUF and F16 vision -projector. Both files, the base CUDA image, and the llama.cpp source are pinned. -The model runs entirely on the RTX 3090, with one inference slot and a 131,072-token context. -Codex compacts at 96,000 tokens to leave room for tool results and output. -The KV cache uses Q8; the single slot leaves more VRAM for vision and runtime -working memory. Additional requests queue so simultaneous generation cannot -reduce the active request's decode rate. An 8 GiB host-memory prompt cache -preserves evicted contexts between coding, reviewer, and helper requests. -The patched server also reserves a separate 2 GiB cache for prefixes before -the first user message, including system instructions and tool definitions. -A second checkpoint at 4,096 tokens retains the shared base when later workspace -metadata changes; shorter headers need only their complete checkpoint. -It captures these from actual requests; there are no copied Codex or Claude -prompts to update. Prefix states survive conversation-cache eviction and remain -available after restoration. Only complete token matches restore these states, -including the recurrent and MTP state. Prefix variants have their own bounded -LRU eviction; first use, unmatched tokens, and server restarts still need prefill. -The container has an 18 GiB RAM limit for the runtime and both caches. -Native MTP drafts up to three tokens, which the target model verifies. The draft -KV cache also uses Q8. Target weights, target KV precision, context length, and -reasoning defaults remain unchanged. The build replaces only the server library; -inference libraries and embedded UI assets come from the pinned base image. -Medium reasoning is enabled for coding, using Qwen's recommended thinking-mode -sampling parameters. This adds -reasoning tokens before answers; the earlier latency baseline disabled thinking. - -Models live in `/srv/clover/Media/AI/LLM/Qwen3.8-27B`, also visible on the Mac as -`/Volumes/clover/Media/AI/LLM/Qwen3.8-27B`. Download once on Zenith: - -```sh -python3 service/local-ai/download.py -``` - -The downloader resumes partial transfers, verifies size and SHA-256, and never -replaces an existing file that fails verification. No model download or GPU -service runs in rehearsal VMs. `STUDIO_LOCAL_AI=true` enables the physical host. - -## API and deployment - -The authenticated API exposes `/v1/responses`, `/v1/messages`, -`/v1/chat/completions`, and `/v1/models`. `/health` is public for deployment -checks. Use `Authorization: Bearer ` or Anthropic's `x-api-key` header. -`api_key` is a generated Nomad service secret; don't put it in Git or shell -arguments. - -The production endpoint is `https://ai.paperclover.net`. The -`STUDIO_LOCAL_AI=true` host environment in `nixos/zenith.nix` enables it. -The 3090 has room for one model instance. Stop any GPU preview before deploying -production; download the wrappers again when their endpoint changes. - -`gateway.py` uses Python's standard library and starts llama.cpp on a container -loopback port. Anthropic system-text blocks are combined into one system message; -Chat Completions pass through unchanged. For -Responses, it collects developer/system instructions at the start of the -prompt, maps tool namespaces to flat names, restores returned namespace -identities, and represents raw custom tools as JSON functions. The raw tool -grammar remains in the tool description. Built-in hosted tools such as OpenAI -web search aren't implemented; unsupported tool types return an explicit error. -Codex's hosted web search is disabled in the wrapper. - -## Coding clients - -Download `snow-codex` or `snow-claude` from the dashboard's **AI / MCP → AI** -tab. They are standalone Bash wrappers around the standard installed Codex and -Claude Code clients; client updates require no wrapper rebuild. Make each -download executable and place it on your PATH. The first run asks for the API -key from the same tab and saves it with mode 0600 under -`${XDG_CONFIG_HOME:-~/.config}/snowglobe-ai/api-key`. No checkout or patched -client is required. The downloads contain endpoint and model settings, never -the API key. The `ai` group opens AI / MCP and grants API-key access and downloads; -infrastructure administrators retain access. Role previews can view the page -and downloads, but cannot access the API key. - -`dashboard/ai/launch.sh` owns the wrapper template. The download endpoint -incorporates the live service hostname and canonical model catalog. -Codex starts with `--approve-for-me`, retaining its workspace sandbox and -automatic approval reviewer; Claude starts with `--permission-mode auto`, -running classification against the same local endpoint. Both use medium -reasoning. Claude's credential store is isolated from ordinary Claude login, -so using Snow Globe does not log the user's subscription out. - -Qwen's raw thinking is streamed as Responses reasoning text, including the -`content_index` required by stock Codex. To display it in the terminal: - -```sh -snow-codex -c show_raw_agent_reasoning=true -c hide_agent_reasoning=false -``` - -Reasoning summaries remain disabled because the model emits raw thinking rather -than a separate summary. `tools/verify-local-ai.py` checks actual Codex app-server -reasoning deltas arrive before the answer; stored reasoning in a completed -response alone does not establish that live display works. The production fix -was checked with stock Codex 0.161.0: 237 live reasoning deltas arrived before -the answer, the interactive terminal updated during thinking, both coding -fixtures passed, and automatic approval authorized its temporary write in -21.4 seconds. - -API and event-stream waits allow eight hours. Claude's byte-stream watchdog -has a 30-minute maximum, kept alive by ten-second server SSE pings. Stock -Codex 0.160.0 retains a hardcoded 90-second approval-review deadline that -provider settings cannot extend. The earlier patched build survived a -95-second forced review delay, but the shipped wrappers intentionally use -standard clients as requested. Queueing can therefore still cause a stock -Codex approval review to time out. - -Override the endpoint or key with `SNOWGLOBE_AI_URL` or `SNOWGLOBE_AI_KEY`. -Explicit client arguments can override wrapper defaults. Existing MCP servers, -hooks, and ordinary commands keep their normal network access. Model inference -and approval classification use Snow Globe; cloud plugins, Apps, automatic -memories, and nonessential client telemetry are disabled for this profile. - -## Verification - -```sh -python3 -m unittest discover -s service/local-ai -p 'test_*.py' -v -python3 tools/verify-local-ai.py --url https://YOUR-ENDPOINT -``` - -The downloadable wrappers were verified against production on 2026-10-05 -with stock Codex 0.160.0 and Claude Code 2.1.289. Both clients fixed the median -fixture and passed all eight unchanged tests, taking 189.3 and 209.4 seconds -while sharing the single slot. Stock Codex's actual automatic approval review -authorized an outside-workspace fixture write in 37.5 seconds. The temporary -file was removed. Download tests also exercised first-run key entry through a -terminal, mode 0600 storage, argument preservation, auth isolation, and catalog -cleanup. Unsigned, cross-origin, and view-as requests were denied; the returned -key authenticated successfully with the live model. - -The second command launches both actual clients concurrently in disposable -fixtures. Each must fix two median bugs, preserve input, pass four tests, and -leave the tests unchanged. It also exercises an actual Codex approval review and removes its temporary -file. JSONL evidence is saved in the printed temporary directory. An optional -`--approval-delay 95` probe demonstrates the stock reviewer deadline; it is -expected to fail with stock Codex. - -The deployed 128K configuration was checked on 2026-10-05 with a synthetic -122,265-token request. It recovered facts from the beginning, middle, and end -through a namespaced tool call. A 122,384-token continuation consumed the tool -result correctly and reused 122,332 cached tokens. A subsequent 122,409-token -request reused 122,359 tokens and generated 384 tokens at 24.21 tokens/second, -with a maximum streamed token gap of 45 ms. Thinking was disabled only for -these retrieval and throughput probes. The server measured 144.4 seconds of -cold prefill at 846.8 tokens/second. The cached stream's first token took 26.1 -seconds while coding clients shared the slot; that elapsed time includes -queueing. The allocation used 20.87 GiB VRAM and peaked at 9.79 GiB host RAM, -with zero OOM events, cgroup limit hits, or restarts throughout verification. - -Both actual clients then passed the eight unchanged coding fixture tests with -medium reasoning enabled. Codex took 285.4 seconds and Claude 343.5 seconds -while competing with the context probe. This checks real harness compatibility; -the simple retrieval probe does not establish coding quality at 122K tokens. -The actual approval-review check passed in 131.0 seconds, including a deliberate -95-second hold beyond stock Codex's deadline. Its authorized write and read-back -succeeded, and the temporary file was removed. - -On 2026-10-05, a synthetic 60,756-token Responses request recovered distinct -facts from the beginning, middle, and end through a namespaced tool call. A -60,875-token continuation consumed the tool result correctly, with 60,823 -cached tokens. A subsequent 60,900-token request generated 384 tokens at -30.75 tokens/second; the largest streamed token gap was 44 ms. These checks -used the earlier non-thinking preset. They validate context handling and one -simple retrieval task, not general coding quality at that length. - -Cold processing of roughly 60K input tokens took about a minute. End-to-end -latency also included waits behind other active requests. The live Codex -session separately reached about 48K input tokens at 33–34 generated -tokens/second, with recent-prefix prefill below one second. Its compaction -requests completed successfully. The earlier 64K limit and 48K compaction threshold -kept space for tool results, reasoning, and output without enlarging the cold -prefill workload further. - -The cache is bounded RAM storage, with no timed expiry or persistence across -server restarts. In an earlier cross-context probe, 15,912 of 15,916 tokens were -restored after two competing contexts, returning in 1.05 seconds versus 23.99 -seconds without the host cache. Distinct large contexts can still evict one -another. The Anthropic endpoint reports `cache_read_input_tokens` and remaining -uncached `input_tokens`; Responses reports `input_tokens_details.cached_tokens`. -There are no API cache charges. - -The previous two-slot allocation used about 20.9 GiB VRAM; one 64K slot used -about 18.4 GiB. [Qwen's model card](https://huggingface.co/Qwen/Qwen3.8-27B) describes -medium effort as the accuracy/speed balance and cautions that reduced reasoning -can increase total agent time through retries. - -## Ajax investigation - -As of 2026-10-05, [PewDiePie's official Ajax page](https://data.pewdiepie.com/) -says the model will release when ready. It describes an ablated Qwen3.5-9B -fine-tune for [Odysseus](https://github.com/odysseus-dev/odysseus), but does not -provide verified model weights or a model license to evaluate yet. - -When official weights exist, inspect the model card, license, file formats, -revision, and hashes before downloading. Use the known llama.cpp runtime with -GGUF data; don't execute model-repository code or load pickle-based weights. -Trial inference should have a read-only filesystem and model mount, no network, -no capabilities, no host credentials or private data, process/memory limits, -and only disposable scratch storage. Tool-use trials belong in a disposable VM -with synthetic files and credentials, separate from the personal coding profile. -Evaluate coding, tool arguments, prompt injection, unauthorized writes, and -attempted outbound requests against the base model. Passing those tests is -evidence about observed behavior, not proof that an unrestricted agent is safe. - -## Runtime tuning (2026-10-07) - -Measurements on the production RTX 3090, using 256 greedy decode tokens with the -same target weights, Q8 KV cache, and context size: - -| Input context | Stock runtime | MTP, three Q8 draft tokens | -| --- | ---: | ---: | -| 16,384 tokens | 39.1 tok/s | 92.0 tok/s | -| 65,536 tokens | 30.3 tok/s | 73.3 tok/s | - -These are warm synthetic decode rates, not a promise for every coding task. -The n-gram draft trial did not improve throughput. MTP's target verifies every -accepted draft token; it does not substitute a smaller model's answers. -Cold prefill remains about 1,200 tok/s at 16K and 990 tok/s at 64K. Caching avoids -processing unchanged prefixes, but new long inputs still pay that cost. - -The production checks exercise stock Codex and Claude Code editing the same -median fixture, all eight unchanged tests, Codex's automatic approval reviewer, -vision, and three-fact retrieval in a 121,816-token input. They establish these -paths remain functional, not a broad model-quality evaluation. The initial -MTP/prefix-cache check with stock Codex 0.161.0 and Claude Code 2.1.293 took 68.6 and 120.4 -seconds while sharing the slot. All 30 other production containers retained -their identities; the AI allocation recorded no OOM events, limit hits, or -restarts during verification. - -`tools/benchmark-local-ai.py` measures decode, natural coding/reasoning workloads, -client switching, changing system metadata, retrieval, and vision. Run inside -the production container, where `LLAMA_API_KEY` is already inherited, against -the backend on `http://127.0.0.1:8081`; it emits timings without prompts or keys: - -```sh -python3 /tmp/benchmark-local-ai.py --suite cache --require-prefix-cache -python3 /tmp/benchmark-local-ai.py --suite decode --contexts 16384 65536 -python3 /tmp/benchmark-local-ai.py --suite retrieval --contexts 122000 -python3 /tmp/benchmark-local-ai.py --suite vision -``` - -The prefix test alternates two system headers and long unrelated user histories, -then changes a trailing system fact. Both restores must reuse at least 4,096 -tokens and answer from the current header. Production retained 4,114 tokens -across client switches and 4,096 after the system fact changed, with the new -fact answered correctly. Repeating the conversation retained 24,135 tokens -and completed in 0.335 seconds, including generation. diff --git a/tools/mac-domains.py b/tools/mac-domains.py deleted file mode 100644 index 40cd7d594e4dca011fdae9ea83b783eab398434c..0000000000000000000000000000000000000000 --- a/tools/mac-domains.py +++ /dev/null @@ -1,248 +0,0 @@ -#!/usr/bin/env python3 -"""Route .studio.test through the rehearsal tunnels on this Mac; stop restores the prior settings.""" - -import argparse -import asyncio -import base64 -import hashlib -import json -import os -from pathlib import Path -import signal -import socket -import ssl -import subprocess -import sys -import time - -STATE = Path("/var/db/snowglobe-test") -RESOLVER = Path("/etc/resolver/studio.test") -HOSTS = Path("/etc/hosts") -KEYCHAIN = "/Library/Keychains/System.keychain" -FORWARDS = ((80, 27080), (443, 27443)) - - -def local_hosts(content): - lines = [] - aliases = [] - for line in content.splitlines(keepends=True): - names, _, comment = line.partition("#") - fields = names.split() - if len(fields) < 2 or not any(name.endswith(".studio.test") for name in fields[1:]): - lines.append(line) - continue - aliases.extend(name for name in fields[1:] if name.endswith(".studio.test")) - remaining = [name for name in fields[1:] if not name.endswith(".studio.test")] - if remaining: - lines.append(" ".join([fields[0], *remaining]) + (" #" + comment.rstrip() if comment else "") + "\n") - elif comment: - lines.append("#" + comment.rstrip() + "\n") - if aliases: - if lines and not lines[-1].endswith("\n"): - lines[-1] += "\n" - lines.append("127.0.0.1 " + " ".join(dict.fromkeys(aliases)) + "\n") - return "".join(lines) - - -class DNSRelay(asyncio.DatagramProtocol): - def __init__(self): - self.requests = set() - - def connection_made(self, transport): - self.transport = transport - - def datagram_received(self, data, address): - if len(data) < 12 or len(self.requests) >= 64: - return - request = asyncio.create_task(asyncio.wait_for(self.forward(data, address), timeout=3)) - self.requests.add(request) - request.add_done_callback(self.finished) - - def finished(self, request): - self.requests.discard(request) - if not request.cancelled(): - request.exception() - - async def forward(self, data, address): - writer = None - try: - reader, writer = await asyncio.open_connection("127.0.0.1", 53153) - writer.write(len(data).to_bytes(2, "big") + data) - await writer.drain() - length = int.from_bytes(await reader.readexactly(2), "big") - response = await reader.readexactly(length) - if len(response) >= 12 and response[:2] == data[:2]: - self.transport.sendto(response, address) - finally: - if writer: - writer.close() - - -async def relay(listeners): - async def connect(reader, writer, upstream): - peer_writer = None - try: - peer_reader, peer_writer = await asyncio.open_connection("127.0.0.1", upstream) - - async def pipe(source, destination): - while data := await source.read(65536): - destination.write(data) - await destination.drain() - if destination.can_write_eof(): - destination.write_eof() - - await asyncio.gather(pipe(reader, peer_writer), pipe(peer_reader, writer)) - except (OSError, asyncio.CancelledError): - pass - finally: - writer.close() - if peer_writer: - peer_writer.close() - - servers = [] - for listener, upstream in listeners: - servers.append(await asyncio.start_server( - lambda reader, writer, port=upstream: connect(reader, writer, port), sock=listener, - )) - transport, _ = await asyncio.get_running_loop().create_datagram_endpoint( - DNSRelay, local_addr=("127.0.0.1", 53153), - ) - try: - await asyncio.gather(*(server.serve_forever() for server in servers)) - finally: - transport.close() - - -def stop(): - record = STATE / "settings.json" - if not record.exists(): - print("Local test routing is already stopped.") - return - settings = json.loads(record.read_text()) - for name, saved in settings["files"].items(): - path = Path(name) - current = hashlib.sha256(path.read_bytes()).hexdigest() if path.exists() else None - before = hashlib.sha256(base64.b64decode(saved["before"])).hexdigest() if saved["before"] is not None else None - if current not in (before, saved["applied"]): - raise SystemExit(f"{path} changed after setup. Restore it using {record}; nothing was overwritten.") - if pid := settings.get("pid"): - command = subprocess.run(["ps", "-p", str(pid), "-o", "command="], capture_output=True, text=True).stdout - if str(Path(__file__).resolve()) + " --relay " in command: - try: - os.kill(pid, signal.SIGTERM) - except ProcessLookupError: - pass - for name, saved in settings["files"].items(): - path = Path(name) - if saved["before"] is None: - path.unlink(missing_ok=True) - else: - path.write_bytes(base64.b64decode(saved["before"])) - if settings.get("addedCA"): - subprocess.run(["security", "remove-trusted-cert", "-d", str(STATE / "ca.crt")], check=True) - subprocess.run(["security", "delete-certificate", "-Z", settings["sha1"], KEYCHAIN], check=True) - subprocess.run(["dscacheutil", "-flushcache"], check=True) - subprocess.run(["killall", "-HUP", "mDNSResponder"], check=True) - record.unlink() - print("Previous DNS, hosts entries, and certificate trust restored.") - - -def start(certificate): - if (STATE / "settings.json").exists(): - raise SystemExit("Local test routing is already configured. Run stop before starting it again.") - uid, gid = int(os.environ["SUDO_UID"]), int(os.environ["SUDO_GID"]) - if uid == 0: - raise SystemExit("Run this with sudo from your normal Mac account.") - for _, port in FORWARDS: - with socket.create_connection(("127.0.0.1", port), timeout=3): - pass - answer = subprocess.check_output(["dig", "+tcp", "@127.0.0.1", "-p", "53153", "globe.studio.test", "+short"]) - if answer.strip() != b"127.0.0.1": - raise SystemExit("The rehearsal DNS tunnel is unavailable. Start its SSH forwards first.") - context = ssl.create_default_context(cafile=str(certificate)) - with socket.create_connection(("127.0.0.1", 27443), timeout=5) as upstream: - with context.wrap_socket(upstream, server_hostname="globe.studio.test"): - pass - for port, _ in FORWARDS: - with socket.socket() as probe: - probe.bind(("127.0.0.1", port)) - with socket.socket(type=socket.SOCK_DGRAM) as probe: - probe.bind(("127.0.0.1", 53153)) - pem = certificate.read_bytes() - der = subprocess.check_output(["openssl", "x509", "-outform", "DER"], input=pem) - existing = subprocess.run(["security", "find-certificate", "-a", "-p", KEYCHAIN], capture_output=True).stdout - sha1 = hashlib.sha1(der).hexdigest().upper() - installed = False - for part in existing.split(b"-----END CERTIFICATE-----")[:-1]: - found = subprocess.check_output(["openssl", "x509", "-outform", "DER"], input=part + b"-----END CERTIFICATE-----\n") - installed = installed or found == der - trusted = subprocess.run(["security", "verify-cert", "-c", str(certificate), "-p", "ssl"], capture_output=True).returncode == 0 - if installed and not trusted: - raise SystemExit("The rehearsal CA has existing custom trust settings. Enable its SSL trust in Keychain Access before setup.") - STATE.mkdir(mode=0o700, exist_ok=True) - os.chmod(STATE, 0o700) - (STATE / "ca.crt").write_bytes(pem) - replacements = {HOSTS: local_hosts(HOSTS.read_text()).encode(), - RESOLVER: b"nameserver 127.0.0.1\nport 53153\n"} - settings = {"files": {}, "addedCA": False, "sha1": sha1} - for path, content in replacements.items(): - settings["files"][str(path)] = { - "before": base64.b64encode(path.read_bytes()).decode() if path.exists() else None, - "applied": hashlib.sha256(content).hexdigest(), - } - record = STATE / "settings.json" - record.write_text(json.dumps(settings, indent=2) + "\n") - os.chmod(record, 0o600) - try: - for path, content in replacements.items(): - path.parent.mkdir(exist_ok=True) - path.write_bytes(content) - os.chmod(path, 0o644) - if not trusted: - subprocess.run(["security", "add-trusted-cert", "-d", "-r", "trustRoot", "-p", "ssl", "-k", KEYCHAIN, str(STATE / "ca.crt")], check=True) - settings["addedCA"] = True - record.write_text(json.dumps(settings, indent=2) + "\n") - with (STATE / "relay.log").open("ab") as log: - process = subprocess.Popen([sys.executable, str(Path(__file__).resolve()), "--relay", str(uid), str(gid)], - stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) - settings["pid"] = process.pid - record.write_text(json.dumps(settings, indent=2) + "\n") - time.sleep(0.3) - if process.poll() is not None: - raise RuntimeError(f"The HTTPS relay did not start. See {STATE / 'relay.log'}.") - subprocess.run(["dscacheutil", "-flushcache"], check=True) - subprocess.run(["killall", "-HUP", "mDNSResponder"], check=True) - subprocess.run(["curl", "--max-time", "8", "--fail", "--silent", "--show-error", - "--output", "/dev/null", "https://globe.studio.test/"], check=True) - except BaseException: - stop() - raise - print("Local .studio.test routing is active. Open https://globe.studio.test/.") - - -if __name__ == "__main__": - if sys.platform != "darwin" or os.geteuid() != 0: - raise SystemExit("This setup needs sudo on the Mac for its resolver, certificate trust, and loopback ports 80/443.") - if len(sys.argv) == 4 and sys.argv[1] == "--relay": - listeners = [] - for port, upstream in FORWARDS: - listener = socket.socket() - listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - listener.bind(("127.0.0.1", port)) - listener.listen() - listeners.append((listener, upstream)) - os.setgroups([]) - os.setgid(int(sys.argv[3])) - os.setuid(int(sys.argv[2])) - asyncio.run(relay(listeners)) - else: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("action", choices=("start", "stop")) - parser.add_argument("certificate", type=Path, nargs="?") - args = parser.parse_args() - if args.action == "stop": - stop() - elif args.certificate: - start(args.certificate.resolve()) - else: - parser.error("start needs the rehearsal's public CA certificate") diff --git a/tools/verify-local-ai.py b/tools/verify-local-ai.py deleted file mode 100644 index 75d7779fb44ae941648e396cb536575ce8110ab2..0000000000000000000000000000000000000000 --- a/tools/verify-local-ai.py +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env python3 -"""Exercise both real coding clients and Codex's automatic approval reviewer.""" -import argparse -import concurrent.futures -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -import json -import os -import selectors -from pathlib import Path -import subprocess -import tempfile -import threading -import time -import urllib.error -import urllib.request - -BUG = '''def summarize(values): - ordered = sorted(values) - return {"count": len(values), "median": ordered[len(ordered) // 2]} -''' -TESTS = '''import unittest -from stats import summarize -class SummaryTests(unittest.TestCase): - def test_odd(self): - self.assertEqual(summarize([9, 1, 5]), {"count": 3, "median": 5}) - def test_even(self): - self.assertEqual(summarize([9, 1, 5, 3]), {"count": 4, "median": 4}) - def test_empty(self): - self.assertEqual(summarize([]), {"count": 0, "median": None}) - def test_input_unchanged(self): - data = [9, 1, 5] - summarize(data) - self.assertEqual(data, [9, 1, 5]) -''' -PROMPT = "Fix stats.py so median is correct for even-length lists and empty input returns a null median. Preserve the caller's list. Run python3 -m unittest -v. Do not change tests. Keep the implementation small." - - -def run_client(root, provider, prompt, extra_env=None, label=None): - env = os.environ.copy() - env["CODEX_HOME"] = str(root / "codex-home") - env.update(extra_env or {}) - args = ["-p", prompt, "--output-format", "stream-json", "--verbose", "--max-turns", "12"] if provider == "claude" else ["exec", "--json", "--skip-git-repo-check", prompt] - started = time.monotonic() - log = root / ((label or provider) + ".jsonl") - with log.open("w") as out: - process = subprocess.run([str(Path(os.environ.get("SNOWGLOBE_AI_BIN_DIR", Path.home() / ".local/bin")) / ("snow-" + provider)), *args], cwd=root / provider, env=env, stdin=subprocess.DEVNULL, stdout=out, stderr=subprocess.STDOUT, timeout=8 * 60 * 60) - assert process.returncode == 0, f"{provider} failed; inspect {log}" - print(f"{label or provider}: completed in {time.monotonic() - started:.1f}s", flush=True) - return log - - -def verify_reasoning(root, endpoint): - env = os.environ.copy() - env.update(CODEX_HOME=str(root / "codex-home"), SNOWGLOBE_AI_URL=endpoint) - wrapper = Path(os.environ.get("SNOWGLOBE_AI_BIN_DIR", Path.home() / ".local/bin")) / "snow-codex" - with (root / "codex-reasoning-stderr.log").open("w") as errors, (root / "codex-reasoning.jsonl").open("wb") as log: - process = subprocess.Popen([str(wrapper), "-c", "show_raw_agent_reasoning=true", "-c", "hide_agent_reasoning=false", "app-server"], cwd=root / "codex", env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=errors, bufsize=0) - with selectors.DefaultSelector() as selector: - selector.register(process.stdout, selectors.EVENT_READ) - process.stdin.write(json.dumps({"id": 0, "method": "initialize", "params": {"clientInfo": {"name": "snowglobe_reasoning_test", "version": "1.0"}}}).encode() + b"\n") - reasoning = 0 - answer = "" - deadline = time.monotonic() + 240 - try: - while time.monotonic() < deadline: - if not selector.select(1): - continue - line = process.stdout.readline() - assert line, "Codex app-server closed before completing the reasoning test" - log.write(line) - event = json.loads(line) - assert "error" not in event, event.get("error") - if event.get("id") == 0: - process.stdin.write(b'{"method":"initialized","params":{}}\n') - process.stdin.write(json.dumps({"id": 1, "method": "thread/start", "params": {"cwd": str(root / "codex")}}).encode() + b"\n") - elif event.get("id") == 1: - process.stdin.write(json.dumps({"id": 2, "method": "turn/start", "params": {"threadId": event["result"]["thread"]["id"], "input": [{"type": "text", "text": "Do not use tools. Find the smallest positive integer divisible by every integer from 1 through 12. Give the integer and a concise justification."}]}}).encode() + b"\n") - method = event.get("method") - if method == "item/reasoning/textDelta": - reasoning += bool(event["params"]["delta"]) - elif method == "item/agentMessage/delta": - assert reasoning > 0, "Codex received no live reasoning before the answer" - answer += event["params"]["delta"] - elif method == "turn/completed": - assert event["params"]["turn"]["status"] == "completed" - assert reasoning > 0 and "27720" in answer.replace(",", ""), "Reasoning fixture failed" - print(f"Codex streamed {reasoning} reasoning deltas before its answer", flush=True) - break - else: - raise TimeoutError("Codex reasoning stream timed out") - finally: - process.stdin.close() - try: - process.wait(timeout=5) - except subprocess.TimeoutExpired: - process.terminate() - process.wait(timeout=5) - - -def review_delay_test(root, delay, endpoint): - delayed = threading.Event() - class Proxy(BaseHTTPRequestHandler): - def log_message(self, *args): - pass - def do_POST(self): - body = self.rfile.read(int(self.headers.get("Content-Length", "0"))) - data = json.loads(body) - properties = data.get("text", {}).get("format", {}).get("schema", {}).get("properties", {}) - if "risk_level" in properties and "outcome" in properties and not delayed.is_set(): - delayed.set() - print(f"Holding an actual approval-review request for {delay}s", flush=True) - time.sleep(delay) - headers = {k: v for k, v in self.headers.items() if k.lower() not in {"host", "connection", "content-length", "accept-encoding"}} - try: - with urllib.request.urlopen(urllib.request.Request(endpoint + self.path, data=body, headers=headers), timeout=28800) as response: - self.send_response(response.status) - self.send_header("Content-Type", response.headers.get("Content-Type", "application/json")) - self.send_header("Connection", "close") - self.end_headers() - while chunk := response.read1(65536): - self.wfile.write(chunk) - self.wfile.flush() - except (BrokenPipeError, ConnectionResetError): - pass - except urllib.error.HTTPError as error: - self.send_response(error.code) - self.end_headers() - self.wfile.write(error.read()) - except (urllib.error.URLError, OSError): - self.send_error(502) - server = ThreadingHTTPServer(("127.0.0.1", 0), Proxy) - threading.Thread(target=server.serve_forever, daemon=True).start() - target_dir = Path.home() / ".local/share/snowglobe-ai/evals" - target_dir.mkdir(parents=True, exist_ok=True) - target = target_dir / (root.name + ".txt") - prompt = f"Use exec_command to create {target} containing exactly approval works. This path is outside the writable workspace. Request sandbox_permissions require_escalated with a justification: this temporary fixture write is explicitly authorized. Then read the file back and report its contents. Do not bypass a rejected action." - try: - run_client(root, "codex", prompt, {"SNOWGLOBE_AI_URL": f"http://127.0.0.1:{server.server_port}"}, "codex-delayed-approval") - assert delayed.is_set(), "The client did not make an approval-review request." - assert target.read_text() == "approval works" - print("Automatic approval authorized the fixture write", flush=True) - finally: - server.shutdown() - server.server_close() - target.unlink(missing_ok=True) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Test Snow Globe with real coding clients") - parser.add_argument("--approval-delay", type=int, default=0) - parser.add_argument("--url", default=os.environ.get("SNOWGLOBE_AI_URL")) - args = parser.parse_args() - if not args.url: - parser.error("Provide --url with the Snow Globe endpoint.") - root = Path(tempfile.mkdtemp(prefix="snowglobe-ai-verify-")) - (root / "codex-home").mkdir() - print(f"Evidence: {root}", flush=True) - for provider in ("claude", "codex"): - work = root / provider - work.mkdir() - (work / "stats.py").write_text(BUG) - (work / "test_stats.py").write_text(TESTS) - with concurrent.futures.ThreadPoolExecutor(2) as pool: - futures = [pool.submit(run_client, root, provider, PROMPT) for provider in ("claude", "codex")] - for future in futures: - future.result() - for provider in ("claude", "codex"): - work = root / provider - assert (work / "test_stats.py").read_text() == TESTS, f"{provider} changed the tests" - subprocess.run(["python3", "-m", "unittest", "-v"], cwd=work, check=True) - assert args.approval_delay >= 0, "Delay must be nonnegative." - verify_reasoning(root, args.url) - review_delay_test(root, args.approval_delay, args.url) - print("Real coding, reasoning, and approval checks passed", flush=True)