| 1 | amends "../../config/Service.pkl" |
| 2 | |
| 3 | import "../../config/site.pkl" as site |
| 4 | import "pkl:json" |
| 5 | |
| 6 | local model = (new json.Parser {}).parse(read("catalog.json")).models[0] |
| 7 | local weights = (new json.Parser {}).parse(read("models.json")) |
| 8 | |
| 9 | meta { name = "Local AI"; launcher = false; access = "admin" } |
| 10 | enabled = (read?("env:STUDIO_LOCAL_AI") ?? "false") == "true" |
| 11 | rollout = "simple" |
| 12 | stageIsolation = "fresh" |
| 13 | prepare = "prepare.py" |
| 14 | healthyDeadline = "30m" |
| 15 | healthRestartGrace = "30m" |
| 16 | secrets { ["api_key"] {}; ["cache_key"] {} } |
| 17 | |
| 18 | container { |
| 19 | build = "." |
| 20 | cpu = 4000 |
| 21 | memory = 26624 |
| 22 | entrypoint = "/usr/bin/python3.12" |
| 23 | http { |
| 24 | containerPort = 8080 |
| 25 | subdomain = "ai" |
| 26 | checkPath = "/health" |
| 27 | } |
| 28 | volumes { |
| 29 | ["/config/gateway.py"] { config = "gateway.py" } |
| 30 | ["/cache/hdd"] {} |
| 31 | ["/cache/ssd"] { src = if (site.preview) null else site.localAiSsdCache } |
| 32 | ["/models"] { src = "\(site.mediaRoot)/AI/LLM/\(weights.directory)"; readOnly = true } |
| 33 | // The host's NVIDIA libraries link to immutable Nix store paths. |
| 34 | ["/nvidia/lib"] { src = "/run/opengl-driver/lib"; readOnly = true } |
| 35 | ["/nix/store"] { src = "/nix/store"; readOnly = true } |
| 36 | } |
| 37 | devices { "/dev/nvidia0"; "/dev/nvidiactl"; "/dev/nvidia-uvm" } |
| 38 | tmpfs { "/dev/shm:rw,size=10g" } |
| 39 | env { |
| 40 | ["LD_LIBRARY_PATH"] = "/nvidia/lib:/usr/local/cuda/lib64" |
| 41 | ["VLLM_API_KEY"] = "${secret.own.api_key}" |
| 42 | ["STUDIO_AI_CACHE_KEY"] = "${secret.own.cache_key}" |
| 43 | ["PYTHONHASHSEED"] = "0" |
| 44 | ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:False" |
| 45 | ["MODEL"] = "/models" |
| 46 | ["CTX"] = "long" |
| 47 | ["MAX_LEN"] = model.context_window.toString() |
| 48 | ["SPEC"] = "mtp" |
| 49 | ["DRAFT_TOKENS"] = "3" |
| 50 | ["PREFIX_CACHE"] = "1" |
| 51 | ["TOOLS"] = "1" |
| 52 | ["VISION"] = "1" |
| 53 | ["VISION_OFFLOAD"] = "1" |
| 54 | ["GPU_UTIL"] = "0.93" |
| 55 | ["MAX_SEQS"] = "8" |
| 56 | ["API_SERVERS"] = "1" |
| 57 | ["HOST"] = "127.0.0.1" |
| 58 | ["PORT"] = "8081" |
| 59 | ["VLLM_MAMBA_ALIGN_KEEP_CHECKPOINTS"] = "1" |
| 60 | ["FLASHINFER_DISABLE_VERSION_CHECK"] = "1" |
| 61 | ["EXTRA_ARGS"] = "--kv-transfer-config {\"kv_connector\":\"OffloadingConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"spec_name\":\"TieringOffloadingSpec\",\"cpu_bytes_to_use\":8589934592,\"offload_prompt_only\":false,\"secondary_tiers\":[{\"type\":\"studio_disk\",\"ssd_dir\":\"/cache/ssd\",\"hdd_dir\":\"/cache/hdd\",\"ssd_bytes\":17179869184,\"hdd_bytes\":274877906944}]}} --limit-mm-per-prompt {\"image\":{\"count\":10}} --default-chat-template-kwargs {\"enable_thinking\":true,\"reasoning_effort\":\"\(model.default_reasoning_level)\"} --generation-config vllm --override-generation-config {\"temperature\":1.0,\"top_p\":0.95,\"top_k\":20,\"min_p\":0,\"presence_penalty\":0,\"repetition_penalty\":1.0} --disable-uvicorn-access-log" |
| 62 | } |
| 63 | args { "/config/gateway.py" } |
| 64 | } |