1{ config, lib, pkgs, ... }:
2let
3 dashboard = pkgs.callPackage ./dashboard.nix { };
4 dashboardPort = "7072";
5 internalPort = 8448;
6 proxyToken = "/var/lib/studio/dashboard-proxy.token";
7 hostTools = lib.fileset.toSource {
8 root = ../.;
9 fileset = lib.fileset.unions [ ../tools/dashboard-host.py ../tools/shale-labels.py ../config/shale.json ../dashboard/src/host-protocol.json ../tools/vms.py ../tools/vm-ssh.py ../tools/dashboard-run.py ../tools/release.py ../guest/protocol.json ];
10 };
11 nativePkl = pkgs.callPackage ./pkl.nix { };
12 screenPython = pkgs.python3.withPackages (packages: [ packages.pygobject3 packages.gst-python ]);
13 screenPlugins = (with pkgs.gst_all_1; [ (lib.getLib gstreamer) gst-plugins-base gst-plugins-good gst-plugins-bad gst-plugins-ugly gst-plugins-rs ]) ++ [ (lib.getLib pkgs.libnice) ];
14 secureOvmf = pkgs.OVMF.override {
15 secureBoot = true;
16 msVarsTemplate = true;
17 tpmSupport = true;
18 };
19 secureAavmf = pkgs.runCommand "aavmf-secure-2025.02" {
20 src = pkgs.fetchurl {
21 url = "https://deb.debian.org/debian/pool/main/e/edk2/qemu-efi-aarch64_2025.02-8+deb13u1_all.deb";
22 sha256 = "a00b2411a79c8aeafd95a7c868ac3cd1aab592f1af9fff965f02d81a625276ed";
23 };
24 nativeBuildInputs = [ pkgs.dpkg ];
25 } ''
26 dpkg-deb -x "$src" unpacked
27 mkdir -p "$out"
28 install -m444 unpacked/usr/share/AAVMF/AAVMF_CODE.secboot.fd "$out/code.fd"
29 install -m444 unpacked/usr/share/AAVMF/AAVMF_VARS.ms.fd "$out/vars.fd"
30 '';
31 windowsArmQemu = (pkgs.qemu.override {
32 minimal = true;
33 vncSupport = true;
34 tpmSupport = true;
35 enableBlobs = true;
36 hostCpuTargets = [ "aarch64-softmmu" ];
37 }).overrideAttrs (old: {
38 version = "11.1.2";
39 src = pkgs.fetchurl {
40 url = "https://download.qemu.org/qemu-11.1.2.tar.xz";
41 sha256 = "05l0hfx3kdkl719xqdps2nqcb5h2zn79nmri68qvw65vwj0mc6vk";
42 };
43 patches = [];
44 nativeBuildInputs = old.nativeBuildInputs ++ (with pkgs.python3Packages; [ qemu-qmp setuptools wheel pip ]);
45 buildInputs = old.buildInputs ++ [ pkgs.pixman pkgs.libtasn1 ];
46 });
47in
48{
49 nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ "nomad" "nvidia-x11" "nvidia-kernel-modules" ];
50
51 networking.firewall = {
52 enable = true;
53 allowedTCPPorts = [ 22 ];
54 allowedUDPPortRanges = [{ from = 50000; to = 50031; }];
55 interfaces.podman0.allowedTCPPorts = [ 443 internalPort 10428 15432 18428 ];
56 interfaces.podman0.allowedTCPPortRanges = [{ from = 20000; to = 32000; }];
57 interfaces.tailscale0.allowedTCPPorts = [ 80 443 445 8081 8082 ];
58 };
59
60 services.openssh.enable = true;
61 services.caddy = {
62 enable = true;
63 package = pkgs.caddy.withPlugins {
64 plugins = [ "github.com/caddyserver/replace-response@v0.0.0-20250618171559-80962887e4c6" ];
65 hash = "sha256-2tcvl9ZDpe7SAy3tZCPNwmw0KoIIakqggVaRtVAg9R0=";
66 };
67 extraConfig = "import /var/lib/caddy/routes.caddy";
68 virtualHosts = {
69 ":8082".extraConfig = "reverse_proxy 127.0.0.1:4646";
70 };
71 };
72 systemd.services.caddy.environment = {
73 OTEL_SERVICE_NAME = "studio-caddy";
74 OTEL_EXPORTER_OTLP_TRACES_ENDPOINT = "http://127.0.0.1:10428/insert/opentelemetry/v1/traces";
75 OTEL_TRACES_SAMPLER = "parentbased_traceidratio";
76 OTEL_TRACES_SAMPLER_ARG = "0.25";
77 };
78
79 boot.supportedFilesystems = [ "zfs" ];
80 virtualisation.podman.enable = true;
81 virtualisation.libvirtd.enable = true;
82 virtualisation.libvirtd.qemu.swtpm.enable = true;
83 systemd.services.podman.environment.LOGGING = "--log-level=warn";
84
85 services.nomad = {
86 enable = true;
87 enableDocker = false;
88 dropPrivileges = false;
89 extraSettingsPlugins = [ pkgs.nomad-driver-podman ];
90 extraPackages = [ pkgs.cni-plugins ];
91 settings = {
92 datacenter = "clover";
93 bind_addr = "0.0.0.0";
94 addresses.http = "127.0.0.1";
95 advertise.http = "127.0.0.1";
96 server = {
97 enabled = true;
98 bootstrap_expect = 1;
99 };
100 acl.enabled = true;
101 client = {
102 enabled = true;
103 host_network.loopback.cidr = "127.0.0.1/8";
104 };
105 plugin.nomad-driver-podman.config.socket_path = "unix:///run/podman/podman.sock";
106 telemetry.collection_interval = "15s";
107 ui.enabled = true;
108 };
109 };
110
111 systemd.services.studio-log-shipper = {
112 wantedBy = [ "multi-user.target" ];
113 after = [ "nomad.service" ];
114 wants = [ "nomad.service" ];
115 unitConfig.ConditionPathExists = "/opt/studio/current/tools/log-shipper.py";
116 path = [ pkgs.systemd ];
117 serviceConfig = {
118 ExecStart = "${pkgs.python3}/bin/python3 /opt/studio/current/tools/log-shipper.py";
119 Restart = "always";
120 RestartSec = 5;
121 };
122 };
123
124 systemd.services.nomad = {
125 after = [ "podman.socket" ];
126 requires = [ "podman.socket" ];
127 };
128
129 systemd.services.studio-router = {
130 wantedBy = [ "multi-user.target" ];
131 wants = [ "nomad.service" "caddy.service" ];
132 after = [ "nomad.service" "caddy.service" ];
133 unitConfig.ConditionPathExists = "/var/lib/studio/router.token";
134 environment = {
135 STUDIO_DOMAIN = config.environment.variables.STUDIO_DOMAIN;
136 STUDIO_DASHBOARD_PORT = dashboardPort;
137 STUDIO_INTERNAL_PORT = toString internalPort;
138 STUDIO_PROXY_TOKEN_FILE = proxyToken;
139 };
140 serviceConfig = {
141 ExecStart = "${pkgs.python3}/bin/python3 ${../tools/router.py}";
142 Restart = "always";
143 RestartSec = 5;
144 };
145 };
146
147 users.groups.studio-dashboard = { };
148 users.users.studio-dashboard = {
149 isSystemUser = true;
150 group = "studio-dashboard";
151 };
152 systemd.services.studio-vm-screen = {
153 wantedBy = [ "multi-user.target" ];
154 wants = [ "studio-host.service" ];
155 after = [ "studio-host.service" ];
156 environment = {
157 GST_PLUGIN_SYSTEM_PATH_1_0 = lib.makeSearchPath "lib/gstreamer-1.0" screenPlugins;
158 GI_TYPELIB_PATH = lib.makeSearchPath "lib/girepository-1.0" (map lib.getLib ([ pkgs.gst_all_1.gstreamer ] ++ screenPlugins));
159 LD_LIBRARY_PATH = "/run/opengl-driver/lib";
160 STUDIO_VM_GUEST_PROTOCOL = toString ../guest/protocol.json;
161 };
162 serviceConfig = {
163 ExecStart = "${screenPython}/bin/python3 ${../tools/vm-screen.py}";
164 User = "studio-dashboard";
165 Group = "studio-dashboard";
166 RuntimeDirectory = "studio-vm-screen";
167 RuntimeDirectoryMode = "0700";
168 RuntimeDirectoryPreserve = "yes";
169 UMask = "0077";
170 ProtectSystem = "strict";
171 ProtectHome = true;
172 PrivateTmp = true;
173 NoNewPrivileges = true;
174 CapabilityBoundingSet = "";
175 MemoryMax = "2G";
176 TasksMax = 128;
177 DevicePolicy = "closed";
178 DeviceAllow = lib.optional (builtins.elem "nvidia" config.services.xserver.videoDrivers) "char-nvidia* rw";
179 RestrictAddressFamilies = [ "AF_UNIX" "AF_INET" "AF_INET6" "AF_NETLINK" ];
180 Restart = "always";
181 RestartSec = 2;
182 };
183 };
184 systemd.services.studio-host = {
185 wantedBy = [ "multi-user.target" ];
186 wants = [ "podman.socket" ];
187 after = [ "podman.socket" ];
188 path = [ pkgs.zfs pkgs.util-linux pkgs.python3 pkgs.libvirt pkgs.qemu pkgs.gptfdisk pkgs.cdrkit pkgs.openssl pkgs.openssh pkgs.podman pkgs.nomad pkgs.systemd pkgs.rsync pkgs.nixos-rebuild nativePkl ]
189 ++ lib.optional (builtins.elem "nvidia" config.services.xserver.videoDrivers) (lib.getBin config.hardware.nvidia.package);
190 environment = (lib.filterAttrs (name: _: lib.hasPrefix "STUDIO_" name) config.environment.variables) // {
191 STUDIO_POOL = lib.attrByPath [ "STUDIO_POOL" ] "studio-demo" config.environment.variables;
192 STUDIO_DASHBOARD_USER = "studio-dashboard";
193 STUDIO_VM_ACCEL = lib.attrByPath [ "STUDIO_VM_ACCEL" ] "kvm" config.environment.variables;
194 };
195 serviceConfig = {
196 ExecStart = "${pkgs.python3}/bin/python3 ${hostTools}/tools/dashboard-host.py";
197 Type = "notify";
198 Group = "studio-dashboard";
199 StateDirectory = "studio/host";
200 StateDirectoryMode = "0700";
201 RuntimeDirectory = "studio-host";
202 RuntimeDirectoryMode = "0750";
203 RuntimeDirectoryPreserve = "yes";
204 UMask = "0077";
205 ProtectSystem = "strict";
206 ReadWritePaths = [ "/srv/vm" "-/srv/prod/shale/data" "/var/lib/studio/dashboard" ] ++ lib.optionals (config.networking.hostName != "clover-demo") [ "/srv/clover/Media/vm" ];
207 ProtectHome = true;
208 PrivateTmp = true;
209 NoNewPrivileges = true;
210 CapabilityBoundingSet = [ "CAP_SYS_ADMIN" "CAP_DAC_READ_SEARCH" "CAP_DAC_OVERRIDE" ];
211 MemoryMax = "256M";
212 TasksMax = 64;
213 DevicePolicy = "closed";
214 DeviceAllow = [ "/dev/zfs rw" "char-pts rw" ] ++ lib.optional (builtins.elem "nvidia" config.services.xserver.videoDrivers) "char-nvidia* rw";
215 RestrictAddressFamilies = [ "AF_UNIX" "AF_INET" ];
216 IPAddressDeny = "any";
217 IPAddressAllow = [ "127.0.0.1/32" ];
218 Restart = "always";
219 RestartSec = 2;
220 };
221 };
222 systemd.services.studio-dashboard = let
223 environment = (lib.filterAttrs (name: _: lib.hasPrefix "STUDIO_" name) config.environment.variables) // {
224 STUDIO_DATA_DIR = "/data";
225 STUDIO_PROXY_TOKEN_FILE = "/run/secrets/dashboard-proxy.token";
226 STUDIO_NOMAD_TOKEN_FILE = "/run/secrets/dashboard-nomad.token";
227 STUDIO_CA_BUNDLE = "/run/secrets/ca-bundle.crt";
228 STUDIO_INDEX_DIR = "/data/index";
229 STUDIO_INTERNAL_URL = "https://dashboard.internal.${config.environment.variables.STUDIO_DOMAIN}:${toString internalPort}";
230 STUDIO_FILES_URL = "https://file.${config.environment.variables.STUDIO_DOMAIN}";
231 STUDIO_AUTH_REQUIRED = "1";
232 STUDIO_AUTH_RP_ID = "auth.${config.environment.variables.STUDIO_DOMAIN}";
233 STUDIO_FILE_ORIGIN = "https://file.${config.environment.variables.STUDIO_DOMAIN}";
234 STUDIO_PUBLIC_ORIGIN = "https://snowglobe.${config.environment.variables.STUDIO_DOMAIN}";
235 STUDIO_JELLYFIN_URL = "https://jelly.${config.environment.variables.STUDIO_DOMAIN}";
236 STUDIO_SHALE_URL = "https://shale.${config.environment.variables.STUDIO_DOMAIN}";
237 STUDIO_PUBLISHED_ROOT = "/srv/clover/Published";
238 STUDIO_DOMAIN = config.environment.variables.STUDIO_DOMAIN;
239 STUDIO_POOL = lib.attrByPath [ "STUDIO_POOL" ] "studio-demo" config.environment.variables;
240 STUDIO_NODE_NAME = config.networking.hostName;
241 STUDIO_MEDIA_READ_ONLY = lib.attrByPath [ "STUDIO_MEDIA_READ_ONLY" ] "false" config.environment.variables;
242 STUDIO_CLOVER_READ_ONLY = lib.attrByPath [ "STUDIO_CLOVER_READ_ONLY" ] "false" config.environment.variables;
243 };
244 in {
245 inherit environment;
246 wantedBy = [ "multi-user.target" ];
247 wants = [ "nomad.service" "studio-router.service" "studio-host.service" "studio-vm-screen.service" ];
248 after = [ "nomad.service" "studio-host.service" "studio-vm-screen.service" ];
249 unitConfig.ConditionPathExists = [ "/var/lib/studio/dashboard.token" "/var/lib/studio/ca-bundle.crt" "/opt/studio/current" ];
250 path = [ pkgs.podman pkgs.coreutils pkgs.curl pkgs.openssl pkgs.systemd ];
251 serviceConfig = {
252 Type = "notify";
253 NotifyAccess = "all";
254 ExecStart = pkgs.writeShellScript "studio-dashboard-start" ''
255 uid=$(id -u studio-dashboard)
256 gid=$(id -g studio-dashboard)
257 optional=()
258 if test -d /srv/prod/ytdl/data && test -d '/srv/clover/Documents/Config/Youtube Downloader'; then
259 optional+=(--volume=/srv/prod/ytdl/data:/srv/prod/ytdl/data:rw
260 --env=STUDIO_YT_STATE=/srv/prod/ytdl/data
261 '--env=STUDIO_YT_CONFIG=/srv/clover/Documents/Config/Youtube Downloader'
262 --env=STUDIO_YT_MEDIA=/srv/clover/Media)
263 fi
264 exec podman run --rm --replace --name=studio-dashboard --pull=never \
265 --sdnotify=conmon --cgroups=no-conmon --log-driver=journald \
266 --user="$uid:$gid" \
267 --group-add=${toString config.users.groups.chloe.gid} --network=podman \
268 --read-only --read-only-tmpfs=false --cap-drop=ALL --security-opt=no-new-privileges \
269 --memory=2g --cpus=4 --pids-limit=256 --cpu-shares=1024 \
270 --publish=127.0.0.1:${dashboardPort}:7072 \
271 --tmpfs=/tmp:rw,noexec,nosuid,nodev,size=256m,mode=1777 \
272 --tmpfs=/run:rw,noexec,nosuid,nodev,size=16m,mode=755 \
273 --env=HOME=/data --env=XDG_CACHE_HOME=/data/cache \
274 --volume=/var/lib/studio/dashboard:/data:rw \
275 --volume=/run/studio-host:/run/studio-host:ro \
276 --volume=/run/studio-vm-screen:/run/studio-vm-screen:ro \
277 --volume=${proxyToken}:/run/secrets/dashboard-proxy.token:ro \
278 --volume=/var/lib/studio/dashboard.token:/run/secrets/dashboard-nomad.token:ro \
279 --volume=/var/lib/studio/ca-bundle.crt:/run/secrets/ca-bundle.crt:ro \
280 --mount=type=bind,src=/srv/clover,dst=/srv/clover,bind-nonrecursive,bind-propagation=rslave,ro="$STUDIO_CLOVER_READ_ONLY" \
281 --mount=type=bind,src=/srv/clover/Media,dst=/srv/clover/Media,bind-nonrecursive,bind-propagation=rslave,ro="$STUDIO_MEDIA_READ_ONLY" \
282 ${lib.concatMapStringsSep " " (name: lib.escapeShellArg "--env=${name}") (builtins.attrNames environment)} \
283 ${lib.concatMapStringsSep " " (name: lib.escapeShellArg "--add-host=${name}.${config.environment.variables.STUDIO_DOMAIN}:host-gateway") [ "dashboard.internal" "auth" "jelly" "db" "shale" ]} \
284 "''${optional[@]}" ${lib.escapeShellArg "${dashboard.image.imageName}:${dashboard.image.imageTag}"}
285 '';
286 ExecStop = "${pkgs.podman}/bin/podman stop --ignore --time=8 studio-dashboard";
287 ExecStopPost = "${pkgs.podman}/bin/podman rm --ignore --force studio-dashboard";
288 CPUWeight = 1000;
289 KillMode = "mixed";
290 TimeoutStartSec = 180;
291 TimeoutStopSec = 30;
292 Restart = "always";
293 RestartSec = 5;
294 };
295 preStart = ''
296 umask 0077
297 if ! test -e ${proxyToken}; then
298 openssl rand -hex 32 > ${proxyToken}.pending
299 mv ${proxyToken}.pending ${proxyToken}
300 fi
301 chown root:studio-dashboard ${proxyToken} /var/lib/studio/dashboard.token
302 chmod 0640 ${proxyToken} /var/lib/studio/dashboard.token
303 chown -R studio-dashboard:studio-dashboard /var/lib/studio/dashboard
304 systemd-tmpfiles --create --prefix=/srv/prod/ytdl/data
305 if ! podman image exists ${lib.escapeShellArg "${dashboard.image.imageName}:${dashboard.image.imageTag}"}; then
306 podman load --quiet --input ${dashboard.image}
307 fi
308 '';
309 postStart = ''
310 for attempt in $(seq 1 30); do
311 if printf 'header = "Studio-Proxy-Token: %s"\n' "$(cat ${proxyToken})" | curl --config - --fail --silent http://127.0.0.1:${dashboardPort}/ >/dev/null; then
312 exit 0
313 fi
314 sleep 1
315 done
316 exit 1
317 '';
318 };
319
320 systemd.tmpfiles.rules = [
321 "d /srv/prod 0755 root root -"
322 "A+ /srv/prod/shale/data - - - - u:root:rwX,d:u:root:rwx"
323 "d /srv/staging 0755 root root -"
324 "d /srv/vm 0755 root root -"
325 "d /srv/logs 0770 clo chloe -"
326 "A+ /srv/prod/ytdl/data - - - - g:chloe:rwX,m::rwX,d:g:chloe:rwx,d:m::rwx"
327 "d /var/lib/studio 0700 root root -"
328 "d /var/lib/studio/dashboard 0700 studio-dashboard studio-dashboard -"
329 "d /var/lib/studio/routes 0700 root root -"
330 "d /var/lib/caddy/studio 0755 caddy caddy -"
331 "f /var/lib/caddy/routes.caddy 0640 caddy caddy -"
332 "d /var/lib/swtpm-localca 0750 tss tss -"
333 ];
334
335 environment.variables.STUDIO_NODE_NAME = config.networking.hostName;
336 environment.variables.STUDIO_DOMAIN = lib.mkDefault "studio.test";
337 environment.variables.STUDIO_VM_SECURE_OVMF_CODE = secureOvmf.firmware;
338 environment.variables.STUDIO_VM_SECURE_OVMF_VARS = secureOvmf.variablesMs;
339 environment.variables.STUDIO_VM_SECURE_AAVMF_CODE = "${secureAavmf}/code.fd";
340 environment.variables.STUDIO_VM_SECURE_AAVMF_VARS = "${secureAavmf}/vars.fd";
341 environment.variables.STUDIO_VM_WINDOWS_ARM_EMULATOR = "${windowsArmQemu}/bin/qemu-system-aarch64";
342 environment.systemPackages = with pkgs; [ acl curl jq openssl python3 rsync zfs qemu ] ++ [ nativePkl ];
343}