1// ffmpeg/ffprobe/magick-based processors: duration, dimensions, optimized
2// image subsets, av1+opus+dash streaming variants, and h264 hls. bodies are
3// ported unchanged from the old file-scan.ts.
4const ffprobeBin = testProgram("ffprobe", "--help");
5const ffmpegBin = testProgram("ffmpeg", "--help");
6
7const ffmpegOptions = ["-hide_banner", "-loglevel", "warning"];
8
9const procDuration: Processor = {
10 name: "duration",
11 version: 1,
12 title: "calculate duration",
13 enable: ffprobeBin !== null,
14 include: rules.extsDuration,
15 cores: 1,
16 async run({ path, mediaFile, signal }) {
17 const stdout = await subprocess.execute([
18 ffprobeBin!,
19 "-v",
20 "error",
21 "-show_entries",
22 "format=duration",
23 "-of",
24 "default=noprint_wrappers=1:nokey=1",
25 path.toString(),
26 ], { signal });
27
28 const duration = parseFloat(stdout.trim());
29 if (Number.isNaN(duration)) {
30 throw new Error("Could not extract duration from " + stdout);
31 }
32 mediaFiles.setDuration(mediaFile, Math.ceil(duration));
33 },
34};
35
36const procDimensions: Processor = {
37 name: "dimensions",
38 version: 2,
39 title: "calculate dimensions",
40 enable: ffprobeBin != null,
41 include: rules.extsDimensions,
42 cores: 1,
43 async run({ path, mediaFile, signal }) {
44 const { ext } = path;
45
46 let dimensions;
47
48 if (ext === ".svg") {
49 // Parse out of text data
50 const content = await path.read("utf-8");
51 const widthMatch = content.match(/width="(\d+)"/);
52 const heightMatch = content.match(/height="(\d+)"/);
53
54 if (widthMatch && heightMatch) {
55 dimensions = `${widthMatch[1]}x${heightMatch[1]}`;
56 }
57 } else if (rules.extsImage.has(ext)) {
58 // Use magick to observe streams
59 const stdout = await subprocess.execute([
60 "magick",
61 "identify",
62 "-auto-orient",
63 "-format",
64 "%w %h",
65 path.toString(),
66 ], { signal });
67 const [w, h] = stdout.split(" ").map((x) => Number(x));
68 if (w && h) {
69 dimensions = w + "x" + h;
70 }
71 } else {
72 // Use ffprobe to observe streams
73 const stdout = await subprocess.execute([
74 "ffprobe",
75 "-v",
76 "error",
77 "-select_streams",
78 "v:0",
79 "-show_entries",
80 "stream=width,height:stream_side_data=rotation",
81 "-of",
82 "json",
83 path.toString(),
84 ], { signal });
85 const result = JSON.parse(stdout);
86 const stream = result.streams[0];
87 if (stream) {
88 let width = UNWRAP(stream.width);
89 let height = UNWRAP(stream.height);
90 // phone videos store the sensor's landscape frame plus a display
91 // matrix; everything downstream (scale filters, aspect-ratio css)
92 // wants the rotated display dimensions.
93 const rotation = (stream.side_data_list ?? [])
94 .find((s: any) => typeof s.rotation === "number")?.rotation ?? 0;
95 if (Math.abs(rotation) % 180 === 90) {
96 [width, height] = [height, width];
97 }
98 dimensions = width + "x" + height;
99 }
100 }
101
102 mediaFiles.setDimensions(mediaFile, dimensions ?? "");
103 },
104};
105
106const procImageSubsets: Processor = {
107 name: "image-subsets",
108 version: 3,
109 title: "encode image subsets",
110 include: rules.extsImage,
111 depends: [procDimensions.name],
112 async run({ path, mediaFile, node, signal }) {
113 const dims = mediaFiles.parseDimensions(mediaFile);
114 // dimensions could not be extracted; nothing to scale.
115 if (!dims) return;
116 const { width, height } = dims;
117 const targetSizes = transcodeRules.imageSizes.filter((w) => w < width);
118 const baseStatus = node.text;
119
120 using stack = new DisposableStack();
121 for (const size of targetSizes) {
122 const { w, h } = resizeDimensions(width, height, size);
123 for (const { ext, args } of transcodeRules.imagePresets) {
124 node.text = baseStatus + ` (${w}x${h}, ${ext.slice(1).toUpperCase()})`;
125
126 stack.use(
127 await derived.produce({
128 mediaFile,
129 node,
130 cores: 2,
131 subkey: `${size}${ext}`,
132 async producer(dir) {
133 await subprocess.execute([
134 ffmpegBin!,
135 ...ffmpegOptions,
136 "-i",
137 path.toString(),
138 "-vf",
139 `scale=${w}:${h}:force_original_aspect_ratio=increase,crop=${w}:${h}`,
140 ...args,
141 dir.join(`${size}${ext}`).toString(),
142 ], { signal });
143 },
144 }),
145 );
146 }
147 }
148
149 stack.move();
150 },
151};
152
153const videoArgsCache = new async.OnceMap(transcodeRules.getVideoInputArgs);
154
155const qualityMap: Record<string, string> = {
156 u: "ultra-high",
157 h: "high",
158 m: "medium",
159 l: "low",
160 d: "data-saving",
161};
162
163function skipVideo(mediaFile: MediaFile) {
164 if ((mediaFile.duration ?? 0) < 5) return true;
165 if (!mediaFile.dimensions) return true;
166 if (rules.processDenyList.has(mediaFile.path)) return true;
167 return false;
168}
169
170const procVideos = transcodeRules.videoFormats.map<Processor>((preset) => ({
171 name: `av1-${preset.id}`,
172 version: 4,
173 title: `encode av1 ${UNWRAP(qualityMap[UNWRAP(preset.id[1])])}`,
174 include: rules.extsVideo,
175 enable: ffmpegBin != null,
176 depends: [procDuration.name, procDimensions.name],
177 async run({ path, mediaFile, node, signal }) {
178 if (skipVideo(mediaFile)) return;
179
180 await derived.produce({
181 mediaFile,
182 node,
183 cores: 4,
184 subkey: `av1-${preset.id}`,
185 async producer(dir) {
186 const input = await videoArgsCache.getOrRun(path);
187 const dimensions = mediaFiles.parseDimensions(mediaFile);
188 ASSERT(dimensions);
189 const args = transcodeRules.getAv1VideoArgs(
190 preset,
191 dimensions,
192 UNWRAP(input.video, "frick on " + path + JSON.stringify(input)),
193 dir,
194 );
195
196 await ffmpeg.spawn({
197 ffmpeg: ffmpegBin!,
198 progress: node,
199 args,
200 cwd: dir.toString(),
201 signal,
202 });
203 },
204 });
205 },
206}));
207
208const procVideoAudios = transcodeRules.audioFormats.map<Processor>((preset) => ({
209 name: `opus-${preset.id}`,
210 version: 3,
211 title: `encode opus ${UNWRAP(qualityMap[UNWRAP(preset.id)])}`,
212 include: rules.extsVideo,
213 enable: ffmpegBin != null,
214 depends: [procDuration.name, procDimensions.name],
215 async run({ path, mediaFile, node, signal }) {
216 if (skipVideo(mediaFile)) return;
217 await derived.produce({
218 mediaFile,
219 node,
220 cores: 4,
221 subkey: `opus-${preset.id}`,
222 async producer(dir) {
223 const input = await videoArgsCache.getOrRun(path);
224 ASSERT(input.video);
225 if (!input.audio) return;
226 const args = transcodeRules.getOpusAudioArgs(preset, input.audio, dir);
227
228 await ffmpeg.spawn({
229 ffmpeg: ffmpegBin!,
230 progress: node,
231 args,
232 cwd: dir.toString(),
233 signal,
234 });
235 },
236 });
237 },
238}));
239
240const procDash: Processor = {
241 name: "dash",
242 version: 7,
243 title: "encode mpeg-dash",
244 include: rules.extsVideo,
245 enable: ffmpegBin != null,
246 depends: [...procVideos, ...procVideoAudios].map((x) => x.name),
247 async run({ path, mediaFile, node, signal }) {
248 if (skipVideo(mediaFile)) return;
249 await derived.produce({
250 mediaFile,
251 node,
252 cores: 1,
253 subkey: `dash-av1`,
254 async producer(dir) {
255 const input = await videoArgsCache.getOrRun(path);
256 const videos = transcodeRules.videoFormats.map(
257 (preset) =>
258 UNWRAP(dir.parent).join(
259 `av1-${preset.id}`,
260 transcodeRules.av1FileName,
261 ),
262 );
263 const audios = input.audio
264 ? transcodeRules.audioFormats.map(
265 (preset) =>
266 UNWRAP(dir.parent).join(
267 `opus-${preset.id}`,
268 transcodeRules.opusFileName,
269 ),
270 )
271 : [];
272 const args = transcodeRules.getMpegDashArgs(
273 videos,
274 audios,
275 dir,
276 );
277
278 await dir.join("d").makeDir();
279
280 await ffmpeg.spawn({
281 ffmpeg: ffmpegBin!,
282 progress: node,
283 args,
284 cwd: dir.toString(),
285 signal,
286 });
287 },
288 });
289 },
290};
291
292const procH264Hls: Processor = {
293 name: "h264-hls",
294 version: 1,
295 title: "encode h.264 hls",
296 include: rules.extsVideo,
297 enable: ffmpegBin != null,
298 depends: [procDuration.name, procDimensions.name],
299 async run({ path, mediaFile, node, signal }) {
300 if (skipVideo(mediaFile)) return;
301
302 await derived.produce({
303 mediaFile,
304 node,
305 cores: 8,
306 subkey: `hls`,
307 async producer(dir) {
308 const input = await videoArgsCache.getOrRun(path);
309 const args = transcodeRules.getH264HlsArgs(input, dir);
310
311 await ffmpeg.spawn({
312 ffmpeg: ffmpegBin!,
313 progress: node,
314 args,
315 cwd: dir.toString(),
316 signal,
317 });
318 },
319 });
320 },
321};
322
323function resizeDimensions(w: number, h: number, desiredWidth: number) {
324 ASSERT(desiredWidth < w, `${desiredWidth} < ${w}`);
325 return { w: desiredWidth, h: Math.floor((h / w) * desiredWidth) };
326}
327
328function testProgram(name: string, helpArgument: string) {
329 // spawnSync does not throw on a missing binary; it reports `error`
330 const result = child_process.spawnSync(name, [helpArgument]);
331 if (!result.error) return name;
332 console.warn(`Missing or corrupt executable '${name}'`);
333 return null;
334}
335
336export const processors: Processor[] = [
337 procDimensions,
338 procDuration,
339 procImageSubsets,
340 ...procVideos,
341 ...procVideoAudios,
342 procDash,
343 procH264Hls,
344];
345
346import * as mediaFiles from "#src/file-viewer/models/media-files.ts";
347import type { MediaFile } from "#src/file-viewer/models/media-files.ts";
348import { ASSERT, UNWRAP } from "@clo/lib/assert";
349import * as async from "@clo/lib/async";
350import * as subprocess from "@clo/lib/subprocess";
351import * as ffmpeg from "@clo/lib/subprocess/ffmpeg";
352import * as child_process from "node:child_process";
353
354import * as derived from "#src/file-viewer/models/derived.ts";
355import * as rules from "#src/file-viewer/rules.ts";
356import * as transcodeRules from "#src/file-viewer/transcode-rules.ts";
357import type { Processor } from "../registry.ts";