1/**
2 * The import alarm job's git client (DESIGN.md §2.3 Import).
3 *
4 * A minimal **smart-HTTP protocol v0 client**: fetches and parses the
5 * `info/refs?service=git-upload-pack` advertisement, then POSTs a
6 * `git-upload-pack` request (wants + optional `deepen`, no haves,
7 * immediate `done`, **no side-band** — so the pack follows the `NAK`
8 * verbatim) and returns the raw pack bytes for the Repo DO to feed
9 * through the same `ingestPack` path as a push, with the same 50 MiB cap
10 * (documented v1 limit: small / depth-limited imports only).
11 *
12 * Network I/O goes through the Effect `HttpClient` service; callers
13 * provide `FetchHttpClient.layer` (see `RepoObject.ts`'s alarm handler).
14 */
15import * as Effect from "effect/Effect";
16import * as Schema from "effect/Schema";
17import * as HttpClient from "effect/http/HttpClient";
18import * as HttpClientRequest from "effect/http/HttpClientRequest";
19import { concatBytes, isOid, ZERO_OID } from "../Protocol/ObjectCodec.ts";
20import {
21 flushPkt,
22 pktPayloadText,
23 pktText,
24 readPktLineAt,
25} from "../Protocol/Pkt.ts";
26
27// ─────────────────────────────────────────────────────────────────────────────
28// Errors & shapes
29// ─────────────────────────────────────────────────────────────────────────────
30
31/**
32 * Raised when the source remote misbehaves: HTTP failures, non-200s,
33 * malformed advertisements, oversize packs, or an `ERR` pkt from the
34 * remote. The Repo DO maps this onto the repo's import job state.
35 */
36export class ImportClientError extends Schema.TaggedError<ImportClientError>()(
37 "ImportClientError",
38 { reason: Schema.String },
39) {}
40
41/** The import source, as accepted by `POST /api/v1/repos/import`. */
42export interface ImportSource {
43 /** Smart-HTTP base URL of the source repository (with or without `.git`). */
44 readonly url: string;
45 /** Restrict the import to a single ref (full or short name). */
46 readonly ref?: string | undefined;
47 /** Depth-limit the imported history (`deepen <n>`). */
48 readonly depth?: number | undefined;
49}
50
51/** One ref parsed from the remote advertisement. */
52export interface RemoteRef {
53 readonly name: string;
54 readonly oid: string;
55 /** Peeled target from a `<name>^{}` line (annotated tags). */
56 readonly peeled?: string | undefined;
57}
58
59/** A parsed v0 `info/refs` advertisement. */
60export interface RemoteAdvertisement {
61 readonly refs: ReadonlyArray<RemoteRef>;
62 /**
63 * The remote HEAD's symref target (e.g. `refs/heads/main`) from the
64 * `symref=HEAD:...` capability, or `null` when not advertised.
65 */
66 readonly head: string | null;
67 /** The remote's advertised capability set. */
68 readonly capabilities: ReadonlySet<string>;
69}
70
71/** The result of {@link runImport}: everything the Repo DO needs to ingest. */
72export interface ImportResult {
73 /** The refs to create locally (already filtered by `source.ref`). */
74 readonly refs: ReadonlyArray<RemoteRef>;
75 /** Short default-branch name derived from the remote HEAD, or `null`. */
76 readonly defaultBranch: string | null;
77 /**
78 * The raw pack bytes, or `undefined` when the remote is empty (nothing
79 * to ingest).
80 */
81 readonly pack: Uint8Array | undefined;
82 /** `shallow <oid>` boundary commits reported by the remote. */
83 readonly shallow: ReadonlyArray<string>;
84}
85
86/** Options of {@link runImport}. */
87export interface ImportOptions {
88 readonly source: ImportSource;
89 /**
90 * Hard cap on the fetched pack size (bytes).
91 * @default 52_428_800 (50 MiB, the same cap as push ingest)
92 */
93 readonly maxPackBytes?: number | undefined;
94}
95
96const DEFAULT_MAX_PACK = 50 * 1024 * 1024;
97
98// ─────────────────────────────────────────────────────────────────────────────
99// Advertisement
100// ─────────────────────────────────────────────────────────────────────────────
101
102const normalizeBase = (url: string): string => url.replace(/\/+$/, "");
103
104/** Decodes all pkt-lines of a buffer into text/flush tokens. */
105type Token =
106 | { readonly _tag: "line"; readonly text: string }
107 | { readonly _tag: "flush" };
108
109const tokenize = (
110 buf: Uint8Array,
111): Effect.Effect<ReadonlyArray<Token>, ImportClientError> =>
112 Effect.suspend(() => {
113 const out: Array<Token> = [];
114 let pos = 0;
115 while (pos < buf.length) {
116 const r = readPktLineAt(buf, pos);
117 if (r._tag === "incomplete") {
118 return Effect.fail(
119 new ImportClientError({ reason: "truncated advertisement" }),
120 );
121 }
122 if (r._tag === "invalid") {
123 return Effect.fail(new ImportClientError({ reason: r.reason }));
124 }
125 if (r.pkt._tag === "flush") {
126 out.push({ _tag: "flush" });
127 } else if (r.pkt._tag === "data") {
128 out.push({ _tag: "line", text: pktPayloadText(r.pkt.payload) });
129 }
130 pos = r.next;
131 }
132 return Effect.succeed(out);
133 });
134
135/**
136 * Parses a v0 `git-upload-pack` advertisement body (including the
137 * `# service=` prelude). Exposed for unit testing against fixture bytes.
138 */
139export const parseAdvertisement = Effect.fn(function* (body: Uint8Array) {
140 const tokens = yield* tokenize(body);
141 const refs: Array<RemoteRef> = [];
142 const peeled = new Map<string, string>();
143 let capabilities: ReadonlySet<string> = new Set();
144 let head: string | null = null;
145 let sawCaps = false;
146
147 for (const token of tokens) {
148 if (token._tag === "flush") continue;
149 const text = token.text;
150 if (text.startsWith("#")) continue; // "# service=git-upload-pack"
151 if (text.startsWith("ERR ")) {
152 return yield* new ImportClientError({
153 reason: `remote error: ${text.slice(4)}`,
154 });
155 }
156 let line = text;
157 if (!sawCaps) {
158 // First ref line carries "\0cap1 cap2 ..."
159 const nul = text.indexOf("\0");
160 if (nul !== -1) {
161 const caps = new Set(
162 text
163 .slice(nul + 1)
164 .split(" ")
165 .filter(Boolean),
166 );
167 capabilities = caps;
168 for (const cap of caps) {
169 if (cap.startsWith("symref=HEAD:")) {
170 head = cap.slice("symref=HEAD:".length);
171 }
172 }
173 line = text.slice(0, nul);
174 sawCaps = true;
175 }
176 }
177 const space = line.indexOf(" ");
178 if (space !== 40 || !isOid(line.slice(0, 40))) continue;
179 const oid = line.slice(0, 40);
180 const name = line.slice(space + 1);
181 if (oid === ZERO_OID && name === "capabilities^{}") {
182 continue; // empty repo advertisement
183 }
184 if (name.endsWith("^{}")) {
185 peeled.set(name.slice(0, -3), oid);
186 } else {
187 refs.push({ name, oid });
188 }
189 }
190
191 const withPeeled = refs.map((ref) => {
192 const p = peeled.get(ref.name);
193 return p === undefined ? ref : { ...ref, peeled: p };
194 });
195
196 return {
197 refs: withPeeled,
198 head,
199 capabilities,
200 } satisfies RemoteAdvertisement;
201});
202
203/**
204 * Fetches and parses the remote's `info/refs?service=git-upload-pack`
205 * advertisement.
206 */
207export const fetchAdvertisement = Effect.fn(function* (url: string) {
208 const client = yield* HttpClient.HttpClient;
209 const response = yield* client
210 .get(`${normalizeBase(url)}/info/refs?service=git-upload-pack`)
211 .pipe(
212 Effect.mapError(
213 (error) =>
214 new ImportClientError({ reason: `info/refs failed: ${error}` }),
215 ),
216 );
217 if (response.status !== 200) {
218 return yield* new ImportClientError({
219 reason: `info/refs returned HTTP ${response.status}`,
220 });
221 }
222 const body = yield* response.arrayBuffer.pipe(
223 Effect.mapError(
224 (error) =>
225 new ImportClientError({ reason: `info/refs body read: ${error}` }),
226 ),
227 );
228 return yield* parseAdvertisement(new Uint8Array(body));
229});
230
231// ─────────────────────────────────────────────────────────────────────────────
232// upload-pack POST
233// ─────────────────────────────────────────────────────────────────────────────
234
235/** Builds the v0 upload-pack request body: wants, optional deepen, done. */
236export const buildUploadPackRequest = (
237 wants: ReadonlyArray<string>,
238 options?: { readonly depth?: number | undefined },
239): Uint8Array => {
240 const parts: Array<Uint8Array> = [];
241 wants.forEach((oid, index) => {
242 parts.push(
243 index === 0
244 ? pktText(`want ${oid} ofs-delta agent=alchemy-git-service/1`)
245 : pktText(`want ${oid}`),
246 );
247 });
248 if (options?.depth !== undefined) {
249 parts.push(pktText(`deepen ${options.depth}`));
250 }
251 parts.push(flushPkt);
252 parts.push(pktText("done"));
253 return concatBytes(parts);
254};
255
256/**
257 * The parsed upload-pack response: the shallow boundary lines (when a
258 * `deepen` was requested) and the raw pack bytes that follow the `NAK`.
259 */
260export interface UploadPackResponse {
261 readonly shallow: ReadonlyArray<string>;
262 readonly pack: Uint8Array;
263}
264
265/**
266 * Splits a (non-side-band) upload-pack response into its shallow section /
267 * NAK and the trailing raw pack. Exposed for unit testing.
268 */
269export const parseUploadPackResponse = Effect.fn(function* (body: Uint8Array) {
270 const shallow: Array<string> = [];
271 let pos = 0;
272 for (;;) {
273 if (pos >= body.length) {
274 return yield* new ImportClientError({
275 reason: "upload-pack response ended before NAK/ACK",
276 });
277 }
278 const r = readPktLineAt(body, pos);
279 if (r._tag === "incomplete" || r._tag === "invalid") {
280 return yield* new ImportClientError({
281 reason:
282 r._tag === "invalid" ? r.reason : "truncated upload-pack response",
283 });
284 }
285 pos = r.next;
286 if (r.pkt._tag !== "data") continue; // flush after the shallow section
287 const text = pktPayloadText(r.pkt.payload);
288 if (text.startsWith("shallow ")) {
289 const oid = text.slice("shallow ".length);
290 if (isOid(oid)) shallow.push(oid);
291 continue;
292 }
293 if (text.startsWith("unshallow ")) continue;
294 if (text.startsWith("ERR ")) {
295 return yield* new ImportClientError({
296 reason: `remote error: ${text.slice(4)}`,
297 });
298 }
299 if (text === "NAK" || text.startsWith("ACK ")) {
300 // Pack bytes follow verbatim (no side-band was requested).
301 return {
302 shallow,
303 pack: body.subarray(pos),
304 } satisfies UploadPackResponse;
305 }
306 // Ignore anything else (progress from misbehaving servers).
307 }
308});
309
310/**
311 * POSTs the upload-pack request and returns the shallow lines + raw pack.
312 */
313export const fetchPack = Effect.fn(function* (
314 url: string,
315 wants: ReadonlyArray<string>,
316 options?: {
317 readonly depth?: number | undefined;
318 readonly maxPackBytes?: number | undefined;
319 },
320) {
321 const client = yield* HttpClient.HttpClient;
322 const maxPack = options?.maxPackBytes ?? DEFAULT_MAX_PACK;
323 const request = HttpClientRequest.post(
324 `${normalizeBase(url)}/git-upload-pack`,
325 ).pipe(
326 HttpClientRequest.setHeaders({
327 "content-type": "application/x-git-upload-pack-request",
328 accept: "application/x-git-upload-pack-result",
329 }),
330 HttpClientRequest.bodyUint8Array(
331 buildUploadPackRequest(wants, { depth: options?.depth }),
332 ),
333 );
334 const response = yield* client
335 .execute(request)
336 .pipe(
337 Effect.mapError(
338 (error) =>
339 new ImportClientError({ reason: `git-upload-pack failed: ${error}` }),
340 ),
341 );
342 if (response.status !== 200) {
343 return yield* new ImportClientError({
344 reason: `git-upload-pack returned HTTP ${response.status}`,
345 });
346 }
347 const body = yield* response.arrayBuffer.pipe(
348 Effect.mapError(
349 (error) =>
350 new ImportClientError({
351 reason: `git-upload-pack body read: ${error}`,
352 }),
353 ),
354 );
355 if (body.byteLength > maxPack + 65536) {
356 return yield* new ImportClientError({
357 reason: `source pack exceeds the ${maxPack} byte import cap`,
358 });
359 }
360 const parsed = yield* parseUploadPackResponse(new Uint8Array(body));
361 if (parsed.pack.byteLength > maxPack) {
362 return yield* new ImportClientError({
363 reason: `source pack exceeds the ${maxPack} byte import cap`,
364 });
365 }
366 return parsed;
367});
368
369// ─────────────────────────────────────────────────────────────────────────────
370// Orchestration
371// ─────────────────────────────────────────────────────────────────────────────
372
373/** Resolves `source.ref` (full or short name) against advertised refs. */
374const selectRefs = (
375 advertised: ReadonlyArray<RemoteRef>,
376 ref: string | undefined,
377): ReadonlyArray<RemoteRef> | undefined => {
378 if (ref === undefined) {
379 return advertised.filter(
380 (r) =>
381 r.name.startsWith("refs/heads/") || r.name.startsWith("refs/tags/"),
382 );
383 }
384 const candidates = [ref, `refs/heads/${ref}`, `refs/tags/${ref}`];
385 const match = advertised.find((r) => candidates.includes(r.name));
386 return match === undefined ? undefined : [match];
387};
388
389/**
390 * Runs the full import client flow: advertisement → ref selection →
391 * upload-pack POST. Returns everything the Repo DO's alarm job needs to
392 * ingest the history and create the refs. Requires `HttpClient.HttpClient`
393 * (provide `FetchHttpClient.layer`).
394 */
395export const runImport = Effect.fn(function* (options: ImportOptions) {
396 const { source } = options;
397 const advertisement = yield* fetchAdvertisement(source.url);
398
399 const selected = selectRefs(advertisement.refs, source.ref);
400 if (selected === undefined) {
401 return yield* new ImportClientError({
402 reason: `ref '${source.ref}' not found on the remote`,
403 });
404 }
405
406 const defaultBranch =
407 advertisement.head !== null && advertisement.head.startsWith("refs/heads/")
408 ? advertisement.head.slice("refs/heads/".length)
409 : null;
410
411 if (selected.length === 0) {
412 // Empty remote — nothing to fetch.
413 return {
414 refs: [],
415 defaultBranch,
416 pack: undefined,
417 shallow: [],
418 } satisfies ImportResult;
419 }
420
421 // Want the peeled targets too? No — wanting the tag object oid pulls the
422 // whole chain; peeled lines are advisory.
423 const wants = Array.from(new Set(selected.map((r) => r.oid)));
424 const { shallow, pack } = yield* fetchPack(source.url, wants, {
425 depth: source.depth,
426 maxPackBytes: options.maxPackBytes,
427 });
428
429 return {
430 refs: selected,
431 defaultBranch,
432 pack,
433 shallow,
434 } satisfies ImportResult;
435});