1import { SourceControlProviderKind } from "@t3tools/contracts";
2import * as Clock from "effect/Clock";
3import * as Context from "effect/Context";
4import * as Data from "effect/Data";
5import * as Duration from "effect/Duration";
6import * as Effect from "effect/Effect";
7import * as Layer from "effect/Layer";
8import * as Option from "effect/Option";
9import * as Redacted from "effect/Redacted";
10import * as Schema from "effect/Schema";
11import * as Semaphore from "effect/Semaphore";
12import { HttpClient, HttpClientRequest, type HttpClientResponse } from "effect/http";
13
14import { collectUint8StreamText } from "@t3tools/provider-core/server/collectStreamText";
15import * as GitHubCredentials from "./GitHubCredentials.ts";
16import * as GitHubQuota from "./GitHubQuota.ts";
17import * as SourceControlRateLimit from "@t3tools/source-control-core/server/SourceControlRateLimit";
18
19const DEFAULT_TIMEOUT = Duration.seconds(30);
20const DEFAULT_MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
21/** Polite to GitHub's secondary limits, which punish bursts of concurrent requests. */
22const CONCURRENCY = 8;
23const API_VERSION = "2022-11-28";
24
25/**
26 * A credential already verified for one host. Every request made under it must target that
27 * host, so a page cannot read one account's data with another's token mid-flight. Server-local:
28 * never put its value in RPC payloads or cache keys.
29 */
30export const PinnedGitHubCredential = Context.Reference<{
31 readonly host: string;
32 readonly token: Redacted.Redacted<string>;
33 readonly credentialFingerprint: string;
34} | null>("@t3tools/source-control-github/server/GitHubApi/PinnedGitHubCredential", {
35 defaultValue: () => null,
36});
37
38export class GitHubApiRequestError extends Schema.TaggedError<GitHubApiRequestError>()(
39 "GitHubApiRequestError",
40 { host: Schema.String, operation: Schema.String, cause: Schema.Defect() },
41) {
42 override get message(): string {
43 return `Could not reach GitHub at ${this.host}.`;
44 }
45}
46
47export class GitHubApiAuthenticationError extends Schema.TaggedError<GitHubApiAuthenticationError>()(
48 "GitHubApiAuthenticationError",
49 { host: Schema.String, operation: Schema.String },
50) {
51 override get message(): string {
52 return `GitHub refused the credential for ${this.host}. Run \`gh auth login --hostname ${this.host}\` and retry.`;
53 }
54}
55
56export class GitHubApiRateLimitError extends Schema.TaggedError<GitHubApiRateLimitError>()(
57 "GitHubApiRateLimitError",
58 {
59 host: Schema.String,
60 operation: Schema.String,
61 retryAt: Schema.optionalKey(Schema.Finite),
62 },
63) {
64 override get message(): string {
65 return "GitHub API rate limit exceeded.";
66 }
67}
68
69export class GitHubApiNotFoundError extends Schema.TaggedError<GitHubApiNotFoundError>()(
70 "GitHubApiNotFoundError",
71 { host: Schema.String, operation: Schema.String },
72) {
73 override get message(): string {
74 return "GitHub could not find the requested resource, or the credential cannot see it.";
75 }
76}
77
78/**
79 * GitHub answered with a failure that is none of the above. `githubErrors` carries GitHub's own
80 * error messages, from a GraphQL `errors` list or a REST `message`/`errors` body: they name the
81 * field and the reason ("A pull request already exists for acme:feature"), never a token.
82 */
83export class GitHubApiResponseError extends Schema.TaggedError<GitHubApiResponseError>()(
84 "GitHubApiResponseError",
85 {
86 host: Schema.String,
87 operation: Schema.String,
88 status: Schema.Int,
89 githubErrors: Schema.optionalKey(Schema.Array(Schema.String)),
90 },
91) {
92 override get message(): string {
93 return this.githubErrors !== undefined && this.githubErrors.length > 0
94 ? `GitHub returned an error: ${this.githubErrors.join("; ")}`
95 : `GitHub returned HTTP ${this.status}.`;
96 }
97}
98
99export type GitHubApiError =
100 | GitHubCredentials.GitHubCredentialUnavailableError
101 | GitHubApiRequestError
102 | GitHubApiAuthenticationError
103 | GitHubApiRateLimitError
104 | GitHubApiNotFoundError
105 | GitHubApiResponseError
106 | SourceControlRateLimit.SourceControlRateLimitPausedError;
107
108export interface GitHubRestResponse {
109 readonly status: number;
110 readonly headers: Readonly<Record<string, string | undefined>>;
111 readonly body: string;
112 readonly truncated: boolean;
113 /** The body was not valid UTF-8, which a raw file read takes to mean binary. */
114 readonly invalidUtf8: boolean;
115}
116
117export interface GitHubRestInput {
118 readonly host: string;
119 readonly operation: string;
120 readonly method?: "GET" | "POST" | "PUT" | "PATCH" | "DELETE";
121 /** Relative to the API root, e.g. `repos/acme/web/pulls/7`; query string included. */
122 readonly path: string;
123 readonly body?: unknown;
124 /** Defaults to `application/vnd.github+json`. */
125 readonly accept?: string;
126 /** Revalidates a cached answer. A 304 is returned rather than failed, and is free. */
127 readonly ifNoneMatch?: string;
128 readonly maxResponseBytes?: number;
129 /** Defaults to 30 seconds; a whole pull request's patch may need longer. */
130 readonly timeout?: Duration.Input;
131 /** Overrides `SourceControlRateLimit.Interactive` for this one request. */
132 readonly allowReserve?: boolean;
133}
134
135export interface GitHubGraphQlInput {
136 readonly host: string;
137 readonly operation: string;
138 readonly query: string;
139 readonly variables?: Readonly<Record<string, unknown>>;
140 readonly allowReserve?: boolean;
141 readonly maxResponseBytes?: number;
142}
143
144export class GitHubApi extends Context.Service<
145 GitHubApi,
146 {
147 /** The raw JSON body of a successful GraphQL answer, with `rateLimit` recorded. */
148 readonly graphql: (input: GitHubGraphQlInput) => Effect.Effect<string, GitHubApiError>;
149 readonly rest: (input: GitHubRestInput) => Effect.Effect<GitHubRestResponse, GitHubApiError>;
150 /** The credential a request to `host` would carry right now. */
151 readonly credential: (
152 host: string,
153 ) => Effect.Effect<
154 { readonly token: Redacted.Redacted<string>; readonly fingerprint: string },
155 GitHubApiError
156 >;
157 }
158>()("@t3tools/source-control-github/server/GitHubApi") {}
159
160function normalizeHost(host: string): string {
161 return host.trim().toLowerCase();
162}
163
164/**
165 * Where the API for a host lives. github.com and GHE.com data residency serve it from an `api.`
166 * subdomain; GitHub Enterprise Server serves it under `/api` on the instance itself.
167 */
168export function gitHubApiUrls(host: string): { readonly rest: string; readonly graphql: string } {
169 const normalized = normalizeHost(host);
170 if (normalized === "github.com") {
171 return { rest: "https://api.github.com", graphql: "https://api.github.com/graphql" };
172 }
173 if (normalized.endsWith(".ghe.com")) {
174 return {
175 rest: `https://api.${normalized}`,
176 graphql: `https://api.${normalized}/graphql`,
177 };
178 }
179 return { rest: `https://${normalized}/api/v3`, graphql: `https://${normalized}/api/graphql` };
180}
181
182/** GraphQL documents longer than this are cut in traces; the hash still identifies them. */
183const TRACED_QUERY_MAX_CHARS = 4_000;
184
185/** A stable short id for a GraphQL document, so traces group by query without its text. */
186function queryHash(query: string): string {
187 let hash = 0x811c9dc5;
188 for (let index = 0; index < query.length; index++) {
189 hash ^= query.charCodeAt(index);
190 hash = Math.imul(hash, 0x01000193);
191 }
192 return (hash >>> 0).toString(16).padStart(8, "0");
193}
194
195/** The numeric rate-limit headers GitHub sends, as span attributes. */
196function rateLimitAttributes(
197 headers: Readonly<Record<string, string | undefined>>,
198): Record<string, number | string> {
199 const attributes: Record<string, number | string> = {};
200 for (const [header, name] of [
201 ["x-ratelimit-limit", "github.ratelimit.limit"],
202 ["x-ratelimit-remaining", "github.ratelimit.remaining"],
203 ["x-ratelimit-used", "github.ratelimit.used"],
204 ["x-ratelimit-reset", "github.ratelimit.reset"],
205 ] as const) {
206 const value = Number(headers[header]);
207 if (headers[header] !== undefined && Number.isFinite(value)) attributes[name] = value;
208 }
209 const resource = headers["x-ratelimit-resource"];
210 if (resource !== undefined) attributes["github.ratelimit.resource"] = resource;
211 return attributes;
212}
213
214/** The pause GitHub asked for, from `retry-after` or the primary limit's reset. */
215function retryAtFrom(
216 headers: Readonly<Record<string, string | undefined>>,
217 now: number,
218): number | undefined {
219 const fromRetryAfter = SourceControlRateLimit.retryAtFromHeader(headers["retry-after"], now);
220 if (fromRetryAfter !== undefined) return fromRetryAfter;
221 const reset = Number(headers["x-ratelimit-reset"]) * 1_000;
222 return Number.isFinite(reset) && reset > now ? reset : undefined;
223}
224
225const decodeGraphQlErrors = Schema.decodeUnknownOption(
226 Schema.fromJsonString(
227 Schema.Struct({
228 errors: Schema.NonEmptyArray(
229 Schema.Struct({
230 type: Schema.optional(Schema.String),
231 message: Schema.optional(Schema.String),
232 }),
233 ),
234 }),
235 ),
236);
237
238/** A REST failure body: `{ message, errors: [{ message } | { resource, field, code }] }`. */
239const decodeRestErrors = Schema.decodeUnknownOption(
240 Schema.fromJsonString(
241 Schema.Struct({
242 message: Schema.optional(Schema.String),
243 errors: Schema.optional(
244 Schema.Array(
245 Schema.Union([
246 Schema.String,
247 Schema.Struct({
248 message: Schema.optional(Schema.String),
249 field: Schema.optional(Schema.String),
250 code: Schema.optional(Schema.String),
251 }),
252 ]),
253 ),
254 ),
255 }),
256 ),
257);
258
259function restErrorMessages(body: string): ReadonlyArray<string> | undefined {
260 return Option.match(decodeRestErrors(body), {
261 onNone: () => undefined,
262 onSome: (decoded) => {
263 const details = (decoded.errors ?? []).flatMap((error) =>
264 typeof error === "string"
265 ? [error]
266 : error.message !== undefined
267 ? [error.message]
268 : error.field !== undefined && error.code !== undefined
269 ? [`${error.field} ${error.code}`]
270 : [],
271 );
272 const messages = [...(decoded.message === undefined ? [] : [decoded.message]), ...details];
273 return messages.length > 0 ? messages : undefined;
274 },
275 });
276}
277
278/** What one GitHub answer means, decided once from its status, headers and body. */
279type Answer = Data.TaggedEnum<{
280 Ok: {};
281 RateLimited: {};
282 Unauthorized: {};
283 NotFound: {};
284 Failed: { readonly messages: ReadonlyArray<string> | undefined };
285}>;
286const Answer = Data.taggedEnum<Answer>();
287
288/**
289 * GitHub reports a failed GraphQL document with HTTP 200 and an `errors` list, so a GraphQL body
290 * is read for its error types as well as its status. `gh api graphql` failed those too, and
291 * callers rely on that to fall back to narrower reads.
292 */
293function classify(input: {
294 readonly status: number;
295 readonly headers: Readonly<Record<string, string | undefined>>;
296 readonly body: string;
297 readonly graphql: boolean;
298 readonly acceptNotModified: boolean;
299}): Answer {
300 const { status, headers, body } = input;
301 const errors = input.graphql
302 ? Option.getOrUndefined(decodeGraphQlErrors(body))?.errors
303 : undefined;
304 const types = errors?.flatMap((error) => (error.type === undefined ? [] : [error.type])) ?? [];
305 const messages = errors?.flatMap((error) => (error.message === undefined ? [] : [error.message]));
306 if (
307 status === 429 ||
308 types.includes("RATE_LIMITED") ||
309 // An exhausted GraphQL quota can answer HTTP 200 with an untyped "API rate limit already
310 // exceeded" error; the headers say the same thing.
311 (input.graphql && errors !== undefined && headers["x-ratelimit-remaining"] === "0") ||
312 (input.graphql &&
313 messages !== undefined &&
314 messages.some((message) => /rate limit (already )?exceeded/i.test(message))) ||
315 (status === 403 &&
316 (headers["x-ratelimit-remaining"] === "0" ||
317 headers["retry-after"] !== undefined ||
318 /rate limit/i.test(body)))
319 ) {
320 return Answer.RateLimited();
321 }
322 if (status === 401) return Answer.Unauthorized();
323 if (
324 status === 404 ||
325 (errors !== undefined && errors.every((error) => error.type === "NOT_FOUND"))
326 ) {
327 return Answer.NotFound();
328 }
329 if (errors !== undefined) return Answer.Failed({ messages });
330 if ((status >= 200 && status < 300) || (status === 304 && input.acceptNotModified)) {
331 return Answer.Ok();
332 }
333 return Answer.Failed({ messages: input.graphql ? undefined : restErrorMessages(body) });
334}
335
336/** @public Service construction is part of the canonical Effect module API. */
337export const make = Effect.gen(function* () {
338 const httpClient = yield* HttpClient.HttpClient;
339 const credentials = yield* GitHubCredentials.GitHubCredentials;
340 const quota = yield* GitHubQuota.GitHubQuota;
341 const limits = yield* SourceControlRateLimit.SourceControlRateLimit;
342 const gate = yield* Semaphore.make(CONCURRENCY);
343
344 const credential: GitHubApi["Service"]["credential"] = Effect.fn("GitHubApi.credential")(
345 function* (host) {
346 const normalized = normalizeHost(host);
347 const pinned = yield* PinnedGitHubCredential;
348 if (pinned !== null) {
349 // A pinned page only ever talks to the host it verified.
350 if (pinned.host !== normalized) {
351 return yield* new GitHubApiAuthenticationError({
352 host: normalized,
353 operation: "credential",
354 });
355 }
356 return { token: pinned.token, fingerprint: pinned.credentialFingerprint };
357 }
358 const held = yield* credentials.get(normalized);
359 return { token: held.token, fingerprint: held.fingerprint };
360 },
361 );
362
363 /**
364 * Sends one request under the host's rate-limit pause and fails a refusal with the error that
365 * says why. `onSuccess` sees the answer only once it is known to be one.
366 */
367 const send = Effect.fn("GitHubApi.send")(function* (input: {
368 readonly host: string;
369 readonly operation: string;
370 readonly request: HttpClientRequest.HttpClientRequest;
371 readonly maxResponseBytes: number;
372 readonly timeout?: Duration.Input | undefined;
373 readonly allowReserve: boolean;
374 readonly acceptNotModified: boolean;
375 /** Reads the body for GraphQL `errors`, which GitHub sends with HTTP 200. */
376 readonly graphql?: boolean;
377 }) {
378 const host = normalizeHost(input.host);
379 // Only the path: a query string can carry a SHA or a branch, and never needs to be in a trace.
380 yield* Effect.annotateCurrentSpan({
381 "github.host": host,
382 "github.operation": input.operation,
383 "github.kind": input.graphql === true ? "graphql" : "rest",
384 "http.request.method": input.request.method,
385 "url.path": new URL(input.request.url).pathname,
386 });
387 const { token, fingerprint } = yield* credential(host);
388 const scope = yield* SourceControlRateLimit.CredentialScope;
389 // REST spends `core` and GraphQL its own quota, each with its own reserve. Every REST path
390 // this reads today is `core`; a `search/` read would need its own resource here. A refusal
391 // still pauses the whole host, the key PullRequestService records its own backoff under.
392 const resource = input.graphql === true ? "graphql" : "core";
393 const key = { provider: SourceControlProviderKind.make("github"), host };
394 const run = Effect.gen(function* () {
395 const lease = yield* quota
396 .admit(host, resource, { allowReserve: input.allowReserve })
397 .pipe(
398 Effect.andThen(limits.check(key, input.allowReserve ? { allowPaused: true } : undefined)),
399 )
400 .pipe(
401 Effect.tapError((paused) =>
402 Effect.annotateCurrentSpan({
403 "github.paused": true,
404 "github.retry_at": paused.retryAt,
405 }),
406 ),
407 );
408 // One deadline covers the headers and the body: a host that answers headers and then stalls
409 // must not hold a slot of the shared gate for undici's own five-minute body timeout.
410 const { response, collected } = yield* Effect.gen(function* () {
411 const response: HttpClientResponse.HttpClientResponse = yield* httpClient
412 .execute(
413 input.request.pipe(
414 HttpClientRequest.bearerToken(Redacted.value(token)),
415 HttpClientRequest.setHeaders({
416 "x-github-api-version": API_VERSION,
417 "user-agent": "t3code",
418 }),
419 ),
420 )
421 .pipe(
422 // `GitHubApi.send` is the request's span. The client's own span would add `url.full`
423 // and `url.query`, which can carry SHAs and branch names, so it is off for GitHub.
424 Effect.provideService(HttpClient.TracerDisabledWhen, () => true),
425 );
426 const collected = yield* collectUint8StreamText({
427 stream: response.stream,
428 maxBytes: input.maxResponseBytes,
429 }).pipe(
430 // 204, 304 and many refusals carry no body at all, which is an empty answer.
431 Effect.catchIf(
432 (error) => error.reason._tag === "EmptyBodyError",
433 () => Effect.succeed({ text: "", truncated: false, invalidUtf8: false }),
434 ),
435 );
436 return { response, collected };
437 }).pipe(
438 Effect.timeout(input.timeout ?? DEFAULT_TIMEOUT),
439 Effect.mapError(
440 (cause) => new GitHubApiRequestError({ host, operation: input.operation, cause }),
441 ),
442 );
443 const headers = response.headers;
444 const status = response.status;
445 yield* quota.observe(host, headers);
446 yield* Effect.annotateCurrentSpan({
447 "http.response.status_code": status,
448 ...rateLimitAttributes(headers),
449 });
450 const context = { host, operation: input.operation };
451 return yield* Answer.$match(
452 classify({
453 status,
454 headers,
455 body: collected.text,
456 graphql: input.graphql === true,
457 acceptNotModified: input.acceptNotModified,
458 }),
459 {
460 Ok: () =>
461 limits.recordSuccess({ ...key, lease }).pipe(
462 Effect.as({
463 status,
464 headers,
465 body: collected.text,
466 truncated: collected.truncated,
467 invalidUtf8: collected.invalidUtf8,
468 }),
469 ),
470 RateLimited: () =>
471 Effect.gen(function* () {
472 const retryAt = retryAtFrom(headers, yield* Clock.currentTimeMillis);
473 yield* limits.recordRateLimit({ ...key, lease, retryAt });
474 yield* Effect.annotateCurrentSpan({
475 "github.rate_limited": true,
476 ...(retryAt === undefined ? {} : { "github.retry_at": retryAt }),
477 });
478 return yield* new GitHubApiRateLimitError({
479 ...context,
480 ...(retryAt === undefined ? {} : { retryAt }),
481 });
482 }),
483 // The source may hold a newer token than the one that was refused.
484 Unauthorized: () =>
485 credentials
486 .invalidate(host)
487 .pipe(Effect.andThen(Effect.fail(new GitHubApiAuthenticationError(context)))),
488 NotFound: () => Effect.fail(new GitHubApiNotFoundError(context)),
489 Failed: ({ messages }) =>
490 Effect.fail(
491 new GitHubApiResponseError({
492 ...context,
493 status,
494 ...(messages === undefined ? {} : { githubErrors: messages }),
495 }),
496 ),
497 },
498 );
499 });
500 // The rate-limit scope follows the credential, so a pause on one account never blocks another.
501 return yield* gate
502 .withPermit(run)
503 .pipe(Effect.provideService(SourceControlRateLimit.CredentialScope, scope || fingerprint));
504 });
505
506 const rest: GitHubApi["Service"]["rest"] = (input) => {
507 const url = `${gitHubApiUrls(input.host).rest}/${input.path.replace(/^\/+/, "")}`;
508 const base = HttpClientRequest.make(input.method ?? "GET")(url).pipe(
509 HttpClientRequest.setHeader("accept", input.accept ?? "application/vnd.github+json"),
510 );
511 const withEtag =
512 input.ifNoneMatch === undefined
513 ? base
514 : base.pipe(HttpClientRequest.setHeader("if-none-match", input.ifNoneMatch));
515 const request =
516 input.body === undefined
517 ? withEtag
518 : withEtag.pipe(HttpClientRequest.bodyJsonUnsafe(input.body));
519 return SourceControlRateLimit.Interactive.pipe(
520 Effect.flatMap((interactive) =>
521 send({
522 host: input.host,
523 operation: input.operation,
524 request,
525 maxResponseBytes: input.maxResponseBytes ?? DEFAULT_MAX_RESPONSE_BYTES,
526 timeout: input.timeout,
527 allowReserve: input.allowReserve ?? interactive,
528 acceptNotModified: input.ifNoneMatch !== undefined,
529 }),
530 ),
531 );
532 };
533
534 const graphql: GitHubApi["Service"]["graphql"] = Effect.fn("GitHubApi.graphql")(
535 function* (input) {
536 const host = normalizeHost(input.host);
537 const allowReserve = input.allowReserve ?? (yield* SourceControlRateLimit.Interactive);
538 // The document, never its variables: user text (bodies, search terms) travels as variables.
539 yield* Effect.annotateCurrentSpan({
540 "github.operation": input.operation,
541 "github.graphql.query_hash": queryHash(input.query),
542 "github.graphql.query":
543 input.query.length > TRACED_QUERY_MAX_CHARS
544 ? `${input.query.slice(0, TRACED_QUERY_MAX_CHARS)}…`
545 : input.query,
546 });
547 const response = yield* send({
548 host,
549 operation: input.operation,
550 request: HttpClientRequest.post(gitHubApiUrls(host).graphql).pipe(
551 HttpClientRequest.acceptJson,
552 HttpClientRequest.bodyJsonUnsafe({
553 query: input.query,
554 variables: input.variables ?? {},
555 }),
556 ),
557 maxResponseBytes: input.maxResponseBytes ?? DEFAULT_MAX_RESPONSE_BYTES,
558 allowReserve,
559 acceptNotModified: false,
560 graphql: true,
561 });
562 // A body cut at the byte cap hides any `errors` past the cut and cannot be decoded.
563 if (response.truncated) {
564 return yield* new GitHubApiResponseError({
565 host,
566 operation: input.operation,
567 status: response.status,
568 });
569 }
570 return response.body;
571 },
572 );
573
574 return GitHubApi.of({ graphql, rest, credential });
575});
576
577export const layer = Layer.effect(GitHubApi, make);
578
579/**
580 * The transport with the credential, quota reserve and rate-limit pause it reads through. An
581 * application builds this once so every GitHub reader shares one view of each quota per host.
582 */
583export const layerWithDependencies = layer.pipe(
584 Layer.provideMerge(GitHubCredentials.layer),
585 Layer.provideMerge(GitHubQuota.layer),
586 Layer.provideMerge(SourceControlRateLimit.layer),
587);