1import * as dataform from "@distilled.cloud/gcp/dataform_v1";
2import * as Data from "effect/Data";
3import * as Effect from "effect/Effect";
4import * as Schedule from "effect/Schedule";
5import { Unowned } from "../../AdoptPolicy.ts";
6import { isResolved } from "../../Diff.ts";
7import * as Provider from "../../Provider.ts";
8import { Resource } from "../../Resource.ts";
9import { tagRecord } from "../../Tags.ts";
10import { GcpEnvironment } from "../Environment.ts";
11import {
12 createInternalLabels,
13 diffLabels,
14 hasAlchemyLabels,
15 toLabels,
16} from "../Labels.ts";
17import type { Providers } from "../Providers.ts";
18import {
19 expandParent,
20 hasAlchemyLabelMap,
21 listReleaseConfigs,
22 listRepositories,
23 listWorkflowConfigs,
24 listWorkflowInvocations,
25 listWorkspaces,
26 locationParent,
27 normalizeLocation,
28 parseResourceName,
29 replaceOnIdentity,
30 retryTransient,
31 sameJson,
32 sameText,
33 toPhysicalId,
34 updateMaskOf,
35 userLabels,
36 waitForOperation,
37 waitUntilExists,
38 waitUntilGone,
39} from "./internal.ts";
40
41export type GitRemoteSettings = {
42 /** Git remote URL. */
43 url?: string;
44 /** Default branch name. Defaults to `main` when omitted. */
45 defaultBranch?: string;
46 /**
47 * Secret Manager secret version used as a Git token
48 * (`projects/{project}/secrets/{secret}/versions/{version}`).
49 */
50 authenticationTokenSecretVersion?: string;
51 /**
52 * Cloud Build `GitRepositoryLink` used for machine credentials
53 * (`projects/{project}/locations/{location}/connections/{connection}/gitRepositoryLinks/{link}`).
54 */
55 gitRepositoryLink?: string;
56 /** SSH host public key. */
57 hostPublicKey?: string;
58 /**
59 * Secret Manager secret version holding the SSH private key
60 * (`projects/{project}/secrets/{secret}/versions/{version}`).
61 */
62 userPrivateKeySecretVersion?: string;
63};
64
65export type WorkspaceCompilationOverrides = {
66 /** Default database (Google Cloud project id). */
67 defaultDatabase?: string;
68 /** Suffix appended to schema (BigQuery dataset) names. */
69 schemaSuffix?: string;
70 /** Prefix prepended to table names. */
71 tablePrefix?: string;
72};
73
74export type RepositoryProps = {
75 /**
76 * Repository id (the `{repository}` segment of
77 * `projects/{project}/locations/{location}/repositories/{repository}`).
78 * If omitted, a unique RFC1035 name is generated. Immutable — changing
79 * it replaces the repository.
80 */
81 repositoryId?: string;
82 /**
83 * Region (`us-central1`, …). Immutable — changing it replaces the
84 * repository. `US-CENTRAL1` is accepted and normalized to `us-central1`.
85 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
86 */
87 location?: string;
88 /**
89 * User-friendly display name. Defaults to the repository id.
90 */
91 displayName?: string;
92 /**
93 * User labels. Alchemy ownership labels are merged in automatically.
94 */
95 labels?: Record<string, string>;
96 /**
97 * Parent folder or team folder
98 * (`projects/{project}/locations/{location}/folders/{folder}` or
99 * `.../teamFolders/{teamFolder}`). Changing it moves the repository.
100 */
101 containingFolder?: string;
102 /**
103 * Git remote. Omit for a Dataform-managed (non-linked) repository.
104 */
105 gitRemoteSettings?: GitRemoteSettings;
106 /**
107 * Secret Manager secret version interpolated into `.npmrc`
108 * (`projects/{project}/secrets/{secret}/versions/{version}`).
109 */
110 npmrcEnvironmentVariablesSecretVersion?: string;
111 /**
112 * Service account used to run workflow invocations.
113 */
114 serviceAccount?: string;
115 /**
116 * Workspace compilation overrides for `dataform.json` defaults.
117 */
118 workspaceCompilationOverrides?: WorkspaceCompilationOverrides;
119 /**
120 * Customer-managed KMS key. Immutable after create.
121 */
122 kmsKeyName?: string;
123 /**
124 * Grant `roles/dataform.admin` to the caller on create.
125 * @default true
126 */
127 setAuthenticatedUserAdmin?: boolean;
128};
129
130export type Repository = Resource<
131 "GCP.Dataform.Repository",
132 RepositoryProps,
133 {
134 /** Full resource name `projects/{project}/locations/{location}/repositories/{repository}`. */
135 name: string;
136 /** Repository id (last path segment). */
137 repositoryId: string;
138 /** Project id. */
139 project: string;
140 /** Location id. */
141 location: string;
142 /** User-friendly display name. */
143 displayName: string | undefined;
144 /** User labels (Alchemy ownership labels stripped). */
145 labels: Record<string, string>;
146 /** Parent folder or team folder, if any. */
147 containingFolder: string | undefined;
148 /** Associated team folder, if any. */
149 teamFolderName: string | undefined;
150 /** Git remote URL, if linked. */
151 gitRemoteUrl: string | undefined;
152 /** Effective default Git branch. */
153 gitDefaultBranch: string | undefined;
154 /** Service account for workflow invocations. */
155 serviceAccount: string | undefined;
156 /** Customer-managed KMS key, if set. */
157 kmsKeyName: string | undefined;
158 /** RFC3339 creation timestamp. */
159 createTime: string | undefined;
160 },
161 never,
162 Providers
163>;
164
165/**
166 * A Dataform Git repository — the container for workspaces, release
167 * configs, workflow configs, and workflow invocations.
168 *
169 * Changing `repositoryId`, `location`, or `kmsKeyName` replaces the
170 * repository. Display name, labels, Git remote, service account, npmrc
171 * secret, and compilation overrides update in place. `containingFolder`
172 * is applied with MoveRepository.
173 *
174 * ### Creating a Repository
175 * **Example:** Generated name
176 * ```typescript
177 * const repo = yield* GCP.Dataform.Repository("Analytics", {
178 * labels: { env: "test" },
179 * });
180 * ```
181 *
182 * **Example:** Named repository
183 * ```typescript
184 * const repo = yield* GCP.Dataform.Repository("Analytics", {
185 * repositoryId: "analytics",
186 * location: "us-central1",
187 * displayName: "analytics",
188 * });
189 * ```
190 *
191 * @resource
192 * @category Dataform
193 */
194export const Repository = Resource<Repository>("GCP.Dataform.Repository");
195
196export class RepositoryNotResolved extends Data.TaggedError(
197 "GCP.Dataform.RepositoryNotResolved",
198)<{
199 name: string;
200}> {}
201
202const resourceName = (
203 project: string,
204 location: string,
205 repositoryId: string,
206) => `${locationParent(project, location)}/repositories/${repositoryId}`;
207
208const gitOf = (
209 settings: GitRemoteSettings | undefined,
210): dataform.GitRemoteSettings | undefined => {
211 if (settings === undefined) return undefined;
212 const ssh =
213 settings.hostPublicKey !== undefined ||
214 settings.userPrivateKeySecretVersion !== undefined
215 ? {
216 hostPublicKey: settings.hostPublicKey,
217 userPrivateKeySecretVersion: settings.userPrivateKeySecretVersion,
218 }
219 : undefined;
220 return {
221 url: settings.url,
222 defaultBranch: settings.defaultBranch,
223 authenticationTokenSecretVersion: settings.authenticationTokenSecretVersion,
224 gitRepositoryLink: settings.gitRepositoryLink,
225 sshAuthenticationConfig: ssh,
226 };
227};
228
229const toAttrs = (
230 repo: dataform.Repository,
231 project: string,
232 region: string,
233) => {
234 const name = repo.name ?? "";
235 const parsed = parseResourceName(name, "repositories", region);
236 return {
237 name,
238 repositoryId: parsed.id,
239 project: parsed.project || project,
240 location: parsed.location,
241 displayName: repo.displayName,
242 labels: userLabels(repo.labels),
243 containingFolder: repo.containingFolder,
244 teamFolderName: repo.teamFolderName,
245 gitRemoteUrl: repo.gitRemoteSettings?.url,
246 gitDefaultBranch:
247 repo.gitRemoteSettings?.effectiveDefaultBranch ??
248 repo.gitRemoteSettings?.defaultBranch,
249 serviceAccount: repo.serviceAccount,
250 kmsKeyName: repo.kmsKeyName,
251 createTime: repo.createTime,
252 };
253};
254
255const getByName = (name: string) =>
256 name.length === 0 || name.includes("//")
257 ? Effect.succeed(undefined)
258 : dataform
259 .getProjectsLocationsRepositories({ name })
260 .pipe(Effect.catchTag("NotFound", () => Effect.succeed(undefined)));
261
262export const RepositoryProvider = () =>
263 Provider.succeed(Repository, {
264 stables: [
265 "name",
266 "repositoryId",
267 "project",
268 "location",
269 "kmsKeyName",
270 "createTime",
271 ],
272
273 diff: Effect.fn(function* ({ news, olds, output }) {
274 if (!isResolved(news)) return undefined;
275 const previousKey = olds?.kmsKeyName ?? output?.kmsKeyName;
276 return replaceOnIdentity({
277 previousId: olds?.repositoryId ?? output?.repositoryId,
278 nextId: news.repositoryId ?? olds?.repositoryId ?? output?.repositoryId,
279 previousLocation: olds?.location ?? output?.location,
280 nextLocation: news.location ?? olds?.location ?? output?.location,
281 extra:
282 previousKey !== undefined &&
283 news.kmsKeyName !== undefined &&
284 news.kmsKeyName !== previousKey,
285 });
286 }),
287
288 read: Effect.fn(function* ({ id, olds, output }) {
289 const env = yield* GcpEnvironment.current;
290 const location = normalizeLocation(
291 olds?.location ?? output?.location,
292 env.region,
293 );
294 const repositoryId = yield* toPhysicalId(
295 id,
296 olds?.repositoryId,
297 output?.repositoryId,
298 );
299 const name =
300 output?.name ?? resourceName(env.project, location, repositoryId);
301 const existing = yield* getByName(name);
302 if (existing === undefined) return undefined;
303 const attrs = toAttrs(existing, env.project, env.region);
304 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels)))
305 ? attrs
306 : Unowned(attrs);
307 }),
308
309 list: () =>
310 Effect.gen(function* () {
311 const env = yield* GcpEnvironment.current;
312 const repos = yield* listRepositories(env.project, env.region);
313 return repos
314 .filter((repo) => hasAlchemyLabelMap(repo.labels))
315 .map((repo) => toAttrs(repo, env.project, env.region));
316 }),
317
318 reconcile: Effect.fn(function* ({ id, news, output }) {
319 const env = yield* GcpEnvironment.current;
320 const location = normalizeLocation(
321 news.location ?? output?.location,
322 env.region,
323 );
324 const repositoryId = yield* toPhysicalId(
325 id,
326 news.repositoryId,
327 output?.repositoryId,
328 );
329 const name = resourceName(env.project, location, repositoryId);
330 const desiredLabels = {
331 ...toLabels(news.labels),
332 ...(yield* createInternalLabels(id)),
333 };
334 const displayName = news.displayName ?? repositoryId;
335 const gitRemoteSettings = gitOf(news.gitRemoteSettings);
336 const containingFolder =
337 news.containingFolder !== undefined
338 ? news.containingFolder.includes("/")
339 ? news.containingFolder
340 : expandParent(
341 news.containingFolder,
342 env.project,
343 location,
344 "folders",
345 )
346 : undefined;
347
348 let current = yield* getByName(output?.name ?? name);
349
350 if (current === undefined) {
351 const created = yield* retryTransient(
352 dataform.createProjectsLocationsRepositories({
353 parent: locationParent(env.project, location),
354 repositoryId,
355 body: {
356 displayName,
357 labels: desiredLabels,
358 containingFolder,
359 gitRemoteSettings,
360 npmrcEnvironmentVariablesSecretVersion:
361 news.npmrcEnvironmentVariablesSecretVersion,
362 serviceAccount: news.serviceAccount,
363 workspaceCompilationOverrides: news.workspaceCompilationOverrides,
364 kmsKeyName: news.kmsKeyName,
365 setAuthenticatedUserAdmin: news.setAuthenticatedUserAdmin ?? true,
366 },
367 }),
368 ).pipe(Effect.catchTag("Conflict", () => getByName(name)));
369 current = created ?? undefined;
370 if (current === undefined) {
371 current = yield* waitUntilExists(getByName(name), name);
372 }
373 }
374
375 if (current === undefined) {
376 return yield* new RepositoryNotResolved({ name });
377 }
378
379 const currentName = current.name ?? name;
380 const observedLabels = tagRecord(current.labels);
381 const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
382 const labelsChanged = upsert.length > 0 || removed.length > 0;
383 const displayChanged = !sameText(current.displayName, displayName);
384 const gitChanged = !sameJson(
385 current.gitRemoteSettings
386 ? {
387 url: current.gitRemoteSettings.url,
388 defaultBranch: current.gitRemoteSettings.defaultBranch,
389 authenticationTokenSecretVersion:
390 current.gitRemoteSettings.authenticationTokenSecretVersion,
391 gitRepositoryLink: current.gitRemoteSettings.gitRepositoryLink,
392 sshAuthenticationConfig:
393 current.gitRemoteSettings.sshAuthenticationConfig,
394 }
395 : undefined,
396 gitRemoteSettings,
397 );
398 const npmrcChanged = !sameText(
399 current.npmrcEnvironmentVariablesSecretVersion,
400 news.npmrcEnvironmentVariablesSecretVersion,
401 );
402 const serviceAccountChanged = !sameText(
403 current.serviceAccount,
404 news.serviceAccount,
405 );
406 const overridesChanged = !sameJson(
407 current.workspaceCompilationOverrides,
408 news.workspaceCompilationOverrides,
409 );
410
411 if (
412 labelsChanged ||
413 displayChanged ||
414 gitChanged ||
415 npmrcChanged ||
416 serviceAccountChanged ||
417 overridesChanged
418 ) {
419 current = yield* retryTransient(
420 dataform.patchProjectsLocationsRepositories({
421 name: currentName,
422 updateMask: updateMaskOf(
423 labelsChanged ? "labels" : undefined,
424 displayChanged ? "displayName" : undefined,
425 gitChanged ? "gitRemoteSettings" : undefined,
426 npmrcChanged
427 ? "npmrcEnvironmentVariablesSecretVersion"
428 : undefined,
429 serviceAccountChanged ? "serviceAccount" : undefined,
430 overridesChanged ? "workspaceCompilationOverrides" : undefined,
431 ),
432 body: {
433 displayName,
434 labels: desiredLabels,
435 gitRemoteSettings,
436 npmrcEnvironmentVariablesSecretVersion:
437 news.npmrcEnvironmentVariablesSecretVersion,
438 serviceAccount: news.serviceAccount,
439 workspaceCompilationOverrides: news.workspaceCompilationOverrides,
440 },
441 }),
442 );
443 }
444
445 const observedFolder = current.containingFolder ?? "";
446 const desiredFolder = containingFolder ?? "";
447 if (desiredFolder !== observedFolder) {
448 const moved = yield* retryTransient(
449 dataform.moveProjectsLocationsRepositories({
450 name: currentName,
451 body: { destinationContainingFolder: desiredFolder },
452 }),
453 );
454 yield* waitForOperation(moved);
455 current = (yield* getByName(currentName)) ?? current;
456 }
457
458 return toAttrs(current, env.project, env.region);
459 }),
460
461 delete: Effect.fn(function* ({ output }) {
462 if (!output.name) return;
463 const parent = output.name;
464 const children = [
465 ...(yield* listWorkspaces(parent)),
466 ...(yield* listWorkflowInvocations(parent)),
467 ...(yield* listWorkflowConfigs(parent)),
468 ...(yield* listReleaseConfigs(parent)),
469 ];
470 yield* Effect.forEach(
471 children,
472 (child) => {
473 const name = child.name ?? "";
474 if (name.includes("/workspaces/")) {
475 return dataform
476 .deleteProjectsLocationsRepositoriesWorkspaces({ name })
477 .pipe(
478 Effect.catchTag(["NotFound", "BadRequest"], () => Effect.void),
479 );
480 }
481 if (name.includes("/workflowInvocations/")) {
482 return dataform
483 .deleteProjectsLocationsRepositoriesWorkflowInvocations({
484 name,
485 })
486 .pipe(
487 Effect.catchTag(["NotFound", "BadRequest"], () => Effect.void),
488 );
489 }
490 if (name.includes("/workflowConfigs/")) {
491 return dataform
492 .deleteProjectsLocationsRepositoriesWorkflowConfigs({ name })
493 .pipe(
494 Effect.catchTag(["NotFound", "BadRequest"], () => Effect.void),
495 );
496 }
497 if (name.includes("/releaseConfigs/")) {
498 return dataform
499 .deleteProjectsLocationsRepositoriesReleaseConfigs({ name })
500 .pipe(
501 Effect.catchTag(["NotFound", "BadRequest"], () => Effect.void),
502 );
503 }
504 return Effect.void;
505 },
506 { concurrency: 4 },
507 );
508 yield* retryTransient(
509 dataform.deleteProjectsLocationsRepositories({
510 name: output.name,
511 force: true,
512 }),
513 ).pipe(
514 Effect.retry({
515 while: (error) => error._tag === "Conflict",
516 times: 8,
517 schedule: Schedule.spaced("2 seconds"),
518 }),
519 Effect.catchTag("NotFound", () => Effect.void),
520 );
521 yield* waitUntilGone(getByName(output.name));
522 }),
523 });