1import * as GCP from "@/GCP";
2import { GcpEnvironment } from "@/GCP/Environment";
3import * as Output from "@/Output";
4import * as Test from "@/Test/Alchemy";
5import * as dataplex from "@distilled.cloud/gcp/dataplex_v1";
6import { expect } from "alchemy-test";
7import * as Effect from "effect/Effect";
8import { MinimumLogLevel } from "effect/References";
9import * as Schedule from "effect/Schedule";
10import { withDataplexSlot } from "./quota.ts";
11
12const { test } = Test.make({ providers: GCP.providers() });
13
14const logLevel = Effect.provideService(
15 MinimumLogLevel,
16 process.env.DEBUG ? "Debug" : "Info",
17);
18
19const runLifecycle = !process.env.FAST;
20
21const waitUntilGone = (name: string) =>
22 dataplex.getProjectsLocationsDataScans({ name }).pipe(
23 Effect.as("found" as const),
24 Effect.catchTag("NotFound", () => Effect.succeed("gone" as const)),
25 Effect.repeat({
26 schedule: Schedule.spaced("2 seconds"),
27 until: (status) => status === "gone",
28 times: 10,
29 }),
30 );
31
32test.provider(
33 "getProjectsLocationsDataScans on a missing scan fails with a typed tag",
34 (stack) =>
35 Effect.gen(function* () {
36 const { project } = yield* GcpEnvironment.current;
37 yield* stack.destroy();
38
39 const error = yield* Effect.flip(
40 dataplex.getProjectsLocationsDataScans({
41 name: `projects/${project}/locations/us-central1/dataScans/alchemy-missing-scan`,
42 }),
43 );
44 expect(error._tag).toEqual("NotFound");
45
46 yield* stack.destroy();
47 }).pipe(logLevel, withDataplexSlot),
48 { tags: ["provider:gcp", "provider:gcp:dataplex", "live"], timeout: 900_000 },
49);
50
51test.provider.skipIf(!runLifecycle)(
52 "create, update, and delete a data scan",
53 (stack) =>
54 Effect.gen(function* () {
55 yield* stack.destroy();
56
57 const created = yield* stack.deploy(
58 Effect.gen(function* () {
59 const dataset = yield* GCP.BigQuery.Dataset("ScanData", {
60 location: "US-CENTRAL1",
61 forceDestroy: true,
62 });
63 const table = yield* GCP.BigQuery.Table("Orders", {
64 datasetId: dataset.datasetId,
65 schema: [{ name: "id", type: "STRING" }],
66 });
67 return yield* GCP.Dataplex.DataScan("OrdersProfile", {
68 location: "us-central1",
69 displayName: "orders profile",
70 description: "scan a",
71 labels: { env: "test" },
72 data: {
73 resource: Output.interpolate`//bigquery.googleapis.com/projects/${table.project}/datasets/${table.datasetId}/tables/${table.tableId}`,
74 },
75 dataProfileSpec: { samplingPercent: 100 },
76 });
77 }),
78 );
79
80 expect(created.name).toContain("/dataScans/");
81 expect(created.dataScanId).toEqual(expect.any(String));
82 expect(created.location).toEqual("us-central1");
83 expect(created.displayName).toEqual("orders profile");
84 expect(created.labels).toMatchObject({ env: "test" });
85
86 const fetched = yield* dataplex.getProjectsLocationsDataScans({
87 name: created.name,
88 view: "FULL",
89 });
90 expect(fetched.name).toEqual(created.name);
91 expect(fetched.labels?.env).toEqual("test");
92 expect(
93 Object.keys(fetched.labels ?? {}).some((key) =>
94 key.startsWith("alchemy-"),
95 ),
96 ).toEqual(true);
97
98 const updated = yield* stack.deploy(
99 Effect.gen(function* () {
100 const dataset = yield* GCP.BigQuery.Dataset("ScanData", {
101 location: "US-CENTRAL1",
102 forceDestroy: true,
103 });
104 const table = yield* GCP.BigQuery.Table("Orders", {
105 datasetId: dataset.datasetId,
106 schema: [{ name: "id", type: "STRING" }],
107 });
108 return yield* GCP.Dataplex.DataScan("OrdersProfile", {
109 dataScanId: created.dataScanId,
110 location: "us-central1",
111 displayName: "orders profile prod",
112 description: "scan b",
113 labels: { env: "prod", team: "data" },
114 data: {
115 resource: Output.interpolate`//bigquery.googleapis.com/projects/${table.project}/datasets/${table.datasetId}/tables/${table.tableId}`,
116 },
117 dataProfileSpec: { samplingPercent: 50 },
118 });
119 }),
120 );
121
122 expect(updated.name).toEqual(created.name);
123 expect(updated.displayName).toEqual("orders profile prod");
124 expect(updated.labels).toMatchObject({ env: "prod", team: "data" });
125
126 yield* stack.destroy();
127 const gone = yield* waitUntilGone(created.name);
128 expect(gone).toEqual("gone");
129 }).pipe(logLevel, withDataplexSlot),
130 { tags: ["provider:gcp", "provider:gcp:dataplex", "live"], timeout: 900_000 },
131);