1import * as dlp from "@distilled.cloud/gcp/dlp_v2";
2import * as Data from "effect/Data";
3import * as Effect from "effect/Effect";
4import * as Schedule from "effect/Schedule";
5import { Unowned } from "../../AdoptPolicy.ts";
6import { isResolved } from "../../Diff.ts";
7import * as Provider from "../../Provider.ts";
8import { Resource } from "../../Resource.ts";
9import { GcpEnvironment } from "../Environment.ts";
10import { createInternalLabels } from "../Labels.ts";
11import type { Providers } from "../Providers.ts";
12import {
13 collectPages,
14 encodeOwnership,
15 hasHybridOwnership,
16 hybridLabels,
17 jsonEqual,
18 lastSegment,
19 projectOf,
20 replaceOnIdentity,
21 toResourceId,
22} from "./internal.ts";
23
24export type RiskAnalysisJobConfig = dlp.GooglePrivacyDlpV2RiskAnalysisJobConfig;
25
26export type DlpJobProps = {
27 /**
28 * Job id (the `{dlpJob}` segment of `projects/{project}/dlpJobs/{dlpJob}`).
29 * If omitted, a unique name is generated. Must match `[a-zA-Z0-9_-]+`
30 * and is at most 100 characters. Immutable — changing it replaces the
31 * job.
32 */
33 jobId?: string;
34 /**
35 * Inspect-job configuration. Mutually exclusive with `riskJob`.
36 * Hybrid storage is the recommended test shape: Alchemy stamps
37 * ownership into `hybridOptions.labels` for `list` / nuke.
38 */
39 inspectJob?: dlp.GooglePrivacyDlpV2InspectJobConfig;
40 /**
41 * Risk-analysis job configuration. Mutually exclusive with
42 * `inspectJob`.
43 */
44 riskJob?: RiskAnalysisJobConfig;
45};
46
47export type DlpJob = Resource<
48 "GCP.DLP.DlpJob",
49 DlpJobProps,
50 {
51 /** Full resource name `projects/{project}/dlpJobs/{id}`. */
52 name: string;
53 /** Job id (last path segment). */
54 jobId: string;
55 /** Project id. */
56 project: string;
57 /** Job type (`INSPECT_JOB` or `RISK_ANALYSIS_JOB`). */
58 type: string | undefined;
59 /** Job state (`PENDING`, `RUNNING`, `DONE`, …). */
60 state: string | undefined;
61 /** Inspect-job snapshot, if this is an inspect job. */
62 inspectJob: dlp.GooglePrivacyDlpV2InspectJobConfig | undefined;
63 /** Trigger that created the job, if any. */
64 jobTriggerName: string | undefined;
65 /** RFC3339 creation timestamp. */
66 createTime: string | undefined;
67 /** RFC3339 start timestamp. */
68 startTime: string | undefined;
69 /** RFC3339 end timestamp. */
70 endTime: string | undefined;
71 },
72 never,
73 Providers
74>;
75
76/**
77 * A project-scoped Cloud DLP inspect or risk-analysis job.
78 *
79 * DLP jobs have no labels or description field. For inspect jobs Alchemy
80 * stamps ownership into `storageConfig.hybridOptions.labels` so `list` /
81 * nuke can find them. Jobs are not updatable — changing `jobId`,
82 * `inspectJob`, or `riskJob` replaces the job.
83 *
84 * ### Creating a DLP Job
85 * **Example:** Hybrid inspect job
86 * ```typescript
87 * const job = yield* GCP.DLP.DlpJob("Scan", {
88 * inspectJob: {
89 * inspectConfig: { infoTypes: [{ name: "EMAIL_ADDRESS" }] },
90 * storageConfig: { hybridOptions: { description: "hybrid scan" } },
91 * },
92 * });
93 * ```
94 *
95 * @resource
96 * @category DLP
97 */
98export const DlpJob = Resource<DlpJob>("GCP.DLP.DlpJob");
99
100export class DlpJobNotResolved extends Data.TaggedError(
101 "GCP.DLP.DlpJobNotResolved",
102)<{
103 name: string;
104}> {}
105
106const MAX_JOB_ID_LENGTH = 64;
107
108const stripJobPrefix = (jobId: string) => jobId.replace(/^[ir]-/, "");
109
110const prefixedJobId = (jobId: string, risk: boolean) => {
111 const bare = stripJobPrefix(jobId).slice(0, MAX_JOB_ID_LENGTH - 2);
112 return `${risk ? "r" : "i"}-${bare}`;
113};
114
115const resourceName = (project: string, jobId: string, risk: boolean) =>
116 `projects/${project}/dlpJobs/${prefixedJobId(jobId, risk)}`;
117
118const observedInspectJob = (job: dlp.GooglePrivacyDlpV2DlpJob) =>
119 job.inspectDetails?.requestedOptions?.jobConfig;
120
121const observedHybridLabels = (job: dlp.GooglePrivacyDlpV2DlpJob) =>
122 observedInspectJob(job)?.storageConfig?.hybridOptions?.labels;
123
124const stampInspectJob = (
125 inspectJob: dlp.GooglePrivacyDlpV2InspectJobConfig | undefined,
126 ownership: Record<string, string>,
127): dlp.GooglePrivacyDlpV2InspectJobConfig | undefined => {
128 if (inspectJob === undefined) return undefined;
129 const hybrid = inspectJob.storageConfig?.hybridOptions;
130 const storage = inspectJob.storageConfig;
131 const usingOtherStorage =
132 storage?.cloudStorageOptions !== undefined ||
133 storage?.bigQueryOptions !== undefined ||
134 storage?.datastoreOptions !== undefined;
135 if (usingOtherStorage && hybrid === undefined) return inspectJob;
136 return {
137 ...inspectJob,
138 storageConfig: {
139 ...storage,
140 hybridOptions: {
141 ...hybrid,
142 labels: {
143 ...hybrid?.labels,
144 ...hybridLabels(ownership),
145 },
146 description: encodeOwnership(ownership, hybrid?.description),
147 },
148 },
149 };
150};
151
152const toAttrs = (job: dlp.GooglePrivacyDlpV2DlpJob, project: string) => {
153 const name = job.name ?? "";
154 return {
155 name,
156 jobId: lastSegment(name),
157 project: projectOf(name) || project,
158 type: job.type,
159 state: job.state,
160 inspectJob: observedInspectJob(job),
161 jobTriggerName: job.jobTriggerName,
162 createTime: job.createTime,
163 startTime: job.startTime,
164 endTime: job.endTime,
165 };
166};
167
168const getByName = (name: string) =>
169 name.length === 0
170 ? Effect.succeed(undefined)
171 : dlp
172 .getProjectsDlpJobs({ name })
173 .pipe(Effect.catchTag("NotFound", () => Effect.succeed(undefined)));
174
175export const DlpJobProvider = () =>
176 Provider.succeed(DlpJob, {
177 stables: ["name", "jobId", "project", "type", "createTime"],
178
179 diff: Effect.fn(function* ({ news, olds, output }) {
180 if (!isResolved(news)) return undefined;
181 const previous = olds?.jobId ?? output?.jobId;
182 const idChanged =
183 previous !== undefined &&
184 news.jobId !== undefined &&
185 news.jobId !== previous;
186 const inspectChanged =
187 olds !== undefined && !jsonEqual(olds.inspectJob, news.inspectJob);
188 const riskChanged =
189 olds !== undefined && !jsonEqual(olds.riskJob, news.riskJob);
190 return replaceOnIdentity(idChanged || inspectChanged || riskChanged);
191 }),
192
193 read: Effect.fn(function* ({ id, olds, output }) {
194 const env = yield* GcpEnvironment.current;
195 const jobId = yield* toResourceId(
196 id,
197 olds?.jobId !== undefined ? stripJobPrefix(olds.jobId) : undefined,
198 output?.jobId !== undefined ? stripJobPrefix(output.jobId) : undefined,
199 );
200 const risk =
201 olds?.riskJob !== undefined || output?.type === "RISK_ANALYSIS_JOB";
202 const name = output?.name ?? resourceName(env.project, jobId, risk);
203 const existing = yield* getByName(name);
204 if (existing === undefined) return undefined;
205 const attrs = toAttrs(existing, env.project);
206 if (output !== undefined && output.name === attrs.name) return attrs;
207 const labels = observedHybridLabels(existing);
208 return hasHybridOwnership(labels) ? attrs : Unowned(attrs);
209 }),
210
211 list: () =>
212 Effect.gen(function* () {
213 const env = yield* GcpEnvironment.current;
214 const items = yield* collectPages(
215 dlp.listProjectsDlpJobs.pages({
216 parent: `projects/${env.project}`,
217 pageSize: 100,
218 type: "INSPECT_JOB",
219 }),
220 (page) => page.jobs,
221 ).pipe(
222 Effect.catchTag("NotFound", () =>
223 Effect.succeed([] as dlp.GooglePrivacyDlpV2DlpJob[]),
224 ),
225 );
226 return items
227 .filter((job) => hasHybridOwnership(observedHybridLabels(job)))
228 .map((job) => toAttrs(job, env.project));
229 }),
230
231 reconcile: Effect.fn(function* ({ id, news, output }) {
232 const env = yield* GcpEnvironment.current;
233 const jobId = yield* toResourceId(
234 id,
235 news.jobId !== undefined ? stripJobPrefix(news.jobId) : undefined,
236 output?.jobId !== undefined ? stripJobPrefix(output.jobId) : undefined,
237 );
238 const risk = news.riskJob !== undefined;
239 const name = resourceName(env.project, jobId, risk);
240 const ownership = yield* createInternalLabels(id);
241 const inspectJob = stampInspectJob(news.inspectJob, ownership);
242
243 let current = yield* getByName(output?.name ?? name);
244
245 if (current === undefined) {
246 const created = yield* dlp
247 .createProjectsDlpJobs({
248 parent: `projects/${env.project}`,
249 body: {
250 jobId: prefixedJobId(jobId, risk),
251 inspectJob,
252 riskJob: news.riskJob,
253 },
254 })
255 .pipe(Effect.catchTag("Conflict", () => getByName(name)));
256 current = created ?? undefined;
257 }
258
259 if (current === undefined) {
260 return yield* new DlpJobNotResolved({ name });
261 }
262
263 if (
264 inspectJob !== undefined &&
265 observedHybridLabels(current) === undefined
266 ) {
267 current = yield* getByName(current.name ?? name).pipe(
268 Effect.filterOrFail(
269 (job) =>
270 job !== undefined &&
271 (observedHybridLabels(job) !== undefined ||
272 job.state === "FAILED" ||
273 job.state === "DONE" ||
274 job.state === "CANCELED"),
275 () => new DlpJobNotResolved({ name }),
276 ),
277 Effect.retry({
278 while: (error) => error._tag === "GCP.DLP.DlpJobNotResolved",
279 times: 8,
280 schedule: Schedule.spaced("1 second"),
281 }),
282 Effect.catchTag("GCP.DLP.DlpJobNotResolved", () =>
283 Effect.succeed(current),
284 ),
285 );
286 }
287
288 if (current === undefined) {
289 return yield* new DlpJobNotResolved({ name });
290 }
291
292 return toAttrs(current, env.project);
293 }),
294
295 delete: Effect.fn(function* ({ output }) {
296 yield* dlp
297 .deleteProjectsDlpJobs({ name: output.name })
298 .pipe(Effect.catchTag("NotFound", () => Effect.void));
299 }),
300 });