1import * as AWS from "@/AWS";
2import { Dataset } from "@/AWS/DataBrew";
3import { Bucket } from "@/AWS/S3";
4import * as Test from "@/Test/Alchemy";
5import * as databrew from "@distilled.cloud/aws/databrew";
6import { expect } from "alchemy-test";
7import * as Effect from "effect/Effect";
8
9const { test } = Test.make({ providers: AWS.providers() });
10
11const getDataset = (name: string) =>
12 databrew
13 .describeDataset({ Name: name })
14 .pipe(
15 Effect.catchTag("ResourceNotFoundException", () =>
16 Effect.succeed(undefined),
17 ),
18 );
19
20test.provider(
21 "create, update, delete DataBrew dataset",
22 (stack) =>
23 Effect.gen(function* () {
24 yield* stack.destroy();
25
26 const created = yield* stack.deploy(
27 Effect.gen(function* () {
28 const bucket = yield* Bucket("DataBucket", { forceDestroy: true });
29 const dataset = yield* Dataset("Sales", {
30 format: "CSV",
31 formatOptions: { csv: { delimiter: ",", headerRow: true } },
32 input: {
33 s3InputDefinition: {
34 bucket: bucket.bucketName,
35 key: "raw/sales.csv",
36 },
37 },
38 tags: { Environment: "test" },
39 });
40 return { bucket, dataset };
41 }),
42 );
43
44 expect(created.dataset.datasetName).toBeDefined();
45 expect(created.dataset.datasetArn).toContain(
46 `:dataset/${created.dataset.datasetName}`,
47 );
48
49 // out-of-band verification
50 const observed = yield* getDataset(created.dataset.datasetName);
51 expect(observed?.Name).toEqual(created.dataset.datasetName);
52 expect(observed?.Format).toEqual("CSV");
53 expect(observed?.Input.S3InputDefinition?.Bucket).toEqual(
54 created.bucket.bucketName,
55 );
56 expect(observed?.Input.S3InputDefinition?.Key).toEqual("raw/sales.csv");
57 expect(observed?.FormatOptions?.Csv?.Delimiter).toEqual(",");
58 expect(observed?.Tags?.["alchemy::id"]).toBeDefined();
59 expect(observed?.Tags?.Environment).toEqual("test");
60
61 // update: different key + delimiter + tag change
62 yield* stack.deploy(
63 Effect.gen(function* () {
64 const bucket = yield* Bucket("DataBucket", { forceDestroy: true });
65 const dataset = yield* Dataset("Sales", {
66 format: "CSV",
67 formatOptions: { csv: { delimiter: ";", headerRow: false } },
68 input: {
69 s3InputDefinition: {
70 bucket: bucket.bucketName,
71 key: "raw/sales-v2.csv",
72 },
73 },
74 tags: { Environment: "staging" },
75 });
76 return { dataset };
77 }),
78 );
79
80 const reobserved = yield* getDataset(created.dataset.datasetName);
81 expect(reobserved?.Input.S3InputDefinition?.Key).toEqual(
82 "raw/sales-v2.csv",
83 );
84 expect(reobserved?.FormatOptions?.Csv?.Delimiter).toEqual(";");
85 expect(reobserved?.Tags?.Environment).toEqual("staging");
86
87 yield* stack.destroy();
88 const gone = yield* getDataset(created.dataset.datasetName);
89 expect(gone).toBeUndefined();
90 }),
91 {
92 tags: ["provider:aws", "provider:aws:databrew", "provider:aws:s3", "live"],
93 timeout: 120_000,
94 },
95);
96
97test.provider(
98 "changing the dataset name replaces the dataset",
99 (stack) =>
100 Effect.gen(function* () {
101 yield* stack.destroy();
102
103 const nameA = "alchemy-test-databrew-dataset-a";
104 const nameB = "alchemy-test-databrew-dataset-b";
105
106 const first = yield* stack.deploy(
107 Effect.gen(function* () {
108 const bucket = yield* Bucket("DataBucket", { forceDestroy: true });
109 const dataset = yield* Dataset("Named", {
110 datasetName: nameA,
111 format: "CSV",
112 input: {
113 s3InputDefinition: {
114 bucket: bucket.bucketName,
115 key: "raw/data.csv",
116 },
117 },
118 });
119 return { bucket, dataset };
120 }),
121 );
122 expect(first.dataset.datasetName).toEqual(nameA);
123
124 const second = yield* stack.deploy(
125 Effect.gen(function* () {
126 const bucket = yield* Bucket("DataBucket", { forceDestroy: true });
127 const dataset = yield* Dataset("Named", {
128 datasetName: nameB,
129 format: "CSV",
130 input: {
131 s3InputDefinition: {
132 bucket: bucket.bucketName,
133 key: "raw/data.csv",
134 },
135 },
136 });
137 return { dataset };
138 }),
139 );
140 expect(second.dataset.datasetName).toEqual(nameB);
141
142 // the old dataset is gone, the new one exists
143 expect(yield* getDataset(nameA)).toBeUndefined();
144 expect((yield* getDataset(nameB))?.Name).toEqual(nameB);
145
146 yield* stack.destroy();
147 expect(yield* getDataset(nameB)).toBeUndefined();
148 }),
149 {
150 tags: ["provider:aws", "provider:aws:databrew", "provider:aws:s3", "live"],
151 timeout: 120_000,
152 },
153);