1import * as AWS from "@/AWS";
2import * as Test from "@/Test/Alchemy";
3import * as iam from "@distilled.cloud/aws/iam";
4import * as lf from "@distilled.cloud/aws/lakeformation";
5import * as sts from "@distilled.cloud/aws/sts";
6import { describe, expect } from "alchemy-test";
7import * as Effect from "effect/Effect";
8import * as Stream from "effect/Stream";
9
10const { test } = Test.make({ providers: AWS.providers() });
11
12/**
13 * LF-tag operations and permission grants require the caller to be a data
14 * lake administrator. Each stack bootstraps the caller as admin via
15 * `DataLakeSettings` (managed additively + restored on destroy). Derive the
16 * caller's IAM role ARN (with path — `GetRole` returns it, the assumed-role
17 * STS ARN does not carry the path).
18 */
19const callerPrincipalArn = Effect.gen(function* () {
20 const identity = yield* sts.getCallerIdentity({});
21 const arn = identity.Arn!;
22 const match = /assumed-role\/([^/]+)\//.exec(arn);
23 if (match === null) return arn;
24 const role = yield* iam.getRole({ RoleName: match[1]! });
25 return role.Role.Arn;
26});
27
28const adminIds = Effect.gen(function* () {
29 const settings = (yield* lf.getDataLakeSettings({})).DataLakeSettings ?? {};
30 return (settings.DataLakeAdmins ?? [])
31 .map((p) => p.DataLakePrincipalIdentifier)
32 .filter((id): id is string => id !== undefined);
33});
34
35/** Raw (unfiltered) permission entries a principal holds on a database. */
36const principalDatabasePermissions = (
37 principal: string,
38 databaseName: string,
39) =>
40 Effect.gen(function* () {
41 const pages = yield* lf.listPermissions
42 .pages({ Resource: { Database: { Name: databaseName } } })
43 .pipe(Stream.runCollect);
44 return Array.from(pages)
45 .flatMap((p) => p.PrincipalResourcePermissions ?? [])
46 .filter((e) => e.Principal?.DataLakePrincipalIdentifier === principal)
47 .flatMap((e) => e.Permissions ?? [])
48 .sort();
49 });
50
51const trustPolicy: AWS.IAM.PolicyDocument = {
52 Version: "2012-10-17",
53 Statement: [
54 {
55 Effect: "Allow",
56 Principal: { Service: "glue.amazonaws.com" },
57 Action: ["sts:AssumeRole"],
58 },
59 ],
60};
61
62// All three tests mutate the account-level DataLakeSettings singleton
63// (adding the caller / a scratch role as admin) — run them sequentially so
64// concurrent read-modify-write cycles cannot clobber each other.
65describe.sequential(
66 "LakeFormation",
67 {
68 tags: [
69 "provider:aws",
70 "provider:aws:iam",
71 "provider:aws:lakeformation",
72 "live",
73 ],
74 },
75 () => {
76 test.provider(
77 "DataLakeSettings adds admins additively and restores on destroy",
78 (stack) =>
79 Effect.gen(function* () {
80 yield* stack.destroy();
81 const beforeAdmins = yield* adminIds;
82
83 const created = yield* stack.deploy(
84 Effect.gen(function* () {
85 const role = yield* AWS.IAM.Role("LfAdminRole", {
86 assumeRolePolicyDocument: trustPolicy,
87 });
88 const settings = yield* AWS.LakeFormation.DataLakeSettings(
89 "Settings",
90 { dataLakeAdmins: [role.roleArn] },
91 );
92 return { role, settings };
93 }),
94 );
95
96 expect(created.settings.catalogId).toBeDefined();
97 expect(created.settings.dataLakeAdmins).toContain(
98 created.role.roleArn,
99 );
100 expect(created.settings.managedAdmins).toEqual([
101 created.role.roleArn,
102 ]);
103
104 // out-of-band: our admin was added, pre-existing admins survived
105 const duringAdmins = yield* adminIds;
106 expect(duringAdmins).toContain(created.role.roleArn);
107 for (const admin of beforeAdmins) {
108 expect(duringAdmins).toContain(admin);
109 }
110
111 // destroy — only the admin we added is removed
112 yield* stack.destroy();
113 const afterAdmins = yield* adminIds;
114 expect(afterAdmins).not.toContain(created.role.roleArn);
115 for (const admin of beforeAdmins) {
116 expect(afterAdmins).toContain(admin);
117 }
118 }),
119 { timeout: 180_000 },
120 );
121
122 test.provider(
123 "Permissions grants, updates, and revokes on a Glue database",
124 (stack) =>
125 Effect.gen(function* () {
126 yield* stack.destroy();
127 const admin = yield* callerPrincipalArn;
128
129 const created = yield* stack.deploy(
130 Effect.gen(function* () {
131 const settings = yield* AWS.LakeFormation.DataLakeSettings(
132 "Admin",
133 { dataLakeAdmins: [admin] },
134 );
135 const database = yield* AWS.Glue.Database("LfDb", {});
136 const role = yield* AWS.IAM.Role("LfAnalyst", {
137 assumeRolePolicyDocument: trustPolicy,
138 });
139 const grant = yield* AWS.LakeFormation.Permissions("AnalystDb", {
140 // consume a settings output so admin bootstrap deploys first
141 // and is destroyed last
142 catalogId: settings.catalogId,
143 principal: role.roleArn,
144 resource: { database: { name: database.databaseName } },
145 permissions: ["CREATE_TABLE", "DESCRIBE"],
146 });
147 return { settings, database, role, grant };
148 }),
149 );
150
151 expect(created.grant.permissions).toEqual([
152 "CREATE_TABLE",
153 "DESCRIBE",
154 ]);
155 expect(created.grant.resource.Database?.Name).toEqual(
156 created.database.databaseName,
157 );
158
159 // out-of-band verification (raw entries, not principal-filtered)
160 const observed = yield* principalDatabasePermissions(
161 created.role.roleArn,
162 created.database.databaseName,
163 );
164 expect(observed).toEqual(["CREATE_TABLE", "DESCRIBE"]);
165
166 // update — swap CREATE_TABLE for ALTER (grant + revoke delta)
167 const updated = yield* stack.deploy(
168 Effect.gen(function* () {
169 const settings = yield* AWS.LakeFormation.DataLakeSettings(
170 "Admin",
171 { dataLakeAdmins: [admin] },
172 );
173 const database = yield* AWS.Glue.Database("LfDb", {});
174 const role = yield* AWS.IAM.Role("LfAnalyst", {
175 assumeRolePolicyDocument: trustPolicy,
176 });
177 const grant = yield* AWS.LakeFormation.Permissions("AnalystDb", {
178 catalogId: settings.catalogId,
179 principal: role.roleArn,
180 resource: { database: { name: database.databaseName } },
181 permissions: ["ALTER", "DESCRIBE"],
182 });
183 return { settings, database, role, grant };
184 }),
185 );
186
187 expect(updated.grant.permissions).toEqual(["ALTER", "DESCRIBE"]);
188 const reobserved = yield* principalDatabasePermissions(
189 created.role.roleArn,
190 created.database.databaseName,
191 );
192 expect(reobserved).toEqual(["ALTER", "DESCRIBE"]);
193
194 // remove the grant from the stack — permissions are revoked while
195 // the database still exists, so we can verify out-of-band
196 yield* stack.deploy(
197 Effect.gen(function* () {
198 const settings = yield* AWS.LakeFormation.DataLakeSettings(
199 "Admin",
200 { dataLakeAdmins: [admin] },
201 );
202 const database = yield* AWS.Glue.Database("LfDb", {});
203 const role = yield* AWS.IAM.Role("LfAnalyst", {
204 assumeRolePolicyDocument: trustPolicy,
205 });
206 return { settings, database, role };
207 }),
208 );
209
210 const revoked = yield* principalDatabasePermissions(
211 created.role.roleArn,
212 created.database.databaseName,
213 );
214 expect(revoked).toEqual([]);
215
216 yield* stack.destroy();
217 }),
218 { tags: ["provider:aws:glue"], timeout: 240_000 },
219 );
220
221 test.provider(
222 "LFTag lifecycle and LFTagAssociation on a Glue database",
223 (stack) =>
224 Effect.gen(function* () {
225 yield* stack.destroy();
226 const admin = yield* callerPrincipalArn;
227
228 const created = yield* stack.deploy(
229 Effect.gen(function* () {
230 const settings = yield* AWS.LakeFormation.DataLakeSettings(
231 "Admin",
232 { dataLakeAdmins: [admin] },
233 );
234 const tag = yield* AWS.LakeFormation.LFTag("EnvTag", {
235 catalogId: settings.catalogId,
236 tagKey: "alchemy-lf-env",
237 tagValues: ["dev", "prod"],
238 });
239 const database = yield* AWS.Glue.Database("LfTagDb", {});
240 const association = yield* AWS.LakeFormation.LFTagAssociation(
241 "DbEnv",
242 {
243 catalogId: settings.catalogId,
244 resource: { database: { name: database.databaseName } },
245 lfTags: [{ tagKey: tag.tagKey, tagValues: ["dev"] }],
246 },
247 );
248 return { settings, tag, database, association };
249 }),
250 );
251
252 expect(created.tag.tagKey).toEqual("alchemy-lf-env");
253 expect([...created.tag.tagValues].sort()).toEqual(["dev", "prod"]);
254
255 // out-of-band: tag definition + assignment on the database
256 const observedTag = yield* lf.getLFTag({ TagKey: "alchemy-lf-env" });
257 expect([...(observedTag.TagValues ?? [])].sort()).toEqual([
258 "dev",
259 "prod",
260 ]);
261 const observedAssignment = yield* lf.getResourceLFTags({
262 Resource: {
263 Database: { Name: created.database.databaseName },
264 },
265 });
266 expect(
267 observedAssignment.LFTagOnDatabase?.find(
268 (t) => t.TagKey === "alchemy-lf-env",
269 )?.TagValues,
270 ).toEqual(["dev"]);
271
272 // update — add a tag value and move the assignment onto it
273 yield* stack.deploy(
274 Effect.gen(function* () {
275 const settings = yield* AWS.LakeFormation.DataLakeSettings(
276 "Admin",
277 { dataLakeAdmins: [admin] },
278 );
279 const tag = yield* AWS.LakeFormation.LFTag("EnvTag", {
280 catalogId: settings.catalogId,
281 tagKey: "alchemy-lf-env",
282 tagValues: ["dev", "prod", "staging"],
283 });
284 const database = yield* AWS.Glue.Database("LfTagDb", {});
285 const association = yield* AWS.LakeFormation.LFTagAssociation(
286 "DbEnv",
287 {
288 catalogId: settings.catalogId,
289 resource: { database: { name: database.databaseName } },
290 lfTags: [{ tagKey: tag.tagKey, tagValues: ["staging"] }],
291 },
292 );
293 return { settings, tag, database, association };
294 }),
295 );
296
297 const updatedTag = yield* lf.getLFTag({ TagKey: "alchemy-lf-env" });
298 expect([...(updatedTag.TagValues ?? [])].sort()).toEqual([
299 "dev",
300 "prod",
301 "staging",
302 ]);
303 const updatedAssignment = yield* lf.getResourceLFTags({
304 Resource: {
305 Database: { Name: created.database.databaseName },
306 },
307 });
308 expect(
309 updatedAssignment.LFTagOnDatabase?.find(
310 (t) => t.TagKey === "alchemy-lf-env",
311 )?.TagValues,
312 ).toEqual(["staging"]);
313
314 // remove tag + association from the stack (settings stay, so the
315 // caller is still admin and can verify the deletion out-of-band —
316 // GetLFTag as a non-admin is AccessDenied, not EntityNotFound)
317 yield* stack.deploy(
318 Effect.gen(function* () {
319 const settings = yield* AWS.LakeFormation.DataLakeSettings(
320 "Admin",
321 { dataLakeAdmins: [admin] },
322 );
323 return { settings };
324 }),
325 );
326 const goneTag = yield* lf
327 .getLFTag({ TagKey: "alchemy-lf-env" })
328 .pipe(
329 Effect.catchTag("EntityNotFoundException", () =>
330 Effect.succeed(undefined),
331 ),
332 );
333 expect(goneTag).toBeUndefined();
334
335 yield* stack.destroy();
336 }),
337 { tags: ["provider:aws:glue"], timeout: 240_000 },
338 );
339
340 test.provider(
341 "LFTagExpression, DataCellsFilter, and OptIn lifecycle",
342 (stack) =>
343 Effect.gen(function* () {
344 yield* stack.destroy();
345 const admin = yield* callerPrincipalArn;
346
347 const build = (options: {
348 expressionValues: string[];
349 rowFilterExpression?: string;
350 }) =>
351 Effect.gen(function* () {
352 const settings = yield* AWS.LakeFormation.DataLakeSettings(
353 "Admin",
354 { dataLakeAdmins: [admin] },
355 );
356 const tag = yield* AWS.LakeFormation.LFTag("ExprTag", {
357 catalogId: settings.catalogId,
358 tagKey: "alchemy-lf-expr",
359 tagValues: ["a", "b"],
360 });
361 const expression = yield* AWS.LakeFormation.LFTagExpression(
362 "Expr",
363 {
364 catalogId: settings.catalogId,
365 name: "alchemy-lf-expression",
366 description: "alchemy test expression",
367 expression: [
368 { tagKey: tag.tagKey, tagValues: options.expressionValues },
369 ],
370 },
371 );
372 const database = yield* AWS.Glue.Database("FilterDb", {});
373 const table = yield* AWS.Glue.Table("FilterTable", {
374 databaseName: database.databaseName,
375 tableType: "EXTERNAL_TABLE",
376 storageDescriptor: {
377 location: "s3://example-bucket/lf-filter/",
378 inputFormat:
379 "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat",
380 outputFormat:
381 "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat",
382 serdeInfo: {
383 serializationLibrary:
384 "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe",
385 },
386 columns: [
387 { name: "id", type: "string" },
388 { name: "email", type: "string" },
389 ],
390 },
391 });
392 const filter = yield* AWS.LakeFormation.DataCellsFilter(
393 "NoEmail",
394 {
395 tableCatalogId: settings.catalogId,
396 databaseName: database.databaseName,
397 tableName: table.tableName,
398 name: "alchemy-no-email",
399 excludedColumnNames: ["email"],
400 rowFilter:
401 options.rowFilterExpression !== undefined
402 ? { filterExpression: options.rowFilterExpression }
403 : undefined,
404 },
405 );
406 const role = yield* AWS.IAM.Role("OptInRole", {
407 assumeRolePolicyDocument: trustPolicy,
408 });
409 const optIn = yield* AWS.LakeFormation.OptIn("DbOptIn", {
410 principal: role.roleArn,
411 resource: {
412 // consume settings.catalogId so admin bootstrap deploys first
413 database: {
414 catalogId: settings.catalogId,
415 name: database.databaseName,
416 },
417 },
418 });
419 return {
420 settings,
421 tag,
422 expression,
423 database,
424 table,
425 filter,
426 role,
427 optIn,
428 };
429 });
430
431 const created = yield* stack.deploy(
432 build({ expressionValues: ["a"] }),
433 );
434
435 expect(created.expression.name).toEqual("alchemy-lf-expression");
436 expect(created.expression.expression).toEqual([
437 { tagKey: "alchemy-lf-expr", tagValues: ["a"] },
438 ]);
439 expect(created.filter.name).toEqual("alchemy-no-email");
440 expect(created.optIn.principal).toEqual(created.role.roleArn);
441
442 // out-of-band verification
443 const observedExpr = yield* lf.getLFTagExpression({
444 Name: "alchemy-lf-expression",
445 });
446 expect(observedExpr.Expression?.[0]?.TagKey).toEqual(
447 "alchemy-lf-expr",
448 );
449 expect(observedExpr.Expression?.[0]?.TagValues).toEqual(["a"]);
450 const observedFilter = yield* lf.getDataCellsFilter({
451 TableCatalogId: created.filter.tableCatalogId,
452 DatabaseName: created.database.databaseName,
453 TableName: created.table.tableName,
454 Name: "alchemy-no-email",
455 });
456 expect(
457 observedFilter.DataCellsFilter?.ColumnWildcard?.ExcludedColumnNames,
458 ).toEqual(["email"]);
459 const observedOptIns = yield* lf.listLakeFormationOptIns({
460 Principal: {
461 DataLakePrincipalIdentifier: created.role.roleArn,
462 },
463 Resource: {
464 Database: { Name: created.database.databaseName },
465 },
466 });
467 expect(
468 observedOptIns.LakeFormationOptInsInfoList?.length,
469 ).toBeGreaterThanOrEqual(1);
470
471 // update — swap the expression's tag values and add a row filter
472 yield* stack.deploy(
473 build({ expressionValues: ["b"], rowFilterExpression: "id='x'" }),
474 );
475
476 const updatedExpr = yield* lf.getLFTagExpression({
477 Name: "alchemy-lf-expression",
478 });
479 expect(updatedExpr.Expression?.[0]?.TagValues).toEqual(["b"]);
480 const updatedFilter = yield* lf.getDataCellsFilter({
481 TableCatalogId: created.filter.tableCatalogId,
482 DatabaseName: created.database.databaseName,
483 TableName: created.table.tableName,
484 Name: "alchemy-no-email",
485 });
486 expect(
487 updatedFilter.DataCellsFilter?.RowFilter?.FilterExpression,
488 ).toEqual("id='x'");
489
490 // remove everything but the admin bootstrap so deletion can be
491 // verified out-of-band while the caller is still an admin
492 yield* stack.deploy(
493 Effect.gen(function* () {
494 const settings = yield* AWS.LakeFormation.DataLakeSettings(
495 "Admin",
496 { dataLakeAdmins: [admin] },
497 );
498 return { settings };
499 }),
500 );
501
502 const goneExpr = yield* lf
503 .getLFTagExpression({ Name: "alchemy-lf-expression" })
504 .pipe(
505 Effect.catchTag("EntityNotFoundException", () =>
506 Effect.succeed(undefined),
507 ),
508 );
509 expect(goneExpr).toBeUndefined();
510 const goneOptIns = yield* lf.listLakeFormationOptIns({
511 Principal: {
512 DataLakePrincipalIdentifier: created.role.roleArn,
513 },
514 });
515 expect(goneOptIns.LakeFormationOptInsInfoList ?? []).toHaveLength(0);
516
517 yield* stack.destroy();
518 }),
519 { tags: ["provider:aws:glue"], timeout: 240_000 },
520 );
521 },
522);