1import * as securitylake from "@distilled.cloud/aws/securitylake";
2import * as Data from "effect/Data";
3import type * as Duration from "effect/Duration";
4import * as Effect from "effect/Effect";
5import { Unowned } from "../../AdoptPolicy.ts";
6import * as Provider from "../../Provider.ts";
7import { Resource } from "../../Resource.ts";
8import { createInternalTags, diffTags, hasAlchemyTags } from "../../Tags.ts";
9import { toWireDays } from "../../Util/Duration.ts";
10import { AWSEnvironment } from "../Environment.ts";
11import type { Providers } from "../Providers.ts";
12import {
13 readSecurityLakeTags,
14 retryWhileConflict,
15 toTagList,
16} from "./internal.ts";
17
18/**
19 * Encryption settings for a Security Lake Region.
20 */
21export interface DataLakeEncryptionConfiguration {
22 /**
23 * The KMS key ID used to encrypt objects at rest, or `S3_MANAGED_KEY` for
24 * SSE-S3 (the default).
25 * @default "S3_MANAGED_KEY"
26 */
27 kmsKeyId?: string;
28}
29
30/**
31 * When objects expire out of the data lake.
32 */
33export interface DataLakeLifecycleExpiration {
34 /**
35 * How long after creation objects are deleted. Accepts any
36 * `Duration.Input` (e.g. `"365 days"`, `Duration.days(365)`; a bare
37 * number is milliseconds); the wire unit is whole days.
38 */
39 days?: Duration.Input;
40}
41
42/**
43 * A storage-class transition rule.
44 */
45export interface DataLakeLifecycleTransition {
46 /** The S3 storage class to transition into (e.g. `GLACIER`, `ONEZONE_IA`). */
47 storageClass?: string;
48 /**
49 * How long after creation objects transition. Accepts any
50 * `Duration.Input` (e.g. `"30 days"`, `Duration.days(30)`; a bare
51 * number is milliseconds); the wire unit is whole days.
52 */
53 days?: Duration.Input;
54}
55
56/**
57 * Lifecycle management (retention + storage-class transitions) for a Region.
58 */
59export interface DataLakeLifecycleConfiguration {
60 /** When objects expire out of the data lake. */
61 expiration?: DataLakeLifecycleExpiration;
62 /** Storage-class transition rules. */
63 transitions?: DataLakeLifecycleTransition[];
64}
65
66/**
67 * Rollup-Region replication for a contributing Region.
68 */
69export interface DataLakeReplicationConfiguration {
70 /** The rollup Regions that this Region's data is replicated into. */
71 regions?: string[];
72 /** IAM role ARN Security Lake assumes to replicate objects. */
73 roleArn?: string;
74}
75
76/**
77 * Per-Region configuration of the Security Lake data lake.
78 */
79export interface DataLakeRegionConfiguration {
80 /** The Region in which Security Lake is enabled. */
81 region: string;
82 /** Encryption at rest for this Region. */
83 encryptionConfiguration?: DataLakeEncryptionConfiguration;
84 /** Retention and storage-class transitions for this Region. */
85 lifecycleConfiguration?: DataLakeLifecycleConfiguration;
86 /** Rollup-Region replication for this Region. */
87 replicationConfiguration?: DataLakeReplicationConfiguration;
88}
89
90export interface DataLakeProps {
91 /**
92 * The Regions to enable Security Lake in, each with its own encryption,
93 * lifecycle, and replication settings. Adding a Region enables Security Lake
94 * there; removing a previously configured Region disables it there.
95 */
96 configurations: DataLakeRegionConfiguration[];
97
98 /**
99 * ARN of the IAM role used by the Security Lake metastore manager (the
100 * `AmazonSecurityLakeMetaStoreManagerV2` role) to populate the Glue
101 * metastore with partition updates.
102 */
103 metaStoreManagerRoleArn: string;
104
105 /**
106 * Tags applied to the data lake. Alchemy ownership tags are merged in
107 * automatically so the data lake can be recognized on subsequent runs.
108 */
109 tags?: Record<string, string>;
110}
111
112/** Attributes of the data lake in one Region. */
113export interface DataLakeRegionAttributes {
114 /** ARN of the data lake in this Region. */
115 dataLakeArn: string;
116 /** The Region. */
117 region: string;
118 /** ARN of the S3 bucket Security Lake created for this Region. */
119 s3BucketArn: string | undefined;
120 /** Creation status of the data lake in this Region. */
121 createStatus: string | undefined;
122}
123
124/** @resource */
125export interface DataLake extends Resource<
126 "AWS.SecurityLake.DataLake",
127 DataLakeProps,
128 {
129 /** ARN of the data lake in the current (deployment) Region. */
130 dataLakeArn: string;
131 /** Every Region the data lake is enabled in. */
132 regions: string[];
133 /** Per-Region attributes (ARN, S3 bucket, status). */
134 dataLakes: DataLakeRegionAttributes[];
135 },
136 never,
137 Providers
138> {}
139
140/**
141 * The Amazon Security Lake data lake — the account-wide singleton that
142 * onboards the account to Security Lake. Enabling it creates S3 buckets,
143 * registers them with Lake Formation, and configures the Glue metastore in
144 * every configured Region.
145 *
146 * This is a heavyweight, account-wide resource: enabling/disabling Security
147 * Lake affects the whole account, and the S3 buckets it creates are retained
148 * after the data lake is deleted.
149 *
150 * ### Enabling Security Lake
151 * **Example:** Single-Region data lake
152 * ```typescript
153 * const lake = yield* SecurityLake.DataLake("Lake", {
154 * configurations: [{ region: "us-west-2" }],
155 * metaStoreManagerRoleArn: metastoreRole.roleArn,
156 * });
157 * ```
158 *
159 * **Example:** Lifecycle management and KMS encryption
160 * ```typescript
161 * const lake = yield* SecurityLake.DataLake("Lake", {
162 * configurations: [
163 * {
164 * region: "us-west-2",
165 * encryptionConfiguration: { kmsKeyId: key.keyId },
166 * lifecycleConfiguration: {
167 * expiration: { days: "365 days" },
168 * transitions: [{ storageClass: "ONEZONE_IA", days: "30 days" }],
169 * },
170 * },
171 * ],
172 * metaStoreManagerRoleArn: metastoreRole.roleArn,
173 * tags: { team: "security" },
174 * });
175 * ```
176 */
177const DataLakeResource = Resource<DataLake>("AWS.SecurityLake.DataLake");
178
179export { DataLakeResource as DataLake };
180
181/**
182 * The data lake failed to reach `COMPLETED` in one of the configured Regions.
183 */
184export class DataLakeCreateFailed extends Data.TaggedError(
185 "DataLakeCreateFailed",
186)<{
187 readonly region: string;
188 readonly reason: string | undefined;
189 readonly code: string | undefined;
190}> {}
191
192// Convert a per-Region prop configuration (Duration-typed lifecycle days)
193// into the wire shape the Security Lake API expects (whole days).
194const toWireConfiguration = (
195 config: DataLakeRegionConfiguration,
196): securitylake.DataLakeConfiguration => ({
197 region: config.region,
198 encryptionConfiguration: config.encryptionConfiguration,
199 lifecycleConfiguration: config.lifecycleConfiguration
200 ? {
201 expiration: config.lifecycleConfiguration.expiration
202 ? { days: toWireDays(config.lifecycleConfiguration.expiration.days) }
203 : undefined,
204 transitions: config.lifecycleConfiguration.transitions?.map(
205 (transition) => ({
206 storageClass: transition.storageClass,
207 days: toWireDays(transition.days),
208 }),
209 ),
210 }
211 : undefined,
212 replicationConfiguration: config.replicationConfiguration,
213});
214
215// Normalize a per-Region config for observed↔desired comparison. AWS reports
216// SSE-S3 as `S3_MANAGED_KEY`, so an unset kmsKeyId compares equal to it.
217const normalizeConfig = (config: {
218 encryptionConfiguration?: securitylake.DataLakeEncryptionConfiguration;
219 lifecycleConfiguration?: securitylake.DataLakeLifecycleConfiguration;
220 replicationConfiguration?: securitylake.DataLakeReplicationConfiguration;
221}) =>
222 JSON.stringify({
223 kmsKeyId: config.encryptionConfiguration?.kmsKeyId ?? "S3_MANAGED_KEY",
224 expirationDays: config.lifecycleConfiguration?.expiration?.days,
225 transitions: [...(config.lifecycleConfiguration?.transitions ?? [])]
226 .map((t) => ({ storageClass: t.storageClass, days: t.days }))
227 .sort((l, r) =>
228 (l.storageClass ?? "").localeCompare(r.storageClass ?? ""),
229 ),
230 replicationRegions: [
231 ...(config.replicationConfiguration?.regions ?? []),
232 ].sort(),
233 replicationRoleArn: config.replicationConfiguration?.roleArn,
234 });
235
236const buildRegionAttrs = (
237 lake: securitylake.DataLakeResource,
238): DataLakeRegionAttributes => ({
239 dataLakeArn: lake.dataLakeArn,
240 region: lake.region,
241 s3BucketArn: lake.s3BucketArn,
242 createStatus: lake.createStatus,
243});
244
245const buildAttrs = (
246 lakes: securitylake.DataLakeResource[],
247 currentRegion: string,
248) => {
249 const sorted = [...lakes].sort((l, r) => l.region.localeCompare(r.region));
250 const current = sorted.find((lake) => lake.region === currentRegion);
251 return {
252 dataLakeArn: (current ?? sorted[0]!).dataLakeArn,
253 regions: sorted.map((lake) => lake.region),
254 dataLakes: sorted.map(buildRegionAttrs),
255 };
256};
257
258// All data lakes for the account across every Region.
259const observeDataLakes = securitylake
260 .listDataLakes({})
261 .pipe(Effect.map((response) => [...(response.dataLakes ?? [])]));
262
263// Observation for read/list: an account that never onboarded Security Lake
264// rejects listDataLakes with AccessDeniedException — that means "no data
265// lake", not a failure.
266const observeDataLakesIfOnboarded = observeDataLakes.pipe(
267 Effect.catchTag(["AccessDeniedException", "UnauthorizedException"], () =>
268 Effect.succeed([] as securitylake.DataLakeResource[]),
269 ),
270);
271
272export const DataLakeProvider = () =>
273 Provider.effect(
274 DataLakeResource,
275 Effect.gen(function* () {
276 // Bounded wait (~90s) for every desired Region to finish onboarding.
277 // A FAILED createStatus is surfaced as a typed error; still-PENDING
278 // Regions after the window are returned as-is (the next reconcile
279 // converges) rather than hanging.
280 const awaitDataLakeReady = Effect.fn(function* (regions: string[]) {
281 let lakes: securitylake.DataLakeResource[] = [];
282 for (let attempt = 0; attempt < 18; attempt++) {
283 lakes = yield* observeDataLakes;
284 const failed = lakes.find(
285 (lake) =>
286 regions.includes(lake.region) && lake.createStatus === "FAILED",
287 );
288 if (failed) {
289 return yield* Effect.fail(
290 new DataLakeCreateFailed({
291 region: failed.region,
292 reason: failed.updateStatus?.exception?.reason,
293 code: failed.updateStatus?.exception?.code,
294 }),
295 );
296 }
297 const allReady = regions.every((region) =>
298 lakes.some(
299 (lake) =>
300 lake.region === region && lake.createStatus === "COMPLETED",
301 ),
302 );
303 if (allReady) return lakes;
304 yield* Effect.sleep("5 seconds");
305 }
306 return lakes;
307 });
308
309 return {
310 read: Effect.fn(function* ({ id }) {
311 const { region } = yield* AWSEnvironment.current;
312 const lakes = yield* observeDataLakesIfOnboarded;
313 if (lakes.length === 0) return undefined;
314 const attrs = buildAttrs(lakes, region);
315 const tags = yield* readSecurityLakeTags(attrs.dataLakeArn);
316 return (yield* hasAlchemyTags(id, tags)) ? attrs : Unowned(attrs);
317 }),
318
319 // The data lake is an account-wide singleton spanning Regions.
320 list: () =>
321 Effect.gen(function* () {
322 const { region } = yield* AWSEnvironment.current;
323 const lakes = yield* observeDataLakesIfOnboarded;
324 return lakes.length > 0 ? [buildAttrs(lakes, region)] : [];
325 }),
326
327 reconcile: Effect.fn(function* ({ id, news, output, session }) {
328 const { region } = yield* AWSEnvironment.current;
329 const internalTags = yield* createInternalTags(id);
330 const desiredTags = { ...news.tags, ...internalTags };
331 const desiredConfigs = news.configurations.map(toWireConfiguration);
332 const desiredRegions = desiredConfigs.map((c) => c.region);
333
334 // 1. OBSERVE — cloud state is authoritative.
335 let lakes = yield* observeDataLakes;
336 const observedByRegion = new Map(
337 lakes.map((lake) => [lake.region, lake]),
338 );
339
340 // 2. ENSURE — createDataLake is additive per Region, so enabling
341 // missing Regions covers both greenfield onboarding and Region
342 // expansion. A ConflictException is a race with a concurrent
343 // enable — fall through to observation.
344 const missing = desiredConfigs.filter(
345 (config) => !observedByRegion.has(config.region),
346 );
347 if (missing.length > 0) {
348 yield* securitylake
349 .createDataLake({
350 configurations: missing,
351 metaStoreManagerRoleArn: news.metaStoreManagerRoleArn,
352 tags: toTagList(desiredTags),
353 })
354 .pipe(
355 Effect.catchTag("ConflictException", () => Effect.succeed({})),
356 );
357 }
358
359 // 3. SYNC per-Region settings — observed ↔ desired delta only.
360 const changed = desiredConfigs.filter((config) => {
361 const observed = observedByRegion.get(config.region);
362 return (
363 observed !== undefined &&
364 normalizeConfig(config) !== normalizeConfig(observed)
365 );
366 });
367 if (changed.length > 0) {
368 yield* securitylake.updateDataLake({
369 configurations: changed,
370 metaStoreManagerRoleArn: news.metaStoreManagerRoleArn,
371 });
372 }
373
374 // 3b. SYNC Region removals — only Regions this resource previously
375 // managed (recorded in output) are disabled; foreign Regions that
376 // happen to be enabled are left alone.
377 const staleRegions = (output?.regions ?? []).filter(
378 (managed) =>
379 !desiredRegions.includes(managed) &&
380 observedByRegion.has(managed),
381 );
382 if (staleRegions.length > 0) {
383 yield* securitylake
384 .deleteDataLake({ regions: staleRegions })
385 .pipe(retryWhileConflict);
386 }
387
388 // 3c. Wait (bounded) for onboarding to complete in every Region.
389 lakes = yield* awaitDataLakeReady(desiredRegions);
390 if (lakes.length === 0) {
391 return yield* Effect.fail(
392 new DataLakeCreateFailed({
393 region: desiredRegions[0] ?? region,
394 reason: "data lake not observable after create",
395 code: undefined,
396 }),
397 );
398 }
399 const managed = lakes.filter((lake) =>
400 desiredRegions.includes(lake.region),
401 );
402 const attrs = buildAttrs(
403 managed.length > 0 ? managed : lakes,
404 region,
405 );
406
407 // 3d. SYNC tags — diff against OBSERVED cloud tags.
408 const observedTags = yield* readSecurityLakeTags(attrs.dataLakeArn);
409 const { upsert, removed } = diffTags(observedTags, desiredTags);
410 if (upsert.length > 0) {
411 yield* securitylake.tagResource({
412 resourceArn: attrs.dataLakeArn,
413 tags: upsert.map((t) => ({ key: t.Key, value: t.Value })),
414 });
415 }
416 if (removed.length > 0) {
417 yield* securitylake.untagResource({
418 resourceArn: attrs.dataLakeArn,
419 tagKeys: removed,
420 });
421 }
422
423 // 4. RETURN fresh attributes.
424 yield* session.note(attrs.dataLakeArn);
425 return attrs;
426 }),
427
428 // Disabling Security Lake retains the S3 buckets it created (AWS
429 // behavior); only log collection and the lake configuration go away.
430 delete: Effect.fn(function* ({ output }) {
431 if (output.regions.length === 0) return;
432 yield* securitylake.deleteDataLake({ regions: output.regions }).pipe(
433 retryWhileConflict,
434 Effect.catchTag("ResourceNotFoundException", () => Effect.void),
435 );
436 }),
437 };
438 }),
439 );