1import * as GCP from "@/GCP";
2import * as Test from "@/Test/Alchemy";
3import * as kafka from "@distilled.cloud/gcp/managedkafka_v1";
4import { expect } from "alchemy-test";
5import * as Effect from "effect/Effect";
6import { MinimumLogLevel } from "effect/References";
7import * as Schedule from "effect/Schedule";
8import { GcpEnvironment } from "@/GCP/Environment";
9import { withKafkaClusterSlot } from "./quota.ts";
10
11const { test } = Test.make({ providers: GCP.providers() });
12
13const logLevel = Effect.provideService(
14 MinimumLogLevel,
15 process.env.DEBUG ? "Debug" : "Info",
16);
17
18// Kafka and Connect clusters take ~30 minutes to provision.
19const runLifecycle = !!process.env.GCP_TEST_SLOW && !process.env.FAST;
20
21const waitUntilGone = (name: string) =>
22 kafka.getProjectsLocationsConnectClustersConnectors({ name }).pipe(
23 Effect.as("found" as const),
24 Effect.catchTag("NotFound", () => Effect.succeed("gone" as const)),
25 Effect.repeat({
26 schedule: Schedule.spaced("1 second"),
27 until: (status) => status === "gone",
28 times: 10,
29 }),
30 );
31
32test.provider(
33 "getProjectsLocationsConnectClustersConnectors on a missing connector fails with a typed tag",
34 (stack) =>
35 Effect.gen(function* () {
36 const { project } = yield* GcpEnvironment.current;
37
38 yield* stack.destroy();
39
40 const error = yield* Effect.flip(
41 kafka.getProjectsLocationsConnectClustersConnectors({
42 name: `projects/${project}/locations/us-central1/connectClusters/alchemy-missing/connectors/alchemy-missing`,
43 }),
44 );
45 expect(error._tag).toEqual("NotFound");
46
47 yield* stack.destroy();
48 }).pipe(logLevel),
49 {
50 tags: ["provider:gcp", "provider:gcp:managedkafka", "live"],
51 timeout: 90_000,
52 },
53);
54
55test.provider.skipIf(!runLifecycle)(
56 "create, update, and delete a kafka connector",
57 (stack) =>
58 Effect.gen(function* () {
59 const { project } = yield* GcpEnvironment.current;
60
61 yield* stack.destroy();
62
63 const created = yield* stack.deploy(
64 Effect.gen(function* () {
65 const cluster = yield* GCP.ManagedKafka.Cluster("Brokers", {
66 location: "us-central1",
67 labels: { env: "test" },
68 });
69 const connect = yield* GCP.ManagedKafka.ConnectCluster("Connect", {
70 kafkaCluster: cluster.name,
71 labels: { env: "test" },
72 });
73 const connector = yield* GCP.ManagedKafka.ConnectClustersConnector(
74 "Sink",
75 {
76 connectCluster: connect.name,
77 configs: {
78 "connector.class":
79 "com.google.pubsub.kafka.sink.CloudPubSubSinkConnector",
80 "tasks.max": "1",
81 topics: "orders",
82 "cps.project": project,
83 "cps.topic": "alchemy-managedkafka-sink",
84 },
85 },
86 );
87 return { cluster, connect, connector };
88 }),
89 );
90
91 expect(created.connector.name).toContain("/connectors/");
92 expect(created.connector.configs["connector.class"]).toContain("PubSub");
93
94 const fetched =
95 yield* kafka.getProjectsLocationsConnectClustersConnectors({
96 name: created.connector.name,
97 });
98 expect(fetched.name).toEqual(created.connector.name);
99
100 const updated = yield* stack.deploy(
101 Effect.gen(function* () {
102 const cluster = yield* GCP.ManagedKafka.Cluster("Brokers", {
103 clusterId: created.cluster.clusterId,
104 location: "us-central1",
105 labels: { env: "test" },
106 });
107 const connect = yield* GCP.ManagedKafka.ConnectCluster("Connect", {
108 connectClusterId: created.connect.connectClusterId,
109 kafkaCluster: cluster.name,
110 labels: { env: "test" },
111 });
112 const connector = yield* GCP.ManagedKafka.ConnectClustersConnector(
113 "Sink",
114 {
115 connectCluster: connect.name,
116 connectorId: created.connector.connectorId,
117 configs: {
118 "connector.class":
119 "com.google.pubsub.kafka.sink.CloudPubSubSinkConnector",
120 "tasks.max": "2",
121 topics: "orders",
122 "cps.project": project,
123 "cps.topic": "alchemy-managedkafka-sink",
124 },
125 },
126 );
127 return { cluster, connect, connector };
128 }),
129 );
130 expect(updated.connector.connectorId).toEqual(
131 created.connector.connectorId,
132 );
133 expect(updated.connector.configs["tasks.max"]).toEqual("2");
134
135 yield* stack.destroy();
136
137 const gone = yield* waitUntilGone(created.connector.name);
138 expect(gone).toEqual("gone");
139 }).pipe(logLevel, withKafkaClusterSlot),
140 {
141 tags: ["provider:gcp", "provider:gcp:managedkafka", "live"],
142 timeout: 10_800_000,
143 retry: 0,
144 },
145);