1import * as AWS from "@/AWS";
2import * as Core from "@/Test/Core";
3import * as Test from "@/Test/Alchemy";
4import { describe, expect } from "alchemy-test";
5import * as Data from "effect/Data";
6import * as Effect from "effect/Effect";
7import * as Schedule from "effect/Schedule";
8import * as HttpClient from "effect/http/HttpClient";
9import * as HttpClientRequest from "effect/http/HttpClientRequest";
10import BedrockTestFunctionLive, { BedrockTestFunction } from "./handler";
11
12const testOptions = { providers: AWS.providers() };
13const { test, beforeAll, afterAll } = Test.make(testOptions);
14const sharedStack = Core.scratchStack(testOptions, "BedrockBindings");
15
16// Lambda function URL cold-start (DNS, IAM propagation, init) can take well
17// over 60s on a fresh deploy under parallel-suite load. Budget ~150s of
18// readiness polling.
19const readinessPolicy = Schedule.max([
20 Schedule.fixed("2 seconds"),
21 Schedule.recurs(75),
22]);
23
24let baseUrl: string;
25
26class TransientUpstream extends Data.TaggedError("TransientUpstream")<{
27 readonly status: number;
28 readonly body: string;
29}> {}
30
31// The shared Lambda fixture occasionally answers a transient 5xx under
32// parallel load (cold re-init, IAM propagation, Bedrock throttling surfaced
33// as a defect). Retry 5xx only; a genuine 4xx/assertion failure surfaces
34// immediately.
35const send = (request: HttpClientRequest.HttpClientRequest) =>
36 HttpClient.execute(request).pipe(
37 Effect.flatMap((response) =>
38 response.status >= 500
39 ? response.text.pipe(
40 Effect.flatMap((body) =>
41 Effect.fail(
42 new TransientUpstream({ status: response.status, body }),
43 ),
44 ),
45 )
46 : Effect.succeed(response),
47 ),
48 Effect.retry({
49 while: (e) => e._tag === "TransientUpstream",
50 schedule: Schedule.max([
51 Schedule.exponential("1 second"),
52 Schedule.recurs(5),
53 ]),
54 }),
55 );
56
57describe(
58 "Bedrock Bindings",
59 {
60 tags: [
61 "provider:aws",
62 "provider:aws:bedrock",
63 "provider:aws:lambda",
64 "live",
65 ],
66 },
67 () => {
68 beforeAll(
69 Effect.gen(function* () {
70 yield* Effect.logInfo(
71 "Bedrock test setup: destroying previous resources",
72 );
73 yield* sharedStack.destroy();
74
75 yield* Effect.logInfo("Bedrock test setup: deploying fixture");
76 const { functionUrl } = yield* sharedStack.deploy(
77 Effect.gen(function* () {
78 return yield* BedrockTestFunction;
79 }).pipe(Effect.provide(BedrockTestFunctionLive)),
80 );
81
82 expect(functionUrl).toBeTruthy();
83 baseUrl = functionUrl!.replace(/\/+$/, "");
84 const readinessUrl = `${baseUrl}/ping`;
85
86 yield* Effect.logInfo(
87 `Bedrock test setup: probing readiness at ${readinessUrl}`,
88 );
89 yield* HttpClient.get(readinessUrl).pipe(
90 Effect.flatMap((response) =>
91 response.status === 200
92 ? Effect.succeed(response)
93 : Effect.fail(
94 new Error(`Function not ready: ${response.status}`),
95 ),
96 ),
97 Effect.tapError((error) =>
98 Effect.logWarning(
99 `Bedrock test setup: fixture not ready yet (${String(error)})`,
100 ),
101 ),
102 Effect.retry({ schedule: readinessPolicy }),
103 );
104 }),
105 { timeout: 240_000 },
106 );
107
108 afterAll.skipIf(!!process.env.NO_DESTROY)(sharedStack.destroy(), {
109 timeout: 120_000,
110 });
111
112 describe("CountTokens", () => {
113 test.provider(
114 "counts input tokens without invoking the model",
115 (_stack) =>
116 Effect.gen(function* () {
117 const response = (yield* send(
118 HttpClientRequest.get(`${baseUrl}/count-tokens`),
119 ).pipe(Effect.flatMap((r) => r.json))) as {
120 inputTokens: number;
121 };
122
123 expect(response.inputTokens).toBeGreaterThan(0);
124 }),
125 { timeout: 120_000 },
126 );
127 });
128
129 describe("Rerank", () => {
130 test.provider(
131 "reranks inline documents by relevance to the query",
132 (_stack) =>
133 Effect.gen(function* () {
134 const response = (yield* send(
135 HttpClientRequest.get(`${baseUrl}/rerank`),
136 ).pipe(Effect.flatMap((r) => r.json))) as {
137 results: Array<{ index: number; relevanceScore: number }>;
138 };
139
140 expect(response.results).toHaveLength(2);
141 // The alchemy document must outrank the banana document.
142 expect(response.results[0]!.index).toBe(0);
143 expect(response.results[0]!.relevanceScore).toBeGreaterThan(
144 response.results[1]!.relevanceScore,
145 );
146 }),
147 { timeout: 120_000 },
148 );
149 });
150
151 describe("InvokeAgent", () => {
152 test.provider(
153 "invokes the bound agent alias and streams a completion",
154 (_stack) =>
155 Effect.gen(function* () {
156 const response = (yield* send(
157 HttpClientRequest.get(`${baseUrl}/invoke-agent`),
158 ).pipe(Effect.flatMap((r) => r.json))) as {
159 sessionId: string;
160 chunkEvents: number;
161 text: string;
162 };
163
164 expect(response.sessionId.length).toBeGreaterThan(0);
165 expect(response.chunkEvents).toBeGreaterThan(0);
166 // Assert a non-empty completion — never exact model output.
167 expect(response.text.trim().length).toBeGreaterThan(0);
168 }),
169 { timeout: 120_000 },
170 );
171 });
172
173 describe("GetAgentMemory + DeleteAgentMemory", () => {
174 test.provider(
175 "reads and clears the agent's long-term memory for a memory id",
176 (_stack) =>
177 Effect.gen(function* () {
178 const response = (yield* send(
179 HttpClientRequest.get(`${baseUrl}/agent-memory`),
180 ).pipe(Effect.flatMap((r) => r.json))) as {
181 memoryContents: number;
182 deleted: boolean;
183 };
184
185 // Session summaries generate asynchronously long after a session
186 // ends, so a fresh agent has none — the route proves the IAM
187 // grants and the read+delete round-trip, not summary generation.
188 expect(response.memoryContents).toBeGreaterThanOrEqual(0);
189 expect(response.deleted).toBe(true);
190 }),
191 { timeout: 120_000 },
192 );
193 });
194
195 describe("Converse", () => {
196 test.provider(
197 "converses with the bound model and returns a completion",
198 (_stack) =>
199 Effect.gen(function* () {
200 const response = (yield* send(
201 HttpClientRequest.get(`${baseUrl}/converse`),
202 ).pipe(Effect.flatMap((r) => r.json))) as {
203 text: string;
204 stopReason: string;
205 outputTokens: number;
206 };
207
208 // Assert a non-empty completion — never exact model output (LLMs
209 // are nondeterministic even at temperature 0).
210 expect(response.text.trim().length).toBeGreaterThan(0);
211 expect(["end_turn", "max_tokens"]).toContain(response.stopReason);
212 expect(response.outputTokens).toBeGreaterThan(0);
213 }),
214 { timeout: 120_000 },
215 );
216 });
217
218 describe("ConverseStream", () => {
219 test.provider(
220 "streams a conversation as typed events",
221 (_stack) =>
222 Effect.gen(function* () {
223 const response = (yield* send(
224 HttpClientRequest.get(`${baseUrl}/converse-stream`),
225 ).pipe(Effect.flatMap((r) => r.json))) as {
226 text: string;
227 deltaEvents: number;
228 totalEvents: number;
229 stopReason: string | undefined;
230 };
231
232 // Assert a non-empty streamed completion — never exact model output.
233 expect(response.text.trim().length).toBeGreaterThan(0);
234 expect(response.deltaEvents).toBeGreaterThan(0);
235 expect(response.totalEvents).toBeGreaterThan(response.deltaEvents);
236 expect(["end_turn", "max_tokens"]).toContain(response.stopReason);
237 }),
238 { timeout: 120_000 },
239 );
240 });
241
242 describe("InvokeModelWithResponseStream", () => {
243 test.provider(
244 "streams raw payload chunks from the bound model",
245 (_stack) =>
246 Effect.gen(function* () {
247 const response = (yield* send(
248 HttpClientRequest.get(`${baseUrl}/invoke-model-stream`),
249 ).pipe(Effect.flatMap((r) => r.json))) as {
250 contentType: string;
251 chunkEvents: number;
252 text: string;
253 };
254
255 expect(response.contentType).toBe("application/json");
256 expect(response.chunkEvents).toBeGreaterThan(0);
257 // Assert a non-empty streamed completion — never exact model output.
258 expect(response.text.trim().length).toBeGreaterThan(0);
259 }),
260 { timeout: 120_000 },
261 );
262 });
263
264 describe("InvokeModel", () => {
265 test.provider(
266 "invokes the bound model with a raw payload and streams the response",
267 (_stack) =>
268 Effect.gen(function* () {
269 const response = (yield* send(
270 HttpClientRequest.get(`${baseUrl}/invoke-model`),
271 ).pipe(Effect.flatMap((r) => r.json))) as {
272 contentType: string;
273 text: string;
274 stopReason: string;
275 };
276
277 expect(response.contentType).toBe("application/json");
278 // Assert a non-empty completion — never exact model output.
279 expect(response.text.trim().length).toBeGreaterThan(0);
280 expect(["end_turn", "max_tokens"]).toContain(response.stopReason);
281 }),
282 { timeout: 120_000 },
283 );
284 });
285 },
286);