1import { afterAll, assert, describe, it } from "@effect/vitest"
2import * as Context from "effect/Context"
3import * as Effect from "effect/Effect"
4import * as Cookies from "effect/http/Cookies"
5import * as HttpEffect from "effect/http/HttpEffect"
6import * as HttpMiddleware from "effect/http/HttpMiddleware"
7import * as HttpPlatform from "effect/http/HttpPlatform"
8import * as HttpServer from "effect/http/HttpServer"
9import type { HttpServerRequest } from "effect/http/HttpServerRequest"
10import * as HttpServerResponse from "effect/http/HttpServerResponse"
11import * as Latch from "effect/Latch"
12import type * as Scope from "effect/Scope"
13import * as Stream from "effect/Stream"
14
15const bigText = "All work and no play makes Jack a dull boy. ".repeat(100)
16const bigJson = JSON.stringify({ text: bigText })
17
18const bigJsonApp = Effect.succeed(HttpServerResponse.text(bigJson, { contentType: "application/json" }))
19
20type App = Effect.Effect<HttpServerResponse.HttpServerResponse, any, HttpServerRequest>
21type CompressionOptions = Parameters<typeof HttpMiddleware.compression>[0]
22
23const get = (
24 handler: (request: Request) => Promise<Response>,
25 headers?: Record<string, string>,
26 method?: string
27) =>
28 handler(
29 new Request("http://localhost/", {
30 method: method ?? "GET",
31 ...(headers === undefined ? {} : { headers })
32 })
33 )
34
35const decompress = (data: ArrayBuffer, format: "gzip" | "deflate"): Promise<string> =>
36 new Response(new Blob([data]).stream().pipeThrough(new DecompressionStream(format))).text()
37
38const platformContext = (compression: HttpPlatform.Compression): Context.Context<HttpPlatform.HttpPlatform> =>
39 Context.make(
40 HttpPlatform.HttpPlatform,
41 HttpPlatform.HttpPlatform.of({
42 platform: "web",
43 compression,
44 fileResponse: () => Effect.die("not implemented"),
45 fileWebResponse: () => Effect.die("not implemented")
46 })
47 )
48
49// A platform stub used to exercise negotiation against algorithms the Web
50// implementation does not support. It only marks the chosen algorithm.
51const negotiationPlatformContext = (algorithms: ReadonlyArray<HttpPlatform.CompressionAlgorithm>) =>
52 platformContext({
53 algorithms: new Set(algorithms),
54 compressResponse: (response, algorithm) =>
55 Effect.succeed(HttpServerResponse.setHeader(response, "content-encoding", algorithm))
56 })
57
58const makeHandler = (
59 app: App,
60 options?: CompressionOptions,
61 context?: Context.Context<HttpPlatform.HttpPlatform>
62) => {
63 const self = app as Effect.Effect<HttpServerResponse.HttpServerResponse, any, HttpServerRequest | Scope.Scope>
64 const middleware = HttpMiddleware.compression(options)
65 if (context !== undefined) {
66 return HttpEffect.toWebHandlerWith<HttpPlatform.HttpPlatform, HttpServerRequest | Scope.Scope>(context)(
67 self,
68 middleware
69 )
70 }
71 const { dispose, handler } = HttpEffect.toWebHandlerLayer(self, HttpServer.layerServices, { middleware })
72 disposers.push(dispose)
73 return handler
74}
75
76const disposers: Array<() => Promise<void>> = []
77
78afterAll(() => Promise.all(disposers.map((dispose) => dispose())))
79
80const randomBytes = (length: number): Uint8Array => {
81 const data = new Uint8Array(length)
82 let state = 0x9e3779b9
83 for (let i = 0; i < length; i++) {
84 state ^= state << 13
85 state ^= state >>> 17
86 state ^= state << 5
87 data[i] = state & 0xff
88 }
89 return data
90}
91
92describe("HttpCompression", () => {
93 for (
94 const [name, makeResponse] of [
95 ["Stream", () => HttpServerResponse.stream(Stream.succeed(new TextEncoder().encode(bigJson)))],
96 ["Uint8Array", () => HttpServerResponse.uint8Array(new TextEncoder().encode(bigJson))],
97 ["Raw", () => HttpServerResponse.raw(new Response(bigJson).body, { contentType: "text/plain" })]
98 ] as const
99 ) {
100 it(`preserves a Content-Type header override when compressing web ${name} responses`, async () => {
101 const original = HttpServerResponse.setHeader(makeResponse(), "content-type", "application/json")
102 assert.strictEqual(original.headers["content-type"], "application/json")
103 const handler = makeHandler(Effect.succeed(original))
104
105 const compressed = await get(handler, { "accept-encoding": "gzip" })
106 assert.strictEqual(compressed.headers.get("content-encoding"), "gzip")
107 assert.strictEqual(await decompress(await compressed.arrayBuffer(), "gzip"), bigJson)
108 assert.strictEqual(compressed.headers.get("content-type"), original.headers["content-type"])
109 })
110 }
111
112 it("preserves a header-only Content-Type when compressing web Raw responses", async () => {
113 const handler = makeHandler(
114 Effect.sync(() =>
115 HttpServerResponse.raw(new Response(bigJson).body, { headers: { "content-type": "application/json" } })
116 )
117 )
118 const uncompressed = await get(handler, { "accept-encoding": "identity" })
119 assert.strictEqual(uncompressed.headers.get("content-type"), "application/json")
120 assert.strictEqual(await uncompressed.text(), bigJson)
121
122 const compressed = await get(handler, { "accept-encoding": "gzip" })
123 assert.strictEqual(compressed.headers.get("content-encoding"), "gzip")
124 assert.strictEqual(await decompress(await compressed.arrayBuffer(), "gzip"), bigJson)
125 assert.strictEqual(compressed.headers.get("content-type"), uncompressed.headers.get("content-type"))
126 })
127
128 it("compresses a compressible response with gzip", async () => {
129 const response = await get(makeHandler(bigJsonApp), { "accept-encoding": "gzip" })
130 assert.strictEqual(response.status, 200)
131 assert.strictEqual(response.headers.get("content-encoding"), "gzip")
132 assert.strictEqual(response.headers.get("vary"), "Accept-Encoding")
133 assert.strictEqual(response.headers.get("content-length"), null)
134 const text = await decompress(await response.arrayBuffer(), "gzip")
135 assert.strictEqual(text, bigJson)
136 })
137
138 it("compresses with deflate when requested", async () => {
139 const response = await get(makeHandler(bigJsonApp), { "accept-encoding": "deflate" })
140 assert.strictEqual(response.headers.get("content-encoding"), "deflate")
141 const text = await decompress(await response.arrayBuffer(), "deflate")
142 assert.strictEqual(text, bigJson)
143 })
144
145 it("server preference order picks br over gzip when supported", async () => {
146 const response = await get(
147 makeHandler(bigJsonApp, undefined, negotiationPlatformContext(["gzip", "deflate", "br", "zstd"])),
148 { "accept-encoding": "gzip, br" }
149 )
150 assert.strictEqual(response.headers.get("content-encoding"), "br")
151 })
152
153 it("client q-values decide acceptability only, server order decides ranking", async () => {
154 const response = await get(
155 makeHandler(bigJsonApp, undefined, negotiationPlatformContext(["gzip", "deflate", "br", "zstd"])),
156 { "accept-encoding": "gzip;q=1, br;q=0.5" }
157 )
158 assert.strictEqual(response.headers.get("content-encoding"), "br")
159 })
160
161 it("q=0 makes a coding unacceptable", async () => {
162 const response = await get(makeHandler(bigJsonApp), { "accept-encoding": "gzip;q=0" })
163 assert.strictEqual(response.headers.get("content-encoding"), null)
164 assert.strictEqual(response.headers.get("vary"), "Accept-Encoding")
165 assert.strictEqual(await response.text(), bigJson)
166 })
167
168 it("falls through server order past q=0 codings", async () => {
169 const response = await get(makeHandler(bigJsonApp), { "accept-encoding": "gzip;q=0, deflate" })
170 assert.strictEqual(response.headers.get("content-encoding"), "deflate")
171 })
172
173 it("wildcard matches unlisted codings", async () => {
174 const response = await get(makeHandler(bigJsonApp), { "accept-encoding": "*" })
175 assert.strictEqual(response.headers.get("content-encoding"), "gzip")
176 })
177
178 it("wildcard with q=0 disables unlisted codings", async () => {
179 const response = await get(makeHandler(bigJsonApp), { "accept-encoding": "*;q=0" })
180 assert.strictEqual(response.status, 200)
181 assert.strictEqual(response.headers.get("content-encoding"), null)
182 assert.strictEqual(await response.text(), bigJson)
183 })
184
185 it("sends identity with 200 when nothing is acceptable, never 406", async () => {
186 const response = await get(makeHandler(bigJsonApp), { "accept-encoding": "br, identity;q=0" })
187 assert.strictEqual(response.status, 200)
188 assert.strictEqual(response.headers.get("content-encoding"), null)
189 assert.strictEqual(response.headers.get("vary"), "Accept-Encoding")
190 assert.strictEqual(await response.text(), bigJson)
191 })
192
193 it("treats a malformed Accept-Encoding as absent", async () => {
194 const response = await get(makeHandler(bigJsonApp), { "accept-encoding": "gzip;q=oops" })
195 assert.strictEqual(response.headers.get("content-encoding"), null)
196 assert.strictEqual(await response.text(), bigJson)
197 })
198
199 it("treats an out-of-range q-value as absent", async () => {
200 const response = await get(makeHandler(bigJsonApp), { "accept-encoding": "gzip;q=1.5" })
201 assert.strictEqual(response.headers.get("content-encoding"), null)
202 assert.strictEqual(await response.text(), bigJson)
203 })
204
205 it("does not compress without an Accept-Encoding header", async () => {
206 const response = await get(makeHandler(bigJsonApp))
207 assert.strictEqual(response.headers.get("content-encoding"), null)
208 assert.strictEqual(response.headers.get("vary"), "Accept-Encoding")
209 assert.strictEqual(await response.text(), bigJson)
210 })
211
212 it("skips bodies below minSize but still sets Vary", async () => {
213 const app = Effect.succeed(HttpServerResponse.text("{}", { contentType: "application/json" }))
214 const response = await get(makeHandler(app), { "accept-encoding": "gzip" })
215 assert.strictEqual(response.headers.get("content-encoding"), null)
216 assert.strictEqual(response.headers.get("vary"), "Accept-Encoding")
217 assert.strictEqual(response.headers.get("content-length"), "2")
218 assert.strictEqual(await response.text(), "{}")
219 })
220
221 it("respects a custom minSize", async () => {
222 const app = Effect.succeed(HttpServerResponse.text("{}", { contentType: "application/json" }))
223 const response = await get(makeHandler(app, { minSize: 1 }), { "accept-encoding": "gzip" })
224 assert.strictEqual(response.headers.get("content-encoding"), "gzip")
225 })
226
227 it("ignores malformed content lengths when checking minSize", async () => {
228 for (const contentLength of ["1e3", "1.5"]) {
229 const app = Effect.succeed(
230 HttpServerResponse.stream(Stream.succeed(new TextEncoder().encode(bigJson)), {
231 contentType: "application/json",
232 headers: { "content-length": contentLength }
233 })
234 )
235 const response = await get(makeHandler(app), { "accept-encoding": "gzip" })
236 assert.strictEqual(response.headers.get("content-encoding"), "gzip")
237 }
238 })
239
240 it("skips non-compressible content types without Vary", async () => {
241 const app = Effect.succeed(
242 HttpServerResponse.uint8Array(randomBytes(2048), { contentType: "image/png" })
243 )
244 const response = await get(makeHandler(app), { "accept-encoding": "gzip" })
245 assert.strictEqual(response.headers.get("content-encoding"), null)
246 assert.strictEqual(response.headers.get("vary"), null)
247 })
248
249 it("supports a custom compressible predicate", async () => {
250 const app = Effect.succeed(
251 HttpServerResponse.uint8Array(randomBytes(2048), { contentType: "image/png" })
252 )
253 const response = await get(
254 makeHandler(app, { compressible: (contentType) => contentType === "image/png" }),
255 { "accept-encoding": "gzip" }
256 )
257 assert.strictEqual(response.headers.get("content-encoding"), "gzip")
258 })
259
260 it("leaves already-encoded responses untouched", async () => {
261 const app = Effect.succeed(
262 HttpServerResponse.text(bigJson, {
263 contentType: "application/json",
264 headers: { "content-encoding": "br" }
265 })
266 )
267 const response = await get(makeHandler(app), { "accept-encoding": "gzip" })
268 assert.strictEqual(response.headers.get("content-encoding"), "br")
269 assert.strictEqual(response.headers.get("vary"), null)
270 assert.strictEqual(await response.text(), bigJson)
271 })
272
273 it("does not stamp headers when a Raw body cannot be transformed", async () => {
274 const app = Effect.succeed(
275 HttpServerResponse.raw(new Response(null), {
276 contentType: "application/json",
277 headers: { etag: "\"abc\"" }
278 })
279 )
280 const response = await get(makeHandler(app), { "accept-encoding": "gzip" })
281 assert.strictEqual(response.headers.get("content-encoding"), null)
282 assert.strictEqual(response.headers.get("vary"), null)
283 assert.strictEqual(response.headers.get("etag"), "\"abc\"")
284 assert.strictEqual(await response.text(), "")
285 })
286
287 it("strips a Content-Encoding: identity opt-out before sending", async () => {
288 const app = Effect.succeed(
289 HttpServerResponse.text(bigJson, {
290 contentType: "application/json",
291 headers: { "content-encoding": "identity" }
292 })
293 )
294 const response = await get(makeHandler(app), { "accept-encoding": "gzip" })
295 assert.strictEqual(response.headers.get("content-encoding"), null)
296 assert.strictEqual(await response.text(), bigJson)
297 })
298
299 it("honors Cache-Control: no-transform", async () => {
300 const app = Effect.succeed(
301 HttpServerResponse.text(bigJson, {
302 contentType: "application/json",
303 headers: { "cache-control": "public, no-transform" }
304 })
305 )
306 const response = await get(makeHandler(app), { "accept-encoding": "gzip" })
307 assert.strictEqual(response.headers.get("content-encoding"), null)
308 assert.strictEqual(await response.text(), bigJson)
309 })
310
311 it("leaves 204, 304, and 206 responses untouched", async () => {
312 for (const status of [204, 304]) {
313 const response = await get(
314 makeHandler(Effect.succeed(HttpServerResponse.empty({ status }))),
315 { "accept-encoding": "gzip" }
316 )
317 assert.strictEqual(response.status, status)
318 assert.strictEqual(response.headers.get("content-encoding"), null)
319 assert.strictEqual(response.headers.get("vary"), null)
320 }
321 const partial = await get(
322 makeHandler(Effect.succeed(HttpServerResponse.text(bigJson, {
323 status: 206,
324 contentType: "application/json"
325 }))),
326 { "accept-encoding": "gzip" }
327 )
328 assert.strictEqual(partial.status, 206)
329 assert.strictEqual(partial.headers.get("content-encoding"), null)
330 assert.strictEqual(await partial.text(), bigJson)
331 })
332
333 it("mirrors GET headers for HEAD requests without a body", async () => {
334 const response = await get(makeHandler(bigJsonApp), { "accept-encoding": "gzip" }, "HEAD")
335 assert.strictEqual(response.headers.get("content-encoding"), "gzip")
336 assert.strictEqual(response.headers.get("vary"), "Accept-Encoding")
337 assert.strictEqual(await response.text(), "")
338 })
339
340 it("compresses unknown-length streams incrementally", async () => {
341 const latch = Latch.makeUnsafe(false)
342 const first = randomBytes(1 << 17)
343 const second = randomBytes(1 << 10)
344 const app = Effect.succeed(HttpServerResponse.stream(
345 Stream.concat(
346 Stream.succeed(first),
347 Stream.concat(Stream.drain(Stream.fromEffect(latch.await)), Stream.succeed(second))
348 ),
349 { contentType: "text/event-stream" }
350 ))
351 const response = await get(makeHandler(app), { "accept-encoding": "gzip" })
352 assert.strictEqual(response.headers.get("content-encoding"), "gzip")
353 assert.strictEqual(response.headers.get("content-length"), null)
354 const reader = response.body!.getReader()
355 const chunks: Array<Uint8Array> = []
356 // the first compressed chunk must arrive while the source stream is
357 // still open, i.e. before the latch is released
358 const initial = await reader.read()
359 assert.isFalse(initial.done)
360 chunks.push(initial.value!)
361 latch.openUnsafe()
362 let result = await reader.read()
363 while (!result.done) {
364 chunks.push(result.value!)
365 result = await reader.read()
366 }
367 const decompressed = new Uint8Array(
368 await new Response(
369 new Blob(chunks as Array<BlobPart>).stream().pipeThrough(new DecompressionStream("gzip"))
370 ).arrayBuffer()
371 )
372 assert.strictEqual(decompressed.length, first.length + second.length)
373 assert.deepStrictEqual(decompressed.slice(0, first.length), first)
374 assert.deepStrictEqual(decompressed.slice(first.length), second)
375 })
376
377 it("weakens a strong ETag on compressed responses", async () => {
378 const app = Effect.succeed(
379 HttpServerResponse.text(bigJson, {
380 contentType: "application/json",
381 headers: { etag: "\"abc\"" }
382 })
383 )
384 const response = await get(makeHandler(app), { "accept-encoding": "gzip" })
385 assert.strictEqual(response.headers.get("etag"), "W/\"abc\"")
386 })
387
388 it("leaves a weak ETag unchanged on compressed responses", async () => {
389 const app = Effect.succeed(
390 HttpServerResponse.text(bigJson, {
391 contentType: "application/json",
392 headers: { etag: "W/\"abc\"" }
393 })
394 )
395 const response = await get(makeHandler(app), { "accept-encoding": "gzip" })
396 assert.strictEqual(response.headers.get("etag"), "W/\"abc\"")
397 })
398
399 it("leaves the ETag of skipped responses unchanged", async () => {
400 const app = Effect.succeed(
401 HttpServerResponse.text(bigJson, {
402 contentType: "application/json",
403 headers: { etag: "\"abc\"" }
404 })
405 )
406 const response = await get(makeHandler(app), { "accept-encoding": "identity" })
407 assert.strictEqual(response.headers.get("etag"), "\"abc\"")
408 })
409
410 it("falls through to a supported algorithm when the preferred one is unavailable", async () => {
411 const response = await get(
412 makeHandler(bigJsonApp, { algorithms: ["zstd", "br", "gzip"] }),
413 { "accept-encoding": "zstd, br, gzip" }
414 )
415 assert.strictEqual(response.headers.get("content-encoding"), "gzip")
416 })
417
418 it("appends Accept-Encoding to an existing Vary header", async () => {
419 const app = Effect.succeed(
420 HttpServerResponse.text(bigJson, {
421 contentType: "application/json",
422 headers: { vary: "Origin" }
423 })
424 )
425 const response = await get(makeHandler(app), { "accept-encoding": "gzip" })
426 assert.strictEqual(response.headers.get("vary"), "Origin, Accept-Encoding")
427 })
428
429 it("does not duplicate Accept-Encoding in Vary and leaves Vary: * alone", async () => {
430 const existing = await get(
431 makeHandler(Effect.succeed(
432 HttpServerResponse.text(bigJson, {
433 contentType: "application/json",
434 headers: { vary: "accept-encoding" }
435 })
436 )),
437 { "accept-encoding": "gzip" }
438 )
439 assert.strictEqual(existing.headers.get("vary"), "accept-encoding")
440
441 const star = await get(
442 makeHandler(Effect.succeed(
443 HttpServerResponse.text(bigJson, {
444 contentType: "application/json",
445 headers: { vary: "*" }
446 })
447 )),
448 { "accept-encoding": "gzip" }
449 )
450 assert.strictEqual(star.headers.get("vary"), "*")
451 })
452
453 it("preserves cookies on compressed responses", async () => {
454 const app = Effect.succeed(
455 HttpServerResponse.text(bigJson, {
456 contentType: "application/json",
457 cookies: Cookies.fromSetCookie(["session=abc"])
458 })
459 )
460 const response = await get(makeHandler(app), { "accept-encoding": "gzip" })
461 assert.strictEqual(response.headers.get("content-encoding"), "gzip")
462 assert.deepStrictEqual(response.headers.getSetCookie(), ["session=abc"])
463 })
464})