1import { NodeHttpServer } from "@effect/platform-node"
2import * as NodeHttpPlatform from "@effect/platform-node/NodeHttpPlatform"
3import { assert, describe, it } from "@effect/vitest"
4import * as Effect from "effect/Effect"
5import * as HttpClient from "effect/http/HttpClient"
6import * as HttpEffect from "effect/http/HttpEffect"
7import * as HttpMiddleware from "effect/http/HttpMiddleware"
8import * as HttpPlatform from "effect/http/HttpPlatform"
9import * as HttpRouter from "effect/http/HttpRouter"
10import type { HttpServerRequest } from "effect/http/HttpServerRequest"
11import * as HttpServerResponse from "effect/http/HttpServerResponse"
12import * as Latch from "effect/Latch"
13import * as Layer from "effect/Layer"
14import * as Stream from "effect/Stream"
15import * as Fs from "node:fs"
16import * as Os from "node:os"
17import * as Path from "node:path"
18import { fileURLToPath } from "node:url"
19import * as Zlib from "node:zlib"
20
21const bigJson = JSON.stringify({ text: "All work and no play makes Jack a dull boy. ".repeat(100) })
22const bigJsonApp = Effect.succeed(HttpServerResponse.text(bigJson, { contentType: "application/json" }))
23
24const zstdSupported = typeof Zlib.zstdCompress === "function"
25
26type App = Effect.Effect<HttpServerResponse.HttpServerResponse, any, any>
27type CompressionOptions = Parameters<typeof HttpMiddleware.compression>[0]
28
29const withHandler = async (
30 app: App,
31 options: CompressionOptions,
32 run: (handler: (request: Request) => Promise<Response>) => Promise<void>
33) => {
34 const { dispose, handler } = HttpEffect.toWebHandlerLayer(
35 app as Effect.Effect<HttpServerResponse.HttpServerResponse, never, HttpServerRequest>,
36 NodeHttpPlatform.layer,
37 { middleware: HttpMiddleware.compression(options) }
38 )
39 try {
40 await run(handler)
41 } finally {
42 await dispose()
43 }
44}
45
46const get = (
47 handler: (request: Request) => Promise<Response>,
48 headers?: Record<string, string>
49) => handler(new Request("http://localhost/", headers === undefined ? {} : { headers }))
50
51describe("NodeHttpCompression", () => {
52 for (
53 const [name, makeResponse] of [
54 ["Stream", () => HttpServerResponse.stream(Stream.succeed(new TextEncoder().encode(bigJson)))],
55 ["Uint8Array", () => HttpServerResponse.uint8Array(new TextEncoder().encode(bigJson))]
56 ] as const
57 ) {
58 it(`preserves a Content-Type header override when compressing ${name} responses`, () => {
59 const original = HttpServerResponse.setHeader(makeResponse(), "content-type", "application/json")
60 assert.strictEqual(original.headers["content-type"], "application/json")
61 return withHandler(
62 Effect.succeed(original),
63 undefined,
64 async (handler) => {
65 const compressed = await get(handler, { "accept-encoding": "gzip" })
66 assert.strictEqual(compressed.headers.get("content-encoding"), "gzip")
67 assert.strictEqual(Zlib.gunzipSync(new Uint8Array(await compressed.arrayBuffer())).toString(), bigJson)
68 assert.strictEqual(compressed.headers.get("content-type"), original.headers["content-type"])
69 }
70 )
71 })
72 }
73
74 it.effect("advertises supported algorithms", () =>
75 Effect.gen(function*() {
76 const platform = yield* HttpPlatform.HttpPlatform
77 assert.isTrue(platform.compression.algorithms.has("gzip"))
78 assert.isTrue(platform.compression.algorithms.has("deflate"))
79 assert.isTrue(platform.compression.algorithms.has("br"))
80 assert.strictEqual(platform.compression.algorithms.has("zstd"), zstdSupported)
81 }).pipe(Effect.provide(NodeHttpPlatform.layer)))
82
83 it("compresses one-shot bodies asynchronously with gzip and an exact Content-Length", () =>
84 withHandler(bigJsonApp, undefined, async (handler) => {
85 const response = await get(handler, { "accept-encoding": "gzip" })
86 assert.strictEqual(response.headers.get("content-encoding"), "gzip")
87 assert.strictEqual(response.headers.get("vary"), "Accept-Encoding")
88 const compressed = new Uint8Array(await response.arrayBuffer())
89 assert.strictEqual(response.headers.get("content-length"), compressed.byteLength.toString())
90 assert.strictEqual(Zlib.gunzipSync(compressed).toString(), bigJson)
91 }))
92
93 it("prefers br over gzip in server order", () =>
94 withHandler(bigJsonApp, undefined, async (handler) => {
95 const response = await get(handler, { "accept-encoding": "gzip, br" })
96 assert.strictEqual(response.headers.get("content-encoding"), "br")
97 const compressed = new Uint8Array(await response.arrayBuffer())
98 assert.strictEqual(response.headers.get("content-length"), compressed.byteLength.toString())
99 assert.strictEqual(Zlib.brotliDecompressSync(compressed).toString(), bigJson)
100 }))
101
102 it("compresses with deflate", () =>
103 withHandler(bigJsonApp, undefined, async (handler) => {
104 const response = await get(handler, { "accept-encoding": "deflate" })
105 assert.strictEqual(response.headers.get("content-encoding"), "deflate")
106 const compressed = new Uint8Array(await response.arrayBuffer())
107 assert.strictEqual(response.headers.get("content-length"), compressed.byteLength.toString())
108 assert.strictEqual(Zlib.inflateSync(compressed).toString(), bigJson)
109 }))
110
111 it.skipIf(!zstdSupported)(
112 "compresses with zstd when opted in",
113 () =>
114 withHandler(bigJsonApp, { algorithms: ["zstd", "gzip"] }, async (handler) => {
115 const response = await get(handler, { "accept-encoding": "zstd" })
116 assert.strictEqual(response.headers.get("content-encoding"), "zstd")
117 const compressed = new Uint8Array(await response.arrayBuffer())
118 assert.strictEqual(response.headers.get("content-length"), compressed.byteLength.toString())
119 assert.strictEqual(Zlib.zstdDecompressSync(compressed).toString(), bigJson)
120 })
121 )
122
123 it("excludes zstd unless opted in", () =>
124 withHandler(bigJsonApp, undefined, async (handler) => {
125 const response = await get(handler, { "accept-encoding": "zstd, gzip" })
126 assert.strictEqual(response.headers.get("content-encoding"), "gzip")
127 }))
128
129 it("compresses stream bodies without a Content-Length", () =>
130 withHandler(
131 Effect.succeed(HttpServerResponse.stream(
132 Stream.fromArray([new TextEncoder().encode(bigJson)]),
133 { contentType: "application/json" }
134 )),
135 undefined,
136 async (handler) => {
137 const response = await get(handler, { "accept-encoding": "gzip" })
138 assert.strictEqual(response.headers.get("content-encoding"), "gzip")
139 assert.strictEqual(response.headers.get("content-length"), null)
140 assert.strictEqual(Zlib.gunzipSync(new Uint8Array(await response.arrayBuffer())).toString(), bigJson)
141 }
142 ))
143
144 it("compresses file responses through the streaming path and weakens the ETag", async () => {
145 const path = fileURLToPath(new URL("./NodeHttpCompression.test.ts", import.meta.url))
146 const contents = Fs.readFileSync(path).toString()
147 await withHandler(
148 HttpServerResponse.file(path, { headers: { "content-type": "text/plain" } }),
149 undefined,
150 async (handler) => {
151 const response = await get(handler, { "accept-encoding": "gzip" })
152 assert.strictEqual(response.headers.get("content-encoding"), "gzip")
153 assert.strictEqual(response.headers.get("content-length"), null)
154 assert.isTrue(response.headers.get("etag")!.startsWith("W/"))
155 assert.strictEqual(Zlib.gunzipSync(new Uint8Array(await response.arrayBuffer())).toString(), contents)
156 }
157 )
158 })
159
160 it.effect("preserves the Content-Type of file responses when compressed with Brotli", () =>
161 Effect.gen(function*() {
162 const directory = Fs.mkdtempSync(Path.join(Os.tmpdir(), "effect-http-compression-"))
163 yield* Effect.addFinalizer(() => Effect.sync(() => Fs.rmSync(directory, { recursive: true })))
164 const path = Path.join(directory, "index.html")
165 const contents = `<!doctype html><html><body>${"<p>Hello world</p>".repeat(100)}</body></html>`
166 Fs.writeFileSync(path, contents)
167
168 yield* HttpRouter.add("GET", "/file", HttpServerResponse.file(path)).pipe(
169 (self) => HttpRouter.serve(self, { middleware: HttpMiddleware.compression() }),
170 Layer.build
171 )
172
173 const uncompressed = yield* HttpClient.get("/file", { headers: { "accept-encoding": "identity" } })
174 assert.strictEqual(uncompressed.status, 200)
175 assert.strictEqual(uncompressed.headers["content-encoding"], undefined)
176 assert.strictEqual(uncompressed.headers["content-type"], "text/html")
177 assert.strictEqual(yield* uncompressed.text, contents)
178
179 const compressed = yield* HttpClient.get("/file", { headers: { "accept-encoding": "br" } })
180 assert.strictEqual(compressed.status, 200)
181 assert.strictEqual(compressed.headers["content-encoding"], "br")
182 assert.strictEqual(yield* compressed.text, contents)
183 assert.strictEqual(compressed.headers["content-type"], uncompressed.headers["content-type"])
184 }).pipe(Effect.provide(NodeHttpServer.layerTest)))
185
186 it("does not acquire file bodies for compressed HEAD requests", async () => {
187 let acquired = 0
188 class CountingFile extends File {
189 override stream() {
190 acquired++
191 return super.stream()
192 }
193 }
194 const file = new CountingFile(["abc"], "test.txt")
195 await withHandler(HttpServerResponse.fileWeb(file), { minSize: 0 }, async (handler) => {
196 const head = await handler(
197 new Request("http://localhost/", {
198 method: "HEAD",
199 headers: { "accept-encoding": "gzip" }
200 })
201 )
202 assert.strictEqual(head.status, 200)
203 assert.strictEqual(await head.text(), "")
204 assert.strictEqual(acquired, 0)
205 })
206 })
207
208 it.effect("flushes compressed chunks incrementally over the wire", () =>
209 Effect.gen(function*() {
210 const latch = yield* Latch.make(false)
211 const encoder = new TextEncoder()
212 yield* HttpRouter.add(
213 "GET",
214 "/sse",
215 Effect.succeed(HttpServerResponse.stream(
216 Stream.concat(
217 Stream.succeed(encoder.encode("data: first\n\n")),
218 Stream.concat(
219 Stream.drain(Stream.fromEffect(latch.await)),
220 Stream.succeed(encoder.encode("data: second\n\n"))
221 )
222 ),
223 { contentType: "text/event-stream" }
224 ))
225 ).pipe(
226 (self) => HttpRouter.serve(self, { middleware: HttpMiddleware.compression() }),
227 Layer.build
228 )
229 const client = yield* HttpClient.HttpClient
230 const response = yield* client.get("/sse", { headers: { "accept-encoding": "gzip" } })
231 assert.strictEqual(response.headers["content-encoding"], "gzip")
232 // the source stream only ends after the latch opens, and the latch only
233 // opens once the first decoded event arrives, so this deadlocks unless
234 // compressed chunks flush incrementally
235 const received: Array<string> = []
236 yield* response.stream.pipe(
237 Stream.runForEach((chunk) =>
238 Effect.suspend(() => {
239 received.push(new TextDecoder().decode(chunk))
240 return latch.open
241 })
242 )
243 )
244 assert.strictEqual(received.join(""), "data: first\n\ndata: second\n\n")
245 }).pipe(Effect.provide(NodeHttpServer.layerTest)))
246})