1import * as AWS from "alchemy/AWS";
2import * as Kubernetes from "alchemy/Kubernetes";
3import * as Effect from "effect/Effect";
4import { HyperPodEksInfra } from "./eks-infra.ts";
5
6/**
7 * The HIGH-LEVEL tier: an effectful `Kubernetes.Job` running ON HyperPod
8 * nodes, written in plain Kubernetes vocabulary. The Effect program is
9 * bundled into a generated image (`main: import.meta.url`), and the
10 * HyperPod resources expose everything the job needs as attributes
11 * referenced through the resource graph:
12 *
13 * - `quota.namespace` — the governed `hyperpod-ns-<team>` namespace,
14 * - `quota.queueName` — the team's Kueue LocalQueue, set under the
15 * well-known `KUEUE_QUEUE_NAME_LABEL`,
16 * - `instanceGroups.workers.nodeSelector` — health-checked nodes of the
17 * `workers` group (typed per key; a typo'd group name is a compile
18 * error).
19 *
20 * Swap `run` for a real training/eval harness; bindings (DynamoDB, S3, SQS,
21 * ...) resolve in init and land IAM on the pod-identity role, exactly like
22 * any other Kubernetes Job on EKS.
23 */
24export default Kubernetes.Job(
25 "TrainJob",
26 Effect.gen(function* () {
27 const { eks, hyperpod, researchQuota } = yield* HyperPodEksInfra;
28 return {
29 cluster: eks,
30 main: import.meta.url,
31 namespace: researchQuota.namespace,
32 labels: {
33 [AWS.SageMaker.KUEUE_QUEUE_NAME_LABEL]: researchQuota.queueName,
34 [AWS.SageMaker.KUEUE_PRIORITY_CLASS_LABEL]: "training-priority",
35 },
36 podTemplate: {
37 spec: {
38 nodeSelector: hyperpod.instanceGroups.workers.nodeSelector,
39 },
40 },
41 backoffLimit: 2,
42 };
43 }),
44 Effect.gen(function* () {
45 return {
46 run: Effect.gen(function* () {
47 yield* Effect.log("training step running on a HyperPod node");
48 }),
49 };
50 }),
51);