apiVersion: apps/v1 kind: Deployment metadata: # After the schema. This is the wave the whole ordering exists for: an api pod that starts # against a schema the migration has not applied is the failure every other number here is # arranged to prevent. annotations: argocd.argoproj.io/sync-wave: "1" name: jarvis-api namespace: jarvis spec: replicas: 2 selector: matchLabels: app.kubernetes.io/name: jarvis app.kubernetes.io/component: api # RECREATED, FOR api 1.0.0 ONLY — the release that crosses from the TypeScript images. A rolling # update would keep a TypeScript pod and a .NET pod serving side by side behind one Service, and # their realtime protocols differ under /socket.io/ (Socket.IO against SignalR): every old pod stops # before the first new one starts, a gap of about thirty seconds, once. The next release puts the # rolling update back: # # type: RollingUpdate # rollingUpdate: # maxUnavailable: 0 # never fewer pods than are being served now # maxSurge: 1 # # With `kubectl apply` or Argo's default client-side apply this switches both ways. Under # server-side apply, a field manager that owns `rollingUpdate` makes the API server refuse it. strategy: type: Recreate template: metadata: labels: app.kubernetes.io/name: jarvis app.kubernetes.io/component: api spec: # 90 seconds, and every part of it is accounted for: # 10s preStop, so this pod leaves the Service's endpoints BEFORE it starts draining # 25s RUN_SHUTDOWN_GRACE_SEC — in-flight assistant runs stop and close their transcripts # ~15s agent sockets closed and their rows written, Kestrel, the database pool, Redis # the rest is margin # The default of 30 would SIGKILL the pod in the middle of the run drain, which is the one # thing the drain exists to prevent. terminationGracePeriodSeconds: 90 # Required by the `restricted` Pod Security Standard that 00-namespace.yaml enforces. At pod # level rather than per container, because the agent-releases initContainer below needs it # too — and a missing seccompProfile is not a warning, it is the pod being refused outright. securityContext: seccompProfile: type: RuntimeDefault # SPREAD THE REPLICAS, because nothing else will. # # The scheduler has no reason of its own to put two pods of the same Deployment on different # nodes, and quite often has a reason not to. Two API replicas on one node is a deployment # that reads as highly available in `kubectl get pods` and loses every agent websocket, every # live run and every open shell the moment that one node reboots. The PodDisruptionBudget in # 07-disruption.yaml does not help here: it constrains voluntary evictions, and a node dying # is not one. # # `ScheduleAnyway` rather than `DoNotSchedule`: on a single-node cluster the strict form # leaves the second replica Pending forever, which is a worse first experience than an # unspread pair. On a real cluster it spreads. If yours spans zones, add a second constraint # on `topology.kubernetes.io/zone` — a rack is a failure domain too. topologySpreadConstraints: - maxSkew: 1 topologyKey: kubernetes.io/hostname whenUnsatisfiable: ScheduleAnyway labelSelector: matchLabels: app.kubernetes.io/name: jarvis app.kubernetes.io/component: api containers: - name: api # BY DIGEST. `:stable` is a moving name, so replicas that restart at different times land # on different builds — and a rollback becomes "hope the tag still points where it did". image: git.luxit.be/luxit/jarvis-api@sha256:REPLACE_ME ports: - name: http containerPort: 4000 lifecycle: preStop: exec: # Endpoint removal and SIGTERM race each other, and losing that race is visible: # a prompt that arrives after the api starts draining is persisted and broadcast as # queued, and this pod will never answer it. Ten seconds is comfortably more than # kube-proxy needs to stop sending new connections here. command: ["/bin/sh", "-c", "sleep 10"] env: - name: JARVIS_SKIP_MIGRATIONS # The schema is the migration Job's business, once, before this rolls. N pods each # migrating is N writers racing the same DDL while the previous # generation is still selecting the columns being altered. value: "true" envFrom: - configMapRef: name: jarvis-api-config # The half that is random bytes, minted once by `01-secret-job.yaml`. Kept apart from # the half an operator writes because one of them can be regenerated and the other # cannot: re-running the minter is a no-op by design, which is only safe while nothing # anybody typed lives in the same object. # # `optional: true` so a deployment that mints by hand — or through SealedSecrets, # ExternalSecrets, Vault — does not need this object to exist at all. - secretRef: name: jarvis-generated optional: true # LAST, and the order is the safety property rather than a formatting choice. `envFrom` # resolves in sequence and the last occurrence of a name wins, so anything written by # hand here overrides the generated value — never the other way round. Reversed, an # operator who had already minted a vault key and then applied the Job by reflex would # have every stored credential silently become unreadable. - secretRef: name: jarvis-secrets volumeMounts: - name: agent-releases mountPath: /srv/agent-releases readOnly: true # Liveness NEVER touches the database. A Postgres failover with a database check here # restarts every pod at once, turning a thirty-second blip into a cold start of the whole # deployment at the moment the database can least afford a reconnection stampede. livenessProbe: httpGet: path: /api/health/live port: http periodSeconds: 10 failureThreshold: 3 # Readiness DOES, and it checks Redis too. A pod cut off from Redis keeps answering HTTP # perfectly while its broadcasts reach nobody — the clients queue rather than fail — so it # looks healthier than a pod that is down and is more dangerous. readinessProbe: httpGet: path: /api/health/ready port: http periodSeconds: 5 timeoutSeconds: 3 failureThreshold: 3 # Boot does a run-recovery sweep and reads settings before it listens. The startup probe # is what stops liveness killing it halfway through on a slow database. startupProbe: httpGet: path: /api/health/live port: http periodSeconds: 5 failureThreshold: 30 resources: requests: cpu: 500m # Not generous — honest. A terminal recording holds up to 8 MiB in memory for the life # of the session, and argon2 takes 64 MiB per share verification. # # Attached and uploaded files are the holders that grow, and every one is capped: # uploads being checked and stored, 3 at a time per pod (`Files/UploadSlots.cs`); files # read back, their bytes or a window of their text; images prepared for the model; and # a PDF's text, read in a worker PROCESS of its own (`Files/PdfText.cs`, # `Files/PdfTextWorker.cs`) that is killed past its own limits, so a hostile file costs # that process and not this one. # # MEASURED on this image, in a cgroup like this pod's (lot 11 P, proof 2j): every upload # slot of two pods busy with a 10 MB, 500-page PDF — the most pages the worker reads — # peaked at 619 MiB and 740 MiB of `memory.current`, the API and its workers together, # from 446 and 496 MiB before. The TypeScript image sized this limit: Prisma's JSON # protocol copied an upload's bytes several times over (~1.1 GiB for three at once) # and a PDF worker's buffers could reach ~500 MB, ~2.75 GiB with every holder full. # None of that is in this process; the 3Gi limit stands until it is lowered on purpose # (docs/dotnet/PLAN.md, "Left to Antoine after lot 11"). The request is what the # scheduler sets aside for an ordinary day, not for the worst minute. memory: 1Gi limits: memory: 3Gi securityContext: runAsNonRoot: true runAsUser: 1000 allowPrivilegeEscalation: false capabilities: drop: ["ALL"] initContainers: # The compiled agent binaries, into this pod's own emptyDir. # # PER POD ON PURPOSE. The content is immutable and small, so a copy per pod costs nothing; # a shared RWO PVC would either pin every pod to one node or leave the second in # Multi-Attach error. Leaving this out entirely is supported: enrollment still works and the # installer answers 503 saying no build is published. - name: agent-releases image: git.luxit.be/luxit/jarvis-agent-dist@sha256:REPLACE_ME command: ["/bin/sh", "-c", "cp -a /dist/. /srv/agent-releases/"] # AN initContainer IS A CONTAINER, and the `restricted` Pod Security Standard judges it # exactly like the one below. This block was missing, and the result was not a warning # about the initContainer — it was the WHOLE POD refused, so the Deployment sat at zero # replicas reporting FailedCreate while every other line in this file was correct. securityContext: runAsNonRoot: true runAsUser: 1000 allowPrivilegeEscalation: false readOnlyRootFilesystem: true capabilities: drop: ["ALL"] volumeMounts: - name: agent-releases mountPath: /srv/agent-releases volumes: - name: agent-releases emptyDir: {} --- apiVersion: v1 kind: Service metadata: # With the Deployment it fronts. annotations: argocd.argoproj.io/sync-wave: "1" name: jarvis-api namespace: jarvis spec: type: ClusterIP # STATED, not left to the default, because the reflex is to reach for it the moment websockets # are involved. It would not help: the hard case is co-locating an operator's browser with an # agent's websocket, which arrives separately and whose id is unknown at the browser's handshake. # That is handled in the application, over Redis. See the README. sessionAffinity: None selector: app.kubernetes.io/name: jarvis app.kubernetes.io/component: api ports: - name: http port: 4000 targetPort: http