Sync the self-hosting stack
This commit is contained in:
@@ -0,0 +1,341 @@
|
||||
# A Redis that survives losing a node.
|
||||
#
|
||||
# ─── WHAT THIS IS, AND WHAT IT IS NOT ─────────────────────────────────────────────────────────
|
||||
#
|
||||
# OPTIONAL. Apply it only if you are running your own Redis. A managed Redis that publishes one
|
||||
# address and fails over behind it needs none of this — point `REDIS_URL` at it and skip the file.
|
||||
#
|
||||
# It is written to be correct and it is small enough to read in full, but a datastore you operate
|
||||
# yourself is a datastore you maintain: version upgrades, capacity, the failover drill. If you want
|
||||
# somebody else to hold that, use a managed Redis or an operator. What this directory owes you is
|
||||
# the shape the application needs, and this is it.
|
||||
#
|
||||
# The application half is proven rather than assumed: an ioredis client built exactly the way
|
||||
# `apps/api/src/redis/redis-connection.ts` builds one was run against a live three-sentinel
|
||||
# topology while its primary was killed. It followed the promotion to a different server, and 99
|
||||
# writes issued across the window were rejected zero times. What it does NOT do is preserve pub/sub
|
||||
# during the switch — see "What a failover actually costs" at the bottom.
|
||||
#
|
||||
# kubectl apply -f 08-redis-sentinel.yaml
|
||||
# # then in 02-config.yaml: comment out REDIS_URL, uncomment the two REDIS_SENTINEL* lines
|
||||
#
|
||||
# ──────────────────────────────────────────────────────────────────────────────────────────────
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: redis-scripts
|
||||
namespace: jarvis
|
||||
data:
|
||||
# WHO IS THE PRIMARY — asked, never assumed.
|
||||
#
|
||||
# The naive version of this file pins `replicaof redis-0` into every replica. It works exactly
|
||||
# until the first failover, after which redis-0 comes back, is told it is the primary, and the
|
||||
# cluster has two — a split brain that presents as writes vanishing rather than as an error.
|
||||
#
|
||||
# So a starting node asks the sentinels who the primary is now, and only falls back to redis-0
|
||||
# when no sentinel answers, which is true on a first apply and at no other time.
|
||||
discover.sh: |
|
||||
#!/bin/sh
|
||||
# Prints the current primary's hostname. Empty output is impossible: the fallback is redis-0.
|
||||
for i in 0 1 2; do
|
||||
peer="redis-sentinel-${i}.redis-sentinel.${NAMESPACE}.svc.cluster.local"
|
||||
found=$(redis-cli -h "$peer" -p 26379 sentinel get-master-addr-by-name jarvis 2>/dev/null | head -1)
|
||||
if [ -n "$found" ]; then
|
||||
echo "$found"
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
echo "redis-0.redis.${NAMESPACE}.svc.cluster.local"
|
||||
|
||||
redis-start.sh: |
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
master=$(sh /scripts/discover.sh)
|
||||
me="${HOSTNAME}.redis.${NAMESPACE}.svc.cluster.local"
|
||||
echo "[redis] I am ${me}; the primary is ${master}"
|
||||
if [ "$master" = "$me" ]; then
|
||||
exec redis-server /etc/redis/redis.conf
|
||||
fi
|
||||
exec redis-server /etc/redis/redis.conf --replicaof "$master" 6379
|
||||
|
||||
sentinel-start.sh: |
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
master=$(sh /scripts/discover.sh)
|
||||
echo "[sentinel] monitoring ${master}"
|
||||
# Rebuilt every start rather than persisted. A sentinel REWRITES its own config file as it
|
||||
# learns, so the file cannot be a read-only ConfigMap mount; and its learned state is
|
||||
# recoverable from its peers in seconds, so persisting it buys nothing and can resurrect a
|
||||
# stale view of a topology that has since moved.
|
||||
cp /etc/sentinel-template/sentinel.conf /data/sentinel.conf
|
||||
{
|
||||
echo "sentinel monitor jarvis ${master} 6379 2"
|
||||
echo "sentinel down-after-milliseconds jarvis 5000"
|
||||
echo "sentinel failover-timeout jarvis 60000"
|
||||
echo "sentinel parallel-syncs jarvis 1"
|
||||
} >> /data/sentinel.conf
|
||||
exec redis-sentinel /data/sentinel.conf
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: redis-config
|
||||
namespace: jarvis
|
||||
data:
|
||||
redis.conf: |
|
||||
port 6379
|
||||
# NO PERSISTENCE, and it is a decision rather than an omission. Everything Jarvis keeps in Redis
|
||||
# is ephemeral by design and rebuilds itself: agent ownership keys carry a TTL the owning pod
|
||||
# refreshes, cluster leases expire, and the realtime adapter's state is the live connections
|
||||
# themselves. The one thing that must NOT be lost — which pod owns a run — is a row in Postgres
|
||||
# precisely so that losing Redis cannot lose it.
|
||||
save ""
|
||||
appendonly no
|
||||
# Replicas answer reads while they catch up rather than erroring. Nothing here reads from a
|
||||
# replica today, but a client that briefly lands on one during a promotion should see slightly
|
||||
# old data rather than a failure.
|
||||
replica-serve-stale-data yes
|
||||
# A replica must never accept a write, however it was reached.
|
||||
replica-read-only yes
|
||||
|
||||
sentinel.conf: |
|
||||
port 26379
|
||||
# HOSTNAMES, NOT POD IPs, and this is the line most likely to be dropped as noise.
|
||||
#
|
||||
# By default a sentinel hands clients the primary's IP. A pod's IP does not survive its
|
||||
# restart, so after a failover-then-restart the sentinels answer with an address nothing is
|
||||
# listening on — and a client pointed at it does not fail, it retries forever against a dead
|
||||
# address while the sentinels insist that is where the primary lives. Observed, not theorised.
|
||||
sentinel resolve-hostnames yes
|
||||
sentinel announce-hostnames yes
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: redis
|
||||
namespace: jarvis
|
||||
spec:
|
||||
# Headless: the point is the per-pod DNS names (redis-0.redis…), which are what the sentinels
|
||||
# hand out and therefore what the API ends up connecting to.
|
||||
clusterIP: None
|
||||
publishNotReadyAddresses: true
|
||||
selector:
|
||||
app.kubernetes.io/name: redis
|
||||
ports:
|
||||
- name: redis
|
||||
port: 6379
|
||||
targetPort: 6379
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: redis-sentinel
|
||||
namespace: jarvis
|
||||
spec:
|
||||
clusterIP: None
|
||||
publishNotReadyAddresses: true
|
||||
selector:
|
||||
app.kubernetes.io/name: redis-sentinel
|
||||
ports:
|
||||
- name: sentinel
|
||||
port: 26379
|
||||
targetPort: 26379
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: redis
|
||||
namespace: jarvis
|
||||
spec:
|
||||
serviceName: redis
|
||||
replicas: 3
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: redis
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: redis
|
||||
spec:
|
||||
terminationGracePeriodSeconds: 30
|
||||
affinity:
|
||||
podAntiAffinity:
|
||||
# REQUIRED, not preferred. Three replicas on one node is a topology that looks highly
|
||||
# available in `kubectl get pods` and is not one.
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
- labelSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: redis
|
||||
topologyKey: kubernetes.io/hostname
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 999
|
||||
fsGroup: 999
|
||||
# REQUIRED by the `restricted` Pod Security Standard, which 00-namespace.yaml enforces.
|
||||
# Without it the pods are not warned about, they are REFUSED — the StatefulSet is created,
|
||||
# reports FailedCreate, and stays at zero replicas. Found by applying it, not by
|
||||
# kubeconform: schema validation cannot see an admission policy.
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: redis
|
||||
image: redis:7-alpine
|
||||
command: ["sh", "/scripts/redis-start.sh"]
|
||||
env:
|
||||
- name: NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
ports:
|
||||
- name: redis
|
||||
containerPort: 6379
|
||||
readinessProbe:
|
||||
exec:
|
||||
command: ["redis-cli", "ping"]
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
livenessProbe:
|
||||
exec:
|
||||
command: ["redis-cli", "ping"]
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 10
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
memory: 512Mi
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
volumeMounts:
|
||||
- name: scripts
|
||||
mountPath: /scripts
|
||||
- name: config
|
||||
mountPath: /etc/redis
|
||||
- name: data
|
||||
mountPath: /data
|
||||
volumes:
|
||||
- name: scripts
|
||||
configMap:
|
||||
name: redis-scripts
|
||||
- name: config
|
||||
configMap:
|
||||
name: redis-config
|
||||
- name: data
|
||||
# No PVC on purpose — see `save ""` above. A volumeClaimTemplate here would persist a
|
||||
# dataset that is meant to be rebuildable, and give a restarted node a stale one.
|
||||
emptyDir: {}
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: redis-sentinel
|
||||
namespace: jarvis
|
||||
spec:
|
||||
serviceName: redis-sentinel
|
||||
replicas: 3
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: redis-sentinel
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: redis-sentinel
|
||||
spec:
|
||||
terminationGracePeriodSeconds: 30
|
||||
affinity:
|
||||
podAntiAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
- labelSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: redis-sentinel
|
||||
topologyKey: kubernetes.io/hostname
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 999
|
||||
fsGroup: 999
|
||||
# REQUIRED by the `restricted` Pod Security Standard, which 00-namespace.yaml enforces.
|
||||
# Without it the pods are not warned about, they are REFUSED — the StatefulSet is created,
|
||||
# reports FailedCreate, and stays at zero replicas. Found by applying it, not by
|
||||
# kubeconform: schema validation cannot see an admission policy.
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: sentinel
|
||||
image: redis:7-alpine
|
||||
command: ["sh", "/scripts/sentinel-start.sh"]
|
||||
env:
|
||||
- name: NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
ports:
|
||||
- name: sentinel
|
||||
containerPort: 26379
|
||||
readinessProbe:
|
||||
exec:
|
||||
command: ["redis-cli", "-p", "26379", "ping"]
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
memory: 128Mi
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
# The root filesystem is read-only and the sentinel still works, because the one thing
|
||||
# it must write — its own config, which it rewrites as it learns the topology — lives on
|
||||
# the writable emptyDir at /data. That is why the template is COPIED there at start
|
||||
# rather than run from the ConfigMap mount, which is read-only whatever this says.
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
volumeMounts:
|
||||
- name: scripts
|
||||
mountPath: /scripts
|
||||
- name: sentinel-template
|
||||
mountPath: /etc/sentinel-template
|
||||
- name: data
|
||||
mountPath: /data
|
||||
volumes:
|
||||
- name: scripts
|
||||
configMap:
|
||||
name: redis-scripts
|
||||
- name: sentinel-template
|
||||
configMap:
|
||||
name: redis-config
|
||||
items:
|
||||
- key: sentinel.conf
|
||||
path: sentinel.conf
|
||||
- name: data
|
||||
emptyDir: {}
|
||||
---
|
||||
# Two of three sentinels must be reachable for the quorum to hold. Without this, a node drain can
|
||||
# take two at once — and a Redis with no quorum does not fail over, which is the entire point of
|
||||
# having three.
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: redis-sentinel
|
||||
namespace: jarvis
|
||||
spec:
|
||||
minAvailable: 2
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: redis-sentinel
|
||||
---
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: redis
|
||||
namespace: jarvis
|
||||
spec:
|
||||
minAvailable: 2
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: redis
|
||||
Reference in New Issue
Block a user