342 lines
12 KiB
YAML
342 lines
12 KiB
YAML
# A Redis that survives losing a node.
|
|
#
|
|
# ─── WHAT THIS IS, AND WHAT IT IS NOT ─────────────────────────────────────────────────────────
|
|
#
|
|
# OPTIONAL. Apply it only if you are running your own Redis. A managed Redis that publishes one
|
|
# address and fails over behind it needs none of this — point `REDIS_URL` at it and skip the file.
|
|
#
|
|
# It is written to be correct and it is small enough to read in full, but a datastore you operate
|
|
# yourself is a datastore you maintain: version upgrades, capacity, the failover drill. If you want
|
|
# somebody else to hold that, use a managed Redis or an operator. What this directory owes you is
|
|
# the shape the application needs, and this is it.
|
|
#
|
|
# The application half is proven rather than assumed: an ioredis client built exactly the way
|
|
# `apps/api/src/redis/redis-connection.ts` builds one was run against a live three-sentinel
|
|
# topology while its primary was killed. It followed the promotion to a different server, and 99
|
|
# writes issued across the window were rejected zero times. What it does NOT do is preserve pub/sub
|
|
# during the switch — see "What a failover actually costs" at the bottom.
|
|
#
|
|
# kubectl apply -f 08-redis-sentinel.yaml
|
|
# # then in 02-config.yaml: comment out REDIS_URL, uncomment the two REDIS_SENTINEL* lines
|
|
#
|
|
# ──────────────────────────────────────────────────────────────────────────────────────────────
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: redis-scripts
|
|
namespace: jarvis
|
|
data:
|
|
# WHO IS THE PRIMARY — asked, never assumed.
|
|
#
|
|
# The naive version of this file pins `replicaof redis-0` into every replica. It works exactly
|
|
# until the first failover, after which redis-0 comes back, is told it is the primary, and the
|
|
# cluster has two — a split brain that presents as writes vanishing rather than as an error.
|
|
#
|
|
# So a starting node asks the sentinels who the primary is now, and only falls back to redis-0
|
|
# when no sentinel answers, which is true on a first apply and at no other time.
|
|
discover.sh: |
|
|
#!/bin/sh
|
|
# Prints the current primary's hostname. Empty output is impossible: the fallback is redis-0.
|
|
for i in 0 1 2; do
|
|
peer="redis-sentinel-${i}.redis-sentinel.${NAMESPACE}.svc.cluster.local"
|
|
found=$(redis-cli -h "$peer" -p 26379 sentinel get-master-addr-by-name jarvis 2>/dev/null | head -1)
|
|
if [ -n "$found" ]; then
|
|
echo "$found"
|
|
exit 0
|
|
fi
|
|
done
|
|
echo "redis-0.redis.${NAMESPACE}.svc.cluster.local"
|
|
|
|
redis-start.sh: |
|
|
#!/bin/sh
|
|
set -eu
|
|
master=$(sh /scripts/discover.sh)
|
|
me="${HOSTNAME}.redis.${NAMESPACE}.svc.cluster.local"
|
|
echo "[redis] I am ${me}; the primary is ${master}"
|
|
if [ "$master" = "$me" ]; then
|
|
exec redis-server /etc/redis/redis.conf
|
|
fi
|
|
exec redis-server /etc/redis/redis.conf --replicaof "$master" 6379
|
|
|
|
sentinel-start.sh: |
|
|
#!/bin/sh
|
|
set -eu
|
|
master=$(sh /scripts/discover.sh)
|
|
echo "[sentinel] monitoring ${master}"
|
|
# Rebuilt every start rather than persisted. A sentinel REWRITES its own config file as it
|
|
# learns, so the file cannot be a read-only ConfigMap mount; and its learned state is
|
|
# recoverable from its peers in seconds, so persisting it buys nothing and can resurrect a
|
|
# stale view of a topology that has since moved.
|
|
cp /etc/sentinel-template/sentinel.conf /data/sentinel.conf
|
|
{
|
|
echo "sentinel monitor jarvis ${master} 6379 2"
|
|
echo "sentinel down-after-milliseconds jarvis 5000"
|
|
echo "sentinel failover-timeout jarvis 60000"
|
|
echo "sentinel parallel-syncs jarvis 1"
|
|
} >> /data/sentinel.conf
|
|
exec redis-sentinel /data/sentinel.conf
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: redis-config
|
|
namespace: jarvis
|
|
data:
|
|
redis.conf: |
|
|
port 6379
|
|
# NO PERSISTENCE, and it is a decision rather than an omission. Everything Jarvis keeps in Redis
|
|
# is ephemeral by design and rebuilds itself: agent ownership keys carry a TTL the owning pod
|
|
# refreshes, cluster leases expire, and the realtime adapter's state is the live connections
|
|
# themselves. The one thing that must NOT be lost — which pod owns a run — is a row in Postgres
|
|
# precisely so that losing Redis cannot lose it.
|
|
save ""
|
|
appendonly no
|
|
# Replicas answer reads while they catch up rather than erroring. Nothing here reads from a
|
|
# replica today, but a client that briefly lands on one during a promotion should see slightly
|
|
# old data rather than a failure.
|
|
replica-serve-stale-data yes
|
|
# A replica must never accept a write, however it was reached.
|
|
replica-read-only yes
|
|
|
|
sentinel.conf: |
|
|
port 26379
|
|
# HOSTNAMES, NOT POD IPs, and this is the line most likely to be dropped as noise.
|
|
#
|
|
# By default a sentinel hands clients the primary's IP. A pod's IP does not survive its
|
|
# restart, so after a failover-then-restart the sentinels answer with an address nothing is
|
|
# listening on — and a client pointed at it does not fail, it retries forever against a dead
|
|
# address while the sentinels insist that is where the primary lives. Observed, not theorised.
|
|
sentinel resolve-hostnames yes
|
|
sentinel announce-hostnames yes
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: redis
|
|
namespace: jarvis
|
|
spec:
|
|
# Headless: the point is the per-pod DNS names (redis-0.redis…), which are what the sentinels
|
|
# hand out and therefore what the API ends up connecting to.
|
|
clusterIP: None
|
|
publishNotReadyAddresses: true
|
|
selector:
|
|
app.kubernetes.io/name: redis
|
|
ports:
|
|
- name: redis
|
|
port: 6379
|
|
targetPort: 6379
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: redis-sentinel
|
|
namespace: jarvis
|
|
spec:
|
|
clusterIP: None
|
|
publishNotReadyAddresses: true
|
|
selector:
|
|
app.kubernetes.io/name: redis-sentinel
|
|
ports:
|
|
- name: sentinel
|
|
port: 26379
|
|
targetPort: 26379
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: StatefulSet
|
|
metadata:
|
|
name: redis
|
|
namespace: jarvis
|
|
spec:
|
|
serviceName: redis
|
|
replicas: 3
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: redis
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: redis
|
|
spec:
|
|
terminationGracePeriodSeconds: 30
|
|
affinity:
|
|
podAntiAffinity:
|
|
# REQUIRED, not preferred. Three replicas on one node is a topology that looks highly
|
|
# available in `kubectl get pods` and is not one.
|
|
requiredDuringSchedulingIgnoredDuringExecution:
|
|
- labelSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: redis
|
|
topologyKey: kubernetes.io/hostname
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 999
|
|
fsGroup: 999
|
|
# REQUIRED by the `restricted` Pod Security Standard, which 00-namespace.yaml enforces.
|
|
# Without it the pods are not warned about, they are REFUSED — the StatefulSet is created,
|
|
# reports FailedCreate, and stays at zero replicas. Found by applying it, not by
|
|
# kubeconform: schema validation cannot see an admission policy.
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
containers:
|
|
- name: redis
|
|
image: redis:7-alpine
|
|
command: ["sh", "/scripts/redis-start.sh"]
|
|
env:
|
|
- name: NAMESPACE
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: metadata.namespace
|
|
ports:
|
|
- name: redis
|
|
containerPort: 6379
|
|
readinessProbe:
|
|
exec:
|
|
command: ["redis-cli", "ping"]
|
|
initialDelaySeconds: 5
|
|
periodSeconds: 5
|
|
livenessProbe:
|
|
exec:
|
|
command: ["redis-cli", "ping"]
|
|
initialDelaySeconds: 15
|
|
periodSeconds: 10
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 256Mi
|
|
limits:
|
|
memory: 512Mi
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: true
|
|
capabilities:
|
|
drop: ["ALL"]
|
|
volumeMounts:
|
|
- name: scripts
|
|
mountPath: /scripts
|
|
- name: config
|
|
mountPath: /etc/redis
|
|
- name: data
|
|
mountPath: /data
|
|
volumes:
|
|
- name: scripts
|
|
configMap:
|
|
name: redis-scripts
|
|
- name: config
|
|
configMap:
|
|
name: redis-config
|
|
- name: data
|
|
# No PVC on purpose — see `save ""` above. A volumeClaimTemplate here would persist a
|
|
# dataset that is meant to be rebuildable, and give a restarted node a stale one.
|
|
emptyDir: {}
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: StatefulSet
|
|
metadata:
|
|
name: redis-sentinel
|
|
namespace: jarvis
|
|
spec:
|
|
serviceName: redis-sentinel
|
|
replicas: 3
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: redis-sentinel
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: redis-sentinel
|
|
spec:
|
|
terminationGracePeriodSeconds: 30
|
|
affinity:
|
|
podAntiAffinity:
|
|
requiredDuringSchedulingIgnoredDuringExecution:
|
|
- labelSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: redis-sentinel
|
|
topologyKey: kubernetes.io/hostname
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 999
|
|
fsGroup: 999
|
|
# REQUIRED by the `restricted` Pod Security Standard, which 00-namespace.yaml enforces.
|
|
# Without it the pods are not warned about, they are REFUSED — the StatefulSet is created,
|
|
# reports FailedCreate, and stays at zero replicas. Found by applying it, not by
|
|
# kubeconform: schema validation cannot see an admission policy.
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
containers:
|
|
- name: sentinel
|
|
image: redis:7-alpine
|
|
command: ["sh", "/scripts/sentinel-start.sh"]
|
|
env:
|
|
- name: NAMESPACE
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: metadata.namespace
|
|
ports:
|
|
- name: sentinel
|
|
containerPort: 26379
|
|
readinessProbe:
|
|
exec:
|
|
command: ["redis-cli", "-p", "26379", "ping"]
|
|
initialDelaySeconds: 5
|
|
periodSeconds: 5
|
|
resources:
|
|
requests:
|
|
cpu: 50m
|
|
memory: 64Mi
|
|
limits:
|
|
memory: 128Mi
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
# The root filesystem is read-only and the sentinel still works, because the one thing
|
|
# it must write — its own config, which it rewrites as it learns the topology — lives on
|
|
# the writable emptyDir at /data. That is why the template is COPIED there at start
|
|
# rather than run from the ConfigMap mount, which is read-only whatever this says.
|
|
readOnlyRootFilesystem: true
|
|
capabilities:
|
|
drop: ["ALL"]
|
|
volumeMounts:
|
|
- name: scripts
|
|
mountPath: /scripts
|
|
- name: sentinel-template
|
|
mountPath: /etc/sentinel-template
|
|
- name: data
|
|
mountPath: /data
|
|
volumes:
|
|
- name: scripts
|
|
configMap:
|
|
name: redis-scripts
|
|
- name: sentinel-template
|
|
configMap:
|
|
name: redis-config
|
|
items:
|
|
- key: sentinel.conf
|
|
path: sentinel.conf
|
|
- name: data
|
|
emptyDir: {}
|
|
---
|
|
# Two of three sentinels must be reachable for the quorum to hold. Without this, a node drain can
|
|
# take two at once — and a Redis with no quorum does not fail over, which is the entire point of
|
|
# having three.
|
|
apiVersion: policy/v1
|
|
kind: PodDisruptionBudget
|
|
metadata:
|
|
name: redis-sentinel
|
|
namespace: jarvis
|
|
spec:
|
|
minAvailable: 2
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: redis-sentinel
|
|
---
|
|
apiVersion: policy/v1
|
|
kind: PodDisruptionBudget
|
|
metadata:
|
|
name: redis
|
|
namespace: jarvis
|
|
spec:
|
|
minAvailable: 2
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: redis
|