From 6ddfb1cbdeb08454c66de639f7d17c9fed044aee Mon Sep 17 00:00:00 2001 From: ced4568 Date: Mon, 17 Aug 2026 12:45:16 -0500 Subject: [PATCH] feat: add probe-port-mismatch and enforce-limits to rootnode-real-ops bundle --- scenarios/bundles/rootnode-real-ops.json | 4 +- scenarios/data/rootnode-enforce-limits.json | 69 +++++++++++++++++++ .../data/rootnode-probe-port-mismatch.json | 57 +++++++++++++++ 3 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 scenarios/data/rootnode-enforce-limits.json create mode 100644 scenarios/data/rootnode-probe-port-mismatch.json diff --git a/scenarios/bundles/rootnode-real-ops.json b/scenarios/bundles/rootnode-real-ops.json index 7ebf742..4217914 100644 --- a/scenarios/bundles/rootnode-real-ops.json +++ b/scenarios/bundles/rootnode-real-ops.json @@ -8,6 +8,8 @@ "exam_minutes": 60, "scenario_ids": [ "rootnode-runaway-monitor", - "broken-deployment" + "broken-deployment", + "rootnode-probe-port-mismatch", + "rootnode-enforce-limits" ] } diff --git a/scenarios/data/rootnode-enforce-limits.json b/scenarios/data/rootnode-enforce-limits.json new file mode 100644 index 0000000..51cc55c --- /dev/null +++ b/scenarios/data/rootnode-enforce-limits.json @@ -0,0 +1,69 @@ +{ + "id": "rootnode-enforce-limits", + "title": "Stop the Next Outage: Enforce Resource Defaults", + "category": "Observability & Ops", + "difficulty": "Hard", + "type": "task", + "weight": 10, + "description": "## Stop the Next Outage: Enforce Resource Defaults\n\nThe `node-metrics` incident happened because a workload was deployed with an unbounded resource request. The immediate fix was to right-size that deployment — but the root cause is still live: **nothing stops the next engineer from deploying into the `workloads` namespace without any resource constraints at all**.\n\nThis scenario is about closing that gap. A LimitRange is a namespace-level policy object that automatically injects default resource requests and limits into any container that omits them. One object, applied once, protects every workload in the namespace going forward — without requiring every developer to remember to set resources manually.\n\n**Your task:** Create a LimitRange named `resource-defaults` in the `workloads` namespace that enforces the following defaults for every container:\n\n| Field | Value |\n|---|---|\n| Default CPU limit | `500m` |\n| Default CPU request | `100m` |\n| Default memory limit | `256Mi` |\n| Default memory request | `128Mi` |\n\nOnce the LimitRange is in place, validate that a pod deployed **without any resource block** automatically receives these defaults.\n\n> This is governance, not cleanup. The goal is a policy that survives the next deployment.", + "hints": [ + { + "title": "1. Check the namespace", + "body": "The workloads namespace should already exist. Confirm it is there before applying anything — if it is missing, create it first.", + "command": "kubectl get namespace workloads" + }, + { + "title": "2. Write the LimitRange manifest", + "body": "A LimitRange with type Container and a default/defaultRequest block is what you need. The default field sets limits; defaultRequest sets requests. Save it to a file and apply with kubectl apply -f.", + "command": "kubectl apply -f - </dev/null || echo ''", + "expected_output": "resource-defaults", + "match": "exact" + }, + { + "description": "Default CPU limit is 500m", + "command": "kubectl get limitrange resource-defaults -n workloads -o jsonpath='{.spec.limits[0].default.cpu}' 2>/dev/null || echo ''", + "expected_output": "500m", + "match": "exact" + }, + { + "description": "Default memory limit is 256Mi", + "command": "kubectl get limitrange resource-defaults -n workloads -o jsonpath='{.spec.limits[0].default.memory}' 2>/dev/null || echo ''", + "expected_output": "256Mi", + "match": "exact" + }, + { + "description": "Default CPU request is 100m", + "command": "kubectl get limitrange resource-defaults -n workloads -o jsonpath='{.spec.limits[0].defaultRequest.cpu}' 2>/dev/null || echo ''", + "expected_output": "100m", + "match": "exact" + }, + { + "description": "Default memory request is 128Mi", + "command": "kubectl get limitrange resource-defaults -n workloads -o jsonpath='{.spec.limits[0].defaultRequest.memory}' 2>/dev/null || echo ''", + "expected_output": "128Mi", + "match": "exact" + } + ] + }, + "default_namespace": "workloads", + "teardown_commands": [ + { "command": "kubectl delete limitrange resource-defaults -n workloads --ignore-not-found" }, + { "command": "kubectl delete namespace workloads --ignore-not-found --wait=false" } + ] +} diff --git a/scenarios/data/rootnode-probe-port-mismatch.json b/scenarios/data/rootnode-probe-port-mismatch.json new file mode 100644 index 0000000..a45ee24 --- /dev/null +++ b/scenarios/data/rootnode-probe-port-mismatch.json @@ -0,0 +1,57 @@ +{ + "id": "rootnode-probe-port-mismatch", + "title": "Stuck Not-Ready: The Probe Port Mismatch", + "category": "Observability & Ops", + "difficulty": "Medium", + "type": "task", + "weight": 8, + "description": "## Stuck Not-Ready: The Probe Port Mismatch\n\nA deployment named `api-gateway` was rolled out to the `default` namespace. The pods start, the container runs — but the deployment never reaches Ready. Traffic won't be routed to it, and the on-call alert is firing.\n\nThis is a class of incident that is easy to miss: the container is actually healthy, but Kubernetes disagrees because the readiness probe is checking the **wrong port**. In production this pattern silently blocks rollouts, causes zero-downtime deploys to stall, and is tricky to diagnose if you go straight to logs instead of reading events.\n\n**Your task:** Diagnose why the pods are failing their readiness check, correct the probe configuration, and restore the deployment to **2/2 Ready**.\n\n> Tip: start with pod status and events — the scheduler and kubelet leave a clear trail. Read before you patch.", + "hints": [ + { + "title": "1. See the failure", + "body": "List pods and look at the READY column. Then describe one of the failing pods and read the Events section at the bottom — kubelet reports readiness probe failures there with the exact port it tried.", + "command": "kubectl get pods\nkubectl describe pod -l app=api-gateway" + }, + { + "title": "2. Inspect the probe config", + "body": "Pull the readiness probe definition from the live deployment. You are looking for the httpGet port — compare it to the port nginx actually listens on (80).", + "command": "kubectl get deployment api-gateway -o jsonpath='{.spec.template.spec.containers[0].readinessProbe}'" + }, + { + "title": "3. Patch the probe port", + "body": "Use a JSON patch to correct the readiness probe port to 80. The deployment will trigger a new rollout and the pods should become Ready within a few seconds.", + "command": "kubectl patch deployment api-gateway --type='json' -p='[{\"op\":\"replace\",\"path\":\"/spec/template/spec/containers/0/readinessProbe/httpGet/port\",\"value\":80}]'" + } + ], + "setup_commands": [ + { "command": "kubectl create deployment api-gateway --image=nginx:1.25 --replicas=2" }, + { "command": "kubectl patch deployment api-gateway --type='json' -p='[{\"op\":\"add\",\"path\":\"/spec/template/spec/containers/0/readinessProbe\",\"value\":{\"httpGet\":{\"path\":\"/\",\"port\":8080},\"initialDelaySeconds\":3,\"periodSeconds\":5}}]'" } + ], + "validation": { + "description": "Checks that the probe port is corrected and both replicas are Ready.", + "commands": [ + { + "description": "Readiness probe port is 80", + "command": "kubectl get deployment api-gateway -o jsonpath='{.spec.template.spec.containers[0].readinessProbe.httpGet.port}'", + "expected_output": "80", + "match": "exact" + }, + { + "description": "Deployment has 2 ready replicas", + "command": "kubectl get deployment api-gateway -o jsonpath='{.status.readyReplicas}' 2>/dev/null | grep -v '^$' || echo 0", + "expected_output": "2", + "match": "exact" + }, + { + "description": "No pods stuck in non-Running state", + "command": "kubectl get pods -l app=api-gateway --no-headers 2>/dev/null | awk '{print $3}' | grep -cv 'Running' || true", + "expected_output": "0", + "match": "exact" + } + ] + }, + "default_namespace": "default", + "teardown_commands": [ + { "command": "kubectl delete deployment api-gateway --ignore-not-found" } + ] +}