diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 6df90df98..665188d10 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -42,3 +42,4 @@ - Daby Sow - Nicholas C. M. Fuller - Ruchir Puri +- Hugo Villarreal diff --git a/documentation/library/faults/README.md b/documentation/library/faults/README.md index aff1c1be6..e6007a950 100644 --- a/documentation/library/faults/README.md +++ b/documentation/library/faults/README.md @@ -7,6 +7,7 @@ A fault is a solvable issue injected into an environment to create an incident. | Name | Platform | | --- | --- | +| [Aggressive Scaledown Kubernetes Horizontal Pod Autoscaler](./aggressive-scaledown-kubernetes-horizontal-pod-autoscaler.md) | Kubernetes | | [Cordoned Kubernetes Worker Node](./cordoned-kubernetes-worker-node.md) | Kubernetes | | [Crashing Kubernetes Workload Init Container](./crashing-kubernetes-workload-init-container.md) | Kubernetes | | [Deleted Kubernetes Service](./deleted-kubernetes-service.md) | Kubernetes | diff --git a/documentation/library/faults/aggressive-scaledown-kubernetes-horizontal-pod-autoscaler.md b/documentation/library/faults/aggressive-scaledown-kubernetes-horizontal-pod-autoscaler.md new file mode 100644 index 000000000..cfcacfd28 --- /dev/null +++ b/documentation/library/faults/aggressive-scaledown-kubernetes-horizontal-pod-autoscaler.md @@ -0,0 +1,32 @@ +>[!NOTE] +>This file is autogenerated. Please do not edit. + +# Aggressive Scaledown Kubernetes Horizontal Pod Autoscaler + +| [Source](../../../scenarios/sre/library/indexes/faults/aggressive-scaledown-kubernetes-horizontal-pod-autoscaler.json) | [Schema](../../../schemas/json/faults/aggressive-scaledown-kubernetes-horizontal-pod-autoscaler.json) | [Implementation](../../../scenarios/sre/project/roles/faults/tasks/inject_aggressive_scaledown_kubernetes_horizontal_pod_autoscaler.yaml) | Kubernetes | Deployment, Performance | + +This fault injects a configuration into a horizontal pod autoscaler that causes it to aggressively scale down workloads, keeping replica counts at a minimum even under high demand. + +## Expectation + +The faulted autoscaler will rapidly scale down the workload to its minimum replica count. Under load, the reduced number of pods will be unable to handle incoming traffic, causing request errors and elevated latency. + +### Alerts + +The following alerts are expected to fire when this fault is injected. + +#### Application Alerts + +- KubePodNotReady + +#### Golden Signal Alerts + +- HighRequestErrorRate +- HighRequestLatency + +## Resources + +- https://kubernetes.io/docs/concepts/workloads/autoscaling/ +- https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ +- https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/ +- https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/horizontal-pod-autoscaler-v2/ diff --git a/documentation/library/scenarios/README.md b/documentation/library/scenarios/README.md index 64b9b18a6..73f9a7352 100644 --- a/documentation/library/scenarios/README.md +++ b/documentation/library/scenarios/README.md @@ -47,3 +47,4 @@ _For statistics on the distrubution spread of scenarios, please see [here](./sta | [102](./sre/102.md) | sre | medium | | [105](./sre/105.md) | sre | medium | | [114](./sre/114.md) | sre | low | +| [115](./sre/115.md) | sre | medium | diff --git a/documentation/library/scenarios/sre/115.md b/documentation/library/scenarios/sre/115.md new file mode 100644 index 000000000..2051d085e --- /dev/null +++ b/documentation/library/scenarios/sre/115.md @@ -0,0 +1,50 @@ +> [!NOTE] +> This file is autogenerated. Please do not edit. + +# Scenario 115 + +| [Source](../../../../scenarios/sre/library/indexes/scenarios/115.json) | Medium Complexity | Kubernetes | Deployment, Performance | + +## Description + +This scenario simulates an OpenTelemetry Demo `frontend` service degradation caused by an HPA misconfigured to aggressively scale down pods under high demand. + +## Applications + +- [OpenTelemetry Demo](../../applications/opentelemetry-demo.md) + +## Faults + +- [Aggressive Scaledown Kubernetes Horizontal Pod Autoscaler](../../../../scenarios/sre/docs/faults.md#Aggressive-Scaledown-Kubernetes-Horizontal-Pod-Autoscaler) +- [OpenTelemetry Demo Feature Flag](../../../../scenarios/sre/docs/faults.md#OpenTelemetry-Demo-Feature-Flag) + +## Solution Guides + +Below is a noncomprehensive list of step by step guides which can lead to remediating the one or more problems in the scenario. + +A **solution** represents the process of fixing an issue created by a fault. To remediate the issues in the scenarios, follow the solutions in order of appearance. + +A solution can have multiple **variants**. A variant is a different process of deriving the solution. If multiple variants are presented, pick only one to solve the problem. + +### Solution 1 + +#### Variant 1 + +1. Manually edit the manifest and remove the aggressive scale-down behavior block, restoring the metrics thresholds to appropriate values. +```shell +kubectl -n otel-demo edit horizontalpodautoscaler frontend +``` + +### Solution 2 + +#### Variant 1 + +1. Disable the feature flag (loadGeneratorFloodHomepage) by manually editing the contents of the ConfigMap. +```shell +kubectl -n otel-demo edit configmap flagd-config +``` + +2. Restart all of the Deployment workloads. +```shell +kubectl -n otel-demo rollout restart deployment +``` diff --git a/documentation/library/scenarios/statistics.md b/documentation/library/scenarios/statistics.md index 27ab7e704..3cf98f5d1 100644 --- a/documentation/library/scenarios/statistics.md +++ b/documentation/library/scenarios/statistics.md @@ -7,16 +7,16 @@ | BookInfo | OpenTelemetry Demo | | --- | --- | -| 5 | 35 | +| 5 | 36 | ## Category Distribution | FinOps | SRE | | --- | --- | -| 2 | 38 | +| 2 | 39 | ## Complexity Distribution | Low | Medium | High | | --- | --- | --- | -| 17 | 22 | 1 | +| 17 | 23 | 1 | diff --git a/scenarios/sre/library/indexes/faults/31.json b/scenarios/sre/library/indexes/faults/31.json new file mode 100644 index 000000000..9784bde43 --- /dev/null +++ b/scenarios/sre/library/indexes/faults/31.json @@ -0,0 +1,87 @@ +{ + "$schema": "https://raw.githubusercontent.com/itbench-hub/ITBench/refs/heads/main/schemas/library/index/fault.json", + "alerts": { + "application": [ + "KubePodNotReady" + ], + "goldenSignal": [ + "HighRequestErrorRate", + "HighRequestLatency" + ] + }, + "arguments": { + "jsonSchema": { + "properties": { + "kubernetesObject": { + "properties": { + "apiVersion": { + "enum": [ + "autoscaling/v2" + ], + "type": "string" + }, + "kind": { + "enum": [ + "HorizontalPodAutoscaler" + ], + "type": "string" + }, + "metadata": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata" + ], + "type": "object" + } + }, + "required": [ + "kubernetesObject" + ], + "type": "object" + } + }, + "description": "This fault injects a configuration into a horizontal pod autoscaler that causes it to aggressively scale down workloads, keeping replica counts at a minimum even under high demand.", + "expectation": "The faulted autoscaler will rapidly scale down the workload to its minimum replica count. Under load, the reduced number of pods will be unable to handle incoming traffic, causing request errors and elevated latency.", + "id": "aggressive-scaledown-kubernetes-horizontal-pod-autoscaler", + "index": 31, + "name": "Aggressive Scaledown Kubernetes Horizontal Pod Autoscaler", + "platform": "Kubernetes", + "resources": [ + "https://kubernetes.io/docs/concepts/workloads/autoscaling/", + "https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/", + "https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/", + "https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/horizontal-pod-autoscaler-v2/" + ], + "solutions": { + "templates": [ + { + "steps": [ + { + "command": "kubectl -n {{ args.kubernetesObject.metadata.namespace }} edit horizontalpodautoscaler {{ args.kubernetesObject.metadata.name }}", + "text": "Manually edit the manifest and remove or correct the scale-down behavior block and restore the metrics thresholds to appropriate values." + } + ] + } + ] + }, + "tags": [ + "Deployment", + "Performance" + ] +} diff --git a/scenarios/sre/library/indexes/scenarios/115.json b/scenarios/sre/library/indexes/scenarios/115.json new file mode 100644 index 000000000..ebf4270b0 --- /dev/null +++ b/scenarios/sre/library/indexes/scenarios/115.json @@ -0,0 +1,126 @@ +{ + "$schema": "https://raw.githubusercontent.com/itbench-hub/ITBench/refs/heads/main/schemas/library/index/scenario.json", + "alerts": [ + "KubePodNotReady", + "HighRequestErrorRate", + "HighRequestLatency" + ], + "category": "sre", + "complexity": "medium", + "description": "This scenario simulates an OpenTelemetry Demo `frontend` service degradation caused by an HPA misconfigured to aggressively scale down pods under high demand.", + "disruptions": [ + { + "injections": [ + { + "args": { + "kubernetesObject": { + "apiVersion": "autoscaling/v2", + "kind": "HorizontalPodAutoscaler", + "metadata": { + "name": "frontend", + "namespace": "otel-demo" + } + } + }, + "id": "aggressive-scaledown-kubernetes-horizontal-pod-autoscaler" + }, + { + "args": { + "flag": { + "name": "loadGeneratorFloodHomepage", + "state": "on" + }, + "kubernetesObject": { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": "flagd-config", + "namespace": "otel-demo" + } + } + }, + "id": "opentelemetry-demo-feature-flag" + } + ], + "waitFor": { + "postInjection": [ + { + "args": { + "kubernetesObject": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": "flagd", + "namespace": "otel-demo" + } + } + }, + "id": "restart-kubernetes-workload" + }, + { + "args": { + "kubernetesObject": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": "load-generator", + "namespace": "otel-demo" + } + } + }, + "id": "restart-kubernetes-workload" + } + ] + } + } + ], + "environment": { + "applications": [ + { + "args": { + "autoscaling": { + "kubernetes": { + "hpa": true + } + } + }, + "id": "opentelemetry-demo" + } + ] + }, + "id": 115, + "index": 115, + "platforms": [ + "Kubernetes" + ], + "solutions": [ + [ + { + "steps": [ + { + "command": "kubectl -n otel-demo edit horizontalpodautoscaler frontend", + "text": "Manually edit the manifest and remove the aggressive scale-down behavior block, restoring the metrics thresholds to appropriate values." + } + ] + } + ], + [ + { + "steps": [ + { + "command": "kubectl -n otel-demo edit configmap flagd-config", + "text": "Disable the feature flag (loadGeneratorFloodHomepage) by manually editing the contents of the ConfigMap." + }, + { + "command": "kubectl -n otel-demo rollout restart deployment", + "text": "Restart all of the Deployment workloads." + } + ] + } + ] + ], + "tags": [ + "Deployment", + "Performance" + ] +} diff --git a/scenarios/sre/project/roles/awx/meta/argument_specs.yaml b/scenarios/sre/project/roles/awx/meta/argument_specs.yaml index 790e7dc1b..4504d357d 100644 --- a/scenarios/sre/project/roles/awx/meta/argument_specs.yaml +++ b/scenarios/sre/project/roles/awx/meta/argument_specs.yaml @@ -130,6 +130,7 @@ argument_specs: - 102 - 105 - 114 + - 115 required: true type: int required: true diff --git a/scenarios/sre/project/roles/awx/vars/main/scenario_fault_count.yaml b/scenarios/sre/project/roles/awx/vars/main/scenario_fault_count.yaml index 9ca9709bc..60f09ccb5 100644 --- a/scenarios/sre/project/roles/awx/vars/main/scenario_fault_count.yaml +++ b/scenarios/sre/project/roles/awx/vars/main/scenario_fault_count.yaml @@ -40,3 +40,4 @@ awx_scenario_fault_count: 102: 1 105: 1 114: 1 + 115: 2 diff --git a/scenarios/sre/project/roles/faults/meta/argument_specs.yaml b/scenarios/sre/project/roles/faults/meta/argument_specs.yaml index 8911e1793..70341c884 100644 --- a/scenarios/sre/project/roles/faults/meta/argument_specs.yaml +++ b/scenarios/sre/project/roles/faults/meta/argument_specs.yaml @@ -33,6 +33,7 @@ argument_specs: - invalid-kubernetes-workload-container-command - kubernetes-api-server-request-surge - misconfigured-kubernetes-horizontal-pod-autoscaler + - aggressive-scaledown-kubernetes-horizontal-pod-autoscaler - misconfigured-kubernetes-workload-container-readiness-probe - modified-kubernetes-workload-container-environment-variable - modified-target-port-kubernetes-service diff --git a/scenarios/sre/project/roles/faults/molecule/inject_aggressive_scaledown_kubernetes_horizontal_pod_autoscaler/converge.yml b/scenarios/sre/project/roles/faults/molecule/inject_aggressive_scaledown_kubernetes_horizontal_pod_autoscaler/converge.yml new file mode 100644 index 000000000..1bf08a4d9 --- /dev/null +++ b/scenarios/sre/project/roles/faults/molecule/inject_aggressive_scaledown_kubernetes_horizontal_pod_autoscaler/converge.yml @@ -0,0 +1,20 @@ +--- +- name: Run fault injection test + hosts: + - localhost + tasks: + - name: Import faults role + ansible.builtin.import_role: + name: faults + vars: + faults_cluster: + kubeconfig: "{{ cluster.kubeconfig }}" + faults_injection_tasks: + - id: aggressive-scaledown-kubernetes-horizontal-pod-autoscaler + args: + kubernetesObject: + apiVersion: autoscaling/v2 + kind: HorizontalPodAutoscaler + metadata: + name: test-workload-hpa + namespace: aggressive-scaledown-hpa-test diff --git a/scenarios/sre/project/roles/faults/molecule/inject_aggressive_scaledown_kubernetes_horizontal_pod_autoscaler/create.yml b/scenarios/sre/project/roles/faults/molecule/inject_aggressive_scaledown_kubernetes_horizontal_pod_autoscaler/create.yml new file mode 100644 index 000000000..dadeed95d --- /dev/null +++ b/scenarios/sre/project/roles/faults/molecule/inject_aggressive_scaledown_kubernetes_horizontal_pod_autoscaler/create.yml @@ -0,0 +1,115 @@ +--- +- name: Create test resources + hosts: + - localhost + tasks: + - name: Include tools role variables + ansible.builtin.include_vars: + file: |- + {{ + [ + playbook_dir, + "..", + "..", + "..", + "tools", + "defaults", + "main", + "managers.yaml" + ] | + ansible.builtin.path_join + }} + + - name: Approve pending certificate requests + ansible.builtin.include_role: + name: cluster + tasks_from: approve_certificate_requests + vars: + cluster_files: + kubeconfig: "{{ cluster.kubeconfig }}" + + - name: Install Kubernetes Metrics Server + kubernetes.core.helm: + chart_ref: "{{ tools_managers.kubernetes_metrics_server.helm.chart.reference }}" + chart_repo_url: "{{ tools_managers.kubernetes_metrics_server.helm.chart.repository }}" + chart_version: "{{ tools_managers.kubernetes_metrics_server.helm.chart.version }}" + create_namespace: true + kubeconfig: "{{ cluster.kubeconfig }}" + release_name: aggressive-scaledown-hpa-release + release_namespace: aggressive-scaledown-hpa-test + release_state: present + timeout: 10m0s + values: + apiService: + insecureSkipTLSVerify: false + tls: + type: helm + wait: true + + - name: Create test deployment + kubernetes.core.k8s: + kubeconfig: "{{ cluster.kubeconfig }}" + resource_definition: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: test-workload + namespace: aggressive-scaledown-hpa-test + spec: + replicas: 2 + selector: + matchLabels: + "app.kubernetes.io/name": test-workload + template: + metadata: + labels: + "app.kubernetes.io/name": test-workload + spec: + containers: + - name: server + image: registry.access.redhat.com/ubi10-minimal:10.2-1782798957 + command: + - /bin/sh + args: + - -c + - "sleep infinity" + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 200m + memory: 256Mi + state: present + wait: true + + - name: Create HorizontalPodAutoscaler + kubernetes.core.k8s: + kubeconfig: "{{ cluster.kubeconfig }}" + resource_definition: + apiVersion: autoscaling/v2 + kind: HorizontalPodAutoscaler + metadata: + name: test-workload-hpa + namespace: aggressive-scaledown-hpa-test + spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: test-workload + minReplicas: 1 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 + state: present diff --git a/scenarios/sre/project/roles/faults/molecule/inject_aggressive_scaledown_kubernetes_horizontal_pod_autoscaler/destroy.yml b/scenarios/sre/project/roles/faults/molecule/inject_aggressive_scaledown_kubernetes_horizontal_pod_autoscaler/destroy.yml new file mode 100644 index 000000000..c167bc1fc --- /dev/null +++ b/scenarios/sre/project/roles/faults/molecule/inject_aggressive_scaledown_kubernetes_horizontal_pod_autoscaler/destroy.yml @@ -0,0 +1,23 @@ +--- +- name: Destroy test resources + hosts: + - localhost + tasks: + - name: Uninstall Kubernetes Metrics Server + kubernetes.core.helm: + kubeconfig: "{{ cluster.kubeconfig }}" + release_name: aggressive-scaledown-hpa-release + release_namespace: aggressive-scaledown-hpa-test + release_state: absent + wait: true + + - name: Delete test namespace + kubernetes.core.k8s: + kubeconfig: "{{ cluster.kubeconfig }}" + resource_definition: + apiVersion: v1 + kind: Namespace + metadata: + name: aggressive-scaledown-hpa-test + state: absent + wait: true diff --git a/scenarios/sre/project/roles/faults/molecule/inject_aggressive_scaledown_kubernetes_horizontal_pod_autoscaler/molecule.yml b/scenarios/sre/project/roles/faults/molecule/inject_aggressive_scaledown_kubernetes_horizontal_pod_autoscaler/molecule.yml new file mode 100644 index 000000000..fb3b879bd --- /dev/null +++ b/scenarios/sre/project/roles/faults/molecule/inject_aggressive_scaledown_kubernetes_horizontal_pod_autoscaler/molecule.yml @@ -0,0 +1,27 @@ +--- +dependency: + name: galaxy + +ansible: + env: + ANSIBLE_ROLES_PATH: ../../.. + executor: + backend: ansible-playbook + args: + ansible_playbook: + - --inventory=${MOLECULE_PROJECT_DIRECTORY}/../../../inventory + - --tags=inject_faults,untagged + playbooks: + create: create.yml + converge: converge.yml + destroy: destroy.yml + verify: verify.yml + +scenario: + name: inject_aggressive_scaledown_kubernetes_horizontal_pod_autoscaler + test_sequence: + - syntax + - create + - converge + - verify + - destroy diff --git a/scenarios/sre/project/roles/faults/molecule/inject_aggressive_scaledown_kubernetes_horizontal_pod_autoscaler/verify.yml b/scenarios/sre/project/roles/faults/molecule/inject_aggressive_scaledown_kubernetes_horizontal_pod_autoscaler/verify.yml new file mode 100644 index 000000000..1884490ff --- /dev/null +++ b/scenarios/sre/project/roles/faults/molecule/inject_aggressive_scaledown_kubernetes_horizontal_pod_autoscaler/verify.yml @@ -0,0 +1,57 @@ +--- +- name: Verify that fault injection succeeded + hosts: + - localhost + tasks: + - name: Retrieve HorizontalPodAutoscaler + kubernetes.core.k8s_info: + kubeconfig: "{{ cluster.kubeconfig }}" + api_version: autoscaling/v2 + kind: HorizontalPodAutoscaler + name: test-workload-hpa + namespace: aggressive-scaledown-hpa-test + register: faults_hpa + + - name: Validate that HPA exists + ansible.builtin.assert: + that: + - faults_hpa.resources | ansible.builtin.length == 1 + fail_msg: Unable to find HPA. Fault injection failed. + success_msg: Found HPA. + + - name: Retrieve CPU metric configuration + ansible.builtin.set_fact: + faults_cpu_metric: |- + {{ + faults_hpa.resources[0].spec.metrics | + ansible.builtin.selectattr('resource.name', '==', 'cpu') + }} + + - name: Validate that CPU metric was set to aggressive scale-down threshold + ansible.builtin.assert: + that: + - faults_cpu_metric | ansible.builtin.length == 1 + - faults_cpu_metric[0].resource.target.averageUtilization == 99 + fail_msg: CPU metric was not set to 99%. Fault injection failed. + success_msg: CPU metric was set to 99%. + + - name: Retrieve scale-down behavior configuration + ansible.builtin.set_fact: + faults_scaledown_behavior: "{{ faults_hpa.resources[0].spec.behavior.scaleDown }}" + + - name: Validate that scale-down stabilization window was removed + ansible.builtin.assert: + that: + - faults_scaledown_behavior.stabilizationWindowSeconds == 0 + fail_msg: Scale-down stabilization window was not set to 0. Fault injection failed. + success_msg: Scale-down stabilization window set to 0. + + - name: Validate that scale-down policy allows 100% removal in 15 seconds + ansible.builtin.assert: + that: + - faults_scaledown_behavior.policies | ansible.builtin.length == 1 + - faults_scaledown_behavior.policies[0].type == "Percent" + - faults_scaledown_behavior.policies[0].value == 100 + - faults_scaledown_behavior.policies[0].periodSeconds == 15 + fail_msg: Scale-down policy was not configured for aggressive removal. Fault injection failed. + success_msg: Scale-down policy configured for 100% removal in 15 seconds. diff --git a/scenarios/sre/project/roles/faults/tasks/inject_aggressive_scaledown_kubernetes_horizontal_pod_autoscaler.yaml b/scenarios/sre/project/roles/faults/tasks/inject_aggressive_scaledown_kubernetes_horizontal_pod_autoscaler.yaml new file mode 100644 index 000000000..13d54471f --- /dev/null +++ b/scenarios/sre/project/roles/faults/tasks/inject_aggressive_scaledown_kubernetes_horizontal_pod_autoscaler.yaml @@ -0,0 +1,30 @@ +--- +- name: Include fault argument validation tasks + ansible.builtin.include_tasks: + file: validate_fault_arguments.yaml + +- name: Update autoscaler with aggressive scale-down behavior + kubernetes.core.k8s: + kubeconfig: "{{ faults_cluster.kubeconfig }}" + resource_definition: + apiVersion: "{{ faults_kubernetes_object.apiVersion }}" + kind: "{{ faults_kubernetes_object.kind }}" + metadata: + name: "{{ faults_kubernetes_object.metadata.name }}" + namespace: "{{ faults_kubernetes_object.metadata.namespace }}" + spec: + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 99 + behavior: + scaleDown: + stabilizationWindowSeconds: 0 + policies: + - type: Percent + value: 100 + periodSeconds: 15 + state: patched diff --git a/scenarios/sre/project/roles/faults/vars/main/task_files.yaml b/scenarios/sre/project/roles/faults/vars/main/task_files.yaml index 70e94336d..8684f1052 100644 --- a/scenarios/sre/project/roles/faults/vars/main/task_files.yaml +++ b/scenarios/sre/project/roles/faults/vars/main/task_files.yaml @@ -26,6 +26,8 @@ faults_task_files: injection: inject_kubernetes_api_server_request_surge.yaml misconfigured-kubernetes-horizontal-pod-autoscaler: injection: inject_misconfigured_kubernetes_horizontal_pod_autoscaler.yaml + aggressive-scaledown-kubernetes-horizontal-pod-autoscaler: + injection: inject_aggressive_scaledown_kubernetes_horizontal_pod_autoscaler.yaml misconfigured-kubernetes-workload-container-readiness-probe: injection: inject_misconfigured_kubernetes_workload_container_readiness_probe.yaml modified-kubernetes-workload-container-environment-variable: diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_115/groundtruth.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_115/groundtruth.yaml new file mode 100644 index 000000000..ef5b66f41 --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_115/groundtruth.yaml @@ -0,0 +1,33 @@ +--- +apiVersion: itbench.io/v2 +kind: GroundTruth +metadata: + name: scenario-115 +spec: + alerts: + - labels: {} + name: KubePodNotReady + - labels: {} + name: HighRequestErrorRate + - labels: {} + name: HighRequestLatency + entities: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: frontend + namespace: otel-demo + - apiVersion: autoscaling/v2 + kind: HorizontalPodAutoscaler + metadata: + name: frontend + namespace: otel-demo + solutions: + - - steps: + - command: kubectl -n otel-demo edit horizontalpodautoscaler frontend + text: Manually edit the manifest and remove the aggressive scale-down behavior block, restoring the metrics thresholds to appropriate values. + - - steps: + - command: kubectl -n otel-demo edit configmap flagd-config + text: Disable the feature flag (loadGeneratorFloodHomepage) by manually editing the contents of the ConfigMap. + - command: kubectl -n otel-demo rollout restart deployment + text: Restart all of the Deployment workloads. diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_115/groundtruth_v1.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_115/groundtruth_v1.yaml new file mode 100644 index 000000000..4d177d31e --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_115/groundtruth_v1.yaml @@ -0,0 +1,109 @@ +--- +apiVersion: itbench.io/v1 +kind: GroundTruth +metadata: + name: scenario-115 +spec: + alerts: + - group_id: frontend-pod-1 + id: KubePodNotReady + metadata: + description: Frontend pods are not ready due to the HPA aggressively scaling down replicas under high load + - group_id: frontend-service-1 + id: HighRequestErrorRate + metadata: + description: Request error rate in service frontend in namespace otel-demo is above threshold due to insufficient replicas + - group_id: frontend-service-1 + id: HighRequestLatency + metadata: + description: Request latency in service frontend in namespace otel-demo is above threshold due to insufficient replicas + - group_id: frontend-proxy-service-1 + id: HighRequestErrorRate + metadata: + description: Request error rate in service frontend-proxy in namespace otel-demo is above threshold as errors cascade from frontend + + aliases: + - - frontend-hpa-1 + - frontend-deployment-1 + - frontend-pod-1 + - frontend-service-1 + - - frontend-proxy-service-1 + - frontend-proxy-pod-1 + + fault: + - category: Change + condition: HPA scale-down behavior patched to use stabilizationWindowSeconds=0 and 100% scale-down in 15s with CPU threshold set to 99% + entity: + group_id: frontend-hpa-1 + kind: HorizontalPodAutoscaler + name: frontend + fault_mechanism: hpa_misconfiguration + + groups: + - filter: + - frontend\b + id: frontend-hpa-1 + kind: HorizontalPodAutoscaler + namespace: otel-demo + root_cause: true + + - filter: + - frontend\b + id: frontend-deployment-1 + kind: Deployment + namespace: otel-demo + + - filter: + - ^frontend-[a-z0-9]+-[a-z0-9]+$ + id: frontend-pod-1 + kind: Pod + namespace: otel-demo + + - filter: + - frontend\b + id: frontend-service-1 + kind: Service + namespace: otel-demo + + - filter: + - frontend-proxy\b + id: frontend-proxy-service-1 + kind: Service + namespace: otel-demo + + - filter: + - ^frontend-proxy-[a-z0-9]+-[a-z0-9]+$ + id: frontend-proxy-pod-1 + kind: Pod + namespace: otel-demo + + propagations: + - condition: HPA configured with stabilizationWindowSeconds=0 and 100% scale-down policy at CPU threshold of 99%, causing immediate scale-down to + minReplicas=1 + effect: Frontend deployment is kept at minimum replicas despite high incoming traffic from load generator + source: frontend-hpa-1 + target: frontend-deployment-1 + - condition: Single replica cannot handle the flood of requests from loadGeneratorFloodHomepage feature flag + effect: Frontend pods become overwhelmed and fail readiness checks + source: frontend-deployment-1 + target: frontend-pod-1 + - condition: Insufficient healthy pods to serve incoming traffic + effect: Frontend service experiences elevated error rates and latency + source: frontend-pod-1 + target: frontend-service-1 + - condition: Frontend errors cascade to proxy layer + effect: Frontend-proxy service experiences elevated error rates + source: frontend-service-1 + target: frontend-proxy-service-1 + + recommendedActions: + - solution: + actions: + - Identify the HPA misconfiguration by inspecting the scale-down behavior block + - Remove or correct the aggressive scaleDown behavior and restore appropriate CPU utilization thresholds + id: fix_hpa_scaledown_behavior + - solution: + actions: + - Disable the loadGeneratorFloodHomepage feature flag in the flagd-config ConfigMap to reduce traffic load + - Restart all Deployment workloads to pick up the updated flag configuration + id: disable_load_generator_flag diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_115/scenario.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_115/scenario.yaml new file mode 100644 index 000000000..dfdaf0c31 --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_115/scenario.yaml @@ -0,0 +1,55 @@ +--- +apiVersion: itbench.io/v2 +kind: Scenario +metadata: + name: scenario-115 +spec: + applications: + opentelemetryDemo: + enabled: true + options: + autoscaling: + kubernetes: + hpa: true + faults: + - injections: + - args: + kubernetesObject: + apiVersion: autoscaling/v2 + kind: HorizontalPodAutoscaler + metadata: + name: frontend + namespace: otel-demo + id: aggressive-scaledown-kubernetes-horizontal-pod-autoscaler + - args: + flag: + name: loadGeneratorFloodHomepage + state: "on" + kubernetesObject: + apiVersion: v1 + kind: ConfigMap + metadata: + name: flagd-config + namespace: otel-demo + id: opentelemetry-demo-feature-flag + waitFor: + postInjection: + - args: + kubernetesObject: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: flagd + namespace: otel-demo + id: restart-kubernetes-workload + - args: + kubernetesObject: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: load-generator + namespace: otel-demo + id: restart-kubernetes-workload + tools: + sre: + enabled: true diff --git a/scenarios/sre/project/roles/scenarios/meta/argument_specs.yaml b/scenarios/sre/project/roles/scenarios/meta/argument_specs.yaml index 000c1f10b..cad8b90f2 100644 --- a/scenarios/sre/project/roles/scenarios/meta/argument_specs.yaml +++ b/scenarios/sre/project/roles/scenarios/meta/argument_specs.yaml @@ -46,5 +46,6 @@ argument_specs: - 102 - 105 - 114 + - 115 required: true type: int diff --git a/schemas/json/faults/aggressive-scaledown-kubernetes-horizontal-pod-autoscaler.json b/schemas/json/faults/aggressive-scaledown-kubernetes-horizontal-pod-autoscaler.json new file mode 100644 index 000000000..49052ec9b --- /dev/null +++ b/schemas/json/faults/aggressive-scaledown-kubernetes-horizontal-pod-autoscaler.json @@ -0,0 +1,46 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "kubernetesObject": { + "properties": { + "apiVersion": { + "enum": [ + "autoscaling/v2" + ], + "type": "string" + }, + "kind": { + "enum": [ + "HorizontalPodAutoscaler" + ], + "type": "string" + }, + "metadata": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata" + ], + "type": "object" + } + }, + "required": [ + "kubernetesObject" + ], + "type": "object" +} diff --git a/schemas/json/library/index/scenario.json b/schemas/json/library/index/scenario.json index 2566fce46..ead40a695 100644 --- a/schemas/json/library/index/scenario.json +++ b/schemas/json/library/index/scenario.json @@ -198,6 +198,22 @@ } } }, + { + "if": { + "properties": { + "id": { + "const": "aggressive-scaledown-kubernetes-horizontal-pod-autoscaler" + } + } + }, + "then": { + "properties": { + "args": { + "$ref": "../../faults/aggressive-scaledown-kubernetes-horizontal-pod-autoscaler.json" + } + } + } + }, { "if": { "properties": { @@ -550,6 +566,7 @@ "kubernetes-api-server-request-surge", "unsupported-architecture-kubernetes-workload-container-image", "deleted-kubernetes-service", + "aggressive-scaledown-kubernetes-horizontal-pod-autoscaler", "valkey-workload-changed-password", "cordoned-kubernetes-worker-node", "misconfigured-kubernetes-horizontal-pod-autoscaler",