diff --git a/documentation/library/faults/README.md b/documentation/library/faults/README.md index 840ac9038..25beffef1 100644 --- a/documentation/library/faults/README.md +++ b/documentation/library/faults/README.md @@ -4,10 +4,16 @@ A fault is a solvable issue injected into an environment to create an incident. | Name | Platform | | --- | --- | +| [Breaking Configuration Change](./breaking-configuration-change.md) | Kubernetes | +| [Cluster Availability Loss](./cluster-availability-loss.md) | Kubernetes | | [Cordoned Kubernetes Worker Node](./cordoned-kubernetes-worker-node.md) | Kubernetes | +| [Corrupted Kubernetes Secret Credentials](./corrupted-kubernetes-secret-credentials.md) | Kubernetes | +| [Crashing Kubernetes Controller Workload](./crashing-kubernetes-controller-workload.md) | Kubernetes | | [Crashing Kubernetes Workload Init Container](./crashing-kubernetes-workload-init-container.md) | Kubernetes | | [Deleted Kubernetes Service](./deleted-kubernetes-service.md) | Kubernetes | | [Disabled Istio Ambient Mode Kubernetes Namespace](./disabled-istio-ambient-mode-kubernetes-namespace.md) | Kubernetes | +| [Exhausted Etcd Database Storage](./exhausted-etcd-database-storage.md) | Kubernetes | +| [Failed Release Pipeline Service](./failed-release-pipeline-service.md) | Kubernetes | | [Failing Name Resolution Kubernetes Workload DNS Policy](./failing-name-resolution-kubernetes-workload-dns-policy.md) | Kubernetes | | [Hanging Kubernetes Workload Init Container](./hanging-kubernetes-workload-init-container.md) | Kubernetes | | [Ingress Port Blocking Network Policy](./ingress-port-blocking-network-policy.md) | Kubernetes | @@ -20,6 +26,8 @@ A fault is a solvable issue injected into an environment to create an incident. | [Misconfigured Kubernetes Workload Container Readiness Probe](./misconfigured-kubernetes-workload-container-readiness-probe.md) | Kubernetes | | [Modified Kubernetes Workload Container Environment Variable](./modified-kubernetes-workload-container-environment-variable.md) | Kubernetes | | [Modified Target Port Kubernetes Service](./modified-target-port-kubernetes-service.md) | Kubernetes | +| [Monitoring Probe Failure](./monitoring-probe-failure.md) | Kubernetes | +| [Node Resource Exhaustion](./node-resource-exhaustion.md) | Kubernetes | | [Nonexistent Kubernetes Workload Container Image](./nonexistent-kubernetes-workload-container-image.md) | Kubernetes | | [Nonexistent Kubernetes Workload Node](./nonexistent-kubernetes-workload-node.md) | Kubernetes | | [Nonexistent Kubernetes Workload Persistent Volume Claim](./nonexistent-kubernetes-workload-persistent-volume-claim.md) | Kubernetes | @@ -27,6 +35,7 @@ A fault is a solvable issue injected into an environment to create an incident. | [Priority Kubernetes Workload Priority Preemption](./priority-kubernetes-workload-priority-preemption.md) | Kubernetes | | [Scaled To Zero Kubernetes Workload](./scaled-to-zero-kubernetes-workload.md) | Kubernetes | | [Scheduled Chaos Mesh Experiment](./scheduled-chaos-mesh-experiment.md) | Kubernetes | +| [Stalled Pipeline Controller](./stalled-pipeline-controller.md) | Kubernetes | | [Strict Mutual TLS Istio Service Mesh Enforcement](./strict-mutual-tls-istio-service-mesh-enforcement.md) | Kubernetes | | [Traffic Denying Istio Gateway Authorization Policy](./traffic-denying-istio-gateway-authorization-policy.md) | Kubernetes | | [Unassigned Kubernetes Workload Container Resource Limits](./unassigned-kubernetes-workload-container-resource-limits.md) | Kubernetes | diff --git a/documentation/library/faults/breaking-configuration-change.md b/documentation/library/faults/breaking-configuration-change.md new file mode 100644 index 000000000..e9f3021d2 --- /dev/null +++ b/documentation/library/faults/breaking-configuration-change.md @@ -0,0 +1,27 @@ +>[!NOTE] +>This file is autogenerated. Please do not edit. + +# Breaking Configuration Change + +| [Source](../../../scenarios/sre/project/roles/documentation/files/library/faults/indexes/35.yaml) | [Schema](../../../schemas/json/faults/breaking-configuration-change.json) | [Implementation](../../../scenarios/sre/project/roles/faults/tasks/inject_breaking_configuration_change.yaml) | Kubernetes | Code, Deployment | + +This fault modifies a workload's environment variable to inject a breaking configuration change, simulating real-world incidents where policy updates, ArgoCD sync issues, or operator index corruption cause cascading failures across dependent services. + +## Expectation + +The faulted workload will fail to function correctly due to the misconfigured environment variable. Dependent services will experience errors as the broken configuration propagates through the service chain. + +### Alerts + +The following alerts are expected to fire when this fault is injected. + +#### Golden Signal Alerts + +- HighRequestErrorRate + +## Resources + +- https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/ +- https://kubernetes.io/docs/concepts/configuration/configmap/ +- https://kubernetes.io/docs/tasks/manage-kubernetes-objects/update-api-object-kubectl-patch/ +- https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/ diff --git a/documentation/library/faults/cluster-availability-loss.md b/documentation/library/faults/cluster-availability-loss.md new file mode 100644 index 000000000..059dfdf62 --- /dev/null +++ b/documentation/library/faults/cluster-availability-loss.md @@ -0,0 +1,31 @@ +>[!NOTE] +>This file is autogenerated. Please do not edit. + +# Cluster Availability Loss + +| [Source](../../../scenarios/sre/project/roles/documentation/files/library/faults/indexes/34.yaml) | [Schema](../../../schemas/json/faults/cluster-availability-loss.json) | [Implementation](../../../scenarios/sre/project/roles/faults/tasks/inject_cluster_availability_loss.yaml) | Kubernetes | Deployment, Performance | + +This fault cordons all worker nodes and then evicts the target workload pods, simulating a cluster-wide availability loss where the control plane becomes unable to schedule or maintain workloads. This reproduces real-world incidents where clusters become completely inaccessible or unresponsive. + +## Expectation + +The target workload pods will be evicted and unable to reschedule due to all nodes being cordoned. Services depending on the workload will experience errors. The cluster will appear degraded with multiple pods in Pending state. + +### Alerts + +The following alerts are expected to fire when this fault is injected. + +#### Application Alerts + +- KubePodNotReady + +#### Golden Signal Alerts + +- HighRequestErrorRate + +## Resources + +- https://kubernetes.io/docs/concepts/architecture/nodes/ +- https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/ +- https://kubernetes.io/docs/reference/kubectl/generated/kubectl_cordon/ +- https://kubernetes.io/docs/tasks/debug/debug-cluster/ diff --git a/documentation/library/faults/corrupted-kubernetes-secret-credentials.md b/documentation/library/faults/corrupted-kubernetes-secret-credentials.md new file mode 100644 index 000000000..b964a49ac --- /dev/null +++ b/documentation/library/faults/corrupted-kubernetes-secret-credentials.md @@ -0,0 +1,27 @@ +>[!NOTE] +>This file is autogenerated. Please do not edit. + +# Corrupted Kubernetes Secret Credentials + +| [Source](../../../scenarios/sre/project/roles/documentation/files/library/faults/indexes/36.yaml) | [Schema](../../../schemas/json/faults/corrupted-kubernetes-secret-credentials.json) | [Implementation](../../../scenarios/sre/project/roles/faults/tasks/inject_corrupted_kubernetes_secret_credentials.yaml) | Kubernetes | Authentication, Deployment | + +This fault corrupts a Kubernetes Secret by replacing its data with invalid credentials, simulating real-world incidents where secret rotation failures, registry authentication issues, or Vault access problems cause service outages. + +## Expectation + +Workloads that depend on the corrupted Secret will fail to authenticate with external services (registries, databases, APIs). Pods may crash or enter error states when they attempt to use the invalid credentials. + +### Alerts + +The following alerts are expected to fire when this fault is injected. + +#### Golden Signal Alerts + +- HighRequestErrorRate + +## Resources + +- https://kubernetes.io/docs/concepts/configuration/secret/ +- https://kubernetes.io/docs/tasks/configmap-secret/managing-secret-using-kubectl/ +- https://kubernetes.io/docs/reference/kubernetes-api/config-and-storage-resources/secret-v1/ +- https://kubernetes.io/docs/tasks/debug/debug-application/ diff --git a/documentation/library/faults/crashing-kubernetes-controller-workload.md b/documentation/library/faults/crashing-kubernetes-controller-workload.md new file mode 100644 index 000000000..470cea34b --- /dev/null +++ b/documentation/library/faults/crashing-kubernetes-controller-workload.md @@ -0,0 +1,28 @@ +>[!NOTE] +>This file is autogenerated. Please do not edit. + +# Crashing Kubernetes Controller Workload + +| [Source](../../../scenarios/sre/project/roles/documentation/files/library/faults/indexes/31.yaml) | [Schema](../../../schemas/json/faults/crashing-kubernetes-controller-workload.json) | [Implementation](../../../scenarios/sre/project/roles/faults/tasks/inject_crashing_kubernetes_controller_workload.yaml) | Kubernetes | Deployment, Performance | + +This fault modifies a controller or operator workload's container command to an invalid value, causing the pod to enter a CrashLoopBackOff state. This simulates real-world incidents where controller pods crash due to configuration errors, incompatible upgrades, or corrupted state, leading to service degradation across the cluster. + +## Expectation + +The faulted controller pod(s) will enter CrashLoopBackOff state. Dependent services will experience degradation as the controller is unable to reconcile resources. + +### Alerts + +The following alerts are expected to fire when this fault is injected. + +#### Application Alerts + +- KubePodCrashLooping +- KubePodNotReady + +## Resources + +- https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ +- https://kubernetes.io/docs/concepts/workloads/controllers/deployment/ +- https://kubernetes.io/docs/tasks/debug/debug-application/debug-running-pod/ +- https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/ diff --git a/documentation/library/faults/exhausted-etcd-database-storage.md b/documentation/library/faults/exhausted-etcd-database-storage.md new file mode 100644 index 000000000..0a97f098d --- /dev/null +++ b/documentation/library/faults/exhausted-etcd-database-storage.md @@ -0,0 +1,27 @@ +>[!NOTE] +>This file is autogenerated. Please do not edit. + +# Exhausted Etcd Database Storage + +| [Source](../../../scenarios/sre/project/roles/documentation/files/library/faults/indexes/32.yaml) | [Schema](../../../schemas/json/faults/exhausted-etcd-database-storage.json) | [Implementation](../../../scenarios/sre/project/roles/faults/tasks/inject_exhausted_etcd_database_storage.yaml) | Kubernetes | Performance | + +This fault fills the etcd database storage by creating a large number of ConfigMaps, pushing etcd usage past its quota threshold. This simulates real-world incidents where etcd storage exhaustion causes the API server to reject writes, leading to cluster-wide degradation or complete outage. + +## Expectation + +The etcd database will approach or exceed its storage quota. The Kubernetes API server will begin rejecting mutating requests. New pod scheduling and resource creation will fail. + +### Alerts + +The following alerts are expected to fire when this fault is injected. + +#### Application Alerts + +- KubePodNotReady + +## Resources + +- https://kubernetes.io/docs/tasks/administer-cluster/configure-upgrade-etcd/ +- https://etcd.io/docs/v3.5/op-guide/maintenance/ +- https://kubernetes.io/docs/concepts/overview/components/#etcd +- https://kubernetes.io/docs/tasks/debug/debug-cluster/ diff --git a/documentation/library/faults/failed-release-pipeline-service.md b/documentation/library/faults/failed-release-pipeline-service.md new file mode 100644 index 000000000..90da537dd --- /dev/null +++ b/documentation/library/faults/failed-release-pipeline-service.md @@ -0,0 +1,27 @@ +>[!NOTE] +>This file is autogenerated. Please do not edit. + +# Failed Release Pipeline Service + +| [Source](../../../scenarios/sre/project/roles/documentation/files/library/faults/indexes/38.yaml) | [Schema](../../../schemas/json/faults/failed-release-pipeline-service.json) | [Implementation](../../../scenarios/sre/project/roles/faults/tasks/inject_failed_release_pipeline_service.yaml) | Kubernetes | Code, Deployment | + +This fault modifies a release pipeline service's container to use an invalid image tag, simulating real-world incidents where release signing failures, broken trusted task lists, or release stage blockages prevent software from being released to production. + +## Expectation + +The release pipeline service will fail to start due to the invalid image. Dependent release workflows will stall as the service becomes unavailable. + +### Alerts + +The following alerts are expected to fire when this fault is injected. + +#### Golden Signal Alerts + +- HighRequestErrorRate + +## Resources + +- https://kubernetes.io/docs/concepts/workloads/controllers/deployment/ +- https://kubernetes.io/docs/concepts/containers/images/ +- https://kubernetes.io/docs/tasks/debug/debug-application/ +- https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/ diff --git a/documentation/library/faults/monitoring-probe-failure.md b/documentation/library/faults/monitoring-probe-failure.md new file mode 100644 index 000000000..0931b53f2 --- /dev/null +++ b/documentation/library/faults/monitoring-probe-failure.md @@ -0,0 +1,27 @@ +>[!NOTE] +>This file is autogenerated. Please do not edit. + +# Monitoring Probe Failure + +| [Source](../../../scenarios/sre/project/roles/documentation/files/library/faults/indexes/39.yaml) | [Schema](../../../schemas/json/faults/monitoring-probe-failure.json) | [Implementation](../../../scenarios/sre/project/roles/faults/tasks/inject_monitoring_probe_failure.yaml) | Kubernetes | Deployment, Performance | + +This fault scales down a monitoring or probe deployment to zero replicas, simulating real-world incidents where multiple monitoring probes go down simultaneously, masking real issues and creating blind spots in observability. + +## Expectation + +The monitoring probe workload will be unavailable. Alerts that depend on the probe data will stop firing, creating a false sense of health while actual issues may go undetected. + +### Alerts + +The following alerts are expected to fire when this fault is injected. + +#### Application Alerts + +- KubePodNotReady + +## Resources + +- https://kubernetes.io/docs/concepts/workloads/controllers/deployment/ +- https://kubernetes.io/docs/tasks/debug/debug-application/ +- https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/deployment-v1/ +- https://kubernetes.io/docs/concepts/cluster-administration/monitoring/ diff --git a/documentation/library/faults/node-resource-exhaustion.md b/documentation/library/faults/node-resource-exhaustion.md new file mode 100644 index 000000000..fafa2bede --- /dev/null +++ b/documentation/library/faults/node-resource-exhaustion.md @@ -0,0 +1,27 @@ +>[!NOTE] +>This file is autogenerated. Please do not edit. + +# Node Resource Exhaustion + +| [Source](../../../scenarios/sre/project/roles/documentation/files/library/faults/indexes/37.yaml) | [Schema](../../../schemas/json/faults/node-resource-exhaustion.json) | [Implementation](../../../scenarios/sre/project/roles/faults/tasks/inject_node_resource_exhaustion.yaml) | Kubernetes | Deployment, Performance | + +This fault injects excessive resource requests into a workload, causing nodes to run out of allocatable resources. This simulates real-world incidents where node-level resource exhaustion, compute capacity shortages, or cluster autoscaler failures lead to pods stuck in Pending state and service degradation. + +## Expectation + +The faulted workload pods will consume excessive resources, causing other pods on the same node to be evicted or fail to schedule. The node may become NotReady if resource pressure is severe enough. + +### Alerts + +The following alerts are expected to fire when this fault is injected. + +#### Application Alerts + +- KubePodNotReady + +## Resources + +- https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ +- https://kubernetes.io/docs/tasks/administer-cluster/manage-resources/memory-default-namespace/ +- https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/ +- https://kubernetes.io/docs/tasks/debug/debug-cluster/resource-metrics-pipeline/ diff --git a/documentation/library/faults/stalled-pipeline-controller.md b/documentation/library/faults/stalled-pipeline-controller.md new file mode 100644 index 000000000..3f988d39d --- /dev/null +++ b/documentation/library/faults/stalled-pipeline-controller.md @@ -0,0 +1,27 @@ +>[!NOTE] +>This file is autogenerated. Please do not edit. + +# Stalled Pipeline Controller + +| [Source](../../../scenarios/sre/project/roles/documentation/files/library/faults/indexes/33.yaml) | [Schema](../../../schemas/json/faults/stalled-pipeline-controller.json) | [Implementation](../../../scenarios/sre/project/roles/faults/tasks/inject_stalled_pipeline_controller.yaml) | Kubernetes | Deployment, Performance | + +This fault scales down a pipeline controller deployment to zero replicas, simulating a controller failure that prevents new pipeline runs from being processed. This reproduces real-world incidents where Tekton or similar CI/CD controllers become unavailable, causing pipeline runs to stall in a pending state. + +## Expectation + +New pipeline runs will not be processed and will remain in a pending or queued state. Existing running pipelines may complete but no new work will be picked up by the controller. + +### Alerts + +The following alerts are expected to fire when this fault is injected. + +#### Application Alerts + +- KubePodNotReady + +## Resources + +- https://kubernetes.io/docs/concepts/workloads/controllers/deployment/ +- https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/deployment-v1/ +- https://tekton.dev/docs/pipelines/ +- https://kubernetes.io/docs/tasks/debug/debug-application/ diff --git a/documentation/library/scenarios/README.md b/documentation/library/scenarios/README.md index c3fac2b74..6de37f026 100644 --- a/documentation/library/scenarios/README.md +++ b/documentation/library/scenarios/README.md @@ -45,6 +45,15 @@ | [102](./sre/102.md) | sre | medium | | [105](./sre/105.md) | sre | medium | | [114](./sre/114.md) | sre | low | +| [115](./sre/115.md) | sre | medium | +| [116](./sre/116.md) | sre | high | +| [117](./sre/117.md) | sre | medium | +| [118](./sre/118.md) | sre | high | +| [119](./sre/119.md) | sre | medium | +| [120](./sre/120.md) | sre | medium | +| [121](./sre/121.md) | sre | medium | +| [122](./sre/122.md) | sre | medium | +| [123](./sre/123.md) | sre | low | ## Statistics @@ -52,16 +61,16 @@ | BookInfo | OpenTelemetry Demo | | --- | --- | -| 5 | 35 | +| 5 | 44 | ### Category Distribution | FinOps | SRE | | --- | --- | -| 2 | 38 | +| 2 | 47 | ### Complexity Distribution | Low | Medium | High | | --- | --- | --- | -| 17 | 22 | 1 | +| 18 | 28 | 3 | diff --git a/documentation/library/scenarios/sre/115.md b/documentation/library/scenarios/sre/115.md new file mode 100644 index 000000000..525f4f9a4 --- /dev/null +++ b/documentation/library/scenarios/sre/115.md @@ -0,0 +1,49 @@ +> [!NOTE] +> This file is autogenerated. Please do not edit. + +# Scenario 115 + +| [Source](../../../../scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/115.yaml) | Medium Complexity | Kubernetes | Deployment, Performance | + +## Description + +This scenario simulates OpenTelemetry Demo's `recommendation` service controller crashing due to a corrupted container command, causing the pod to enter CrashLoopBackOff and degrading upstream services that depend on it. + +## Applications + +- [OpenTelemetry Demo](../../applications/opentelemetry-demo.md) + +## Faults + +- [Crashing Kubernetes Controller Workload](../../../../scenarios/sre/docs/faults.md#Crashing-Kubernetes-Controller-Workload) + +## Solution Guides + +Below is a noncomprehensive list of step by step guides which can lead to remediating the one or more problems in the scenario. + +A **solution** represents the process of fixing an issue created by a fault. To remediate the issues in the scenarios, follow the solutions in order of appearance. + +A solution can have multiple **variants**. A variant is a different process of deriving the solution. If multiple variants are presented, pick only one to solve the problem. + +### Solution 1 + +#### Variant 1 + +1. Revert the last change done to the controller manifest. +```shell +kubectl -n otel-demo rollout undo deployment/recommendation +``` + +#### Variant 2 + +1. Manually edit the manifest and replace the container command with the correct value. +```shell +kubectl -n otel-demo edit deployment recommendation +``` + +#### Variant 3 + +1. Check the previous container logs to understand the crash reason. +```shell +kubectl -n otel-demo logs -l app=recommendation -c recommendation --previous +``` diff --git a/documentation/library/scenarios/sre/116.md b/documentation/library/scenarios/sre/116.md new file mode 100644 index 000000000..055e0d1ad --- /dev/null +++ b/documentation/library/scenarios/sre/116.md @@ -0,0 +1,38 @@ +> [!NOTE] +> This file is autogenerated. Please do not edit. + +# Scenario 116 + +| [Source](../../../../scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/116.yaml) | High Complexity | Kubernetes | Performance | + +## Description + +This scenario simulates etcd storage exhaustion by filling the cluster's etcd database with large ConfigMaps, causing the API server to reject writes and degrading cluster-wide operations including the OpenTelemetry Demo application. + +## Applications + +- [OpenTelemetry Demo](../../applications/opentelemetry-demo.md) + +## Faults + +- [Exhausted Etcd Database Storage](../../../../scenarios/sre/docs/faults.md#Exhausted-Etcd-Database-Storage) + +## Solution Guides + +Below is a noncomprehensive list of step by step guides which can lead to remediating the one or more problems in the scenario. + +A **solution** represents the process of fixing an issue created by a fault. To remediate the issues in the scenarios, follow the solutions in order of appearance. + +A solution can have multiple **variants**. A variant is a different process of deriving the solution. If multiple variants are presented, pick only one to solve the problem. + +### Solution 1 + +#### Variant 1 + +1. Compact and defragment the etcd database to reclaim storage space. + +2. Remove unnecessary resources (completed jobs, orphaned ConfigMaps) to reduce etcd storage usage. + +#### Variant 2 + +1. Increase the etcd storage quota if the current limit is too low for the cluster workload. diff --git a/documentation/library/scenarios/sre/117.md b/documentation/library/scenarios/sre/117.md new file mode 100644 index 000000000..fdf81c037 --- /dev/null +++ b/documentation/library/scenarios/sre/117.md @@ -0,0 +1,42 @@ +> [!NOTE] +> This file is autogenerated. Please do not edit. + +# Scenario 117 + +| [Source](../../../../scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/117.yaml) | Medium Complexity | Kubernetes | Deployment, Performance | + +## Description + +This scenario simulates a pipeline controller failure by scaling the OpenTelemetry Demo's `kafka` deployment to zero replicas, causing downstream services that depend on event processing to stall, similar to a Tekton pipeline controller becoming unavailable. + +## Applications + +- [OpenTelemetry Demo](../../applications/opentelemetry-demo.md) + +## Faults + +- [Stalled Pipeline Controller](../../../../scenarios/sre/docs/faults.md#Stalled-Pipeline-Controller) + +## Solution Guides + +Below is a noncomprehensive list of step by step guides which can lead to remediating the one or more problems in the scenario. + +A **solution** represents the process of fixing an issue created by a fault. To remediate the issues in the scenarios, follow the solutions in order of appearance. + +A solution can have multiple **variants**. A variant is a different process of deriving the solution. If multiple variants are presented, pick only one to solve the problem. + +### Solution 1 + +#### Variant 1 + +1. Scale the pipeline controller deployment back to at least 1 replica. +```shell +kubectl -n otel-demo scale deployment/kafka --replicas=1 +``` + +#### Variant 2 + +1. Restart the pipeline controller deployment to recover from a stuck state. +```shell +kubectl -n otel-demo rollout restart deployment/kafka +``` diff --git a/documentation/library/scenarios/sre/118.md b/documentation/library/scenarios/sre/118.md new file mode 100644 index 000000000..c856a0b33 --- /dev/null +++ b/documentation/library/scenarios/sre/118.md @@ -0,0 +1,41 @@ +> [!NOTE] +> This file is autogenerated. Please do not edit. + +# Scenario 118 + +| [Source](../../../../scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/118.yaml) | High Complexity | Kubernetes | Deployment, Performance | + +## Description + +This scenario simulates a cluster availability loss by cordoning worker nodes and evicting the OpenTelemetry Demo's `checkout` service, causing it to become unschedulable and degrading the entire checkout flow. + +## Applications + +- [OpenTelemetry Demo](../../applications/opentelemetry-demo.md) + +## Faults + +- [Cluster Availability Loss](../../../../scenarios/sre/docs/faults.md#Cluster-Availability-Loss) + +## Solution Guides + +Below is a noncomprehensive list of step by step guides which can lead to remediating the one or more problems in the scenario. + +A **solution** represents the process of fixing an issue created by a fault. To remediate the issues in the scenarios, follow the solutions in order of appearance. + +A solution can have multiple **variants**. A variant is a different process of deriving the solution. If multiple variants are presented, pick only one to solve the problem. + +### Solution 1 + +#### Variant 1 + +1. Identify which worker nodes are cordoned. +```shell +kubectl get nodes +``` + +2. Uncordon all affected worker nodes to restore scheduling. + +#### Variant 2 + +1. Investigate the root cause of the cluster unavailability (etcd health, API server logs, node conditions) before uncordoning. diff --git a/documentation/library/scenarios/sre/119.md b/documentation/library/scenarios/sre/119.md new file mode 100644 index 000000000..a952f7524 --- /dev/null +++ b/documentation/library/scenarios/sre/119.md @@ -0,0 +1,42 @@ +> [!NOTE] +> This file is autogenerated. Please do not edit. + +# Scenario 119 + +| [Source](../../../../scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/119.yaml) | Medium Complexity | Kubernetes | Code, Deployment | + +## Description + +This scenario simulates a breaking configuration change on the OpenTelemetry Demo's `cart` service by injecting an invalid environment variable, causing the service to malfunction and degrading the shopping cart workflow. + +## Applications + +- [OpenTelemetry Demo](../../applications/opentelemetry-demo.md) + +## Faults + +- [Breaking Configuration Change](../../../../scenarios/sre/docs/faults.md#Breaking-Configuration-Change) + +## Solution Guides + +Below is a noncomprehensive list of step by step guides which can lead to remediating the one or more problems in the scenario. + +A **solution** represents the process of fixing an issue created by a fault. To remediate the issues in the scenarios, follow the solutions in order of appearance. + +A solution can have multiple **variants**. A variant is a different process of deriving the solution. If multiple variants are presented, pick only one to solve the problem. + +### Solution 1 + +#### Variant 1 + +1. Revert the last change done to the workload manifest. +```shell +kubectl -n otel-demo rollout undo deployment/cart +``` + +#### Variant 2 + +1. Manually edit the manifest and replace the environment variable with the correct value. +```shell +kubectl -n otel-demo edit deployment cart +``` diff --git a/documentation/library/scenarios/sre/120.md b/documentation/library/scenarios/sre/120.md new file mode 100644 index 000000000..0a11ee19e --- /dev/null +++ b/documentation/library/scenarios/sre/120.md @@ -0,0 +1,41 @@ +> [!NOTE] +> This file is autogenerated. Please do not edit. + +# Scenario 120 + +| [Source](../../../../scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/120.yaml) | Medium Complexity | Kubernetes | Authentication, Deployment | + +## Description + +This scenario simulates a credential failure by corrupting the Secret used by the OpenTelemetry Demo's `email` service, causing it to fail when attempting to authenticate with external services. + +## Applications + +- [OpenTelemetry Demo](../../applications/opentelemetry-demo.md) + +## Faults + +- [Corrupted Kubernetes Secret Credentials](../../../../scenarios/sre/docs/faults.md#Corrupted-Kubernetes-Secret-Credentials) + +## Solution Guides + +Below is a noncomprehensive list of step by step guides which can lead to remediating the one or more problems in the scenario. + +A **solution** represents the process of fixing an issue created by a fault. To remediate the issues in the scenarios, follow the solutions in order of appearance. + +A solution can have multiple **variants**. A variant is a different process of deriving the solution. If multiple variants are presented, pick only one to solve the problem. + +### Solution 1 + +#### Variant 1 + +1. Inspect the corrupted secret to identify the invalid data. +```shell +kubectl -n otel-demo get secret email-service-secret -o yaml +``` + +2. Restore the secret with correct credentials from the credential management system. + +#### Variant 2 + +1. Rotate the credentials in the external system and update the Kubernetes Secret to match. diff --git a/documentation/library/scenarios/sre/121.md b/documentation/library/scenarios/sre/121.md new file mode 100644 index 000000000..549875795 --- /dev/null +++ b/documentation/library/scenarios/sre/121.md @@ -0,0 +1,42 @@ +> [!NOTE] +> This file is autogenerated. Please do not edit. + +# Scenario 121 + +| [Source](../../../../scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/121.yaml) | Medium Complexity | Kubernetes | Deployment, Performance | + +## Description + +This scenario simulates node resource exhaustion by injecting excessive resource requests into the OpenTelemetry Demo's `product-catalog` service, causing resource pressure on the node and affecting co-located workloads. + +## Applications + +- [OpenTelemetry Demo](../../applications/opentelemetry-demo.md) + +## Faults + +- [Node Resource Exhaustion](../../../../scenarios/sre/docs/faults.md#Node-Resource-Exhaustion) + +## Solution Guides + +Below is a noncomprehensive list of step by step guides which can lead to remediating the one or more problems in the scenario. + +A **solution** represents the process of fixing an issue created by a fault. To remediate the issues in the scenarios, follow the solutions in order of appearance. + +A solution can have multiple **variants**. A variant is a different process of deriving the solution. If multiple variants are presented, pick only one to solve the problem. + +### Solution 1 + +#### Variant 1 + +1. Revert the resource request changes to restore normal resource usage. +```shell +kubectl -n otel-demo rollout undo deployment/product-catalog +``` + +#### Variant 2 + +1. Manually edit the manifest and set resource requests to appropriate values. +```shell +kubectl -n otel-demo edit deployment product-catalog +``` diff --git a/documentation/library/scenarios/sre/122.md b/documentation/library/scenarios/sre/122.md new file mode 100644 index 000000000..844f0340b --- /dev/null +++ b/documentation/library/scenarios/sre/122.md @@ -0,0 +1,42 @@ +> [!NOTE] +> This file is autogenerated. Please do not edit. + +# Scenario 122 + +| [Source](../../../../scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/122.yaml) | Medium Complexity | Kubernetes | Code, Deployment | + +## Description + +This scenario simulates a release pipeline failure by injecting an invalid image into the OpenTelemetry Demo's `shipping` service, causing it to fail to start and blocking the shipping workflow. + +## Applications + +- [OpenTelemetry Demo](../../applications/opentelemetry-demo.md) + +## Faults + +- [Failed Release Pipeline Service](../../../../scenarios/sre/docs/faults.md#Failed-Release-Pipeline-Service) + +## Solution Guides + +Below is a noncomprehensive list of step by step guides which can lead to remediating the one or more problems in the scenario. + +A **solution** represents the process of fixing an issue created by a fault. To remediate the issues in the scenarios, follow the solutions in order of appearance. + +A solution can have multiple **variants**. A variant is a different process of deriving the solution. If multiple variants are presented, pick only one to solve the problem. + +### Solution 1 + +#### Variant 1 + +1. Revert the deployment to the previous working image. +```shell +kubectl -n otel-demo rollout undo deployment/shipping +``` + +#### Variant 2 + +1. Set the container image to the correct version. +```shell +kubectl -n otel-demo set image deployment/shipping shipping=correct-image:latest +``` diff --git a/documentation/library/scenarios/sre/123.md b/documentation/library/scenarios/sre/123.md new file mode 100644 index 000000000..cb7d5e63b --- /dev/null +++ b/documentation/library/scenarios/sre/123.md @@ -0,0 +1,42 @@ +> [!NOTE] +> This file is autogenerated. Please do not edit. + +# Scenario 123 + +| [Source](../../../../scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/123.yaml) | Low Complexity | Kubernetes | Deployment, Performance | + +## Description + +This scenario simulates a monitoring probe failure by scaling down the OpenTelemetry Demo's `load-generator` deployment to zero, causing the loss of synthetic traffic that monitoring relies on to detect issues. + +## Applications + +- [OpenTelemetry Demo](../../applications/opentelemetry-demo.md) + +## Faults + +- [Monitoring Probe Failure](../../../../scenarios/sre/docs/faults.md#Monitoring-Probe-Failure) + +## Solution Guides + +Below is a noncomprehensive list of step by step guides which can lead to remediating the one or more problems in the scenario. + +A **solution** represents the process of fixing an issue created by a fault. To remediate the issues in the scenarios, follow the solutions in order of appearance. + +A solution can have multiple **variants**. A variant is a different process of deriving the solution. If multiple variants are presented, pick only one to solve the problem. + +### Solution 1 + +#### Variant 1 + +1. Scale the monitoring probe deployment back to at least 1 replica. +```shell +kubectl -n otel-demo scale deployment/load-generator --replicas=1 +``` + +#### Variant 2 + +1. Restart the monitoring probe deployment to recover from a stuck state. +```shell +kubectl -n otel-demo rollout restart deployment/load-generator +``` diff --git a/scenarios/sre/project/roles/documentation/files/library/faults/index.json b/scenarios/sre/project/roles/documentation/files/library/faults/index.json index 65b76308a..4267d1069 100644 --- a/scenarios/sre/project/roles/documentation/files/library/faults/index.json +++ b/scenarios/sre/project/roles/documentation/files/library/faults/index.json @@ -1,13 +1,418 @@ [ + { + "alerts": { + "goldenSignal": [ + "HighRequestErrorRate" + ] + }, + "arguments": { + "jsonSchema": { + "properties": { + "container": { + "properties": { + "name": { + "type": "string" + } + }, + "required": [ + "name" + ], + "type": "object" + }, + "kubernetesObject": { + "properties": { + "apiVersion": { + "enum": [ + "apps/v1" + ], + "type": "string" + }, + "kind": { + "enum": [ + "Deployment", + "StatefulSet" + ], + "type": "string" + }, + "metadata": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata" + ], + "type": "object" + } + }, + "required": [ + "kubernetesObject", + "container" + ], + "type": "object" + } + }, + "description": "This fault modifies a workload's environment variable to inject a breaking configuration change, simulating real-world incidents where policy updates, ArgoCD sync issues, or operator index corruption cause cascading failures across dependent services.", + "expectation": "The faulted workload will fail to function correctly due to the misconfigured environment variable. Dependent services will experience errors as the broken configuration propagates through the service chain.", + "id": "breaking-configuration-change", + "index": 35, + "name": "Breaking Configuration Change", + "platform": "Kubernetes", + "resources": [ + "https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/", + "https://kubernetes.io/docs/concepts/configuration/configmap/", + "https://kubernetes.io/docs/tasks/manage-kubernetes-objects/update-api-object-kubectl-patch/", + "https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/" + ], + "solutions": { + "templates": [ + { + "steps": [ + { + "command": "kubectl -n {{ args.kubernetesObject.metadata.namespace }} rollout undo {{ args.kubernetesObject.kind | lower }}/{{ args.kubernetesObject.metadata.name }}", + "text": "Revert the last change done to the workload manifest." + } + ] + }, + { + "steps": [ + { + "command": "kubectl -n {{ args.kubernetesObject.metadata.namespace }} edit {{ args.kubernetesObject.kind | lower }} {{ args.kubernetesObject.metadata.name }}", + "text": "Manually edit the manifest and replace the environment variable with the correct value." + } + ] + } + ] + }, + "tags": [ + "Code", + "Deployment" + ] + }, + { + "alerts": { + "application": [ + "KubePodNotReady" + ], + "goldenSignal": [ + "HighRequestErrorRate" + ] + }, + "arguments": { + "jsonSchema": { + "properties": { + "kubernetesObject": { + "properties": { + "apiVersion": { + "enum": [ + "apps/v1" + ], + "type": "string" + }, + "kind": { + "enum": [ + "Deployment", + "StatefulSet" + ], + "type": "string" + }, + "metadata": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata" + ], + "type": "object" + } + }, + "required": [ + "kubernetesObject" + ], + "type": "object" + } + }, + "description": "This fault cordons all worker nodes and then evicts the target workload pods, simulating a cluster-wide availability loss where the control plane becomes unable to schedule or maintain workloads. This reproduces real-world incidents where clusters become completely inaccessible or unresponsive.", + "expectation": "The target workload pods will be evicted and unable to reschedule due to all nodes being cordoned. Services depending on the workload will experience errors. The cluster will appear degraded with multiple pods in Pending state.", + "id": "cluster-availability-loss", + "index": 34, + "name": "Cluster Availability Loss", + "platform": "Kubernetes", + "resources": [ + "https://kubernetes.io/docs/concepts/architecture/nodes/", + "https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/", + "https://kubernetes.io/docs/reference/kubectl/generated/kubectl_cordon/", + "https://kubernetes.io/docs/tasks/debug/debug-cluster/" + ], + "solutions": { + "templates": [ + { + "steps": [ + { + "command": "kubectl get nodes", + "text": "Identify which worker nodes are cordoned." + }, + { + "text": "Uncordon all affected worker nodes to restore scheduling." + } + ] + }, + { + "steps": [ + { + "text": "Investigate the root cause of the cluster unavailability (etcd health, API server logs, node conditions) before uncordoning." + } + ] + } + ] + }, + "tags": [ + "Deployment", + "Performance" + ] + }, + { + "alerts": { + "application": [ + "KubePodNotReady" + ] + }, + "arguments": { + "jsonSchema": { + "properties": { + "kubernetesObject": { + "properties": { + "apiVersion": { + "enum": [ + "apps/v1" + ], + "type": "string" + }, + "kind": { + "enum": [ + "Deployment", + "StatefulSet" + ], + "type": "string" + }, + "metadata": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata" + ], + "type": "object" + } + }, + "required": [ + "kubernetesObject" + ], + "type": "object" + } + }, + "description": "This fault places a workload on a node and prevents it from scaling by blocking all new scheduling attempts.", + "expectation": "The faulted pod(s) will enter the `Pending` state due to an `FailedScheduling` warning. Thus, the new pod will not start running.", + "id": "cordoned-kubernetes-worker-node", + "index": 1, + "name": "Cordoned Kubernetes Worker Node", + "platform": "Kubernetes", + "resources": [ + "https://kubernetes.io/docs/concepts/architecture/nodes/", + "https://kubernetes.io/docs/concepts/workloads/", + "https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/", + "https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/" + ], + "solutions": { + "templates": [ + { + "steps": [ + { + "command": "kubectl -n {{ args.kubernetesObject.metadata.namespace }} rollout undo {{ args.kubernetesObject.kind | lower }}/{{ args.kubernetesObject.metadata.name }}", + "text": "Revert the last change done to the manifest." + } + ] + }, + { + "steps": [ + { + "command": "kubectl -n {{ args.kubernetesObject.metadata.namespace }} edit {{ args.kubernetesObject.kind | lower }} {{ args.kubernetesObject.metadata.name }}", + "text": "Manually edit the manifest and replace the node selector with the correct value." + } + ] + }, + { + "steps": [ + { + "command": "kubectl get nodes --selector='node-role.kubernetes.io/node='", + "text": "Retrieve all worker node nodes" + }, + { + "text": "Uncordon all worker nodes which are cordoned." + } + ] + } + ] + }, + "tags": [ + "Deployment", + "Performance" + ] + }, + { + "alerts": { + "goldenSignal": [ + "HighRequestErrorRate" + ] + }, + "arguments": { + "jsonSchema": { + "properties": { + "kubernetesObject": { + "properties": { + "apiVersion": { + "enum": [ + "v1" + ], + "type": "string" + }, + "kind": { + "enum": [ + "Secret" + ], + "type": "string" + }, + "metadata": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata" + ], + "type": "object" + } + }, + "required": [ + "kubernetesObject" + ], + "type": "object" + } + }, + "description": "This fault corrupts a Kubernetes Secret by replacing its data with invalid credentials, simulating real-world incidents where secret rotation failures, registry authentication issues, or Vault access problems cause service outages.", + "expectation": "Workloads that depend on the corrupted Secret will fail to authenticate with external services (registries, databases, APIs). Pods may crash or enter error states when they attempt to use the invalid credentials.", + "id": "corrupted-kubernetes-secret-credentials", + "index": 36, + "name": "Corrupted Kubernetes Secret Credentials", + "platform": "Kubernetes", + "resources": [ + "https://kubernetes.io/docs/concepts/configuration/secret/", + "https://kubernetes.io/docs/tasks/configmap-secret/managing-secret-using-kubectl/", + "https://kubernetes.io/docs/reference/kubernetes-api/config-and-storage-resources/secret-v1/", + "https://kubernetes.io/docs/tasks/debug/debug-application/" + ], + "solutions": { + "templates": [ + { + "steps": [ + { + "command": "kubectl -n {{ args.kubernetesObject.metadata.namespace }} get secret {{ args.kubernetesObject.metadata.name }} -o yaml", + "text": "Inspect the corrupted secret to identify the invalid data." + }, + { + "text": "Restore the secret with correct credentials from the credential management system." + } + ] + }, + { + "steps": [ + { + "text": "Rotate the credentials in the external system and update the Kubernetes Secret to match." + } + ] + } + ] + }, + "tags": [ + "Authentication", + "Deployment" + ] + }, { "alerts": { "application": [ + "KubePodCrashLooping", "KubePodNotReady" ] }, "arguments": { "jsonSchema": { "properties": { + "container": { + "properties": { + "name": { + "type": "string" + } + }, + "required": [ + "name" + ], + "type": "object" + }, "kubernetesObject": { "properties": { "apiVersion": { @@ -48,21 +453,22 @@ } }, "required": [ - "kubernetesObject" + "kubernetesObject", + "container" ], "type": "object" } }, - "description": "This fault places a workload on a node and prevents it from scaling by blocking all new scheduling attempts.", - "expectation": "The faulted pod(s) will enter the `Pending` state due to an `FailedScheduling` warning. Thus, the new pod will not start running.", - "id": "cordoned-kubernetes-worker-node", - "index": 1, - "name": "Cordoned Kubernetes Worker Node", + "description": "This fault modifies a controller or operator workload's container command to an invalid value, causing the pod to enter a CrashLoopBackOff state. This simulates real-world incidents where controller pods crash due to configuration errors, incompatible upgrades, or corrupted state, leading to service degradation across the cluster.", + "expectation": "The faulted controller pod(s) will enter CrashLoopBackOff state. Dependent services will experience degradation as the controller is unable to reconcile resources.", + "id": "crashing-kubernetes-controller-workload", + "index": 31, + "name": "Crashing Kubernetes Controller Workload", "platform": "Kubernetes", "resources": [ - "https://kubernetes.io/docs/concepts/architecture/nodes/", - "https://kubernetes.io/docs/concepts/workloads/", - "https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/", + "https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/", + "https://kubernetes.io/docs/concepts/workloads/controllers/deployment/", + "https://kubernetes.io/docs/tasks/debug/debug-application/debug-running-pod/", "https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/" ], "solutions": { @@ -71,7 +477,7 @@ "steps": [ { "command": "kubectl -n {{ args.kubernetesObject.metadata.namespace }} rollout undo {{ args.kubernetesObject.kind | lower }}/{{ args.kubernetesObject.metadata.name }}", - "text": "Revert the last change done to the manifest." + "text": "Revert the last change done to the controller manifest." } ] }, @@ -79,18 +485,15 @@ "steps": [ { "command": "kubectl -n {{ args.kubernetesObject.metadata.namespace }} edit {{ args.kubernetesObject.kind | lower }} {{ args.kubernetesObject.metadata.name }}", - "text": "Manually edit the manifest and replace the node selector with the correct value." + "text": "Manually edit the manifest and replace the container command with the correct value." } ] }, { "steps": [ { - "command": "kubectl get nodes --selector='node-role.kubernetes.io/node='", - "text": "Retrieve all worker node nodes" - }, - { - "text": "Uncordon all worker nodes which are cordoned." + "command": "kubectl -n {{ args.kubernetesObject.metadata.namespace }} logs -l app={{ args.kubernetesObject.metadata.name }} -c {{ args.container.name }} --previous", + "text": "Check the previous container logs to understand the crash reason." } ] } @@ -338,31 +741,223 @@ "type": "object" } }, - "description": "This fault modifies a namespace, disabling it from being included in Istio's Ambient Mode service mesh.", - "expectation": "No traffic will be routed through the service mesh. This would all the requests to not be processed with the various policies configured in Istio.", - "id": "disabled-istio-ambient-mode-kubernetes-namespace", - "index": 3, - "name": "Disabled Istio Ambient Mode Kubernetes Namespace", + "description": "This fault modifies a namespace, disabling it from being included in Istio's Ambient Mode service mesh.", + "expectation": "No traffic will be routed through the service mesh. This would all the requests to not be processed with the various policies configured in Istio.", + "id": "disabled-istio-ambient-mode-kubernetes-namespace", + "index": 3, + "name": "Disabled Istio Ambient Mode Kubernetes Namespace", + "platform": "Kubernetes", + "resources": [ + "https://istio.io/latest/docs/ambient/overview/", + "https://istio.io/latest/docs/ambient/usage/add-workloads/" + ], + "solutions": { + "templates": [ + { + "steps": [ + { + "command": "kubectl -n {{ args.kubernetesObject.metadata.name }} edit namespace {{ args.kubernetesObject.metadata.name }}-deny", + "text": "Manually edit the manifest and replace the invalid label with the correct value." + } + ] + } + ] + }, + "tags": [ + "Deployment", + "Networking" + ] + }, + { + "alerts": { + "application": [ + "KubePodNotReady" + ] + }, + "arguments": { + "jsonSchema": { + "properties": { + "kubernetesObject": { + "properties": { + "apiVersion": { + "enum": [ + "apps/v1" + ], + "type": "string" + }, + "kind": { + "enum": [ + "StatefulSet" + ], + "type": "string" + }, + "metadata": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata" + ], + "type": "object" + } + }, + "required": [ + "kubernetesObject" + ], + "type": "object" + } + }, + "description": "This fault fills the etcd database storage by creating a large number of ConfigMaps, pushing etcd usage past its quota threshold. This simulates real-world incidents where etcd storage exhaustion causes the API server to reject writes, leading to cluster-wide degradation or complete outage.", + "expectation": "The etcd database will approach or exceed its storage quota. The Kubernetes API server will begin rejecting mutating requests. New pod scheduling and resource creation will fail.", + "id": "exhausted-etcd-database-storage", + "index": 32, + "name": "Exhausted Etcd Database Storage", + "platform": "Kubernetes", + "resources": [ + "https://kubernetes.io/docs/tasks/administer-cluster/configure-upgrade-etcd/", + "https://etcd.io/docs/v3.5/op-guide/maintenance/", + "https://kubernetes.io/docs/concepts/overview/components/#etcd", + "https://kubernetes.io/docs/tasks/debug/debug-cluster/" + ], + "solutions": { + "templates": [ + { + "steps": [ + { + "text": "Compact and defragment the etcd database to reclaim storage space." + }, + { + "text": "Remove unnecessary resources (completed jobs, orphaned ConfigMaps) to reduce etcd storage usage." + } + ] + }, + { + "steps": [ + { + "text": "Increase the etcd storage quota if the current limit is too low for the cluster workload." + } + ] + } + ] + }, + "tags": [ + "Performance" + ] + }, + { + "alerts": { + "goldenSignal": [ + "HighRequestErrorRate" + ] + }, + "arguments": { + "jsonSchema": { + "properties": { + "container": { + "properties": { + "name": { + "type": "string" + } + }, + "required": [ + "name" + ], + "type": "object" + }, + "kubernetesObject": { + "properties": { + "apiVersion": { + "enum": [ + "apps/v1" + ], + "type": "string" + }, + "kind": { + "enum": [ + "Deployment" + ], + "type": "string" + }, + "metadata": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata" + ], + "type": "object" + } + }, + "required": [ + "kubernetesObject", + "container" + ], + "type": "object" + } + }, + "description": "This fault modifies a release pipeline service's container to use an invalid image tag, simulating real-world incidents where release signing failures, broken trusted task lists, or release stage blockages prevent software from being released to production.", + "expectation": "The release pipeline service will fail to start due to the invalid image. Dependent release workflows will stall as the service becomes unavailable.", + "id": "failed-release-pipeline-service", + "index": 38, + "name": "Failed Release Pipeline Service", "platform": "Kubernetes", "resources": [ - "https://istio.io/latest/docs/ambient/overview/", - "https://istio.io/latest/docs/ambient/usage/add-workloads/" + "https://kubernetes.io/docs/concepts/workloads/controllers/deployment/", + "https://kubernetes.io/docs/concepts/containers/images/", + "https://kubernetes.io/docs/tasks/debug/debug-application/", + "https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/" ], "solutions": { "templates": [ { "steps": [ { - "command": "kubectl -n {{ args.kubernetesObject.metadata.name }} edit namespace {{ args.kubernetesObject.metadata.name }}-deny", - "text": "Manually edit the manifest and replace the invalid label with the correct value." + "command": "kubectl -n {{ args.kubernetesObject.metadata.namespace }} rollout undo {{ args.kubernetesObject.kind | lower }}/{{ args.kubernetesObject.metadata.name }}", + "text": "Revert the deployment to the previous working image." + } + ] + }, + { + "steps": [ + { + "command": "kubectl -n {{ args.kubernetesObject.metadata.namespace }} set image {{ args.kubernetesObject.kind | lower }}/{{ args.kubernetesObject.metadata.name }} {{ args.container.name }}=correct-image:latest", + "text": "Set the container image to the correct version." } ] } ] }, "tags": [ - "Deployment", - "Networking" + "Code", + "Deployment" ] }, { @@ -1527,6 +2122,188 @@ "Networking" ] }, + { + "alerts": { + "application": [ + "KubePodNotReady" + ] + }, + "arguments": { + "jsonSchema": { + "properties": { + "kubernetesObject": { + "properties": { + "apiVersion": { + "enum": [ + "apps/v1" + ], + "type": "string" + }, + "kind": { + "enum": [ + "Deployment", + "DaemonSet" + ], + "type": "string" + }, + "metadata": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata" + ], + "type": "object" + } + }, + "required": [ + "kubernetesObject" + ], + "type": "object" + } + }, + "description": "This fault scales down a monitoring or probe deployment to zero replicas, simulating real-world incidents where multiple monitoring probes go down simultaneously, masking real issues and creating blind spots in observability.", + "expectation": "The monitoring probe workload will be unavailable. Alerts that depend on the probe data will stop firing, creating a false sense of health while actual issues may go undetected.", + "id": "monitoring-probe-failure", + "index": 39, + "name": "Monitoring Probe Failure", + "platform": "Kubernetes", + "resources": [ + "https://kubernetes.io/docs/concepts/workloads/controllers/deployment/", + "https://kubernetes.io/docs/tasks/debug/debug-application/", + "https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/deployment-v1/", + "https://kubernetes.io/docs/concepts/cluster-administration/monitoring/" + ], + "solutions": { + "templates": [ + { + "steps": [ + { + "command": "kubectl -n {{ args.kubernetesObject.metadata.namespace }} scale {{ args.kubernetesObject.kind | lower }}/{{ args.kubernetesObject.metadata.name }} --replicas=1", + "text": "Scale the monitoring probe deployment back to at least 1 replica." + } + ] + }, + { + "steps": [ + { + "command": "kubectl -n {{ args.kubernetesObject.metadata.namespace }} rollout restart {{ args.kubernetesObject.kind | lower }}/{{ args.kubernetesObject.metadata.name }}", + "text": "Restart the monitoring probe deployment to recover from a stuck state." + } + ] + } + ] + }, + "tags": [ + "Deployment", + "Performance" + ] + }, + { + "alerts": { + "application": [ + "KubePodNotReady" + ] + }, + "arguments": { + "jsonSchema": { + "properties": { + "kubernetesObject": { + "properties": { + "apiVersion": { + "enum": [ + "apps/v1" + ], + "type": "string" + }, + "kind": { + "enum": [ + "Deployment", + "StatefulSet" + ], + "type": "string" + }, + "metadata": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata" + ], + "type": "object" + } + }, + "required": [ + "kubernetesObject" + ], + "type": "object" + } + }, + "description": "This fault injects excessive resource requests into a workload, causing nodes to run out of allocatable resources. This simulates real-world incidents where node-level resource exhaustion, compute capacity shortages, or cluster autoscaler failures lead to pods stuck in Pending state and service degradation.", + "expectation": "The faulted workload pods will consume excessive resources, causing other pods on the same node to be evicted or fail to schedule. The node may become NotReady if resource pressure is severe enough.", + "id": "node-resource-exhaustion", + "index": 37, + "name": "Node Resource Exhaustion", + "platform": "Kubernetes", + "resources": [ + "https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/", + "https://kubernetes.io/docs/tasks/administer-cluster/manage-resources/memory-default-namespace/", + "https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/", + "https://kubernetes.io/docs/tasks/debug/debug-cluster/resource-metrics-pipeline/" + ], + "solutions": { + "templates": [ + { + "steps": [ + { + "command": "kubectl -n {{ args.kubernetesObject.metadata.namespace }} rollout undo {{ args.kubernetesObject.kind | lower }}/{{ args.kubernetesObject.metadata.name }}", + "text": "Revert the resource request changes to restore normal resource usage." + } + ] + }, + { + "steps": [ + { + "command": "kubectl -n {{ args.kubernetesObject.metadata.namespace }} edit {{ args.kubernetesObject.kind | lower }} {{ args.kubernetesObject.metadata.name }}", + "text": "Manually edit the manifest and set resource requests to appropriate values." + } + ] + } + ] + }, + "tags": [ + "Deployment", + "Performance" + ] + }, { "alerts": { "application": [ @@ -2234,6 +3011,96 @@ "Performance" ] }, + { + "alerts": { + "application": [ + "KubePodNotReady" + ] + }, + "arguments": { + "jsonSchema": { + "properties": { + "kubernetesObject": { + "properties": { + "apiVersion": { + "enum": [ + "apps/v1" + ], + "type": "string" + }, + "kind": { + "enum": [ + "Deployment" + ], + "type": "string" + }, + "metadata": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata" + ], + "type": "object" + } + }, + "required": [ + "kubernetesObject" + ], + "type": "object" + } + }, + "description": "This fault scales down a pipeline controller deployment to zero replicas, simulating a controller failure that prevents new pipeline runs from being processed. This reproduces real-world incidents where Tekton or similar CI/CD controllers become unavailable, causing pipeline runs to stall in a pending state.", + "expectation": "New pipeline runs will not be processed and will remain in a pending or queued state. Existing running pipelines may complete but no new work will be picked up by the controller.", + "id": "stalled-pipeline-controller", + "index": 33, + "name": "Stalled Pipeline Controller", + "platform": "Kubernetes", + "resources": [ + "https://kubernetes.io/docs/concepts/workloads/controllers/deployment/", + "https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/deployment-v1/", + "https://tekton.dev/docs/pipelines/", + "https://kubernetes.io/docs/tasks/debug/debug-application/" + ], + "solutions": { + "templates": [ + { + "steps": [ + { + "command": "kubectl -n {{ args.kubernetesObject.metadata.namespace }} scale {{ args.kubernetesObject.kind | lower }}/{{ args.kubernetesObject.metadata.name }} --replicas=1", + "text": "Scale the pipeline controller deployment back to at least 1 replica." + } + ] + }, + { + "steps": [ + { + "command": "kubectl -n {{ args.kubernetesObject.metadata.namespace }} rollout restart {{ args.kubernetesObject.kind | lower }}/{{ args.kubernetesObject.metadata.name }}", + "text": "Restart the pipeline controller deployment to recover from a stuck state." + } + ] + } + ] + }, + "tags": [ + "Deployment", + "Performance" + ] + }, { "alerts": { "goldenSignal": [ diff --git a/scenarios/sre/project/roles/documentation/files/library/faults/indexes/31.yaml b/scenarios/sre/project/roles/documentation/files/library/faults/indexes/31.yaml new file mode 100644 index 000000000..a6444e0c3 --- /dev/null +++ b/scenarios/sre/project/roles/documentation/files/library/faults/indexes/31.yaml @@ -0,0 +1,80 @@ +--- +alerts: + application: + - KubePodCrashLooping + - KubePodNotReady +arguments: + jsonSchema: + properties: + container: + properties: + name: + type: string + required: + - name + type: object + kubernetesObject: + properties: + apiVersion: + enum: + - apps/v1 + type: string + kind: + enum: + - Deployment + - StatefulSet + type: string + metadata: + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + required: + - apiVersion + - kind + - metadata + type: object + required: + - kubernetesObject + - container + type: object +description: >- + This fault modifies a controller or operator workload's container command to an invalid value, + causing the pod to enter a CrashLoopBackOff state. This simulates real-world incidents where + controller pods crash due to configuration errors, incompatible upgrades, or corrupted state, + leading to service degradation across the cluster. +expectation: >- + The faulted controller pod(s) will enter CrashLoopBackOff state. Dependent services will + experience degradation as the controller is unable to reconcile resources. +name: Crashing Kubernetes Controller Workload +platform: Kubernetes +resources: + - https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ + - https://kubernetes.io/docs/concepts/workloads/controllers/deployment/ + - https://kubernetes.io/docs/tasks/debug/debug-application/debug-running-pod/ + - https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/ +solutions: + templates: + - steps: + - command: >- + kubectl -n {{ args.kubernetesObject.metadata.namespace }} rollout undo + {{ args.kubernetesObject.kind | lower }}/{{ args.kubernetesObject.metadata.name }} + text: Revert the last change done to the controller manifest. + - steps: + - command: >- + kubectl -n {{ args.kubernetesObject.metadata.namespace }} edit + {{ args.kubernetesObject.kind | lower }} {{ args.kubernetesObject.metadata.name }} + text: Manually edit the manifest and replace the container command with the correct value. + - steps: + - command: >- + kubectl -n {{ args.kubernetesObject.metadata.namespace }} logs + -l app={{ args.kubernetesObject.metadata.name }} -c {{ args.container.name }} --previous + text: Check the previous container logs to understand the crash reason. +tags: + - Deployment + - Performance diff --git a/scenarios/sre/project/roles/documentation/files/library/faults/indexes/32.yaml b/scenarios/sre/project/roles/documentation/files/library/faults/indexes/32.yaml new file mode 100644 index 000000000..e921dc6de --- /dev/null +++ b/scenarios/sre/project/roles/documentation/files/library/faults/indexes/32.yaml @@ -0,0 +1,64 @@ +--- +alerts: + application: + - KubePodNotReady +arguments: + jsonSchema: + properties: + kubernetesObject: + properties: + apiVersion: + enum: + - apps/v1 + type: string + kind: + enum: + - StatefulSet + type: string + metadata: + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + required: + - apiVersion + - kind + - metadata + type: object + required: + - kubernetesObject + type: object +description: >- + This fault fills the etcd database storage by creating a large number of ConfigMaps, + pushing etcd usage past its quota threshold. This simulates real-world incidents where + etcd storage exhaustion causes the API server to reject writes, leading to cluster-wide + degradation or complete outage. +expectation: >- + The etcd database will approach or exceed its storage quota. The Kubernetes API server + will begin rejecting mutating requests. New pod scheduling and resource creation will fail. +name: Exhausted Etcd Database Storage +platform: Kubernetes +resources: + - https://kubernetes.io/docs/tasks/administer-cluster/configure-upgrade-etcd/ + - https://etcd.io/docs/v3.5/op-guide/maintenance/ + - https://kubernetes.io/docs/concepts/overview/components/#etcd + - https://kubernetes.io/docs/tasks/debug/debug-cluster/ +solutions: + templates: + - steps: + - text: >- + Compact and defragment the etcd database to reclaim storage space. + - text: >- + Remove unnecessary resources (completed jobs, orphaned ConfigMaps) to reduce + etcd storage usage. + - steps: + - text: >- + Increase the etcd storage quota if the current limit is too low for the + cluster workload. +tags: + - Performance diff --git a/scenarios/sre/project/roles/documentation/files/library/faults/indexes/33.yaml b/scenarios/sre/project/roles/documentation/files/library/faults/indexes/33.yaml new file mode 100644 index 000000000..de502ffac --- /dev/null +++ b/scenarios/sre/project/roles/documentation/files/library/faults/indexes/33.yaml @@ -0,0 +1,66 @@ +--- +alerts: + application: + - KubePodNotReady +arguments: + jsonSchema: + properties: + kubernetesObject: + properties: + apiVersion: + enum: + - apps/v1 + type: string + kind: + enum: + - Deployment + type: string + metadata: + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + required: + - apiVersion + - kind + - metadata + type: object + required: + - kubernetesObject + type: object +description: >- + This fault scales down a pipeline controller deployment to zero replicas, simulating + a controller failure that prevents new pipeline runs from being processed. This + reproduces real-world incidents where Tekton or similar CI/CD controllers become + unavailable, causing pipeline runs to stall in a pending state. +expectation: >- + New pipeline runs will not be processed and will remain in a pending or queued state. + Existing running pipelines may complete but no new work will be picked up by the controller. +name: Stalled Pipeline Controller +platform: Kubernetes +resources: + - https://kubernetes.io/docs/concepts/workloads/controllers/deployment/ + - https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/deployment-v1/ + - https://tekton.dev/docs/pipelines/ + - https://kubernetes.io/docs/tasks/debug/debug-application/ +solutions: + templates: + - steps: + - command: >- + kubectl -n {{ args.kubernetesObject.metadata.namespace }} scale + {{ args.kubernetesObject.kind | lower }}/{{ args.kubernetesObject.metadata.name }} + --replicas=1 + text: Scale the pipeline controller deployment back to at least 1 replica. + - steps: + - command: >- + kubectl -n {{ args.kubernetesObject.metadata.namespace }} rollout restart + {{ args.kubernetesObject.kind | lower }}/{{ args.kubernetesObject.metadata.name }} + text: Restart the pipeline controller deployment to recover from a stuck state. +tags: + - Deployment + - Performance diff --git a/scenarios/sre/project/roles/documentation/files/library/faults/indexes/34.yaml b/scenarios/sre/project/roles/documentation/files/library/faults/indexes/34.yaml new file mode 100644 index 000000000..5c4fb5825 --- /dev/null +++ b/scenarios/sre/project/roles/documentation/files/library/faults/indexes/34.yaml @@ -0,0 +1,67 @@ +--- +alerts: + application: + - KubePodNotReady + goldenSignal: + - HighRequestErrorRate +arguments: + jsonSchema: + properties: + kubernetesObject: + properties: + apiVersion: + enum: + - apps/v1 + type: string + kind: + enum: + - Deployment + - StatefulSet + type: string + metadata: + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + required: + - apiVersion + - kind + - metadata + type: object + required: + - kubernetesObject + type: object +description: >- + This fault cordons all worker nodes and then evicts the target workload pods, + simulating a cluster-wide availability loss where the control plane becomes unable + to schedule or maintain workloads. This reproduces real-world incidents where + clusters become completely inaccessible or unresponsive. +expectation: >- + The target workload pods will be evicted and unable to reschedule due to all nodes + being cordoned. Services depending on the workload will experience errors. The cluster + will appear degraded with multiple pods in Pending state. +name: Cluster Availability Loss +platform: Kubernetes +resources: + - https://kubernetes.io/docs/concepts/architecture/nodes/ + - https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/ + - https://kubernetes.io/docs/reference/kubectl/generated/kubectl_cordon/ + - https://kubernetes.io/docs/tasks/debug/debug-cluster/ +solutions: + templates: + - steps: + - command: kubectl get nodes + text: Identify which worker nodes are cordoned. + - text: Uncordon all affected worker nodes to restore scheduling. + - steps: + - text: >- + Investigate the root cause of the cluster unavailability (etcd health, + API server logs, node conditions) before uncordoning. +tags: + - Deployment + - Performance diff --git a/scenarios/sre/project/roles/documentation/files/library/faults/indexes/35.yaml b/scenarios/sre/project/roles/documentation/files/library/faults/indexes/35.yaml new file mode 100644 index 000000000..668036e5f --- /dev/null +++ b/scenarios/sre/project/roles/documentation/files/library/faults/indexes/35.yaml @@ -0,0 +1,75 @@ +--- +alerts: + goldenSignal: + - HighRequestErrorRate +arguments: + jsonSchema: + properties: + container: + properties: + name: + type: string + required: + - name + type: object + kubernetesObject: + properties: + apiVersion: + enum: + - apps/v1 + type: string + kind: + enum: + - Deployment + - StatefulSet + type: string + metadata: + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + required: + - apiVersion + - kind + - metadata + type: object + required: + - kubernetesObject + - container + type: object +description: >- + This fault modifies a workload's environment variable to inject a breaking + configuration change, simulating real-world incidents where policy updates, + ArgoCD sync issues, or operator index corruption cause cascading failures + across dependent services. +expectation: >- + The faulted workload will fail to function correctly due to the misconfigured + environment variable. Dependent services will experience errors as the broken + configuration propagates through the service chain. +name: Breaking Configuration Change +platform: Kubernetes +resources: + - https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/ + - https://kubernetes.io/docs/concepts/configuration/configmap/ + - https://kubernetes.io/docs/tasks/manage-kubernetes-objects/update-api-object-kubectl-patch/ + - https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/ +solutions: + templates: + - steps: + - command: >- + kubectl -n {{ args.kubernetesObject.metadata.namespace }} rollout undo + {{ args.kubernetesObject.kind | lower }}/{{ args.kubernetesObject.metadata.name }} + text: Revert the last change done to the workload manifest. + - steps: + - command: >- + kubectl -n {{ args.kubernetesObject.metadata.namespace }} edit + {{ args.kubernetesObject.kind | lower }} {{ args.kubernetesObject.metadata.name }} + text: Manually edit the manifest and replace the environment variable with the correct value. +tags: + - Code + - Deployment diff --git a/scenarios/sre/project/roles/documentation/files/library/faults/indexes/36.yaml b/scenarios/sre/project/roles/documentation/files/library/faults/indexes/36.yaml new file mode 100644 index 000000000..cc855210e --- /dev/null +++ b/scenarios/sre/project/roles/documentation/files/library/faults/indexes/36.yaml @@ -0,0 +1,63 @@ +--- +alerts: + goldenSignal: + - HighRequestErrorRate +arguments: + jsonSchema: + properties: + kubernetesObject: + properties: + apiVersion: + enum: + - v1 + type: string + kind: + enum: + - Secret + type: string + metadata: + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + required: + - apiVersion + - kind + - metadata + type: object + required: + - kubernetesObject + type: object +description: >- + This fault corrupts a Kubernetes Secret by replacing its data with invalid + credentials, simulating real-world incidents where secret rotation failures, + registry authentication issues, or Vault access problems cause service outages. +expectation: >- + Workloads that depend on the corrupted Secret will fail to authenticate with + external services (registries, databases, APIs). Pods may crash or enter error + states when they attempt to use the invalid credentials. +name: Corrupted Kubernetes Secret Credentials +platform: Kubernetes +resources: + - https://kubernetes.io/docs/concepts/configuration/secret/ + - https://kubernetes.io/docs/tasks/configmap-secret/managing-secret-using-kubectl/ + - https://kubernetes.io/docs/reference/kubernetes-api/config-and-storage-resources/secret-v1/ + - https://kubernetes.io/docs/tasks/debug/debug-application/ +solutions: + templates: + - steps: + - command: >- + kubectl -n {{ args.kubernetesObject.metadata.namespace }} get secret + {{ args.kubernetesObject.metadata.name }} -o yaml + text: Inspect the corrupted secret to identify the invalid data. + - text: Restore the secret with correct credentials from the credential management system. + - steps: + - text: Rotate the credentials in the external system and update the Kubernetes Secret to match. +tags: + - Authentication + - Deployment diff --git a/scenarios/sre/project/roles/documentation/files/library/faults/indexes/37.yaml b/scenarios/sre/project/roles/documentation/files/library/faults/indexes/37.yaml new file mode 100644 index 000000000..32e1bd41e --- /dev/null +++ b/scenarios/sre/project/roles/documentation/files/library/faults/indexes/37.yaml @@ -0,0 +1,67 @@ +--- +alerts: + application: + - KubePodNotReady +arguments: + jsonSchema: + properties: + kubernetesObject: + properties: + apiVersion: + enum: + - apps/v1 + type: string + kind: + enum: + - Deployment + - StatefulSet + type: string + metadata: + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + required: + - apiVersion + - kind + - metadata + type: object + required: + - kubernetesObject + type: object +description: >- + This fault injects excessive resource requests into a workload, causing nodes + to run out of allocatable resources. This simulates real-world incidents where + node-level resource exhaustion, compute capacity shortages, or cluster autoscaler + failures lead to pods stuck in Pending state and service degradation. +expectation: >- + The faulted workload pods will consume excessive resources, causing other pods + on the same node to be evicted or fail to schedule. The node may become + NotReady if resource pressure is severe enough. +name: Node Resource Exhaustion +platform: Kubernetes +resources: + - https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + - https://kubernetes.io/docs/tasks/administer-cluster/manage-resources/memory-default-namespace/ + - https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/ + - https://kubernetes.io/docs/tasks/debug/debug-cluster/resource-metrics-pipeline/ +solutions: + templates: + - steps: + - command: >- + kubectl -n {{ args.kubernetesObject.metadata.namespace }} rollout undo + {{ args.kubernetesObject.kind | lower }}/{{ args.kubernetesObject.metadata.name }} + text: Revert the resource request changes to restore normal resource usage. + - steps: + - command: >- + kubectl -n {{ args.kubernetesObject.metadata.namespace }} edit + {{ args.kubernetesObject.kind | lower }} {{ args.kubernetesObject.metadata.name }} + text: Manually edit the manifest and set resource requests to appropriate values. +tags: + - Deployment + - Performance diff --git a/scenarios/sre/project/roles/documentation/files/library/faults/indexes/38.yaml b/scenarios/sre/project/roles/documentation/files/library/faults/indexes/38.yaml new file mode 100644 index 000000000..314c828de --- /dev/null +++ b/scenarios/sre/project/roles/documentation/files/library/faults/indexes/38.yaml @@ -0,0 +1,74 @@ +--- +alerts: + goldenSignal: + - HighRequestErrorRate +arguments: + jsonSchema: + properties: + container: + properties: + name: + type: string + required: + - name + type: object + kubernetesObject: + properties: + apiVersion: + enum: + - apps/v1 + type: string + kind: + enum: + - Deployment + type: string + metadata: + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + required: + - apiVersion + - kind + - metadata + type: object + required: + - kubernetesObject + - container + type: object +description: >- + This fault modifies a release pipeline service's container to use an invalid + image tag, simulating real-world incidents where release signing failures, + broken trusted task lists, or release stage blockages prevent software from + being released to production. +expectation: >- + The release pipeline service will fail to start due to the invalid image. + Dependent release workflows will stall as the service becomes unavailable. +name: Failed Release Pipeline Service +platform: Kubernetes +resources: + - https://kubernetes.io/docs/concepts/workloads/controllers/deployment/ + - https://kubernetes.io/docs/concepts/containers/images/ + - https://kubernetes.io/docs/tasks/debug/debug-application/ + - https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/ +solutions: + templates: + - steps: + - command: >- + kubectl -n {{ args.kubernetesObject.metadata.namespace }} rollout undo + {{ args.kubernetesObject.kind | lower }}/{{ args.kubernetesObject.metadata.name }} + text: Revert the deployment to the previous working image. + - steps: + - command: >- + kubectl -n {{ args.kubernetesObject.metadata.namespace }} set image + {{ args.kubernetesObject.kind | lower }}/{{ args.kubernetesObject.metadata.name }} + {{ args.container.name }}=correct-image:latest + text: Set the container image to the correct version. +tags: + - Code + - Deployment diff --git a/scenarios/sre/project/roles/documentation/files/library/faults/indexes/39.yaml b/scenarios/sre/project/roles/documentation/files/library/faults/indexes/39.yaml new file mode 100644 index 000000000..a6f018a2f --- /dev/null +++ b/scenarios/sre/project/roles/documentation/files/library/faults/indexes/39.yaml @@ -0,0 +1,67 @@ +--- +alerts: + application: + - KubePodNotReady +arguments: + jsonSchema: + properties: + kubernetesObject: + properties: + apiVersion: + enum: + - apps/v1 + type: string + kind: + enum: + - Deployment + - DaemonSet + type: string + metadata: + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + required: + - apiVersion + - kind + - metadata + type: object + required: + - kubernetesObject + type: object +description: >- + This fault scales down a monitoring or probe deployment to zero replicas, + simulating real-world incidents where multiple monitoring probes go down + simultaneously, masking real issues and creating blind spots in observability. +expectation: >- + The monitoring probe workload will be unavailable. Alerts that depend on + the probe data will stop firing, creating a false sense of health while + actual issues may go undetected. +name: Monitoring Probe Failure +platform: Kubernetes +resources: + - https://kubernetes.io/docs/concepts/workloads/controllers/deployment/ + - https://kubernetes.io/docs/tasks/debug/debug-application/ + - https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/deployment-v1/ + - https://kubernetes.io/docs/concepts/cluster-administration/monitoring/ +solutions: + templates: + - steps: + - command: >- + kubectl -n {{ args.kubernetesObject.metadata.namespace }} scale + {{ args.kubernetesObject.kind | lower }}/{{ args.kubernetesObject.metadata.name }} + --replicas=1 + text: Scale the monitoring probe deployment back to at least 1 replica. + - steps: + - command: >- + kubectl -n {{ args.kubernetesObject.metadata.namespace }} rollout restart + {{ args.kubernetesObject.kind | lower }}/{{ args.kubernetesObject.metadata.name }} + text: Restart the monitoring probe deployment to recover from a stuck state. +tags: + - Deployment + - Performance diff --git a/scenarios/sre/project/roles/documentation/files/library/scenarios/index.json b/scenarios/sre/project/roles/documentation/files/library/scenarios/index.json index ab3524593..7b7bd0322 100644 --- a/scenarios/sre/project/roles/documentation/files/library/scenarios/index.json +++ b/scenarios/sre/project/roles/documentation/files/library/scenarios/index.json @@ -3056,5 +3056,595 @@ "Deployment", "Networking" ] + }, + { + "alerts": [ + "KubePodCrashLooping", + "KubePodNotReady" + ], + "category": "sre", + "complexity": "medium", + "description": "This scenario simulates OpenTelemetry Demo's `recommendation` service controller crashing due to a corrupted container command, causing the pod to enter CrashLoopBackOff and degrading upstream services that depend on it.", + "disruptions": [ + { + "injections": [ + { + "args": { + "container": { + "name": "recommendation" + }, + "kubernetesObject": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": "recommendation", + "namespace": "otel-demo" + } + } + }, + "id": "crashing-kubernetes-controller-workload" + } + ] + } + ], + "environment": { + "applications": [ + { + "id": "opentelemetry-demo" + } + ] + }, + "id": 115, + "index": 115, + "platforms": [ + "Kubernetes" + ], + "solutions": [ + [ + { + "steps": [ + { + "command": "kubectl -n otel-demo rollout undo deployment/recommendation", + "text": "Revert the last change done to the controller manifest." + } + ] + }, + { + "steps": [ + { + "command": "kubectl -n otel-demo edit deployment recommendation", + "text": "Manually edit the manifest and replace the container command with the correct value." + } + ] + }, + { + "steps": [ + { + "command": "kubectl -n otel-demo logs -l app=recommendation -c recommendation --previous", + "text": "Check the previous container logs to understand the crash reason." + } + ] + } + ] + ], + "tags": [ + "Deployment", + "Performance" + ] + }, + { + "alerts": [ + "KubePodNotReady" + ], + "category": "sre", + "complexity": "high", + "description": "This scenario simulates etcd storage exhaustion by filling the cluster's etcd database with large ConfigMaps, causing the API server to reject writes and degrading cluster-wide operations including the OpenTelemetry Demo application.", + "disruptions": [ + { + "injections": [ + { + "args": { + "kubernetesObject": { + "apiVersion": "apps/v1", + "kind": "StatefulSet", + "metadata": { + "name": "etcd", + "namespace": "otel-demo" + } + } + }, + "id": "exhausted-etcd-database-storage" + } + ] + } + ], + "environment": { + "applications": [ + { + "id": "opentelemetry-demo" + } + ] + }, + "id": 116, + "index": 116, + "platforms": [ + "Kubernetes" + ], + "solutions": [ + [ + { + "steps": [ + { + "text": "Compact and defragment the etcd database to reclaim storage space." + }, + { + "text": "Remove unnecessary resources (completed jobs, orphaned ConfigMaps) to reduce etcd storage usage." + } + ] + }, + { + "steps": [ + { + "text": "Increase the etcd storage quota if the current limit is too low for the cluster workload." + } + ] + } + ] + ], + "tags": [ + "Performance" + ] + }, + { + "alerts": [ + "KubePodNotReady" + ], + "category": "sre", + "complexity": "medium", + "description": "This scenario simulates a pipeline controller failure by scaling the OpenTelemetry Demo's `kafka` deployment to zero replicas, causing downstream services that depend on event processing to stall, similar to a Tekton pipeline controller becoming unavailable.", + "disruptions": [ + { + "injections": [ + { + "args": { + "kubernetesObject": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": "kafka", + "namespace": "otel-demo" + } + } + }, + "id": "stalled-pipeline-controller" + } + ] + } + ], + "environment": { + "applications": [ + { + "id": "opentelemetry-demo" + } + ] + }, + "id": 117, + "index": 117, + "platforms": [ + "Kubernetes" + ], + "solutions": [ + [ + { + "steps": [ + { + "command": "kubectl -n otel-demo scale deployment/kafka --replicas=1", + "text": "Scale the pipeline controller deployment back to at least 1 replica." + } + ] + }, + { + "steps": [ + { + "command": "kubectl -n otel-demo rollout restart deployment/kafka", + "text": "Restart the pipeline controller deployment to recover from a stuck state." + } + ] + } + ] + ], + "tags": [ + "Deployment", + "Performance" + ] + }, + { + "alerts": [ + "KubePodNotReady", + "HighRequestErrorRate" + ], + "category": "sre", + "complexity": "high", + "description": "This scenario simulates a cluster availability loss by cordoning worker nodes and evicting the OpenTelemetry Demo's `checkout` service, causing it to become unschedulable and degrading the entire checkout flow.", + "disruptions": [ + { + "injections": [ + { + "args": { + "kubernetesObject": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": "checkout", + "namespace": "otel-demo" + } + } + }, + "id": "cluster-availability-loss" + } + ] + } + ], + "environment": { + "applications": [ + { + "id": "opentelemetry-demo" + } + ] + }, + "id": 118, + "index": 118, + "platforms": [ + "Kubernetes" + ], + "solutions": [ + [ + { + "steps": [ + { + "command": "kubectl get nodes", + "text": "Identify which worker nodes are cordoned." + }, + { + "text": "Uncordon all affected worker nodes to restore scheduling." + } + ] + }, + { + "steps": [ + { + "text": "Investigate the root cause of the cluster unavailability (etcd health, API server logs, node conditions) before uncordoning." + } + ] + } + ] + ], + "tags": [ + "Deployment", + "Performance" + ] + }, + { + "alerts": [ + "HighRequestErrorRate" + ], + "category": "sre", + "complexity": "medium", + "description": "This scenario simulates a breaking configuration change on the OpenTelemetry Demo's `cart` service by injecting an invalid environment variable, causing the service to malfunction and degrading the shopping cart workflow.", + "disruptions": [ + { + "injections": [ + { + "args": { + "container": { + "name": "cart" + }, + "kubernetesObject": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": "cart", + "namespace": "otel-demo" + } + } + }, + "id": "breaking-configuration-change" + } + ] + } + ], + "environment": { + "applications": [ + { + "id": "opentelemetry-demo" + } + ] + }, + "id": 119, + "index": 119, + "platforms": [ + "Kubernetes" + ], + "solutions": [ + [ + { + "steps": [ + { + "command": "kubectl -n otel-demo rollout undo deployment/cart", + "text": "Revert the last change done to the workload manifest." + } + ] + }, + { + "steps": [ + { + "command": "kubectl -n otel-demo edit deployment cart", + "text": "Manually edit the manifest and replace the environment variable with the correct value." + } + ] + } + ] + ], + "tags": [ + "Code", + "Deployment" + ] + }, + { + "alerts": [ + "HighRequestErrorRate" + ], + "category": "sre", + "complexity": "medium", + "description": "This scenario simulates a credential failure by corrupting the Secret used by the OpenTelemetry Demo's `email` service, causing it to fail when attempting to authenticate with external services.", + "disruptions": [ + { + "injections": [ + { + "args": { + "kubernetesObject": { + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": "email-service-secret", + "namespace": "otel-demo" + } + } + }, + "id": "corrupted-kubernetes-secret-credentials" + } + ] + } + ], + "environment": { + "applications": [ + { + "id": "opentelemetry-demo" + } + ] + }, + "id": 120, + "index": 120, + "platforms": [ + "Kubernetes" + ], + "solutions": [ + [ + { + "steps": [ + { + "command": "kubectl -n otel-demo get secret email-service-secret -o yaml", + "text": "Inspect the corrupted secret to identify the invalid data." + }, + { + "text": "Restore the secret with correct credentials from the credential management system." + } + ] + }, + { + "steps": [ + { + "text": "Rotate the credentials in the external system and update the Kubernetes Secret to match." + } + ] + } + ] + ], + "tags": [ + "Authentication", + "Deployment" + ] + }, + { + "alerts": [ + "KubePodNotReady" + ], + "category": "sre", + "complexity": "medium", + "description": "This scenario simulates node resource exhaustion by injecting excessive resource requests into the OpenTelemetry Demo's `product-catalog` service, causing resource pressure on the node and affecting co-located workloads.", + "disruptions": [ + { + "injections": [ + { + "args": { + "kubernetesObject": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": "product-catalog", + "namespace": "otel-demo" + } + } + }, + "id": "node-resource-exhaustion" + } + ] + } + ], + "environment": { + "applications": [ + { + "id": "opentelemetry-demo" + } + ] + }, + "id": 121, + "index": 121, + "platforms": [ + "Kubernetes" + ], + "solutions": [ + [ + { + "steps": [ + { + "command": "kubectl -n otel-demo rollout undo deployment/product-catalog", + "text": "Revert the resource request changes to restore normal resource usage." + } + ] + }, + { + "steps": [ + { + "command": "kubectl -n otel-demo edit deployment product-catalog", + "text": "Manually edit the manifest and set resource requests to appropriate values." + } + ] + } + ] + ], + "tags": [ + "Deployment", + "Performance" + ] + }, + { + "alerts": [ + "HighRequestErrorRate" + ], + "category": "sre", + "complexity": "medium", + "description": "This scenario simulates a release pipeline failure by injecting an invalid image into the OpenTelemetry Demo's `shipping` service, causing it to fail to start and blocking the shipping workflow.", + "disruptions": [ + { + "injections": [ + { + "args": { + "container": { + "name": "shipping" + }, + "kubernetesObject": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": "shipping", + "namespace": "otel-demo" + } + } + }, + "id": "failed-release-pipeline-service" + } + ] + } + ], + "environment": { + "applications": [ + { + "id": "opentelemetry-demo" + } + ] + }, + "id": 122, + "index": 122, + "platforms": [ + "Kubernetes" + ], + "solutions": [ + [ + { + "steps": [ + { + "command": "kubectl -n otel-demo rollout undo deployment/shipping", + "text": "Revert the deployment to the previous working image." + } + ] + }, + { + "steps": [ + { + "command": "kubectl -n otel-demo set image deployment/shipping shipping=correct-image:latest", + "text": "Set the container image to the correct version." + } + ] + } + ] + ], + "tags": [ + "Code", + "Deployment" + ] + }, + { + "alerts": [ + "KubePodNotReady" + ], + "category": "sre", + "complexity": "low", + "description": "This scenario simulates a monitoring probe failure by scaling down the OpenTelemetry Demo's `load-generator` deployment to zero, causing the loss of synthetic traffic that monitoring relies on to detect issues.", + "disruptions": [ + { + "injections": [ + { + "args": { + "kubernetesObject": { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": "load-generator", + "namespace": "otel-demo" + } + } + }, + "id": "monitoring-probe-failure" + } + ] + } + ], + "environment": { + "applications": [ + { + "id": "opentelemetry-demo" + } + ] + }, + "id": 123, + "index": 123, + "platforms": [ + "Kubernetes" + ], + "solutions": [ + [ + { + "steps": [ + { + "command": "kubectl -n otel-demo scale deployment/load-generator --replicas=1", + "text": "Scale the monitoring probe deployment back to at least 1 replica." + } + ] + }, + { + "steps": [ + { + "command": "kubectl -n otel-demo rollout restart deployment/load-generator", + "text": "Restart the monitoring probe deployment to recover from a stuck state." + } + ] + } + ] + ], + "tags": [ + "Deployment", + "Performance" + ] } ] diff --git a/scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/115.yaml b/scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/115.yaml new file mode 100644 index 000000000..ac89d350e --- /dev/null +++ b/scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/115.yaml @@ -0,0 +1,39 @@ +--- +alerts: + - KubePodCrashLooping + - KubePodNotReady +category: sre +complexity: medium +description: This scenario simulates OpenTelemetry Demo's `recommendation` service controller crashing due to a corrupted container command, causing the pod to + enter CrashLoopBackOff and degrading upstream services that depend on it. +disruptions: + - injections: + - args: + container: + name: recommendation + kubernetesObject: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: recommendation + namespace: otel-demo + id: crashing-kubernetes-controller-workload +environment: + applications: + - id: opentelemetry-demo +id: 115 +platforms: + - Kubernetes +solutions: + - - steps: + - command: kubectl -n otel-demo rollout undo deployment/recommendation + text: Revert the last change done to the controller manifest. + - steps: + - command: kubectl -n otel-demo edit deployment recommendation + text: Manually edit the manifest and replace the container command with the correct value. + - steps: + - command: kubectl -n otel-demo logs -l app=recommendation -c recommendation --previous + text: Check the previous container logs to understand the crash reason. +tags: + - Deployment + - Performance diff --git a/scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/116.yaml b/scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/116.yaml new file mode 100644 index 000000000..6d7a93b30 --- /dev/null +++ b/scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/116.yaml @@ -0,0 +1,31 @@ +--- +alerts: + - KubePodNotReady +category: sre +complexity: high +description: This scenario simulates etcd storage exhaustion by filling the cluster's etcd database with large ConfigMaps, causing the API server to reject + writes and degrading cluster-wide operations including the OpenTelemetry Demo application. +disruptions: + - injections: + - args: + kubernetesObject: + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: etcd + namespace: otel-demo + id: exhausted-etcd-database-storage +environment: + applications: + - id: opentelemetry-demo +id: 116 +platforms: + - Kubernetes +solutions: + - - steps: + - text: Compact and defragment the etcd database to reclaim storage space. + - text: Remove unnecessary resources (completed jobs, orphaned ConfigMaps) to reduce etcd storage usage. + - steps: + - text: Increase the etcd storage quota if the current limit is too low for the cluster workload. +tags: + - Performance diff --git a/scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/117.yaml b/scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/117.yaml new file mode 100644 index 000000000..3613c284e --- /dev/null +++ b/scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/117.yaml @@ -0,0 +1,33 @@ +--- +alerts: + - KubePodNotReady +category: sre +complexity: medium +description: This scenario simulates a pipeline controller failure by scaling the OpenTelemetry Demo's `kafka` deployment to zero replicas, causing downstream + services that depend on event processing to stall, similar to a Tekton pipeline controller becoming unavailable. +disruptions: + - injections: + - args: + kubernetesObject: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: kafka + namespace: otel-demo + id: stalled-pipeline-controller +environment: + applications: + - id: opentelemetry-demo +id: 117 +platforms: + - Kubernetes +solutions: + - - steps: + - command: kubectl -n otel-demo scale deployment/kafka --replicas=1 + text: Scale the pipeline controller deployment back to at least 1 replica. + - steps: + - command: kubectl -n otel-demo rollout restart deployment/kafka + text: Restart the pipeline controller deployment to recover from a stuck state. +tags: + - Deployment + - Performance diff --git a/scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/118.yaml b/scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/118.yaml new file mode 100644 index 000000000..76be370ac --- /dev/null +++ b/scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/118.yaml @@ -0,0 +1,35 @@ +# yaml-language-server: $schema=../schema.json +--- +alerts: + - KubePodNotReady + - HighRequestErrorRate +category: sre +complexity: high +description: This scenario simulates a cluster availability loss by cordoning worker nodes and evicting the OpenTelemetry Demo's `checkout` service, causing it + to become unschedulable and degrading the entire checkout flow. +disruptions: + - injections: + - args: + kubernetesObject: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: checkout + namespace: otel-demo + id: cluster-availability-loss +environment: + applications: + - id: opentelemetry-demo +id: 118 +platforms: + - Kubernetes +solutions: + - - steps: + - command: kubectl get nodes + text: Identify which worker nodes are cordoned. + - text: Uncordon all affected worker nodes to restore scheduling. + - steps: + - text: Investigate the root cause of the cluster unavailability (etcd health, API server logs, node conditions) before uncordoning. +tags: + - Deployment + - Performance diff --git a/scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/119.yaml b/scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/119.yaml new file mode 100644 index 000000000..a4812466f --- /dev/null +++ b/scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/119.yaml @@ -0,0 +1,36 @@ +# yaml-language-server: $schema=../schema.json +--- +alerts: + - HighRequestErrorRate +category: sre +complexity: medium +description: This scenario simulates a breaking configuration change on the OpenTelemetry Demo's `cart` service by injecting an invalid environment variable, + causing the service to malfunction and degrading the shopping cart workflow. +disruptions: + - injections: + - args: + container: + name: cart + kubernetesObject: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: cart + namespace: otel-demo + id: breaking-configuration-change +environment: + applications: + - id: opentelemetry-demo +id: 119 +platforms: + - Kubernetes +solutions: + - - steps: + - command: kubectl -n otel-demo rollout undo deployment/cart + text: Revert the last change done to the workload manifest. + - steps: + - command: kubectl -n otel-demo edit deployment cart + text: Manually edit the manifest and replace the environment variable with the correct value. +tags: + - Code + - Deployment diff --git a/scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/120.yaml b/scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/120.yaml new file mode 100644 index 000000000..507c55c84 --- /dev/null +++ b/scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/120.yaml @@ -0,0 +1,34 @@ +# yaml-language-server: $schema=../schema.json +--- +alerts: + - HighRequestErrorRate +category: sre +complexity: medium +description: This scenario simulates a credential failure by corrupting the Secret used by the OpenTelemetry Demo's `email` service, causing it to fail when + attempting to authenticate with external services. +disruptions: + - injections: + - args: + kubernetesObject: + apiVersion: v1 + kind: Secret + metadata: + name: email-service-secret + namespace: otel-demo + id: corrupted-kubernetes-secret-credentials +environment: + applications: + - id: opentelemetry-demo +id: 120 +platforms: + - Kubernetes +solutions: + - - steps: + - command: kubectl -n otel-demo get secret email-service-secret -o yaml + text: Inspect the corrupted secret to identify the invalid data. + - text: Restore the secret with correct credentials from the credential management system. + - steps: + - text: Rotate the credentials in the external system and update the Kubernetes Secret to match. +tags: + - Authentication + - Deployment diff --git a/scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/121.yaml b/scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/121.yaml new file mode 100644 index 000000000..0621487c3 --- /dev/null +++ b/scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/121.yaml @@ -0,0 +1,34 @@ +# yaml-language-server: $schema=../schema.json +--- +alerts: + - KubePodNotReady +category: sre +complexity: medium +description: This scenario simulates node resource exhaustion by injecting excessive resource requests into the OpenTelemetry Demo's `product-catalog` service, + causing resource pressure on the node and affecting co-located workloads. +disruptions: + - injections: + - args: + kubernetesObject: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: product-catalog + namespace: otel-demo + id: node-resource-exhaustion +environment: + applications: + - id: opentelemetry-demo +id: 121 +platforms: + - Kubernetes +solutions: + - - steps: + - command: kubectl -n otel-demo rollout undo deployment/product-catalog + text: Revert the resource request changes to restore normal resource usage. + - steps: + - command: kubectl -n otel-demo edit deployment product-catalog + text: Manually edit the manifest and set resource requests to appropriate values. +tags: + - Deployment + - Performance diff --git a/scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/122.yaml b/scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/122.yaml new file mode 100644 index 000000000..ab39f09ba --- /dev/null +++ b/scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/122.yaml @@ -0,0 +1,36 @@ +# yaml-language-server: $schema=../schema.json +--- +alerts: + - HighRequestErrorRate +category: sre +complexity: medium +description: This scenario simulates a release pipeline failure by injecting an invalid image into the OpenTelemetry Demo's `shipping` service, causing it to + fail to start and blocking the shipping workflow. +disruptions: + - injections: + - args: + container: + name: shipping + kubernetesObject: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: shipping + namespace: otel-demo + id: failed-release-pipeline-service +environment: + applications: + - id: opentelemetry-demo +id: 122 +platforms: + - Kubernetes +solutions: + - - steps: + - command: kubectl -n otel-demo rollout undo deployment/shipping + text: Revert the deployment to the previous working image. + - steps: + - command: kubectl -n otel-demo set image deployment/shipping shipping=correct-image:latest + text: Set the container image to the correct version. +tags: + - Code + - Deployment diff --git a/scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/123.yaml b/scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/123.yaml new file mode 100644 index 000000000..864b4d795 --- /dev/null +++ b/scenarios/sre/project/roles/documentation/files/library/scenarios/indexes/123.yaml @@ -0,0 +1,34 @@ +# yaml-language-server: $schema=../schema.json +--- +alerts: + - KubePodNotReady +category: sre +complexity: low +description: This scenario simulates a monitoring probe failure by scaling down the OpenTelemetry Demo's `load-generator` deployment to zero, causing the loss + of synthetic traffic that monitoring relies on to detect issues. +disruptions: + - injections: + - args: + kubernetesObject: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: load-generator + namespace: otel-demo + id: monitoring-probe-failure +environment: + applications: + - id: opentelemetry-demo +id: 123 +platforms: + - Kubernetes +solutions: + - - steps: + - command: kubectl -n otel-demo scale deployment/load-generator --replicas=1 + text: Scale the monitoring probe deployment back to at least 1 replica. + - steps: + - command: kubectl -n otel-demo rollout restart deployment/load-generator + text: Restart the monitoring probe deployment to recover from a stuck state. +tags: + - Deployment + - Performance diff --git a/scenarios/sre/project/roles/documentation/files/library/scenarios/schema.json b/scenarios/sre/project/roles/documentation/files/library/scenarios/schema.json index e6d772c4e..bfb3d70a5 100644 --- a/scenarios/sre/project/roles/documentation/files/library/scenarios/schema.json +++ b/scenarios/sre/project/roles/documentation/files/library/scenarios/schema.json @@ -111,6 +111,136 @@ }, "faultItem": { "allOf": [ + { + "if": { + "properties": { + "id": { + "const": "breaking-configuration-change" + } + } + }, + "then": { + "properties": { + "args": { + "properties": { + "container": { + "properties": { + "name": { + "type": "string" + } + }, + "required": [ + "name" + ], + "type": "object" + }, + "kubernetesObject": { + "properties": { + "apiVersion": { + "enum": [ + "apps/v1" + ], + "type": "string" + }, + "kind": { + "enum": [ + "Deployment", + "StatefulSet" + ], + "type": "string" + }, + "metadata": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata" + ], + "type": "object" + } + }, + "required": [ + "kubernetesObject", + "container" + ], + "type": "object" + } + } + } + }, + { + "if": { + "properties": { + "id": { + "const": "cluster-availability-loss" + } + } + }, + "then": { + "properties": { + "args": { + "properties": { + "kubernetesObject": { + "properties": { + "apiVersion": { + "enum": [ + "apps/v1" + ], + "type": "string" + }, + "kind": { + "enum": [ + "Deployment", + "StatefulSet" + ], + "type": "string" + }, + "metadata": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata" + ], + "type": "object" + } + }, + "required": [ + "kubernetesObject" + ], + "type": "object" + } + } + } + }, { "if": { "properties": { @@ -127,14 +257,260 @@ "properties": { "apiVersion": { "enum": [ - "apps/v1" + "apps/v1" + ], + "type": "string" + }, + "kind": { + "enum": [ + "Deployment", + "StatefulSet" + ], + "type": "string" + }, + "metadata": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata" + ], + "type": "object" + } + }, + "required": [ + "kubernetesObject" + ], + "type": "object" + } + } + } + }, + { + "if": { + "properties": { + "id": { + "const": "corrupted-kubernetes-secret-credentials" + } + } + }, + "then": { + "properties": { + "args": { + "properties": { + "kubernetesObject": { + "properties": { + "apiVersion": { + "enum": [ + "v1" + ], + "type": "string" + }, + "kind": { + "enum": [ + "Secret" + ], + "type": "string" + }, + "metadata": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata" + ], + "type": "object" + } + }, + "required": [ + "kubernetesObject" + ], + "type": "object" + } + } + } + }, + { + "if": { + "properties": { + "id": { + "const": "crashing-kubernetes-controller-workload" + } + } + }, + "then": { + "properties": { + "args": { + "properties": { + "container": { + "properties": { + "name": { + "type": "string" + } + }, + "required": [ + "name" + ], + "type": "object" + }, + "kubernetesObject": { + "properties": { + "apiVersion": { + "enum": [ + "apps/v1" + ], + "type": "string" + }, + "kind": { + "enum": [ + "Deployment", + "StatefulSet" + ], + "type": "string" + }, + "metadata": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata" + ], + "type": "object" + } + }, + "required": [ + "kubernetesObject", + "container" + ], + "type": "object" + } + } + } + }, + { + "if": { + "properties": { + "id": { + "const": "crashing-kubernetes-workload-init-container" + } + } + }, + "then": { + "properties": { + "args": { + "properties": { + "kubernetesObject": { + "properties": { + "apiVersion": { + "enum": [ + "apps/v1" + ], + "type": "string" + }, + "kind": { + "enum": [ + "Deployment", + "StatefulSet" + ], + "type": "string" + }, + "metadata": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata" + ], + "type": "object" + } + }, + "required": [ + "kubernetesObject" + ], + "type": "object" + } + } + } + }, + { + "if": { + "properties": { + "id": { + "const": "deleted-kubernetes-service" + } + } + }, + "then": { + "properties": { + "args": { + "properties": { + "kubernetesObject": { + "properties": { + "apiVersion": { + "enum": [ + "v1" ], "type": "string" }, "kind": { "enum": [ - "Deployment", - "StatefulSet" + "Service" ], "type": "string" }, @@ -174,7 +550,7 @@ "if": { "properties": { "id": { - "const": "crashing-kubernetes-workload-init-container" + "const": "disabled-istio-ambient-mode-kubernetes-namespace" } } }, @@ -186,14 +562,13 @@ "properties": { "apiVersion": { "enum": [ - "apps/v1" + "v1" ], "type": "string" }, "kind": { "enum": [ - "Deployment", - "StatefulSet" + "Namespace" ], "type": "string" }, @@ -201,14 +576,10 @@ "properties": { "name": { "type": "string" - }, - "namespace": { - "type": "string" } }, "required": [ - "name", - "namespace" + "name" ], "type": "object" } @@ -233,7 +604,7 @@ "if": { "properties": { "id": { - "const": "deleted-kubernetes-service" + "const": "exhausted-etcd-database-storage" } } }, @@ -245,13 +616,13 @@ "properties": { "apiVersion": { "enum": [ - "v1" + "apps/v1" ], "type": "string" }, "kind": { "enum": [ - "Service" + "StatefulSet" ], "type": "string" }, @@ -291,7 +662,7 @@ "if": { "properties": { "id": { - "const": "disabled-istio-ambient-mode-kubernetes-namespace" + "const": "failed-release-pipeline-service" } } }, @@ -299,17 +670,28 @@ "properties": { "args": { "properties": { + "container": { + "properties": { + "name": { + "type": "string" + } + }, + "required": [ + "name" + ], + "type": "object" + }, "kubernetesObject": { "properties": { "apiVersion": { "enum": [ - "v1" + "apps/v1" ], "type": "string" }, "kind": { "enum": [ - "Namespace" + "Deployment" ], "type": "string" }, @@ -317,10 +699,14 @@ "properties": { "name": { "type": "string" + }, + "namespace": { + "type": "string" } }, "required": [ - "name" + "name", + "namespace" ], "type": "object" } @@ -334,7 +720,8 @@ } }, "required": [ - "kubernetesObject" + "kubernetesObject", + "container" ], "type": "object" } @@ -1130,6 +1517,124 @@ } } }, + { + "if": { + "properties": { + "id": { + "const": "monitoring-probe-failure" + } + } + }, + "then": { + "properties": { + "args": { + "properties": { + "kubernetesObject": { + "properties": { + "apiVersion": { + "enum": [ + "apps/v1" + ], + "type": "string" + }, + "kind": { + "enum": [ + "Deployment", + "DaemonSet" + ], + "type": "string" + }, + "metadata": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata" + ], + "type": "object" + } + }, + "required": [ + "kubernetesObject" + ], + "type": "object" + } + } + } + }, + { + "if": { + "properties": { + "id": { + "const": "node-resource-exhaustion" + } + } + }, + "then": { + "properties": { + "args": { + "properties": { + "kubernetesObject": { + "properties": { + "apiVersion": { + "enum": [ + "apps/v1" + ], + "type": "string" + }, + "kind": { + "enum": [ + "Deployment", + "StatefulSet" + ], + "type": "string" + }, + "metadata": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata" + ], + "type": "object" + } + }, + "required": [ + "kubernetesObject" + ], + "type": "object" + } + } + } + }, { "if": { "properties": { @@ -1596,6 +2101,64 @@ } } }, + { + "if": { + "properties": { + "id": { + "const": "stalled-pipeline-controller" + } + } + }, + "then": { + "properties": { + "args": { + "properties": { + "kubernetesObject": { + "properties": { + "apiVersion": { + "enum": [ + "apps/v1" + ], + "type": "string" + }, + "kind": { + "enum": [ + "Deployment" + ], + "type": "string" + }, + "metadata": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata" + ], + "type": "object" + } + }, + "required": [ + "kubernetesObject" + ], + "type": "object" + } + } + } + }, { "if": { "properties": { @@ -2064,10 +2627,16 @@ }, "id": { "enum": [ + "breaking-configuration-change", + "cluster-availability-loss", "cordoned-kubernetes-worker-node", + "corrupted-kubernetes-secret-credentials", + "crashing-kubernetes-controller-workload", "crashing-kubernetes-workload-init-container", "deleted-kubernetes-service", "disabled-istio-ambient-mode-kubernetes-namespace", + "exhausted-etcd-database-storage", + "failed-release-pipeline-service", "failing-name-resolution-kubernetes-workload-dns-policy", "hanging-kubernetes-workload-init-container", "ingress-port-blocking-network-policy", @@ -2080,6 +2649,8 @@ "misconfigured-kubernetes-workload-container-readiness-probe", "modified-kubernetes-workload-container-environment-variable", "modified-target-port-kubernetes-service", + "monitoring-probe-failure", + "node-resource-exhaustion", "nonexistent-kubernetes-workload-container-image", "nonexistent-kubernetes-workload-node", "nonexistent-kubernetes-workload-persistent-volume-claim", @@ -2087,6 +2658,7 @@ "priority-kubernetes-workload-priority-preemption", "scaled-to-zero-kubernetes-workload", "scheduled-chaos-mesh-experiment", + "stalled-pipeline-controller", "strict-mutual-tls-istio-service-mesh-enforcement", "traffic-denying-istio-gateway-authorization-policy", "unassigned-kubernetes-workload-container-resource-limits", diff --git a/scenarios/sre/project/roles/documentation/templates/library/scenarios/indexes/115.j2 b/scenarios/sre/project/roles/documentation/templates/library/scenarios/indexes/115.j2 new file mode 100644 index 000000000..2166311b7 --- /dev/null +++ b/scenarios/sre/project/roles/documentation/templates/library/scenarios/indexes/115.j2 @@ -0,0 +1,26 @@ +--- +category: sre +complexity: medium +description: >- + This scenario simulates OpenTelemetry Demo's `recommendation` service controller + crashing due to a corrupted container command, causing the pod to enter CrashLoopBackOff + and degrading upstream services that depend on it. +disruptions: + - injections: + - args: + container: + name: recommendation + kubernetesObject: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: recommendation + namespace: {{ applications_managers.opentelemetry_demo.kubernetes.namespace }} + id: crashing-kubernetes-controller-workload +environment: + applications: + - id: opentelemetry-demo +id: 115 +solutionTemplates: + - disruptionIndex: 0 + injectionIndex: 0 diff --git a/scenarios/sre/project/roles/documentation/templates/library/scenarios/indexes/116.j2 b/scenarios/sre/project/roles/documentation/templates/library/scenarios/indexes/116.j2 new file mode 100644 index 000000000..b1b34c0f3 --- /dev/null +++ b/scenarios/sre/project/roles/documentation/templates/library/scenarios/indexes/116.j2 @@ -0,0 +1,24 @@ +--- +category: sre +complexity: high +description: >- + This scenario simulates etcd storage exhaustion by filling the cluster's etcd database + with large ConfigMaps, causing the API server to reject writes and degrading cluster-wide + operations including the OpenTelemetry Demo application. +disruptions: + - injections: + - args: + kubernetesObject: + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: etcd + namespace: {{ applications_managers.opentelemetry_demo.kubernetes.namespace }} + id: exhausted-etcd-database-storage +environment: + applications: + - id: opentelemetry-demo +id: 116 +solutionTemplates: + - disruptionIndex: 0 + injectionIndex: 0 diff --git a/scenarios/sre/project/roles/documentation/templates/library/scenarios/indexes/117.j2 b/scenarios/sre/project/roles/documentation/templates/library/scenarios/indexes/117.j2 new file mode 100644 index 000000000..ce333e2c2 --- /dev/null +++ b/scenarios/sre/project/roles/documentation/templates/library/scenarios/indexes/117.j2 @@ -0,0 +1,24 @@ +--- +category: sre +complexity: medium +description: >- + This scenario simulates a pipeline controller failure by scaling the OpenTelemetry Demo's + `kafka` deployment to zero replicas, causing downstream services that depend on event + processing to stall, similar to a Tekton pipeline controller becoming unavailable. +disruptions: + - injections: + - args: + kubernetesObject: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: kafka + namespace: {{ applications_managers.opentelemetry_demo.kubernetes.namespace }} + id: stalled-pipeline-controller +environment: + applications: + - id: opentelemetry-demo +id: 117 +solutionTemplates: + - disruptionIndex: 0 + injectionIndex: 0 diff --git a/scenarios/sre/project/roles/documentation/templates/library/scenarios/indexes/118.j2 b/scenarios/sre/project/roles/documentation/templates/library/scenarios/indexes/118.j2 new file mode 100644 index 000000000..695e02c89 --- /dev/null +++ b/scenarios/sre/project/roles/documentation/templates/library/scenarios/indexes/118.j2 @@ -0,0 +1,24 @@ +--- +category: sre +complexity: high +description: >- + This scenario simulates a cluster availability loss by cordoning worker nodes and + evicting the OpenTelemetry Demo's `checkout` service, causing it to become unschedulable + and degrading the entire checkout flow. +disruptions: + - injections: + - args: + kubernetesObject: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: checkout + namespace: {{ applications_managers.opentelemetry_demo.kubernetes.namespace }} + id: cluster-availability-loss +environment: + applications: + - id: opentelemetry-demo +id: 118 +solutionTemplates: + - disruptionIndex: 0 + injectionIndex: 0 diff --git a/scenarios/sre/project/roles/documentation/templates/library/scenarios/indexes/119.j2 b/scenarios/sre/project/roles/documentation/templates/library/scenarios/indexes/119.j2 new file mode 100644 index 000000000..e23f126bf --- /dev/null +++ b/scenarios/sre/project/roles/documentation/templates/library/scenarios/indexes/119.j2 @@ -0,0 +1,26 @@ +--- +category: sre +complexity: medium +description: >- + This scenario simulates a breaking configuration change on the OpenTelemetry Demo's + `cart` service by injecting an invalid environment variable, causing the service to + malfunction and degrading the shopping cart workflow. +disruptions: + - injections: + - args: + container: + name: cart + kubernetesObject: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: cart + namespace: {{ applications_managers.opentelemetry_demo.kubernetes.namespace }} + id: breaking-configuration-change +environment: + applications: + - id: opentelemetry-demo +id: 119 +solutionTemplates: + - disruptionIndex: 0 + injectionIndex: 0 diff --git a/scenarios/sre/project/roles/documentation/templates/library/scenarios/indexes/120.j2 b/scenarios/sre/project/roles/documentation/templates/library/scenarios/indexes/120.j2 new file mode 100644 index 000000000..a9fff30eb --- /dev/null +++ b/scenarios/sre/project/roles/documentation/templates/library/scenarios/indexes/120.j2 @@ -0,0 +1,24 @@ +--- +category: sre +complexity: medium +description: >- + This scenario simulates a credential failure by corrupting the Secret used by + the OpenTelemetry Demo's `email` service, causing it to fail when attempting + to authenticate with external services. +disruptions: + - injections: + - args: + kubernetesObject: + apiVersion: v1 + kind: Secret + metadata: + name: email-service-secret + namespace: {{ applications_managers.opentelemetry_demo.kubernetes.namespace }} + id: corrupted-kubernetes-secret-credentials +environment: + applications: + - id: opentelemetry-demo +id: 120 +solutionTemplates: + - disruptionIndex: 0 + injectionIndex: 0 diff --git a/scenarios/sre/project/roles/documentation/templates/library/scenarios/indexes/121.j2 b/scenarios/sre/project/roles/documentation/templates/library/scenarios/indexes/121.j2 new file mode 100644 index 000000000..017176b41 --- /dev/null +++ b/scenarios/sre/project/roles/documentation/templates/library/scenarios/indexes/121.j2 @@ -0,0 +1,24 @@ +--- +category: sre +complexity: medium +description: >- + This scenario simulates node resource exhaustion by injecting excessive resource + requests into the OpenTelemetry Demo's `product-catalog` service, causing resource + pressure on the node and affecting co-located workloads. +disruptions: + - injections: + - args: + kubernetesObject: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: product-catalog + namespace: {{ applications_managers.opentelemetry_demo.kubernetes.namespace }} + id: node-resource-exhaustion +environment: + applications: + - id: opentelemetry-demo +id: 121 +solutionTemplates: + - disruptionIndex: 0 + injectionIndex: 0 diff --git a/scenarios/sre/project/roles/documentation/templates/library/scenarios/indexes/122.j2 b/scenarios/sre/project/roles/documentation/templates/library/scenarios/indexes/122.j2 new file mode 100644 index 000000000..1af59cf2c --- /dev/null +++ b/scenarios/sre/project/roles/documentation/templates/library/scenarios/indexes/122.j2 @@ -0,0 +1,26 @@ +--- +category: sre +complexity: medium +description: >- + This scenario simulates a release pipeline failure by injecting an invalid image + into the OpenTelemetry Demo's `shipping` service, causing it to fail to start + and blocking the shipping workflow. +disruptions: + - injections: + - args: + container: + name: shipping + kubernetesObject: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: shipping + namespace: {{ applications_managers.opentelemetry_demo.kubernetes.namespace }} + id: failed-release-pipeline-service +environment: + applications: + - id: opentelemetry-demo +id: 122 +solutionTemplates: + - disruptionIndex: 0 + injectionIndex: 0 diff --git a/scenarios/sre/project/roles/documentation/templates/library/scenarios/indexes/123.j2 b/scenarios/sre/project/roles/documentation/templates/library/scenarios/indexes/123.j2 new file mode 100644 index 000000000..bc88473f5 --- /dev/null +++ b/scenarios/sre/project/roles/documentation/templates/library/scenarios/indexes/123.j2 @@ -0,0 +1,24 @@ +--- +category: sre +complexity: low +description: >- + This scenario simulates a monitoring probe failure by scaling down the OpenTelemetry + Demo's `load-generator` deployment to zero, causing the loss of synthetic traffic + that monitoring relies on to detect issues. +disruptions: + - injections: + - args: + kubernetesObject: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: load-generator + namespace: {{ applications_managers.opentelemetry_demo.kubernetes.namespace }} + id: monitoring-probe-failure +environment: + applications: + - id: opentelemetry-demo +id: 123 +solutionTemplates: + - disruptionIndex: 0 + injectionIndex: 0 diff --git a/scenarios/sre/project/roles/faults/files/library/map.json b/scenarios/sre/project/roles/faults/files/library/map.json index 24791956e..1a3dd62b5 100644 --- a/scenarios/sre/project/roles/faults/files/library/map.json +++ b/scenarios/sre/project/roles/faults/files/library/map.json @@ -1,7 +1,19 @@ { + "breaking-configuration-change": { + "taskFile": "inject_breaking_configuration_change.yaml" + }, + "cluster-availability-loss": { + "taskFile": "inject_cluster_availability_loss.yaml" + }, "cordoned-kubernetes-worker-node": { "taskFile": "inject_cordoned_kubernetes_worker_node.yaml" }, + "corrupted-kubernetes-secret-credentials": { + "taskFile": "inject_corrupted_kubernetes_secret_credentials.yaml" + }, + "crashing-kubernetes-controller-workload": { + "taskFile": "inject_crashing_kubernetes_controller_workload.yaml" + }, "crashing-kubernetes-workload-init-container": { "taskFile": "inject_crashing_kubernetes_workload_init_container.yaml" }, @@ -11,6 +23,12 @@ "disabled-istio-ambient-mode-kubernetes-namespace": { "taskFile": "inject_disabled_istio_ambient_mode_kubernetes_namespace.yaml" }, + "exhausted-etcd-database-storage": { + "taskFile": "inject_exhausted_etcd_database_storage.yaml" + }, + "failed-release-pipeline-service": { + "taskFile": "inject_failed_release_pipeline_service.yaml" + }, "failing-name-resolution-kubernetes-workload-dns-policy": { "taskFile": "inject_failing_name_resolution_kubernetes_workload_dns_policy.yaml" }, @@ -47,6 +65,12 @@ "modified-target-port-kubernetes-service": { "taskFile": "inject_modified_target_port_kubernetes_service.yaml" }, + "monitoring-probe-failure": { + "taskFile": "inject_monitoring_probe_failure.yaml" + }, + "node-resource-exhaustion": { + "taskFile": "inject_node_resource_exhaustion.yaml" + }, "nonexistent-kubernetes-workload-container-image": { "taskFile": "inject_nonexistent_kubernetes_workload_container_image.yaml" }, @@ -68,6 +92,9 @@ "scheduled-chaos-mesh-experiment": { "taskFile": "inject_scheduled_chaos_mesh_experiment.yaml" }, + "stalled-pipeline-controller": { + "taskFile": "inject_stalled_pipeline_controller.yaml" + }, "strict-mutual-tls-istio-service-mesh-enforcement": { "taskFile": "inject_strict_mutual_tls_istio_service_mesh_enforcement.yaml" }, diff --git a/scenarios/sre/project/roles/faults/meta/argument_specs.yaml b/scenarios/sre/project/roles/faults/meta/argument_specs.yaml index 8911e1793..43ba4b1f7 100644 --- a/scenarios/sre/project/roles/faults/meta/argument_specs.yaml +++ b/scenarios/sre/project/roles/faults/meta/argument_specs.yaml @@ -20,10 +20,16 @@ argument_specs: type: dict id: choices: + - breaking-configuration-change + - cluster-availability-loss - cordoned-kubernetes-worker-node + - corrupted-kubernetes-secret-credentials + - crashing-kubernetes-controller-workload - crashing-kubernetes-workload-init-container - deleted-kubernetes-service - disabled-istio-ambient-mode-kubernetes-namespace + - exhausted-etcd-database-storage + - failed-release-pipeline-service - failing-name-resolution-kubernetes-workload-dns-policy - hanging-kubernetes-workload-init-container - ingress-port-blocking-network-policy @@ -36,6 +42,8 @@ argument_specs: - misconfigured-kubernetes-workload-container-readiness-probe - modified-kubernetes-workload-container-environment-variable - modified-target-port-kubernetes-service + - monitoring-probe-failure + - node-resource-exhaustion - nonexistent-kubernetes-workload-container-image - nonexistent-kubernetes-workload-node - nonexistent-kubernetes-workload-persistent-volume-claim @@ -43,6 +51,7 @@ argument_specs: - priority-kubernetes-workload-priority-preemption - scaled-to-zero-kubernetes-workload - scheduled-chaos-mesh-experiment + - stalled-pipeline-controller - strict-mutual-tls-istio-service-mesh-enforcement - traffic-denying-istio-gateway-authorization-policy - unassigned-kubernetes-workload-container-resource-limits diff --git a/scenarios/sre/project/roles/faults/tasks/inject_breaking_configuration_change.yaml b/scenarios/sre/project/roles/faults/tasks/inject_breaking_configuration_change.yaml new file mode 100644 index 000000000..f41a57313 --- /dev/null +++ b/scenarios/sre/project/roles/faults/tasks/inject_breaking_configuration_change.yaml @@ -0,0 +1,25 @@ +--- +- name: Include fault argument validation tasks + ansible.builtin.include_tasks: + file: validate_fault_arguments.yaml + +- name: Update workload with breaking environment variable + kubernetes.core.k8s: + kubeconfig: "{{ faults_cluster.kubeconfig }}" + resource_definition: + apiVersion: "{{ faults_kubernetes_object.apiVersion }}" + kind: "{{ faults_kubernetes_object.kind }}" + metadata: + name: "{{ faults_kubernetes_object.metadata.name }}" + namespace: "{{ faults_kubernetes_object.metadata.namespace }}" + spec: + template: + spec: + containers: + - name: "{{ faults_container.name }}" + env: + - name: OTEL_SERVICE_NAME + value: "" + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://nonexistent-collector:4317" + state: patched diff --git a/scenarios/sre/project/roles/faults/tasks/inject_cluster_availability_loss.yaml b/scenarios/sre/project/roles/faults/tasks/inject_cluster_availability_loss.yaml new file mode 100644 index 000000000..00e9359bd --- /dev/null +++ b/scenarios/sre/project/roles/faults/tasks/inject_cluster_availability_loss.yaml @@ -0,0 +1,44 @@ +--- +- name: Include fault argument validation tasks + ansible.builtin.include_tasks: + file: validate_fault_arguments.yaml + +- name: Include worker node validation tasks + ansible.builtin.include_tasks: + file: validate_worker_nodes.yaml + +- name: Cordon all worker nodes + ansible.builtin.command: + cmd: >- + kubectl --kubeconfig={{ faults_cluster.kubeconfig }} + cordon {{ item }} + loop: "{{ faults_worker_nodes }}" + loop_control: + label: "{{ item }}" + +- name: Delete target workload pods + ansible.builtin.command: + cmd: >- + kubectl --kubeconfig={{ faults_cluster.kubeconfig }} + -n {{ faults_kubernetes_object.metadata.namespace }} + delete pods -l app={{ faults_kubernetes_object.metadata.name }} + --force --grace-period=0 + +- name: Wait for pods to be in Pending state + kubernetes.core.k8s_info: + kubeconfig: "{{ faults_cluster.kubeconfig }}" + api_version: v1 + kind: Pod + namespace: "{{ faults_kubernetes_object.metadata.namespace }}" + label_selectors: + - "app={{ faults_kubernetes_object.metadata.name }}" + register: faults_pending_pods + until: + - faults_pending_pods.resources | length > 0 + - >- + faults_pending_pods.resources | + selectattr('status.phase', 'equalto', 'Pending') | + list | + length > 0 + retries: 15 + delay: 10 diff --git a/scenarios/sre/project/roles/faults/tasks/inject_corrupted_kubernetes_secret_credentials.yaml b/scenarios/sre/project/roles/faults/tasks/inject_corrupted_kubernetes_secret_credentials.yaml new file mode 100644 index 000000000..80b4bddd5 --- /dev/null +++ b/scenarios/sre/project/roles/faults/tasks/inject_corrupted_kubernetes_secret_credentials.yaml @@ -0,0 +1,27 @@ +--- +- name: Include fault argument validation tasks + ansible.builtin.include_tasks: + file: validate_fault_arguments.yaml + +- name: Corrupt secret data with invalid credentials + kubernetes.core.k8s: + kubeconfig: "{{ faults_cluster.kubeconfig }}" + resource_definition: + apiVersion: v1 + kind: Secret + metadata: + name: "{{ faults_kubernetes_object.metadata.name }}" + namespace: "{{ faults_kubernetes_object.metadata.namespace }}" + type: Opaque + data: + password: "aW52YWxpZC1jcmVkZW50aWFscw==" + username: "aW52YWxpZC11c2Vy" + token: "aW52YWxpZC10b2tlbg==" + state: patched + +- name: Restart pods that use the secret + ansible.builtin.command: + cmd: >- + kubectl --kubeconfig={{ faults_cluster.kubeconfig }} + -n {{ faults_kubernetes_object.metadata.namespace }} + delete pods --all --force --grace-period=0 diff --git a/scenarios/sre/project/roles/faults/tasks/inject_crashing_kubernetes_controller_workload.yaml b/scenarios/sre/project/roles/faults/tasks/inject_crashing_kubernetes_controller_workload.yaml new file mode 100644 index 000000000..8867a473a --- /dev/null +++ b/scenarios/sre/project/roles/faults/tasks/inject_crashing_kubernetes_controller_workload.yaml @@ -0,0 +1,46 @@ +--- +- name: Include fault argument validation tasks + ansible.builtin.include_tasks: + file: validate_fault_arguments.yaml + +- name: Update workload containers + kubernetes.core.k8s: + kubeconfig: "{{ faults_cluster.kubeconfig }}" + resource_definition: + apiVersion: "{{ faults_kubernetes_object.apiVersion }}" + kind: "{{ faults_kubernetes_object.kind }}" + metadata: + name: "{{ faults_kubernetes_object.metadata.name }}" + namespace: "{{ faults_kubernetes_object.metadata.namespace }}" + spec: + template: + spec: + containers: + - name: "{{ faults_container.name }}" + command: + - invalid-command + state: patched + +- name: Wait for pod to enter CrashLoopBackOff + kubernetes.core.k8s_info: + kubeconfig: "{{ faults_cluster.kubeconfig }}" + api_version: v1 + kind: Pod + namespace: "{{ faults_kubernetes_object.metadata.namespace }}" + label_selectors: + - "app={{ faults_kubernetes_object.metadata.name }}" + register: faults_crashing_pods + until: + - faults_crashing_pods.resources | length > 0 + - >- + faults_crashing_pods.resources | + map(attribute='status') | + map(attribute='containerStatuses', default=[]) | + flatten | + selectattr('name', 'equalto', faults_container.name) | + selectattr('state.waiting.reason', 'defined') | + selectattr('state.waiting.reason', 'in', ['CrashLoopBackOff', 'Error']) | + list | + length > 0 + retries: 20 + delay: 15 diff --git a/scenarios/sre/project/roles/faults/tasks/inject_exhausted_etcd_database_storage.yaml b/scenarios/sre/project/roles/faults/tasks/inject_exhausted_etcd_database_storage.yaml new file mode 100644 index 000000000..ff9b8516b --- /dev/null +++ b/scenarios/sre/project/roles/faults/tasks/inject_exhausted_etcd_database_storage.yaml @@ -0,0 +1,36 @@ +--- +- name: Include fault argument validation tasks + ansible.builtin.include_tasks: + file: validate_fault_arguments.yaml + +- name: Create ConfigMaps to fill etcd storage + kubernetes.core.k8s: + kubeconfig: "{{ faults_cluster.kubeconfig }}" + resource_definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: "etcd-fill-{{ item }}" + namespace: "{{ faults_kubernetes_object.metadata.namespace }}" + labels: + app: etcd-storage-fill + data: + payload: "{{ lookup('pipe', 'head -c 900000 /dev/urandom | base64 | head -c 900000') }}" + state: present + loop: "{{ range(1, 201) | list }}" + loop_control: + label: "etcd-fill-{{ item }}" + +- name: Verify etcd storage pressure + kubernetes.core.k8s_info: + kubeconfig: "{{ faults_cluster.kubeconfig }}" + api_version: v1 + kind: ConfigMap + namespace: "{{ faults_kubernetes_object.metadata.namespace }}" + label_selectors: + - app=etcd-storage-fill + register: faults_etcd_fill_result + until: + - faults_etcd_fill_result.resources | length >= 200 + retries: 5 + delay: 10 diff --git a/scenarios/sre/project/roles/faults/tasks/inject_failed_release_pipeline_service.yaml b/scenarios/sre/project/roles/faults/tasks/inject_failed_release_pipeline_service.yaml new file mode 100644 index 000000000..5a2ab123c --- /dev/null +++ b/scenarios/sre/project/roles/faults/tasks/inject_failed_release_pipeline_service.yaml @@ -0,0 +1,44 @@ +--- +- name: Include fault argument validation tasks + ansible.builtin.include_tasks: + file: validate_fault_arguments.yaml + +- name: Update workload with invalid container image + kubernetes.core.k8s: + kubeconfig: "{{ faults_cluster.kubeconfig }}" + resource_definition: + apiVersion: "{{ faults_kubernetes_object.apiVersion }}" + kind: "{{ faults_kubernetes_object.kind }}" + metadata: + name: "{{ faults_kubernetes_object.metadata.name }}" + namespace: "{{ faults_kubernetes_object.metadata.namespace }}" + spec: + template: + spec: + containers: + - name: "{{ faults_container.name }}" + image: "ghcr.io/nonexistent-org/invalid-release-image:v0.0.0" + state: patched + +- name: Wait for pod to fail image pull + kubernetes.core.k8s_info: + kubeconfig: "{{ faults_cluster.kubeconfig }}" + api_version: v1 + kind: Pod + namespace: "{{ faults_kubernetes_object.metadata.namespace }}" + label_selectors: + - "app={{ faults_kubernetes_object.metadata.name }}" + register: faults_release_pods + until: + - faults_release_pods.resources | length > 0 + - >- + faults_release_pods.resources | + map(attribute='status') | + map(attribute='containerStatuses', default=[]) | + flatten | + selectattr('state.waiting.reason', 'defined') | + selectattr('state.waiting.reason', 'in', ['ImagePullBackOff', 'ErrImagePull']) | + list | + length > 0 + retries: 20 + delay: 15 diff --git a/scenarios/sre/project/roles/faults/tasks/inject_monitoring_probe_failure.yaml b/scenarios/sre/project/roles/faults/tasks/inject_monitoring_probe_failure.yaml new file mode 100644 index 000000000..6fe4231fc --- /dev/null +++ b/scenarios/sre/project/roles/faults/tasks/inject_monitoring_probe_failure.yaml @@ -0,0 +1,33 @@ +--- +- name: Include fault argument validation tasks + ansible.builtin.include_tasks: + file: validate_fault_arguments.yaml + +- name: Scale monitoring probe to zero replicas + kubernetes.core.k8s_scale: + api_version: "{{ faults_kubernetes_object.apiVersion }}" + kubeconfig: "{{ faults_cluster.kubeconfig }}" + kind: "{{ faults_kubernetes_object.kind }}" + name: "{{ faults_kubernetes_object.metadata.name }}" + namespace: "{{ faults_kubernetes_object.metadata.namespace }}" + replicas: 0 + wait: true + wait_timeout: 120 + +- name: Verify probe pods are terminated + kubernetes.core.k8s_info: + kubeconfig: "{{ faults_cluster.kubeconfig }}" + api_version: v1 + kind: Pod + namespace: "{{ faults_kubernetes_object.metadata.namespace }}" + label_selectors: + - "app={{ faults_kubernetes_object.metadata.name }}" + register: faults_probe_pods + until: + - >- + faults_probe_pods.resources | + selectattr('status.phase', 'equalto', 'Running') | + list | + length == 0 + retries: 10 + delay: 10 diff --git a/scenarios/sre/project/roles/faults/tasks/inject_node_resource_exhaustion.yaml b/scenarios/sre/project/roles/faults/tasks/inject_node_resource_exhaustion.yaml new file mode 100644 index 000000000..4ed03b8ac --- /dev/null +++ b/scenarios/sre/project/roles/faults/tasks/inject_node_resource_exhaustion.yaml @@ -0,0 +1,43 @@ +--- +- name: Include fault argument validation tasks + ansible.builtin.include_tasks: + file: validate_fault_arguments.yaml + +- name: Inject excessive resource requests + kubernetes.core.k8s: + kubeconfig: "{{ faults_cluster.kubeconfig }}" + resource_definition: + apiVersion: "{{ faults_kubernetes_object.apiVersion }}" + kind: "{{ faults_kubernetes_object.kind }}" + metadata: + name: "{{ faults_kubernetes_object.metadata.name }}" + namespace: "{{ faults_kubernetes_object.metadata.namespace }}" + spec: + template: + spec: + containers: + - name: "{{ faults_kubernetes_object.metadata.name }}" + resources: + requests: + memory: "64Gi" + cpu: "32" + state: patched + +- name: Wait for pods to be in Pending state + kubernetes.core.k8s_info: + kubeconfig: "{{ faults_cluster.kubeconfig }}" + api_version: v1 + kind: Pod + namespace: "{{ faults_kubernetes_object.metadata.namespace }}" + label_selectors: + - "app={{ faults_kubernetes_object.metadata.name }}" + register: faults_resource_pods + until: + - faults_resource_pods.resources | length > 0 + - >- + faults_resource_pods.resources | + selectattr('status.phase', 'equalto', 'Pending') | + list | + length > 0 + retries: 15 + delay: 10 diff --git a/scenarios/sre/project/roles/faults/tasks/inject_stalled_pipeline_controller.yaml b/scenarios/sre/project/roles/faults/tasks/inject_stalled_pipeline_controller.yaml new file mode 100644 index 000000000..86cea6a15 --- /dev/null +++ b/scenarios/sre/project/roles/faults/tasks/inject_stalled_pipeline_controller.yaml @@ -0,0 +1,33 @@ +--- +- name: Include fault argument validation tasks + ansible.builtin.include_tasks: + file: validate_fault_arguments.yaml + +- name: Scale pipeline controller to zero replicas + kubernetes.core.k8s_scale: + api_version: "{{ faults_kubernetes_object.apiVersion }}" + kubeconfig: "{{ faults_cluster.kubeconfig }}" + kind: "{{ faults_kubernetes_object.kind }}" + name: "{{ faults_kubernetes_object.metadata.name }}" + namespace: "{{ faults_kubernetes_object.metadata.namespace }}" + replicas: 0 + wait: true + wait_timeout: 120 + +- name: Verify controller pods are terminated + kubernetes.core.k8s_info: + kubeconfig: "{{ faults_cluster.kubeconfig }}" + api_version: v1 + kind: Pod + namespace: "{{ faults_kubernetes_object.metadata.namespace }}" + label_selectors: + - "app={{ faults_kubernetes_object.metadata.name }}" + register: faults_stalled_controller_pods + until: + - >- + faults_stalled_controller_pods.resources | + selectattr('status.phase', 'equalto', 'Running') | + list | + length == 0 + retries: 10 + delay: 10 diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_115/groundtruth.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_115/groundtruth.yaml new file mode 100644 index 000000000..242d91040 --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_115/groundtruth.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: itbench.io/v2 +kind: GroundTruth +metadata: + name: scenario-115 +spec: + alerts: + - labels: {} + name: KubePodCrashLooping + - labels: {} + name: KubePodNotReady + entities: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: recommendation + namespace: otel-demo + solutions: + - - steps: + - command: kubectl -n otel-demo rollout undo deployment/recommendation + text: Revert the last change done to the controller manifest. + - steps: + - command: kubectl -n otel-demo edit deployment recommendation + text: Manually edit the manifest and replace the container command with the correct value. + - steps: + - command: kubectl -n otel-demo logs -l app=recommendation -c recommendation --previous + text: Check the previous container logs to understand the crash reason. diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_115/groundtruth_v1.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_115/groundtruth_v1.yaml new file mode 100644 index 000000000..84f49a245 --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_115/groundtruth_v1.yaml @@ -0,0 +1,87 @@ +--- +apiVersion: itbench.io/v1 +kind: GroundTruth +metadata: + name: scenario-115 +spec: + alerts: + - group_id: recommendation-pod-1 + id: KubePodCrashLooping + metadata: + description: Pod recommendation in namespace otel-demo is restarting frequently due to invalid container command + - group_id: recommendation-pod-1 + id: KubePodNotReady + metadata: + description: Pod recommendation in namespace otel-demo has been in a non-ready state for longer than 15 minutes + - group_id: frontend-service-1 + id: HighRequestErrorRate + metadata: + description: Request error rate in service frontend in namespace otel-demo is elevated due to recommendation service unavailability + groups: + - id: recommendation-deployment-1 + kind: Deployment + namespace: otel-demo + filter: + - recommendation\b + root_cause: true + - id: recommendation-pod-1 + kind: Pod + namespace: otel-demo + filter: + - recommendation-.* + - id: recommendation-service-1 + kind: Service + namespace: otel-demo + filter: + - recommendation\b + - id: frontend-service-1 + kind: Service + namespace: otel-demo + filter: + - frontend\b + - id: frontend-pod-1 + kind: Pod + namespace: otel-demo + filter: + - frontend-.* + aliases: + - - recommendation-deployment-1 + - recommendation-pod-1 + - - recommendation-service-1 + - recommendation-pod-1 + - - frontend-service-1 + - frontend-pod-1 + propagations: + - source: recommendation-deployment-1 + target: recommendation-pod-1 + condition: Deployment container command modified to invalid value + effect: Pod enters CrashLoopBackOff as container fails to start with invalid entrypoint + - source: recommendation-pod-1 + target: recommendation-service-1 + condition: Recommendation pod is in CrashLoopBackOff with no healthy containers + effect: Recommendation service has no ready endpoints and cannot serve requests + - source: recommendation-service-1 + target: frontend-service-1 + condition: Frontend service depends on recommendation service for product recommendations + effect: Frontend experiences errors when calling recommendation service, increasing error rate + fault: + - category: Modify + condition: Container command replaced with invalid value causing immediate crash on startup + entity: + group_id: recommendation-deployment-1 + kind: Deployment + name: recommendation + fault_mechanism: Container entrypoint corruption simulating controller crash due to bad configuration push or incompatible upgrade + recommendedActions: + - solution: + actions: + - Rollback the deployment using kubectl rollout undo deployment/recommendation -n otel-demo + - Verify pods return to Running state + - Confirm recommendation service has healthy endpoints + id: rollback-deployment + - solution: + actions: + - Check pod logs with kubectl logs -c recommendation --previous to identify crash reason + - Edit the deployment and fix the container command + - Restart the deployment and verify recovery + id: fix-container-command diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_115/scenario.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_115/scenario.yaml new file mode 100644 index 000000000..cb48b8cf8 --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_115/scenario.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: itbench.io/v2 +kind: Scenario +metadata: + name: scenario-115 +spec: + applications: + opentelemetryDemo: + enabled: true + faults: + - injections: + - args: + container: + name: recommendation + kubernetesObject: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: recommendation + namespace: otel-demo + id: crashing-kubernetes-controller-workload + tools: + sre: + enabled: true diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_116/groundtruth.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_116/groundtruth.yaml new file mode 100644 index 000000000..79261568e --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_116/groundtruth.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: itbench.io/v2 +kind: GroundTruth +metadata: + name: scenario-116 +spec: + alerts: + - labels: {} + name: KubePodNotReady + entities: + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: etcd + namespace: otel-demo + solutions: + - - steps: + - text: Compact and defragment the etcd database to reclaim storage space. + - text: Remove unnecessary resources (completed jobs, orphaned ConfigMaps) to reduce etcd storage usage. + - steps: + - text: Increase the etcd storage quota if the current limit is too low for the cluster workload. diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_116/groundtruth_v1.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_116/groundtruth_v1.yaml new file mode 100644 index 000000000..3c2fdf914 --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_116/groundtruth_v1.yaml @@ -0,0 +1,60 @@ +--- +apiVersion: itbench.io/v1 +kind: GroundTruth +metadata: + name: scenario-116 +spec: + alerts: + - group_id: etcd-statefulset-1 + id: KubePodNotReady + metadata: + description: etcd pod in namespace otel-demo is experiencing storage pressure due to excessive ConfigMap creation + groups: + - id: etcd-statefulset-1 + kind: StatefulSet + namespace: otel-demo + filter: + - etcd\b + root_cause: true + - id: etcd-pod-1 + kind: Pod + namespace: otel-demo + filter: + - etcd-.* + - id: configmap-fill-1 + kind: ConfigMap + namespace: otel-demo + filter: + - etcd-fill-.* + aliases: + - - etcd-statefulset-1 + - etcd-pod-1 + propagations: + - source: configmap-fill-1 + target: etcd-statefulset-1 + condition: Large number of ConfigMaps created consuming etcd storage beyond quota threshold + effect: etcd database approaches storage quota limit, API server begins rejecting mutating requests + - source: etcd-statefulset-1 + target: etcd-pod-1 + condition: etcd storage quota exceeded or nearly exceeded + effect: etcd pods report high storage usage, cluster operations degrade as writes are rejected + fault: + - category: Fill + condition: etcd database filled with large ConfigMaps pushing storage past quota threshold + entity: + group_id: configmap-fill-1 + kind: ConfigMap + name: etcd-fill + fault_mechanism: Storage exhaustion via bulk ConfigMap creation simulating real-world etcd quota incidents + recommendedActions: + - solution: + actions: + - Delete the etcd-fill ConfigMaps using kubectl delete configmap -l app=etcd-storage-fill -n otel-demo + - Compact and defragment etcd to reclaim space + - Monitor etcd storage usage returning to normal levels + id: cleanup-configmaps + - solution: + actions: + - Increase etcd storage quota if current limit is insufficient + - Review resource creation patterns to prevent future exhaustion + id: increase-quota diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_116/scenario.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_116/scenario.yaml new file mode 100644 index 000000000..a7bd1a05b --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_116/scenario.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: itbench.io/v2 +kind: Scenario +metadata: + name: scenario-116 +spec: + applications: + opentelemetryDemo: + enabled: true + faults: + - injections: + - args: + kubernetesObject: + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: etcd + namespace: otel-demo + id: exhausted-etcd-database-storage + tools: + sre: + enabled: true diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_117/groundtruth.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_117/groundtruth.yaml new file mode 100644 index 000000000..bcf4bce87 --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_117/groundtruth.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: itbench.io/v2 +kind: GroundTruth +metadata: + name: scenario-117 +spec: + alerts: + - labels: {} + name: KubePodNotReady + entities: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: kafka + namespace: otel-demo + solutions: + - - steps: + - command: kubectl -n otel-demo scale deployment/kafka --replicas=1 + text: Scale the pipeline controller deployment back to at least 1 replica. + - steps: + - command: kubectl -n otel-demo rollout restart deployment/kafka + text: Restart the pipeline controller deployment to recover from a stuck state. diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_117/groundtruth_v1.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_117/groundtruth_v1.yaml new file mode 100644 index 000000000..c8078c7cf --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_117/groundtruth_v1.yaml @@ -0,0 +1,103 @@ +--- +apiVersion: itbench.io/v1 +kind: GroundTruth +metadata: + name: scenario-117 +spec: + alerts: + - group_id: kafka-pod-1 + id: KubePodNotReady + metadata: + description: Kafka pod in namespace otel-demo has been terminated due to deployment scaled to zero replicas + - group_id: accounting-service-1 + id: NoRequestsReceived + metadata: + description: Accounting service receives no events as Kafka message broker is unavailable + - group_id: fraud-service-1 + id: NoRequestsReceived + metadata: + description: Fraud detection service receives no events as Kafka message broker is unavailable + groups: + - id: kafka-deployment-1 + kind: Deployment + namespace: otel-demo + filter: + - kafka\b + root_cause: true + - id: kafka-pod-1 + kind: Pod + namespace: otel-demo + filter: + - kafka-.* + - id: kafka-service-1 + kind: Service + namespace: otel-demo + filter: + - kafka\b + - id: accounting-service-1 + kind: Service + namespace: otel-demo + filter: + - accounting\b + - id: accounting-pod-1 + kind: Pod + namespace: otel-demo + filter: + - accounting-.* + - id: fraud-service-1 + kind: Service + namespace: otel-demo + filter: + - fraud.* + - id: fraud-pod-1 + kind: Pod + namespace: otel-demo + filter: + - fraud.* + aliases: + - - kafka-deployment-1 + - kafka-pod-1 + - - kafka-service-1 + - kafka-pod-1 + - - accounting-service-1 + - accounting-pod-1 + - - fraud-service-1 + - fraud-pod-1 + propagations: + - source: kafka-deployment-1 + target: kafka-pod-1 + condition: Kafka deployment scaled to zero replicas simulating controller failure + effect: All Kafka pods are terminated, message broker becomes completely unavailable + - source: kafka-pod-1 + target: kafka-service-1 + condition: No Kafka pods running to serve as endpoints + effect: Kafka service has no ready endpoints, all connections to the message broker fail + - source: kafka-service-1 + target: accounting-service-1 + condition: Accounting service consumes events from Kafka for order processing + effect: Accounting service cannot consume events, order processing stalls + - source: kafka-service-1 + target: fraud-service-1 + condition: Fraud detection service consumes events from Kafka for transaction analysis + effect: Fraud detection service cannot consume events, transaction monitoring stalls + fault: + - category: Scale + condition: Deployment replicas set to zero causing complete service unavailability + entity: + group_id: kafka-deployment-1 + kind: Deployment + name: kafka + fault_mechanism: Controller unavailability via zero-replica scaling simulating pipeline controller failure + recommendedActions: + - solution: + actions: + - Scale Kafka deployment back using kubectl scale deployment/kafka --replicas=1 -n otel-demo + - Verify Kafka pods are running and ready + - Confirm downstream services resume event processing + id: restore-replicas + - solution: + actions: + - Restart Kafka deployment using kubectl rollout restart deployment/kafka -n otel-demo + - Monitor pod status for successful startup + - Verify event consumers reconnect successfully + id: restart-deployment diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_117/scenario.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_117/scenario.yaml new file mode 100644 index 000000000..fe5054000 --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_117/scenario.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: itbench.io/v2 +kind: Scenario +metadata: + name: scenario-117 +spec: + applications: + opentelemetryDemo: + enabled: true + faults: + - injections: + - args: + kubernetesObject: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: kafka + namespace: otel-demo + id: stalled-pipeline-controller + tools: + sre: + enabled: true diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_118/groundtruth.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_118/groundtruth.yaml new file mode 100644 index 000000000..fa208097f --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_118/groundtruth.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: itbench.io/v2 +kind: GroundTruth +metadata: + name: scenario-118 +spec: + alerts: + - labels: {} + name: KubePodNotReady + - labels: {} + name: HighRequestErrorRate + entities: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: checkout + namespace: otel-demo + solutions: + - - steps: + - command: kubectl get nodes + text: Identify which worker nodes are cordoned. + - text: Uncordon all affected worker nodes to restore scheduling. + - steps: + - text: Investigate the root cause of the cluster unavailability (etcd health, API server logs, node conditions) before uncordoning. diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_118/groundtruth_v1.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_118/groundtruth_v1.yaml new file mode 100644 index 000000000..aa7922d3e --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_118/groundtruth_v1.yaml @@ -0,0 +1,83 @@ +--- +apiVersion: itbench.io/v1 +kind: GroundTruth +metadata: + name: scenario-118 +spec: + alerts: + - group_id: checkout-pod-1 + id: KubePodNotReady + metadata: + description: Pod checkout in namespace otel-demo has been in a non-ready state due to nodes being cordoned and pods evicted to Pending + - group_id: frontend-service-1 + id: HighRequestErrorRate + metadata: + description: Request error rate in service frontend in namespace otel-demo is elevated due to checkout service unavailability + groups: + - id: checkout-deployment-1 + kind: Deployment + namespace: otel-demo + filter: + - checkout\b + root_cause: true + - id: checkout-pod-1 + kind: Pod + namespace: otel-demo + filter: + - checkout-.* + - id: checkout-service-1 + kind: Service + namespace: otel-demo + filter: + - checkout\b + - id: frontend-service-1 + kind: Service + namespace: otel-demo + filter: + - frontend\b + - id: frontend-pod-1 + kind: Pod + namespace: otel-demo + filter: + - frontend-.* + aliases: + - - checkout-deployment-1 + - checkout-pod-1 + - - checkout-service-1 + - checkout-pod-1 + - - frontend-service-1 + - frontend-pod-1 + propagations: + - source: checkout-deployment-1 + target: checkout-pod-1 + condition: Nodes cordoned causing checkout pods to be evicted + effect: Checkout pods stuck in Pending state with no schedulable nodes available + - source: checkout-pod-1 + target: checkout-service-1 + condition: Checkout pods are in Pending state with no healthy containers + effect: Checkout service has no ready endpoints and cannot serve requests + - source: checkout-service-1 + target: frontend-service-1 + condition: Frontend service depends on checkout service for order processing + effect: Frontend experiences errors when calling checkout service, increasing error rate + fault: + - category: Cordon + condition: Nodes cordoned causing pod eviction and inability to reschedule + entity: + group_id: checkout-deployment-1 + kind: Deployment + name: checkout + fault_mechanism: Cluster availability loss via node cordoning simulating node drain or maintenance event leaving pods unschedulable + recommendedActions: + - solution: + actions: + - Identify cordoned nodes using kubectl get nodes and check for SchedulingDisabled status + - Uncordon the affected nodes using kubectl uncordon + - Verify checkout pods transition from Pending to Running state + - Confirm checkout service has healthy endpoints and frontend errors resolve + id: uncordon-nodes + - solution: + actions: + - Add additional nodes to the cluster to provide scheduling capacity + - Verify checkout pods are scheduled and running on new nodes + id: add-cluster-capacity diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_118/scenario.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_118/scenario.yaml new file mode 100644 index 000000000..9a4bc5e05 --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_118/scenario.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: itbench.io/v2 +kind: Scenario +metadata: + name: scenario-118 +spec: + applications: + opentelemetryDemo: + enabled: true + faults: + - injections: + - args: + kubernetesObject: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: checkout + namespace: otel-demo + id: cluster-availability-loss + tools: + sre: + enabled: true diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_119/groundtruth.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_119/groundtruth.yaml new file mode 100644 index 000000000..04ab80ecd --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_119/groundtruth.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: itbench.io/v2 +kind: GroundTruth +metadata: + name: scenario-119 +spec: + alerts: + - labels: {} + name: HighRequestErrorRate + entities: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: cart + namespace: otel-demo + solutions: + - - steps: + - command: kubectl -n otel-demo rollout undo deployment/cart + text: Revert the last change done to the workload manifest. + - steps: + - command: kubectl -n otel-demo edit deployment cart + text: Manually edit the manifest and replace the environment variable with the correct value. diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_119/groundtruth_v1.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_119/groundtruth_v1.yaml new file mode 100644 index 000000000..581891100 --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_119/groundtruth_v1.yaml @@ -0,0 +1,80 @@ +--- +apiVersion: itbench.io/v1 +kind: GroundTruth +metadata: + name: scenario-119 +spec: + alerts: + - group_id: frontend-service-1 + id: HighRequestErrorRate + metadata: + description: Request error rate in service frontend in namespace otel-demo is elevated due to cart service returning errors from corrupted configuration + groups: + - id: cart-deployment-1 + kind: Deployment + namespace: otel-demo + filter: + - cart\b + root_cause: true + - id: cart-pod-1 + kind: Pod + namespace: otel-demo + filter: + - cart-.* + - id: cart-service-1 + kind: Service + namespace: otel-demo + filter: + - cart\b + - id: frontend-service-1 + kind: Service + namespace: otel-demo + filter: + - frontend\b + - id: frontend-pod-1 + kind: Pod + namespace: otel-demo + filter: + - frontend-.* + aliases: + - - cart-deployment-1 + - cart-pod-1 + - - cart-service-1 + - cart-pod-1 + - - frontend-service-1 + - frontend-pod-1 + propagations: + - source: cart-deployment-1 + target: cart-pod-1 + condition: Environment variable OTEL_EXPORTER_OTLP_ENDPOINT set to nonexistent collector address + effect: Cart pod starts but experiences errors when attempting to export telemetry to invalid endpoint + - source: cart-pod-1 + target: cart-service-1 + condition: Cart pod is running but returning error responses due to corrupted configuration + effect: Cart service returns errors to callers on cart operations + - source: cart-service-1 + target: frontend-service-1 + condition: Frontend service depends on cart service for shopping cart functionality + effect: Frontend experiences errors when calling cart service, increasing error rate + fault: + - category: Modify + condition: Environment variable OTEL_EXPORTER_OTLP_ENDPOINT corrupted to point to nonexistent collector + entity: + group_id: cart-deployment-1 + kind: Deployment + name: cart + fault_mechanism: Configuration corruption via environment variable modification simulating bad configuration push or misconfigured deployment + recommendedActions: + - solution: + actions: + - Inspect cart deployment environment variables using kubectl describe deployment/cart -n otel-demo + - Identify the corrupted OTEL_EXPORTER_OTLP_ENDPOINT value + - Fix the environment variable to point to the correct collector endpoint + - Restart the deployment and verify cart service recovers + id: fix-environment-variable + - solution: + actions: + - Rollback the deployment using kubectl rollout undo deployment/cart -n otel-demo + - Verify cart pods return to healthy state + - Confirm frontend error rate returns to normal + id: rollback-deployment diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_119/scenario.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_119/scenario.yaml new file mode 100644 index 000000000..c04c76b65 --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_119/scenario.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: itbench.io/v2 +kind: Scenario +metadata: + name: scenario-119 +spec: + applications: + opentelemetryDemo: + enabled: true + faults: + - injections: + - args: + container: + name: cart + kubernetesObject: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: cart + namespace: otel-demo + id: breaking-configuration-change + tools: + sre: + enabled: true diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_120/groundtruth.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_120/groundtruth.yaml new file mode 100644 index 000000000..c127cd5ae --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_120/groundtruth.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: itbench.io/v2 +kind: GroundTruth +metadata: + name: scenario-120 +spec: + alerts: + - labels: {} + name: HighRequestErrorRate + entities: + - apiVersion: v1 + kind: Secret + metadata: + name: email-service-secret + namespace: otel-demo + solutions: + - - steps: + - command: kubectl -n otel-demo get secret email-service-secret -o yaml + text: Inspect the corrupted secret to identify the invalid data. + - text: Restore the secret with correct credentials from the credential management system. + - steps: + - text: Rotate the credentials in the external system and update the Kubernetes Secret to match. diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_120/groundtruth_v1.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_120/groundtruth_v1.yaml new file mode 100644 index 000000000..2c8535826 --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_120/groundtruth_v1.yaml @@ -0,0 +1,78 @@ +--- +apiVersion: itbench.io/v1 +kind: GroundTruth +metadata: + name: scenario-120 +spec: + alerts: + - group_id: checkout-service-1 + id: HighRequestErrorRate + metadata: + description: Request error rate in service checkout in namespace otel-demo is elevated due to email service authentication failures from corrupted secret + groups: + - id: email-secret-1 + kind: Secret + namespace: otel-demo + filter: + - email-service-secret\b + root_cause: true + - id: email-pod-1 + kind: Pod + namespace: otel-demo + filter: + - email-.* + - id: email-service-1 + kind: Service + namespace: otel-demo + filter: + - email\b + - id: checkout-service-1 + kind: Service + namespace: otel-demo + filter: + - checkout\b + - id: checkout-pod-1 + kind: Pod + namespace: otel-demo + filter: + - checkout-.* + aliases: + - - email-service-1 + - email-pod-1 + - - checkout-service-1 + - checkout-pod-1 + propagations: + - source: email-secret-1 + target: email-pod-1 + condition: Secret email-service-secret credentials replaced with invalid values + effect: Email pod experiences authentication failures when connecting to email backend + - source: email-pod-1 + target: email-service-1 + condition: Email pod cannot authenticate and returns errors + effect: Email service is effectively unavailable, all requests fail with authentication errors + - source: email-service-1 + target: checkout-service-1 + condition: Checkout service depends on email service to send order confirmation emails + effect: Checkout service experiences errors when attempting to send order emails, increasing error rate + fault: + - category: Modify + condition: Secret credentials replaced with invalid values causing authentication failures + entity: + group_id: email-secret-1 + kind: Secret + name: email-service-secret + fault_mechanism: Credential corruption via secret modification simulating accidental secret rotation with wrong values or vault sync failure + recommendedActions: + - solution: + actions: + - Inspect the email-service-secret using kubectl get secret email-service-secret -n otel-demo -o yaml + - Restore the correct credentials from backup or secret management system + - Restart email pods to pick up the corrected secret values + - Verify email service recovers and checkout errors resolve + id: restore-secret-credentials + - solution: + actions: + - Rotate the credentials by generating new valid values in both the secret and the email backend + - Apply the updated secret and restart email deployment + - Confirm end-to-end order email flow works correctly + id: rotate-credentials diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_120/scenario.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_120/scenario.yaml new file mode 100644 index 000000000..a5ae3e922 --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_120/scenario.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: itbench.io/v2 +kind: Scenario +metadata: + name: scenario-120 +spec: + applications: + opentelemetryDemo: + enabled: true + faults: + - injections: + - args: + kubernetesObject: + apiVersion: v1 + kind: Secret + metadata: + name: email-service-secret + namespace: otel-demo + id: corrupted-kubernetes-secret-credentials + tools: + sre: + enabled: true diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_121/groundtruth.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_121/groundtruth.yaml new file mode 100644 index 000000000..1a6619668 --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_121/groundtruth.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: itbench.io/v2 +kind: GroundTruth +metadata: + name: scenario-121 +spec: + alerts: + - labels: {} + name: KubePodNotReady + entities: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: product-catalog + namespace: otel-demo + solutions: + - - steps: + - command: kubectl -n otel-demo rollout undo deployment/product-catalog + text: Revert the resource request changes to restore normal resource usage. + - steps: + - command: kubectl -n otel-demo edit deployment product-catalog + text: Manually edit the manifest and set resource requests to appropriate values. diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_121/groundtruth_v1.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_121/groundtruth_v1.yaml new file mode 100644 index 000000000..63653fa33 --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_121/groundtruth_v1.yaml @@ -0,0 +1,84 @@ +--- +apiVersion: itbench.io/v1 +kind: GroundTruth +metadata: + name: scenario-121 +spec: + alerts: + - group_id: product-catalog-pod-1 + id: KubePodNotReady + metadata: + description: Pod product-catalog in namespace otel-demo has been in a non-ready state due to excessive resource requests preventing scheduling + - group_id: frontend-service-1 + id: HighRequestErrorRate + metadata: + description: Request error rate in service frontend in namespace otel-demo is elevated due to product-catalog service unavailability + groups: + - id: product-catalog-deployment-1 + kind: Deployment + namespace: otel-demo + filter: + - product-catalog\b + root_cause: true + - id: product-catalog-pod-1 + kind: Pod + namespace: otel-demo + filter: + - product-catalog-.* + - id: product-catalog-service-1 + kind: Service + namespace: otel-demo + filter: + - product-catalog\b + - id: frontend-service-1 + kind: Service + namespace: otel-demo + filter: + - frontend\b + - id: frontend-pod-1 + kind: Pod + namespace: otel-demo + filter: + - frontend-.* + aliases: + - - product-catalog-deployment-1 + - product-catalog-pod-1 + - - product-catalog-service-1 + - product-catalog-pod-1 + - - frontend-service-1 + - frontend-pod-1 + propagations: + - source: product-catalog-deployment-1 + target: product-catalog-pod-1 + condition: Deployment resource requests set to 64Gi memory and 32 CPU exceeding all node capacities + effect: Product-catalog pod stuck in Pending state as no node has sufficient resources to schedule it + - source: product-catalog-pod-1 + target: product-catalog-service-1 + condition: Product-catalog pod is in Pending state with no healthy containers + effect: Product-catalog service has no ready endpoints and cannot serve requests + - source: product-catalog-service-1 + target: frontend-service-1 + condition: Frontend service depends on product-catalog service for product listings and details + effect: Frontend experiences errors when calling product-catalog service, increasing error rate + fault: + - category: Modify + condition: Resource requests set to excessive values (64Gi memory, 32 CPU) preventing pod scheduling + entity: + group_id: product-catalog-deployment-1 + kind: Deployment + name: product-catalog + fault_mechanism: Node resource exhaustion via excessive resource requests simulating misconfigured resource limits or capacity planning failure + recommendedActions: + - solution: + actions: + - Check pod events using kubectl describe pod -l app=product-catalog -n otel-demo for scheduling failures + - Identify the excessive resource requests in the deployment spec + - Reduce resource requests to reasonable values (e.g., 256Mi memory, 250m CPU) + - Verify product-catalog pod transitions to Running and service endpoints become available + id: fix-resource-requests + - solution: + actions: + - Rollback the deployment using kubectl rollout undo deployment/product-catalog -n otel-demo + - Verify pod is scheduled and running with previous resource configuration + - Confirm frontend error rate returns to normal + id: rollback-deployment diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_121/scenario.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_121/scenario.yaml new file mode 100644 index 000000000..8a7191fad --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_121/scenario.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: itbench.io/v2 +kind: Scenario +metadata: + name: scenario-121 +spec: + applications: + opentelemetryDemo: + enabled: true + faults: + - injections: + - args: + kubernetesObject: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: product-catalog + namespace: otel-demo + id: node-resource-exhaustion + tools: + sre: + enabled: true diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_122/groundtruth.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_122/groundtruth.yaml new file mode 100644 index 000000000..ae8481d4b --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_122/groundtruth.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: itbench.io/v2 +kind: GroundTruth +metadata: + name: scenario-122 +spec: + alerts: + - labels: {} + name: HighRequestErrorRate + entities: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: shipping + namespace: otel-demo + solutions: + - - steps: + - command: kubectl -n otel-demo rollout undo deployment/shipping + text: Revert the deployment to the previous working image. + - steps: + - command: kubectl -n otel-demo set image deployment/shipping shipping=correct-image:latest + text: Set the container image to the correct version. diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_122/groundtruth_v1.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_122/groundtruth_v1.yaml new file mode 100644 index 000000000..b5ad90a27 --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_122/groundtruth_v1.yaml @@ -0,0 +1,88 @@ +--- +apiVersion: itbench.io/v1 +kind: GroundTruth +metadata: + name: scenario-122 +spec: + alerts: + - group_id: shipping-pod-1 + id: KubePodNotReady + metadata: + description: Pod shipping in namespace otel-demo has been in a non-ready state due to container image pull failure (ImagePullBackOff) + - group_id: shipping-pod-1 + id: KubeContainerWaiting + metadata: + description: Container in pod shipping in namespace otel-demo is stuck in Waiting state due to ImagePullBackOff + - group_id: checkout-service-1 + id: HighRequestErrorRate + metadata: + description: Request error rate in service checkout in namespace otel-demo is elevated due to shipping service unavailability + groups: + - id: shipping-deployment-1 + kind: Deployment + namespace: otel-demo + filter: + - shipping\b + root_cause: true + - id: shipping-pod-1 + kind: Pod + namespace: otel-demo + filter: + - shipping-.* + - id: shipping-service-1 + kind: Service + namespace: otel-demo + filter: + - shipping\b + - id: checkout-service-1 + kind: Service + namespace: otel-demo + filter: + - checkout\b + - id: checkout-pod-1 + kind: Pod + namespace: otel-demo + filter: + - checkout-.* + aliases: + - - shipping-deployment-1 + - shipping-pod-1 + - - shipping-service-1 + - shipping-pod-1 + - - checkout-service-1 + - checkout-pod-1 + propagations: + - source: shipping-deployment-1 + target: shipping-pod-1 + condition: Container image set to nonexistent image tag or registry path + effect: Shipping pod enters ImagePullBackOff as kubelet cannot pull the specified container image + - source: shipping-pod-1 + target: shipping-service-1 + condition: Shipping pod is stuck in ImagePullBackOff with no running containers + effect: Shipping service has no ready endpoints and cannot serve requests + - source: shipping-service-1 + target: checkout-service-1 + condition: Checkout service depends on shipping service for shipping cost calculation and order fulfillment + effect: Checkout service experiences errors when calling shipping service, increasing error rate + fault: + - category: Modify + condition: Container image changed to nonexistent image causing ImagePullBackOff + entity: + group_id: shipping-deployment-1 + kind: Deployment + name: shipping + fault_mechanism: Failed release pipeline via invalid container image simulating bad CI/CD deployment with wrong image tag or deleted registry artifact + recommendedActions: + - solution: + actions: + - Check pod events using kubectl describe pod -l app=shipping -n otel-demo for image pull errors + - Identify the incorrect container image in the deployment spec + - Update the container image to the correct tag using kubectl set image deployment/shipping shipping= -n otel-demo + - Verify shipping pod transitions to Running state + id: fix-container-image + - solution: + actions: + - Rollback the deployment using kubectl rollout undo deployment/shipping -n otel-demo + - Verify shipping pod starts with the previous working image + - Confirm checkout service error rate returns to normal + id: rollback-deployment diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_122/scenario.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_122/scenario.yaml new file mode 100644 index 000000000..6efddcdbd --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_122/scenario.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: itbench.io/v2 +kind: Scenario +metadata: + name: scenario-122 +spec: + applications: + opentelemetryDemo: + enabled: true + faults: + - injections: + - args: + container: + name: shipping + kubernetesObject: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: shipping + namespace: otel-demo + id: failed-release-pipeline-service + tools: + sre: + enabled: true diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_123/groundtruth.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_123/groundtruth.yaml new file mode 100644 index 000000000..a1ca54c80 --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_123/groundtruth.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: itbench.io/v2 +kind: GroundTruth +metadata: + name: scenario-123 +spec: + alerts: + - labels: {} + name: KubePodNotReady + entities: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: load-generator + namespace: otel-demo + solutions: + - - steps: + - command: kubectl -n otel-demo scale deployment/load-generator --replicas=1 + text: Scale the monitoring probe deployment back to at least 1 replica. + - steps: + - command: kubectl -n otel-demo rollout restart deployment/load-generator + text: Restart the monitoring probe deployment to recover from a stuck state. diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_123/groundtruth_v1.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_123/groundtruth_v1.yaml new file mode 100644 index 000000000..dfbb53e6c --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_123/groundtruth_v1.yaml @@ -0,0 +1,72 @@ +--- +apiVersion: itbench.io/v1 +kind: GroundTruth +metadata: + name: scenario-123 +spec: + alerts: + - group_id: load-generator-pod-1 + id: KubePodNotReady + metadata: + description: Pod load-generator in namespace otel-demo has been terminated due to deployment scaled to zero replicas + - group_id: frontend-service-1 + id: NoRequestsReceived + metadata: + description: Frontend service receives no synthetic traffic as load-generator is unavailable, creating a monitoring blind spot + groups: + - id: load-generator-deployment-1 + kind: Deployment + namespace: otel-demo + filter: + - load-generator\b + root_cause: true + - id: load-generator-pod-1 + kind: Pod + namespace: otel-demo + filter: + - load-generator-.* + - id: frontend-service-1 + kind: Service + namespace: otel-demo + filter: + - frontend\b + - id: frontend-pod-1 + kind: Pod + namespace: otel-demo + filter: + - frontend-.* + aliases: + - - load-generator-deployment-1 + - load-generator-pod-1 + - - frontend-service-1 + - frontend-pod-1 + propagations: + - source: load-generator-deployment-1 + target: load-generator-pod-1 + condition: Load-generator deployment scaled to zero replicas + effect: All load-generator pods are terminated, synthetic traffic generation stops completely + - source: load-generator-pod-1 + target: frontend-service-1 + condition: No load-generator pods running to send synthetic monitoring traffic + effect: Frontend service and all downstream services receive no synthetic requests, monitoring probes go silent and real issues may go undetected + fault: + - category: Scale + condition: Deployment replicas set to zero causing complete loss of synthetic monitoring traffic + entity: + group_id: load-generator-deployment-1 + kind: Deployment + name: load-generator + fault_mechanism: Monitoring probe failure via zero-replica scaling simulating accidental scale-down of synthetic traffic generator leaving services unmonitored + recommendedActions: + - solution: + actions: + - Scale load-generator deployment back using kubectl scale deployment/load-generator --replicas=1 -n otel-demo + - Verify load-generator pod starts and begins sending synthetic traffic + - Confirm monitoring dashboards and alerts resume receiving request data + id: restore-replicas + - solution: + actions: + - Restart load-generator deployment using kubectl rollout restart deployment/load-generator -n otel-demo + - Monitor pod status for successful startup + - Verify synthetic traffic is flowing to frontend and downstream services + id: restart-deployment diff --git a/scenarios/sre/project/roles/scenarios/files/scenario_123/scenario.yaml b/scenarios/sre/project/roles/scenarios/files/scenario_123/scenario.yaml new file mode 100644 index 000000000..e43970e4f --- /dev/null +++ b/scenarios/sre/project/roles/scenarios/files/scenario_123/scenario.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: itbench.io/v2 +kind: Scenario +metadata: + name: scenario-123 +spec: + applications: + opentelemetryDemo: + enabled: true + faults: + - injections: + - args: + kubernetesObject: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: load-generator + namespace: otel-demo + id: monitoring-probe-failure + tools: + sre: + enabled: true diff --git a/scenarios/sre/project/roles/scenarios/meta/argument_specs.yaml b/scenarios/sre/project/roles/scenarios/meta/argument_specs.yaml index 000c1f10b..4dcb7a64c 100644 --- a/scenarios/sre/project/roles/scenarios/meta/argument_specs.yaml +++ b/scenarios/sre/project/roles/scenarios/meta/argument_specs.yaml @@ -46,5 +46,14 @@ argument_specs: - 102 - 105 - 114 + - 115 + - 116 + - 117 + - 118 + - 119 + - 120 + - 121 + - 122 + - 123 required: true type: int diff --git a/schemas/json/faults/breaking-configuration-change.json b/schemas/json/faults/breaking-configuration-change.json new file mode 100644 index 000000000..6c1da702d --- /dev/null +++ b/schemas/json/faults/breaking-configuration-change.json @@ -0,0 +1,59 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "container": { + "properties": { + "name": { + "type": "string" + } + }, + "required": [ + "name" + ], + "type": "object" + }, + "kubernetesObject": { + "properties": { + "apiVersion": { + "enum": [ + "apps/v1" + ], + "type": "string" + }, + "kind": { + "enum": [ + "Deployment", + "StatefulSet" + ], + "type": "string" + }, + "metadata": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata" + ], + "type": "object" + } + }, + "required": [ + "kubernetesObject", + "container" + ], + "type": "object" +} diff --git a/schemas/json/faults/cluster-availability-loss.json b/schemas/json/faults/cluster-availability-loss.json new file mode 100644 index 000000000..e19e2a60d --- /dev/null +++ b/schemas/json/faults/cluster-availability-loss.json @@ -0,0 +1,47 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "kubernetesObject": { + "properties": { + "apiVersion": { + "enum": [ + "apps/v1" + ], + "type": "string" + }, + "kind": { + "enum": [ + "Deployment", + "StatefulSet" + ], + "type": "string" + }, + "metadata": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata" + ], + "type": "object" + } + }, + "required": [ + "kubernetesObject" + ], + "type": "object" +} diff --git a/schemas/json/faults/corrupted-kubernetes-secret-credentials.json b/schemas/json/faults/corrupted-kubernetes-secret-credentials.json new file mode 100644 index 000000000..fb99a3a78 --- /dev/null +++ b/schemas/json/faults/corrupted-kubernetes-secret-credentials.json @@ -0,0 +1,46 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "kubernetesObject": { + "properties": { + "apiVersion": { + "enum": [ + "v1" + ], + "type": "string" + }, + "kind": { + "enum": [ + "Secret" + ], + "type": "string" + }, + "metadata": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata" + ], + "type": "object" + } + }, + "required": [ + "kubernetesObject" + ], + "type": "object" +} diff --git a/schemas/json/faults/crashing-kubernetes-controller-workload.json b/schemas/json/faults/crashing-kubernetes-controller-workload.json new file mode 100644 index 000000000..6c1da702d --- /dev/null +++ b/schemas/json/faults/crashing-kubernetes-controller-workload.json @@ -0,0 +1,59 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "container": { + "properties": { + "name": { + "type": "string" + } + }, + "required": [ + "name" + ], + "type": "object" + }, + "kubernetesObject": { + "properties": { + "apiVersion": { + "enum": [ + "apps/v1" + ], + "type": "string" + }, + "kind": { + "enum": [ + "Deployment", + "StatefulSet" + ], + "type": "string" + }, + "metadata": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata" + ], + "type": "object" + } + }, + "required": [ + "kubernetesObject", + "container" + ], + "type": "object" +} diff --git a/schemas/json/faults/exhausted-etcd-database-storage.json b/schemas/json/faults/exhausted-etcd-database-storage.json new file mode 100644 index 000000000..29c1419f2 --- /dev/null +++ b/schemas/json/faults/exhausted-etcd-database-storage.json @@ -0,0 +1,46 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "kubernetesObject": { + "properties": { + "apiVersion": { + "enum": [ + "apps/v1" + ], + "type": "string" + }, + "kind": { + "enum": [ + "StatefulSet" + ], + "type": "string" + }, + "metadata": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata" + ], + "type": "object" + } + }, + "required": [ + "kubernetesObject" + ], + "type": "object" +} diff --git a/schemas/json/faults/failed-release-pipeline-service.json b/schemas/json/faults/failed-release-pipeline-service.json new file mode 100644 index 000000000..d7a9c4653 --- /dev/null +++ b/schemas/json/faults/failed-release-pipeline-service.json @@ -0,0 +1,58 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "container": { + "properties": { + "name": { + "type": "string" + } + }, + "required": [ + "name" + ], + "type": "object" + }, + "kubernetesObject": { + "properties": { + "apiVersion": { + "enum": [ + "apps/v1" + ], + "type": "string" + }, + "kind": { + "enum": [ + "Deployment" + ], + "type": "string" + }, + "metadata": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata" + ], + "type": "object" + } + }, + "required": [ + "kubernetesObject", + "container" + ], + "type": "object" +} diff --git a/schemas/json/faults/monitoring-probe-failure.json b/schemas/json/faults/monitoring-probe-failure.json new file mode 100644 index 000000000..5e86e8063 --- /dev/null +++ b/schemas/json/faults/monitoring-probe-failure.json @@ -0,0 +1,47 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "kubernetesObject": { + "properties": { + "apiVersion": { + "enum": [ + "apps/v1" + ], + "type": "string" + }, + "kind": { + "enum": [ + "Deployment", + "DaemonSet" + ], + "type": "string" + }, + "metadata": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata" + ], + "type": "object" + } + }, + "required": [ + "kubernetesObject" + ], + "type": "object" +} diff --git a/schemas/json/faults/node-resource-exhaustion.json b/schemas/json/faults/node-resource-exhaustion.json new file mode 100644 index 000000000..e19e2a60d --- /dev/null +++ b/schemas/json/faults/node-resource-exhaustion.json @@ -0,0 +1,47 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "kubernetesObject": { + "properties": { + "apiVersion": { + "enum": [ + "apps/v1" + ], + "type": "string" + }, + "kind": { + "enum": [ + "Deployment", + "StatefulSet" + ], + "type": "string" + }, + "metadata": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata" + ], + "type": "object" + } + }, + "required": [ + "kubernetesObject" + ], + "type": "object" +} diff --git a/schemas/json/faults/stalled-pipeline-controller.json b/schemas/json/faults/stalled-pipeline-controller.json new file mode 100644 index 000000000..4481f46a6 --- /dev/null +++ b/schemas/json/faults/stalled-pipeline-controller.json @@ -0,0 +1,46 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "kubernetesObject": { + "properties": { + "apiVersion": { + "enum": [ + "apps/v1" + ], + "type": "string" + }, + "kind": { + "enum": [ + "Deployment" + ], + "type": "string" + }, + "metadata": { + "properties": { + "name": { + "type": "string" + }, + "namespace": { + "type": "string" + } + }, + "required": [ + "name", + "namespace" + ], + "type": "object" + } + }, + "required": [ + "apiVersion", + "kind", + "metadata" + ], + "type": "object" + } + }, + "required": [ + "kubernetesObject" + ], + "type": "object" +}