From 86164e5f5464a66a801062f0408982260281ffe6 Mon Sep 17 00:00:00 2001 From: Jinsoo Heo Date: Thu, 20 Aug 2026 07:08:25 +0900 Subject: [PATCH] feat(validator): add optional startupProbe The chart renders only livenessProbe and readinessProbe, so a slow-starting validator can only be accommodated by raising liveness initialDelaySeconds, which also delays steady-state failure detection. A startupProbe separates the two: while it runs, liveness and readiness are suspended, and once it passes they behave normally. This is not hypothetical. On 2026-08-18 a prod ghost pod took 182s to serve /healthz against the chart default initialDelaySeconds of 60, collected six Unhealthy events and was killed 21s after it came up. The version it replaced took exactly 60s, i.e. it had been clearing the probe with no margin at all. Disabled by default, so existing releases render unchanged (verified: helm template with default values emits no startupProbe). Applies to both the ghost and ghost-vao deployments. Default budget when enabled is failureThreshold 30 x periodSeconds 10 = 5 minutes. --- charts/validator/Chart.yaml | 2 +- charts/validator/README.md | 5 +++-- charts/validator/templates/deployment-vao.yaml | 5 +++++ charts/validator/templates/deployment.yaml | 5 +++++ charts/validator/values.yaml | 16 ++++++++++++++++ 5 files changed, 30 insertions(+), 3 deletions(-) diff --git a/charts/validator/Chart.yaml b/charts/validator/Chart.yaml index f8af759..17dba13 100644 --- a/charts/validator/Chart.yaml +++ b/charts/validator/Chart.yaml @@ -22,7 +22,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.6.11" +version: "0.7.0" # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/validator/README.md b/charts/validator/README.md index 33bbcf2..629ce02 100644 --- a/charts/validator/README.md +++ b/charts/validator/README.md @@ -1,6 +1,6 @@ # validator -![Version: 0.6.11](https://img.shields.io/badge/Version-0.6.11-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.78.5](https://img.shields.io/badge/AppVersion-0.78.5-informational?style=flat-square) +![Version: 0.7.0](https://img.shields.io/badge/Version-0.7.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.78.5](https://img.shields.io/badge/AppVersion-0.78.5-informational?style=flat-square) A Helm chart for deploying Chronicle Validator on Kubernetes @@ -31,7 +31,7 @@ A Helm chart for deploying Chronicle Validator on Kubernetes | ghost.service.trafficDistribution | string | `""` | Optional Kubernetes Service traffic distribution hint, for example PreferSameZone. | | ghost.service.type | string | `"LoadBalancer"` | Type of service for the validator, only `LoadBalancer` supported for now | | ghost.watchdogConfigReg | string | `"0x94Fea534aef6df5cF66C2DAE5CE0A05d10C068F3"` | WATCHDOG onchain config address | -| global | object | `{"affinity":{},"chainId":1,"chainName":"eth","chainTxType":"eip1559","fullnameOverride":"ghost","image":{"pullPolicy":"Always","repository":"ghcr.io/chronicleprotocol/ghost","tag":"0.78.5@sha256:951d711620659f022ad662af5543f825631ec598551b38027c9e54ef748055b6"},"imagePullSecrets":[],"liveness":{"enabled":true,"livenessProbe":{"failureThreshold":4,"httpGet":{"path":"/healthz","port":9100},"initialDelaySeconds":60,"periodSeconds":60}},"logFormat":"text","logLevel":"info","metrics":{"enabled":true,"port":9090},"nameOverride":"","nodeSelector":{},"podAnnotations":{},"podSecurityContext":{},"readiness":{"enabled":true,"readinessProbe":{"failureThreshold":4,"httpGet":{"path":"/healthz","port":9100},"initialDelaySeconds":60,"periodSeconds":60}},"replicaCount":1,"resources":{},"securityContext":{},"serviceAccount":{"annotations":{},"create":true,"name":""},"tolerations":[],"topologySpreadConstraints":[]}` | Global values for the validator chart, values are used across the chart resources | +| global | object | `{"affinity":{},"chainId":1,"chainName":"eth","chainTxType":"eip1559","fullnameOverride":"ghost","image":{"pullPolicy":"Always","repository":"ghcr.io/chronicleprotocol/ghost","tag":"0.78.5@sha256:951d711620659f022ad662af5543f825631ec598551b38027c9e54ef748055b6"},"imagePullSecrets":[],"liveness":{"enabled":true,"livenessProbe":{"failureThreshold":4,"httpGet":{"path":"/healthz","port":9100},"initialDelaySeconds":60,"periodSeconds":60}},"logFormat":"text","logLevel":"info","metrics":{"enabled":true,"port":9090},"nameOverride":"","nodeSelector":{},"podAnnotations":{},"podSecurityContext":{},"readiness":{"enabled":true,"readinessProbe":{"failureThreshold":4,"httpGet":{"path":"/healthz","port":9100},"initialDelaySeconds":60,"periodSeconds":60}},"replicaCount":1,"resources":{},"securityContext":{},"serviceAccount":{"annotations":{},"create":true,"name":""},"startup":{"enabled":false,"startupProbe":{"failureThreshold":30,"httpGet":{"path":"/healthz","port":9100},"periodSeconds":10}},"tolerations":[],"topologySpreadConstraints":[]}` | Global values for the validator chart, values are used across the chart resources | | global.affinity | object | `{}` | pod Affinity spec applied validator | | global.chainId | int | `1` | chain id for the "target" or "main" chain we use for the validator. Can be mainnet ethereum `1` or sepolia ethereum `11155111` | | global.chainName | string | `"eth"` | chain name for the "target" or "main" chain we use for the validator | @@ -52,6 +52,7 @@ A Helm chart for deploying Chronicle Validator on Kubernetes | global.serviceAccount.annotations | object | `{}` | Annotations to add to the service account | | global.serviceAccount.create | bool | `true` | Specifies whether a service account should be created | | global.serviceAccount.name | string | `""` | If not set and create is true, a name is generated using the fullname template | +| global.startup | object | `{"enabled":false,"startupProbe":{"failureThreshold":30,"httpGet":{"path":"/healthz","port":9100},"periodSeconds":10}}` | Startup probe : give a slow-starting validator time to serve /healthz before the liveness probe can kill it. While this probe is running the liveness and readiness probes are suspended, so startup time and steady-state failure detection are tuned independently instead of trading off against each other. Disabled by default to keep existing releases unchanged; enable it for validators whose startup can exceed the liveness initialDelaySeconds. failureThreshold x periodSeconds is the startup budget (default 30 x 10s = 5 minutes). | | global.tolerations | list | `[]` | Tolerations applied validator | | global.topologySpreadConstraints | list | `[]` | Topology spread constraints applied to both ghost and vao pods | | serviceMonitor.annotations | object | `{}` | Additional ServiceMonitor annotations | diff --git a/charts/validator/templates/deployment-vao.yaml b/charts/validator/templates/deployment-vao.yaml index cb56a91..fa27586 100644 --- a/charts/validator/templates/deployment-vao.yaml +++ b/charts/validator/templates/deployment-vao.yaml @@ -68,6 +68,11 @@ spec: protocol: TCP {{- end }} + {{- if and .Values.global.startup .Values.global.startup.enabled }} + startupProbe: + {{- toYaml .Values.global.startup.startupProbe | nindent 12 }} + {{- end }} + {{- if .Values.global.liveness.enabled }} livenessProbe: {{- toYaml .Values.global.liveness.livenessProbe | nindent 12 }} diff --git a/charts/validator/templates/deployment.yaml b/charts/validator/templates/deployment.yaml index 36575aa..48291ef 100644 --- a/charts/validator/templates/deployment.yaml +++ b/charts/validator/templates/deployment.yaml @@ -70,6 +70,11 @@ spec: {{- end }} + {{- if and .Values.global.startup .Values.global.startup.enabled }} + startupProbe: + {{- toYaml .Values.global.startup.startupProbe | nindent 12 }} + {{- end }} + {{- if .Values.global.liveness.enabled }} livenessProbe: {{- toYaml .Values.global.liveness.livenessProbe | nindent 12 }} diff --git a/charts/validator/values.yaml b/charts/validator/values.yaml index 691495a..7bc0860 100644 --- a/charts/validator/values.yaml +++ b/charts/validator/values.yaml @@ -87,6 +87,22 @@ global: periodSeconds: 60 failureThreshold: 4 + # -- Startup probe : give a slow-starting validator time to serve /healthz before the + # liveness probe can kill it. While this probe is running the liveness and readiness + # probes are suspended, so startup time and steady-state failure detection are tuned + # independently instead of trading off against each other. + # Disabled by default to keep existing releases unchanged; enable it for validators + # whose startup can exceed the liveness initialDelaySeconds. + # failureThreshold x periodSeconds is the startup budget (default 30 x 10s = 5 minutes). + startup: + enabled: false + startupProbe: + httpGet: + path: /healthz + port: 9100 + periodSeconds: 10 + failureThreshold: 30 + # -- Metrics port for the validator, used by the service and the service monitor metrics: enabled: true