From 5b5cd4df95c6352038d33d9c142b45737c681e78 Mon Sep 17 00:00:00 2001 From: Kristiyan Ivanov Date: Fri, 9 Oct 2026 22:33:52 +0300 Subject: [PATCH 01/13] fix(provisioning): durable re-provisioning of existing tenants Converge by reading the live Deployment and replacing container env/resources wholesale rather than strategic-merge patching, which produced env entries carrying both value and valueFrom (422 Invalid). Backfill missing keys on an existing tenant Secret instead of failing when a tenant predates newer required keys. Ship entitlement v1.14.2. --- proprietary/entitlement/package.json | 2 +- .../src/provisioning/provisioning.service.ts | 180 +++++++++--------- .../infra/k8s/entitlement-deployment.yaml | 4 +- 3 files changed, 96 insertions(+), 90 deletions(-) diff --git a/proprietary/entitlement/package.json b/proprietary/entitlement/package.json index d8f8bee15..8c0eecb27 100644 --- a/proprietary/entitlement/package.json +++ b/proprietary/entitlement/package.json @@ -1,6 +1,6 @@ { "name": "@app/entitlement", - "version": "1.13.0", + "version": "1.14.2", "private": true, "scripts": { "dev": "nest start --watch", diff --git a/proprietary/entitlement/src/provisioning/provisioning.service.ts b/proprietary/entitlement/src/provisioning/provisioning.service.ts index ab7bac9a0..e606e8136 100644 --- a/proprietary/entitlement/src/provisioning/provisioning.service.ts +++ b/proprietary/entitlement/src/provisioning/provisioning.service.ts @@ -26,16 +26,6 @@ const VALKEY_NAME_PATTERN = /^[a-z][a-z0-9-]{1,38}[a-z0-9]$/; const DEMO_OBSERVABILITY_NAMESPACE = 'demo-observability'; const DEMO_OBSERVABILITY_PROMETHEUS_PORT = 9090; -// Env vars injected only for the demo tenant. On re-provision of a tenant that -// has been un-flagged (isDemo: false), these must be actively removed — a -// strategic-merge env patch only ever adds/updates by name — so the published -// token and OTLP mirror don't linger on a now-ordinary tenant. -const DEMO_ONLY_ENV = [ - 'DEMO_HOSTNAME', - 'PROMETHEUS_METRICS_PUBLIC_TOKEN', - 'OTEL_EXPORTER_OTLP_ENDPOINT', - 'OTEL_EVENTS_ENABLED', -]; // The public SNI host is an opaque, stable label derived from the instance id // rather than its name: this keeps it globally unique on the shared wildcard @@ -1098,51 +1088,69 @@ export class ProvisioningService { // published credential never breaks the demo-observability scrape config. const prometheusMetricsPublicToken = isDemo ? crypto.randomBytes(32).toString('hex') : undefined; + // The full key set the Deployment references (all as non-optional + // secretKeyRefs). Shared by the create and the re-provision backfill paths. + const stringData: Record = { + STORAGE_URL: storageUrl, + // Cloud auth secrets + CLOUD_MODE: 'true', + AUTH_PUBLIC_KEY: this.authPublicKey, + SESSION_SECRET: sessionSecret, + OTEL_INGEST_TOKEN: otelIngestToken, + PROMETHEUS_METRICS_TOKEN: prometheusMetricsToken, + ...(prometheusMetricsPublicToken + ? { PROMETHEUS_METRICS_PUBLIC_TOKEN: prometheusMetricsPublicToken } + : {}), + // Entitlement API config (for workspace management) + ENTITLEMENT_API_URL: this.entitlementApiUrl, + ENTITLEMENT_API_KEY: this.entitlementApiKey, + }; + try { await this.coreApi.createNamespacedSecret({ namespace, - body: { - metadata: { - name: 'db-credentials', - }, - type: 'Opaque', - stringData: { - STORAGE_URL: storageUrl, - // Cloud auth secrets - CLOUD_MODE: 'true', - AUTH_PUBLIC_KEY: this.authPublicKey, - SESSION_SECRET: sessionSecret, - OTEL_INGEST_TOKEN: otelIngestToken, - PROMETHEUS_METRICS_TOKEN: prometheusMetricsToken, - ...(prometheusMetricsPublicToken - ? { PROMETHEUS_METRICS_PUBLIC_TOKEN: prometheusMetricsPublicToken } - : {}), - // Entitlement API config (for workspace management) - ENTITLEMENT_API_URL: this.entitlementApiUrl, - ENTITLEMENT_API_KEY: this.entitlementApiKey, - }, - }, + body: { metadata: { name: 'db-credentials' }, type: 'Opaque', stringData }, }); } catch (error: any) { - if (this.isAlreadyExistsError(error)) { - this.logger.warn(`Secret db-credentials already exists in ${namespace}, continuing...`); - // A retry of a tenant whose secret predates the metrics-endpoint gate can - // reuse a Secret that lacks PROMETHEUS_METRICS_TOKEN. The Deployment - // created below references that key (non-optional), so without it the pod - // fails to start. Backfill it before we get there. - await this.ensureMetricsTokenInSecret(namespace); - if (isDemo) { - // Same backfill for the demo-only published token. - await this.ensureMetricsTokenInSecret(namespace, 'PROMETHEUS_METRICS_PUBLIC_TOKEN'); - } - // For a non-demo tenant the published token is pruned later, after - // createDeployment has settled a spec that no longer references it — - // dropping the Secret key here would leave the running Deployment's - // non-optional secretKeyRef pointing at a missing key. - } else { + if (!this.isAlreadyExistsError(error)) { throw error; } + // Re-provision: the Secret already exists. Backfill any keys it is + // missing. An older tenant's Secret can predate the current schema (e.g. + // lack CLOUD_MODE / AUTH_PUBLIC_KEY / SESSION_SECRET / ENTITLEMENT_API_*), + // and the Deployment references every key as a non-optional secretKeyRef — + // a missing one wedges the new pod in CreateContainerConfigError. Only ADD + // absent keys; never overwrite existing ones, so a running tenant's tokens + // and SESSION_SECRET are not rotated out from under it. (A non-demo + // tenant's stale PROMETHEUS_METRICS_PUBLIC_TOKEN is pruned later in + // createDeployment, once the spec no longer references it.) + this.logger.warn(`Secret db-credentials already exists in ${namespace}, backfilling missing keys...`); + await this.backfillSecretKeys(namespace, stringData); + } + } + + // Adds any of `desired`'s keys that are absent from the db-credentials Secret, + // leaving existing values untouched. Brings an older tenant's Secret up to the + // current key set on re-provision without rotating live credentials. + private async backfillSecretKeys(namespace: string, desired: Record): Promise { + const secret = await this.coreApi.readNamespacedSecret({ name: 'db-credentials', namespace }); + const present = new Set(Object.keys(secret.data ?? {})); + const missing: Record = {}; + for (const [key, value] of Object.entries(desired)) { + if (!present.has(key)) { + missing[key] = value; + } + } + if (Object.keys(missing).length === 0) { + return; } + await this.coreApi.patchNamespacedSecret( + { name: 'db-credentials', namespace, body: { stringData: missing } }, + k8s.setHeaderOptions('Content-Type', k8s.PatchStrategy.MergePatch), + ); + this.logger.log( + `Backfilled missing db-credentials keys in ${namespace}: ${Object.keys(missing).join(', ')}`, + ); } private async createResourceQuota( @@ -1400,48 +1408,46 @@ export class ProvisioningService { try { await this.appsApi.createNamespacedDeployment({ namespace, body }); } catch (error: any) { - if (this.isAlreadyExistsError(error)) { - // Converge an existing Deployment to the current spec instead of - // skipping: a warn-and-continue here left pre-existing tenants without - // later-introduced env/resources (e.g. the demo tenant's published - // metrics token and OTLP mirror), silently diverging from what - // provisioning reported. - // - // Strategic-merge PATCH scoped to the fields this provisioner owns — - // env, resources, replicas — rather than a full replace. It needs no - // resourceVersion (so there's no read→replace 409 race) and leaves - // everything else (pod-template annotations like the metrics-token - // restart marker, the running image) intact, so re-provision neither - // churns the pod nor rolls an out-of-band image back to the DB tag. - // env merges by name (new demo vars added); demo-only vars are - // explicitly deleted when the tenant isn't a demo so a published token - // can't linger. replicas is set back to the desired running count: a - // deprovision that scaled the app to 0 and then failed leaves the - // tenant in 'error' (re-provisionable) at replicas 0, and without this - // the converge would leave it there and waitForDeploymentReady would - // time out. - this.logger.log(`Deployment already exists in ${namespace}, converging app container`); - const container = body.spec!.template!.spec!.containers![0]; - const env: any[] = [...(container.env ?? [])]; - if (!isDemo) { - for (const name of DEMO_ONLY_ENV) { - env.push({ name, $patch: 'delete' }); + if (!this.isAlreadyExistsError(error)) { + throw error; + } + // Converge an existing Deployment to the current spec instead of + // skipping: a warn-and-continue here left pre-existing tenants without + // later-introduced env/resources (e.g. the demo tenant's published + // metrics token and OTLP mirror), silently diverging from what + // provisioning reported. + // + // Read-modify-replace, NOT a strategic-merge patch: a merge patch of the + // env list merges entries by name, so any env var whose source flips + // between `value` and `valueFrom` ends up carrying both, which the API + // rejects with a 422. Replacing the container's env wholesale on the live + // object avoids that, and naturally drops demo-only vars a now-non-demo + // tenant shouldn't keep (the desired env simply omits them). We touch + // only env, resources and replicas — the live image, pod-template + // annotations (e.g. the metrics-token restart marker) and probes are + // preserved. Retry on a 409 from a concurrent status write. + this.logger.log(`Deployment already exists in ${namespace}, converging app container`); + const desired = body.spec!.template!.spec!.containers![0]; + for (let attempt = 0; ; attempt++) { + const existing = await this.appsApi.readNamespacedDeployment({ name: 'betterdb', namespace }); + const c = existing.spec?.template?.spec?.containers?.find((x) => x.name === 'betterdb'); + if (!c) { + throw new Error(`betterdb container not found in ${namespace}`); + } + c.env = desired.env; + c.resources = desired.resources; + existing.spec!.replicas = body.spec!.replicas; + try { + await this.appsApi.replaceNamespacedDeployment({ name: 'betterdb', namespace, body: existing }); + break; + } catch (e: any) { + // isAlreadyExistsError also matches a 409 Conflict (stale + // resourceVersion) on replace; re-read and retry a few times. + if (this.isAlreadyExistsError(e) && attempt < 3) { + continue; } + throw e; } - const patchBody = { - spec: { - replicas: body.spec!.replicas, - template: { - spec: { containers: [{ name: container.name, env, resources: container.resources }] }, - }, - }, - } as unknown as k8s.V1Deployment; - await this.appsApi.patchNamespacedDeployment( - { name: 'betterdb', namespace, body: patchBody }, - k8s.setHeaderOptions('Content-Type', k8s.PatchStrategy.StrategicMergePatch), - ); - } else { - throw error; } } diff --git a/proprietary/infra/k8s/entitlement-deployment.yaml b/proprietary/infra/k8s/entitlement-deployment.yaml index bf2f66650..9d1ad58af 100644 --- a/proprietary/infra/k8s/entitlement-deployment.yaml +++ b/proprietary/infra/k8s/entitlement-deployment.yaml @@ -22,7 +22,7 @@ spec: serviceAccountName: entitlement containers: - name: entitlement - image: 811740411689.dkr.ecr.us-east-1.amazonaws.com/betterdb-entitlement:v1.13.0 + image: 811740411689.dkr.ecr.us-east-1.amazonaws.com/betterdb-entitlement:v1.14.2 imagePullPolicy: Always ports: - containerPort: 3002 @@ -48,7 +48,7 @@ spec: - name: ROUTE53_ZONE_ID value: "Z071377925BCBHU7WUE59" - name: DEFAULT_IMAGE_TAG - value: "v0.47.0" + value: "v0.49.1" - name: RESEND_API_KEY valueFrom: secretKeyRef: From b1dd3c7956ccb3c3c0ccaab435a217c428d630ae Mon Sep 17 00:00:00 2001 From: Kristiyan Ivanov Date: Fri, 9 Oct 2026 22:33:59 +0300 Subject: [PATCH 02/13] feat(demo-observability): clarify scrape vs OTLP, land populated Rename the scrape job betterdb -> betterdb-scrape so it is unambiguous next to the OTLP push series (job=betterdb-monitor); the demo tenant's exposition can take ~40s when it polls unreachable monitored DBs, so set scrape_interval 60s / scrape_timeout 45s. Land visitors directly on Instance Vitals via GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH, with the job variable defaulting to the push path. build-dashboards-configmap.sh applies that default to copies of the shipped pack without forking the JSON. --- .../infra/k8s/demo-observability/README.md | 12 +++---- .../build-dashboards-configmap.sh | 32 +++++++++++++++++++ .../infra/k8s/demo-observability/grafana.yaml | 10 ++++-- .../k8s/demo-observability/prometheus.yaml | 10 +++++- 4 files changed, 54 insertions(+), 10 deletions(-) create mode 100755 proprietary/infra/k8s/demo-observability/build-dashboards-configmap.sh diff --git a/proprietary/infra/k8s/demo-observability/README.md b/proprietary/infra/k8s/demo-observability/README.md index e55ffcdc2..60de58512 100644 --- a/proprietary/infra/k8s/demo-observability/README.md +++ b/proprietary/infra/k8s/demo-observability/README.md @@ -89,11 +89,13 @@ kubectl create secret generic grafana-admin -n demo-observability \ ### 4. Dashboards -The shipped pack is provisioned as-is — no fork of the JSON: +The shipped pack is provisioned with one demo-only override — the `job` +template variable defaults to `betterdb-monitor` (the OTLP push path) so +visitors land on a populated dashboard. The shipped JSON is not edited; the +override is applied to copies. See the script header for details: ```bash -kubectl create configmap grafana-dashboards -n demo-observability \ - --from-file=../../../../deploy/observability/dashboards/ +./build-dashboards-configmap.sh ``` ### 5. Prometheus + Grafana @@ -185,9 +187,7 @@ kubectl rollout restart deployment/prometheus -n demo-observability ### Update dashboards after a pack release ```bash -kubectl create configmap grafana-dashboards -n demo-observability \ - --from-file=../../../../deploy/observability/dashboards/ \ - --dry-run=client -o yaml | kubectl apply -f - +./build-dashboards-configmap.sh kubectl rollout restart deployment/grafana -n demo-observability ``` diff --git a/proprietary/infra/k8s/demo-observability/build-dashboards-configmap.sh b/proprietary/infra/k8s/demo-observability/build-dashboards-configmap.sh new file mode 100755 index 000000000..2f5eec28a --- /dev/null +++ b/proprietary/infra/k8s/demo-observability/build-dashboards-configmap.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Build the grafana-dashboards ConfigMap from the shipped dashboard pack. +# +# One demo-only override: the `job` template variable defaults to +# "betterdb-monitor" (the OTLP push path, ~2s fresh) so a visitor lands on a +# populated dashboard instead of the empty "betterdb-scrape" job they haven't +# selected yet. They can still flip the variable to compare pull vs push. +# +# The shipped JSON under deploy/observability/dashboards/ is NOT edited — +# external users import it with a generic default; the override is applied to +# throwaway copies here. Idempotent: safe for first install and pack updates. +set -euo pipefail + +SRC="$(cd "$(dirname "$0")/../../../../deploy/observability/dashboards" && pwd)" +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT + +for f in "$SRC"/*.json; do + python3 - "$f" "$TMP/$(basename "$f")" <<'PY' +import json, sys +src, dst = sys.argv[1], sys.argv[2] +d = json.load(open(src)) +dash = d["dashboard"] if isinstance(d, dict) and "dashboard" in d else d +for v in dash.get("templating", {}).get("list", []): + if v.get("name") == "job": + v["current"] = {"selected": True, "text": "betterdb-monitor", "value": "betterdb-monitor"} +json.dump(d, open(dst, "w"), indent=2) +PY +done + +kubectl create configmap grafana-dashboards -n demo-observability \ + --from-file="$TMP" --dry-run=client -o yaml | kubectl apply -f - diff --git a/proprietary/infra/k8s/demo-observability/grafana.yaml b/proprietary/infra/k8s/demo-observability/grafana.yaml index 995b7b68e..f53833c2c 100644 --- a/proprietary/infra/k8s/demo-observability/grafana.yaml +++ b/proprietary/infra/k8s/demo-observability/grafana.yaml @@ -78,6 +78,10 @@ spec: env: - name: GF_SERVER_ROOT_URL value: https://demo-grafana.app.betterdb.com + # Land visitors directly on the Instance Vitals dashboard instead of + # the empty Grafana home (one of the provisioned dashboard files). + - name: GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH + value: /var/lib/grafana/dashboards/betterdb-instance-vitals.json # Public read-only access. - name: GF_AUTH_ANONYMOUS_ENABLED value: "true" @@ -152,9 +156,9 @@ spec: - name: dashboard-provider configMap: name: grafana-dashboard-provider - # Created from the shipped dashboard pack: - # kubectl create configmap grafana-dashboards -n demo-observability \ - # --from-file=deploy/observability/dashboards/ + # Built from the shipped dashboard pack by build-dashboards-configmap.sh + # (defaults the `job` variable to the OTLP push path so the landing + # dashboard is populated). See that script and the README. - name: dashboards configMap: name: grafana-dashboards diff --git a/proprietary/infra/k8s/demo-observability/prometheus.yaml b/proprietary/infra/k8s/demo-observability/prometheus.yaml index 339b6d0d4..f07bd6a85 100644 --- a/proprietary/infra/k8s/demo-observability/prometheus.yaml +++ b/proprietary/infra/k8s/demo-observability/prometheus.yaml @@ -34,8 +34,16 @@ data: # The demo tenant's own exposition endpoint, authenticated with the # PRIMARY metrics token (the published public token is for visitors; # rotating it must not break this scrape). - - job_name: 'betterdb' + # The pull/scrape path. Distinct job name from the OTLP-push series + # (which arrive as job="betterdb-monitor", from the app's OTLP + # service.name) so the two ingestion paths are unambiguous. + - job_name: 'betterdb-scrape' metrics_path: '/api/prometheus/metrics' + # The tenant's /metrics can take ~15s when it polls unreachable + # monitored databases, so give the scrape generous headroom over the + # 10s default (timeout must be <= interval). + scrape_interval: 60s + scrape_timeout: 45s authorization: type: Bearer credentials_file: /etc/prometheus-secrets/scrape-token/token From e2035fcf638d504590f4a48611483cf6b704f9ba Mon Sep 17 00:00:00 2001 From: Kristiyan Ivanov Date: Fri, 9 Oct 2026 22:44:23 +0300 Subject: [PATCH 03/13] fix(infra): manage vpc-cni as EKS addon with POD_MTU=1500 The cluster bootstraps a self-managed vpc-cni (it carries Helm chart labels but has no real Helm release), so its config was outside terraform. Adopt it as a managed addon (resolve_conflicts OVERWRITE), pinned to the running v1.20.4-eksbuild.2, and set POD_MTU=1500 so pod interfaces stop inheriting the 9001 ENI MTU: large pod-to-pod responses (the Prometheus /metrics scrape) were hitting a PMTUD blackhole on 1500-only node paths. Node ENIs keep jumbo frames. --- proprietary/infra/terraform/eks.tf | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/proprietary/infra/terraform/eks.tf b/proprietary/infra/terraform/eks.tf index 1d3a63024..c29c7f7bb 100644 --- a/proprietary/infra/terraform/eks.tf +++ b/proprietary/infra/terraform/eks.tf @@ -5,7 +5,7 @@ module "eks" { cluster_name = "${var.project_name}-cluster" cluster_version = "1.31" - vpc_id = module.vpc.vpc_id + vpc_id = module.vpc.vpc_id subnet_ids = concat( module.vpc.private_subnets, [ @@ -20,6 +20,32 @@ module "eks" { # Enable OIDC provider (needed for Karpenter and ALB controller IAM roles) enable_irsa = true + # Manage the VPC CNI as an EKS addon so its config lives in terraform. The + # cluster bootstraps a self-managed vpc-cni at creation (it carries Helm + # chart labels but has no real Helm release); adopting it as a managed addon + # with OVERWRITE is what lets us pin POD_MTU. + # + # POD_MTU=1500 clamps pod interfaces to 1500 bytes while node ENIs keep the + # jumbo default (AWS_VPC_ENI_MTU=9001). Without it pods inherit 9001 and large + # pod-to-pod responses (the ~100KB Prometheus /metrics scrape) hit a PMTUD + # blackhole on node paths that only carry 1500-byte frames: the handshake and + # small requests succeed, then the big data packets are silently dropped until + # timeout. Clamping only the pod MTU fixes it without touching node-level + # jumbo traffic (node-to-node, EBS, VPC endpoints). + cluster_addons = { + vpc-cni = { + addon_version = "v1.20.4-eksbuild.2" # matches the running CNI image + resolve_conflicts_on_create = "OVERWRITE" # adopt the bootstrapped self-managed DaemonSet + resolve_conflicts_on_update = "OVERWRITE" + configuration_values = jsonencode({ + env = { + POD_MTU = "1500" + AWS_VPC_ENI_MTU = "9001" + } + }) + } + } + # Small system node group for Karpenter, CoreDNS, ALB controller eks_managed_node_groups = { system = { From 0df906588ef49e2b56a148651c289c9f81aafe85 Mon Sep 17 00:00:00 2001 From: Kristiyan Ivanov Date: Fri, 9 Oct 2026 23:24:04 +0300 Subject: [PATCH 04/13] feat(dashboards): visualize every pushed metric The shipped pack covered 35 of the 115 metrics the product exports. Add 6 dashboards - Persistence, Keyspace & Eviction, Clients & Pub/Sub, Security & ACL, Vector Search, and Monitor Internals (Node.js/process runtime, poll and GC histograms as heatmaps) - and extend Instance Vitals, Query Patterns, and Anomalies. Every metric now appears on a dashboard (verified 115/115). --- .../dashboards/betterdb-anomalies.json | 81 +- .../dashboards/betterdb-clients.json | 379 ++++++++ .../dashboards/betterdb-instance-vitals.json | 162 +++- .../dashboards/betterdb-keyspace.json | 298 +++++++ .../betterdb-monitor-internals.json | 820 ++++++++++++++++++ .../dashboards/betterdb-persistence.json | 253 ++++++ .../dashboards/betterdb-query-patterns.json | 154 +++- .../dashboards/betterdb-security.json | 292 +++++++ .../dashboards/betterdb-vector.json | 226 +++++ 9 files changed, 2654 insertions(+), 11 deletions(-) create mode 100644 deploy/observability/dashboards/betterdb-clients.json create mode 100644 deploy/observability/dashboards/betterdb-keyspace.json create mode 100644 deploy/observability/dashboards/betterdb-monitor-internals.json create mode 100644 deploy/observability/dashboards/betterdb-persistence.json create mode 100644 deploy/observability/dashboards/betterdb-security.json create mode 100644 deploy/observability/dashboards/betterdb-vector.json diff --git a/deploy/observability/dashboards/betterdb-anomalies.json b/deploy/observability/dashboards/betterdb-anomalies.json index 0bfae6418..2f1affbc0 100644 --- a/deploy/observability/dashboards/betterdb-anomalies.json +++ b/deploy/observability/dashboards/betterdb-anomalies.json @@ -1,7 +1,10 @@ { "uid": "betterdb-anomalies", - "title": "BetterDB · Anomalies", - "tags": ["betterdb", "valkey"], + "title": "BetterDB \u00b7 Anomalies", + "tags": [ + "betterdb", + "valkey" + ], "schemaVersion": 39, "version": 1, "refresh": "30s", @@ -464,6 +467,78 @@ "x": 0, "y": 32 } + }, + { + "id": 11, + "type": "timeseries", + "title": "Correlated groups (total)", + "description": "Active groups of anomalies the correlator has linked together.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_correlated_groups_total{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{connection}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + } + }, + { + "id": 12, + "type": "timeseries", + "title": "Correlated groups by severity", + "description": "Correlated anomaly groups split by severity.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_correlated_groups_by_severity{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{severity}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + } } ] -} +} \ No newline at end of file diff --git a/deploy/observability/dashboards/betterdb-clients.json b/deploy/observability/dashboards/betterdb-clients.json new file mode 100644 index 000000000..a5da87c57 --- /dev/null +++ b/deploy/observability/dashboards/betterdb-clients.json @@ -0,0 +1,379 @@ +{ + "uid": "betterdb-clients", + "title": "BetterDB \u00b7 Clients & Pub/Sub", + "tags": [ + "betterdb", + "valkey" + ], + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { + "from": "now-6h", + "to": "now" + }, + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "ds", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "betterdb", + "value": "betterdb" + }, + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "definition": "label_values(betterdb_memory_used_bytes, job)", + "hide": 0, + "includeAll": false, + "label": "Scrape job", + "multi": false, + "name": "job", + "options": [], + "query": { + "query": "label_values(betterdb_memory_used_bytes, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "name": "connection", + "label": "Connection", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "query": { + "query": "label_values(betterdb_memory_used_bytes{job=~\"$job\"}, connection)", + "refId": "connection" + }, + "includeAll": true, + "multi": true, + "refresh": 2, + "current": { + "text": "All", + "value": "$__all" + } + } + ] + }, + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "Connections (current vs peak)", + "description": "Current connected clients against the peak seen. A current value pinned near the peak hints at connection-pool exhaustion.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_client_connections_current{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{connection}} current" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_client_connections_peak{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{connection}} peak" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + }, + { + "id": 2, + "type": "timeseries", + "title": "Connections by client name", + "description": "Connections grouped by the client-reported name (CLIENT SETNAME).", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_client_connections_by_name{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{client_name}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + } + }, + { + "id": 3, + "type": "timeseries", + "title": "Connections by user", + "description": "Connections grouped by authenticated ACL user.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_client_connections_by_user{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{user}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + } + }, + { + "id": 4, + "type": "timeseries", + "title": "Tracking clients", + "description": "Clients using client-side caching invalidation (CLIENT TRACKING).", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_tracking_clients{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{connection}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + } + }, + { + "id": 5, + "type": "timeseries", + "title": "New connections/sec", + "description": "Connection acceptance rate. A spike can indicate a client that is not pooling.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "rate(betterdb_connections_received_total{job=~\"$job\",connection=~\"$connection\"}[5m])", + "legendFormat": "{{connection}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + } + }, + { + "id": 6, + "type": "timeseries", + "title": "Connected replicas", + "description": "Replicas currently attached to this primary.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_connected_slaves{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{connection}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + } + }, + { + "id": 7, + "type": "timeseries", + "title": "Pub/Sub channels", + "description": "Active publish/subscribe channels.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_pubsub_channels{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{connection}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + } + }, + { + "id": 8, + "type": "timeseries", + "title": "Pub/Sub patterns", + "description": "Active publish/subscribe pattern subscriptions.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_pubsub_patterns{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{connection}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + } + } + ] +} \ No newline at end of file diff --git a/deploy/observability/dashboards/betterdb-instance-vitals.json b/deploy/observability/dashboards/betterdb-instance-vitals.json index fe00c36da..89544ef49 100644 --- a/deploy/observability/dashboards/betterdb-instance-vitals.json +++ b/deploy/observability/dashboards/betterdb-instance-vitals.json @@ -1,7 +1,10 @@ { "uid": "betterdb-instance-vitals", - "title": "BetterDB · Instance Vitals", - "tags": ["betterdb", "valkey"], + "title": "BetterDB \u00b7 Instance Vitals", + "tags": [ + "betterdb", + "valkey" + ], "schemaVersion": 39, "version": 1, "refresh": "30s", @@ -514,6 +517,159 @@ "x": 0, "y": 40 } + }, + { + "id": 12, + "type": "timeseries", + "title": "Network throughput", + "description": "Inbound and outbound kilobytes per second handled by the server.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "KBs", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_instantaneous_input_kbps{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{connection}} in" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_instantaneous_output_kbps{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{connection}} out" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + } + }, + { + "id": 13, + "type": "timeseries", + "title": "Peak memory", + "description": "Highest memory the server has used since start.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_memory_used_peak_bytes{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{connection}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + } + }, + { + "id": 14, + "type": "timeseries", + "title": "Fragmentation (bytes)", + "description": "Absolute bytes of allocator fragmentation (RSS minus used).", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_memory_fragmentation_bytes{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{connection}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + } + }, + { + "id": 15, + "type": "timeseries", + "title": "Commands processed/sec", + "description": "Total command throughput derived from the cumulative counter.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "rate(betterdb_commands_processed_total{job=~\"$job\",connection=~\"$connection\"}[5m])", + "legendFormat": "{{connection}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 56 + } } ] -} +} \ No newline at end of file diff --git a/deploy/observability/dashboards/betterdb-keyspace.json b/deploy/observability/dashboards/betterdb-keyspace.json new file mode 100644 index 000000000..cdb48d235 --- /dev/null +++ b/deploy/observability/dashboards/betterdb-keyspace.json @@ -0,0 +1,298 @@ +{ + "uid": "betterdb-keyspace", + "title": "BetterDB \u00b7 Keyspace & Eviction", + "tags": [ + "betterdb", + "valkey" + ], + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { + "from": "now-6h", + "to": "now" + }, + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "ds", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "betterdb", + "value": "betterdb" + }, + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "definition": "label_values(betterdb_memory_used_bytes, job)", + "hide": 0, + "includeAll": false, + "label": "Scrape job", + "multi": false, + "name": "job", + "options": [], + "query": { + "query": "label_values(betterdb_memory_used_bytes, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "name": "connection", + "label": "Connection", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "query": { + "query": "label_values(betterdb_memory_used_bytes{job=~\"$job\"}, connection)", + "refId": "connection" + }, + "includeAll": true, + "multi": true, + "refresh": 2, + "current": { + "text": "All", + "value": "$__all" + } + } + ] + }, + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "Keys (total)", + "description": "Total keys reported across logical databases.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_keyspace_keys{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{connection}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + }, + { + "id": 2, + "type": "timeseries", + "title": "Keys with a TTL", + "description": "Keys that have an expiry set. A low share of expiring keys with growing memory suggests missing TTLs.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_keyspace_keys_expiring{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{connection}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + } + }, + { + "id": 3, + "type": "timeseries", + "title": "Keys expiring by database", + "description": "Per-database count of keys with an expiry set.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_db_keys_expiring{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{connection}} {{db}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + } + }, + { + "id": 4, + "type": "timeseries", + "title": "Average TTL by database", + "description": "Mean remaining TTL per logical database.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_db_avg_ttl_seconds{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{connection}} {{db}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + } + }, + { + "id": 5, + "type": "timeseries", + "title": "Evictions/sec", + "description": "Keys evicted per second under the maxmemory policy. Sustained evictions mean the working set exceeds the memory limit.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "rate(betterdb_evicted_keys_total{job=~\"$job\",connection=~\"$connection\"}[5m])", + "legendFormat": "{{connection}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + } + }, + { + "id": 6, + "type": "timeseries", + "title": "Expirations/sec", + "description": "Keys expired per second by TTL.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "rate(betterdb_expired_keys_total{job=~\"$job\",connection=~\"$connection\"}[5m])", + "legendFormat": "{{connection}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + } + } + ] +} \ No newline at end of file diff --git a/deploy/observability/dashboards/betterdb-monitor-internals.json b/deploy/observability/dashboards/betterdb-monitor-internals.json new file mode 100644 index 000000000..da3f40578 --- /dev/null +++ b/deploy/observability/dashboards/betterdb-monitor-internals.json @@ -0,0 +1,820 @@ +{ + "uid": "betterdb-monitor-internals", + "title": "BetterDB \u00b7 Monitor Internals", + "tags": [ + "betterdb", + "monitor" + ], + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { + "from": "now-6h", + "to": "now" + }, + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "ds", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "betterdb", + "value": "betterdb" + }, + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "definition": "label_values(betterdb_nodejs_version_info, job)", + "hide": 0, + "includeAll": false, + "label": "Scrape job", + "multi": false, + "name": "job", + "options": [], + "query": { + "query": "label_values(betterdb_nodejs_version_info, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "Process CPU/sec", + "description": "CPU seconds consumed per second by the Monitor process.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "rate(betterdb_process_cpu_user_seconds_total{job=~\"$job\"}[5m])", + "legendFormat": "user" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "rate(betterdb_process_cpu_system_seconds_total{job=~\"$job\"}[5m])", + "legendFormat": "sys" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "rate(betterdb_process_cpu_seconds_total{job=~\"$job\"}[5m])", + "legendFormat": "total" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + }, + { + "id": 2, + "type": "timeseries", + "title": "Process memory", + "description": "Resident, heap, and virtual memory of the Monitor process.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_process_resident_memory_bytes{job=~\"$job\"}", + "legendFormat": "rss" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_process_heap_bytes{job=~\"$job\"}", + "legendFormat": "heap" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_process_virtual_memory_bytes{job=~\"$job\"}", + "legendFormat": "virtual" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + } + }, + { + "id": 3, + "type": "timeseries", + "title": "File descriptors", + "description": "Open file descriptors against the process limit.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_process_open_fds{job=~\"$job\"}", + "legendFormat": "open" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_process_max_fds{job=~\"$job\"}", + "legendFormat": "max" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + } + }, + { + "id": 4, + "type": "stat", + "title": "Process uptime", + "description": "Seconds since the Monitor process started.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "time() - betterdb_process_start_time_seconds{job=~\"$job\"}", + "legendFormat": "uptime" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + } + }, + { + "id": 5, + "type": "timeseries", + "title": "Node heap", + "description": "V8 heap total vs used, plus external (off-heap) memory.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_nodejs_heap_size_total_bytes{job=~\"$job\"}", + "legendFormat": "total" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_nodejs_heap_size_used_bytes{job=~\"$job\"}", + "legendFormat": "used" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_nodejs_external_memory_bytes{job=~\"$job\"}", + "legendFormat": "external" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + } + }, + { + "id": 6, + "type": "timeseries", + "title": "Node heap spaces", + "description": "Per-space V8 heap total, used, and available.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_nodejs_heap_space_size_total_bytes{job=~\"$job\"}", + "legendFormat": "{{space}} total" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_nodejs_heap_space_size_used_bytes{job=~\"$job\"}", + "legendFormat": "{{space}} used" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_nodejs_heap_space_size_available_bytes{job=~\"$job\"}", + "legendFormat": "{{space}} avail" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + } + }, + { + "id": 7, + "type": "timeseries", + "title": "Event loop lag", + "description": "Node.js event-loop lag. Rising p99 lag means the Monitor is CPU-bound and metric collection may slow.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_nodejs_eventloop_lag_seconds{job=~\"$job\"}", + "legendFormat": "current" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_nodejs_eventloop_lag_mean_seconds{job=~\"$job\"}", + "legendFormat": "mean" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_nodejs_eventloop_lag_min_seconds{job=~\"$job\"}", + "legendFormat": "min" + }, + { + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_nodejs_eventloop_lag_max_seconds{job=~\"$job\"}", + "legendFormat": "max" + }, + { + "refId": "E", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_nodejs_eventloop_lag_stddev_seconds{job=~\"$job\"}", + "legendFormat": "stddev" + }, + { + "refId": "F", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_nodejs_eventloop_lag_p50_seconds{job=~\"$job\"}", + "legendFormat": "p50" + }, + { + "refId": "G", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_nodejs_eventloop_lag_p90_seconds{job=~\"$job\"}", + "legendFormat": "p90" + }, + { + "refId": "H", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_nodejs_eventloop_lag_p99_seconds{job=~\"$job\"}", + "legendFormat": "p99" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + } + }, + { + "id": 8, + "type": "timeseries", + "title": "Active handles / requests / resources", + "description": "Libuv active handles, requests, and resources (current and cumulative).", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_nodejs_active_handles{job=~\"$job\"}", + "legendFormat": "handles" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_nodejs_active_handles_total{job=~\"$job\"}", + "legendFormat": "handles total" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_nodejs_active_requests{job=~\"$job\"}", + "legendFormat": "requests" + }, + { + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_nodejs_active_requests_total{job=~\"$job\"}", + "legendFormat": "requests total" + }, + { + "refId": "E", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_nodejs_active_resources{job=~\"$job\"}", + "legendFormat": "resources" + }, + { + "refId": "F", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_nodejs_active_resources_total{job=~\"$job\"}", + "legendFormat": "resources total" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + } + }, + { + "id": 9, + "type": "heatmap", + "title": "GC duration (heatmap)", + "description": "Distribution of garbage-collection pause durations.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "sum by (le) (rate(betterdb_nodejs_gc_duration_seconds_bucket{job=~\"$job\"}[5m]))", + "legendFormat": "{{le}}", + "format": "heatmap" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + } + }, + { + "id": 10, + "type": "timeseries", + "title": "GC average duration", + "description": "Mean GC pause = rate(sum)/rate(count).", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "rate(betterdb_nodejs_gc_duration_seconds_sum{job=~\"$job\"}[5m]) / clamp_min(rate(betterdb_nodejs_gc_duration_seconds_count{job=~\"$job\"}[5m]), 0.0001)", + "legendFormat": "avg" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + } + }, + { + "id": 11, + "type": "heatmap", + "title": "Poll duration (heatmap)", + "description": "Distribution of how long each monitored-instance poll takes.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "sum by (le) (rate(betterdb_poll_duration_seconds_bucket{job=~\"$job\"}[5m]))", + "legendFormat": "{{le}}", + "format": "heatmap" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + } + }, + { + "id": 12, + "type": "timeseries", + "title": "Poll average duration", + "description": "Mean poll duration = rate(sum)/rate(count). Climbing here precedes scrape timeouts.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "rate(betterdb_poll_duration_seconds_sum{job=~\"$job\"}[5m]) / clamp_min(rate(betterdb_poll_duration_seconds_count{job=~\"$job\"}[5m]), 0.0001)", + "legendFormat": "avg" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + } + }, + { + "id": 13, + "type": "timeseries", + "title": "Polls/sec", + "description": "Monitored-instance poll rate.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "rate(betterdb_polls_total{job=~\"$job\"}[5m])", + "legendFormat": "polls" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + } + }, + { + "id": 14, + "type": "timeseries", + "title": "Stale polls", + "description": "Polls currently flagged stale (data older than the poll interval).", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_poll_stale{job=~\"$job\"}", + "legendFormat": "stale" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + } + }, + { + "id": 15, + "type": "timeseries", + "title": "OTLP metric points/sec", + "description": "Rate of OTLP metric points the Monitor exports to the collector.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "rate(betterdb_otlp_metric_points_accepted_total{job=~\"$job\"}[5m])", + "legendFormat": "accepted" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + } + }, + { + "id": 16, + "type": "table", + "title": "Node.js version", + "description": "Runtime version of the Monitor process.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_nodejs_version_info{job=~\"$job\"}", + "legendFormat": "{{version}}", + "format": "table", + "instant": true, + "range": false + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 56 + } + } + ] +} \ No newline at end of file diff --git a/deploy/observability/dashboards/betterdb-persistence.json b/deploy/observability/dashboards/betterdb-persistence.json new file mode 100644 index 000000000..d5140fc00 --- /dev/null +++ b/deploy/observability/dashboards/betterdb-persistence.json @@ -0,0 +1,253 @@ +{ + "uid": "betterdb-persistence", + "title": "BetterDB \u00b7 Persistence", + "tags": [ + "betterdb", + "valkey" + ], + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { + "from": "now-6h", + "to": "now" + }, + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "ds", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "betterdb", + "value": "betterdb" + }, + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "definition": "label_values(betterdb_memory_used_bytes, job)", + "hide": 0, + "includeAll": false, + "label": "Scrape job", + "multi": false, + "name": "job", + "options": [], + "query": { + "query": "label_values(betterdb_memory_used_bytes, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "name": "connection", + "label": "Connection", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "query": { + "query": "label_values(betterdb_memory_used_bytes{job=~\"$job\"}, connection)", + "refId": "connection" + }, + "includeAll": true, + "multi": true, + "refresh": 2, + "current": { + "text": "All", + "value": "$__all" + } + } + ] + }, + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "Changes since last RDB save", + "description": "Writes accumulated since the last successful RDB snapshot. A number that keeps climbing without resetting means snapshots are not completing.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_rdb_changes_since_last_save{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{connection}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + }, + { + "id": 2, + "type": "timeseries", + "title": "Time since last RDB save", + "description": "Seconds since the last successful RDB save. A sawtooth that climbs without dropping means save is failing.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "time() - betterdb_rdb_last_save_timestamp_seconds{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{connection}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + } + }, + { + "id": 3, + "type": "stat", + "title": "RDB last bgsave OK", + "description": "1 when the most recent background RDB save succeeded, 0 on failure.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_rdb_last_bgsave_ok{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{connection}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + } + }, + { + "id": 4, + "type": "stat", + "title": "AOF enabled", + "description": "1 when append-only file persistence is enabled.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_aof_enabled{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{connection}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + } + }, + { + "id": 5, + "type": "stat", + "title": "AOF last rewrite OK", + "description": "1 when the most recent AOF rewrite succeeded, 0 on failure.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_aof_last_bgrewrite_ok{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{connection}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + } + } + ] +} \ No newline at end of file diff --git a/deploy/observability/dashboards/betterdb-query-patterns.json b/deploy/observability/dashboards/betterdb-query-patterns.json index dba6fdf04..69b28db8b 100644 --- a/deploy/observability/dashboards/betterdb-query-patterns.json +++ b/deploy/observability/dashboards/betterdb-query-patterns.json @@ -1,7 +1,10 @@ { "uid": "betterdb-query-patterns", - "title": "BetterDB · Query Patterns", - "tags": ["betterdb", "valkey"], + "title": "BetterDB \u00b7 Query Patterns", + "tags": [ + "betterdb", + "valkey" + ], "schemaVersion": 39, "version": 1, "refresh": "30s", @@ -157,7 +160,7 @@ }, "fieldConfig": { "defaults": { - "unit": "µs", + "unit": "\u00b5s", "custom": { "fillOpacity": 10 } @@ -336,7 +339,7 @@ }, "fieldConfig": { "defaults": { - "unit": "µs" + "unit": "\u00b5s" }, "overrides": [] }, @@ -378,6 +381,147 @@ "x": 0, "y": 32 } + }, + { + "id": 9, + "type": "timeseries", + "title": "Command calls by type", + "description": "Top command types by call rate.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "topk(10, rate(betterdb_commandstats_calls_total{job=~\"$job\",connection=~\"$connection\"}[5m]))", + "legendFormat": "{{command}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + } + }, + { + "id": 10, + "type": "timeseries", + "title": "Large requests logged", + "description": "Count of oversized request payloads recorded in the command log.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_commandlog_large_request{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{connection}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + } + }, + { + "id": 11, + "type": "timeseries", + "title": "Large replies logged", + "description": "Count of oversized reply payloads recorded in the command log.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_commandlog_large_reply{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{connection}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + } + }, + { + "id": 12, + "type": "stat", + "title": "Slowlog last ID", + "description": "Most recent slowlog entry ID; a jump means new slow commands were recorded.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_slowlog_last_id{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{connection}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + } } ] -} +} \ No newline at end of file diff --git a/deploy/observability/dashboards/betterdb-security.json b/deploy/observability/dashboards/betterdb-security.json new file mode 100644 index 000000000..99dccba83 --- /dev/null +++ b/deploy/observability/dashboards/betterdb-security.json @@ -0,0 +1,292 @@ +{ + "uid": "betterdb-security", + "title": "BetterDB \u00b7 Security & ACL", + "tags": [ + "betterdb", + "valkey" + ], + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { + "from": "now-6h", + "to": "now" + }, + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "ds", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "betterdb", + "value": "betterdb" + }, + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "definition": "label_values(betterdb_memory_used_bytes, job)", + "hide": 0, + "includeAll": false, + "label": "Scrape job", + "multi": false, + "name": "job", + "options": [], + "query": { + "query": "label_values(betterdb_memory_used_bytes, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "name": "connection", + "label": "Connection", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "query": { + "query": "label_values(betterdb_memory_used_bytes{job=~\"$job\"}, connection)", + "refId": "connection" + }, + "includeAll": true, + "multi": true, + "refresh": 2, + "current": { + "text": "All", + "value": "$__all" + } + } + ] + }, + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "ACL denials/sec", + "description": "Commands rejected per second by ACL rules. Any sustained rate means a client is attempting unauthorized operations.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "rate(betterdb_acl_denied{job=~\"$job\",connection=~\"$connection\"}[5m])", + "legendFormat": "{{connection}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + }, + { + "id": 2, + "type": "timeseries", + "title": "ACL denials by reason", + "description": "Denial rate split by cause (command, key, channel, auth).", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "rate(betterdb_acl_denied_by_reason{job=~\"$job\",connection=~\"$connection\"}[5m])", + "legendFormat": "{{reason}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + } + }, + { + "id": 3, + "type": "timeseries", + "title": "ACL denials by user", + "description": "Denial rate split by ACL user.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "rate(betterdb_acl_denied_by_user{job=~\"$job\",connection=~\"$connection\"}[5m])", + "legendFormat": "{{username}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + } + }, + { + "id": 4, + "type": "timeseries", + "title": "CVE findings", + "description": "Known CVEs matching the detected server version.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_cve_findings{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{connection}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + } + }, + { + "id": 5, + "type": "stat", + "title": "Known-exploited CVEs (KEV)", + "description": "Count of findings on CISA's Known Exploited Vulnerabilities list. Non-zero warrants urgent patching.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_cve_kev{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{connection}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + } + }, + { + "id": 6, + "type": "stat", + "title": "CVE dataset stale", + "description": "1 when the local CVE dataset is older than the refresh window, so findings may be incomplete.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_cve_dataset_stale{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{connection}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + } + } + ] +} \ No newline at end of file diff --git a/deploy/observability/dashboards/betterdb-vector.json b/deploy/observability/dashboards/betterdb-vector.json new file mode 100644 index 000000000..5716fd305 --- /dev/null +++ b/deploy/observability/dashboards/betterdb-vector.json @@ -0,0 +1,226 @@ +{ + "uid": "betterdb-vector", + "title": "BetterDB \u00b7 Vector Search", + "tags": [ + "betterdb", + "valkey" + ], + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { + "from": "now-6h", + "to": "now" + }, + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "ds", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "betterdb", + "value": "betterdb" + }, + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "definition": "label_values(betterdb_memory_used_bytes, job)", + "hide": 0, + "includeAll": false, + "label": "Scrape job", + "multi": false, + "name": "job", + "options": [], + "query": { + "query": "label_values(betterdb_memory_used_bytes, job)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "name": "connection", + "label": "Connection", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "query": { + "query": "label_values(betterdb_memory_used_bytes{job=~\"$job\"}, connection)", + "refId": "connection" + }, + "includeAll": true, + "multi": true, + "refresh": 2, + "current": { + "text": "All", + "value": "$__all" + } + } + ] + }, + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "Indexed documents", + "description": "Documents indexed per vector index.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_vector_index_docs{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{connection}} {{index}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } + }, + { + "id": 2, + "type": "timeseries", + "title": "Index memory", + "description": "Memory consumed by each vector index.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_vector_index_memory_bytes{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{connection}} {{index}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + } + }, + { + "id": 3, + "type": "timeseries", + "title": "Percent indexed", + "description": "Share of documents fully indexed. A value stuck below 100% means indexing is backlogged.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_vector_index_percent_indexed{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{connection}} {{index}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + } + }, + { + "id": 4, + "type": "timeseries", + "title": "Indexing failures", + "description": "Cumulative indexing failures per index.", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${ds}" + }, + "expr": "betterdb_vector_index_indexing_failures{job=~\"$job\",connection=~\"$connection\"}", + "legendFormat": "{{connection}} {{index}}" + } + ], + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + } + } + ] +} \ No newline at end of file From 4c3d3a6d5a397d7a2de4a31ac0b73d9600b06c51 Mon Sep 17 00:00:00 2001 From: Kristiyan Ivanov Date: Fri, 9 Oct 2026 23:33:57 +0300 Subject: [PATCH 05/13] feat(dashboards): add cross-dashboard nav dropdown Add a tag-based ("betterdb") dashboards link to every dashboard so you can hop between Vitals, Persistence, Keyspace, Clients, Security, Vector, Anomalies, Query Patterns, Cluster Slots, and Monitor Internals from the header. Preserves the selected connection/job/time across jumps. Dynamic by tag, so new betterdb dashboards appear automatically. --- .../dashboards/betterdb-anomalies.json | 15 +++++++++++++ .../dashboards/betterdb-clients.json | 15 +++++++++++++ .../dashboards/betterdb-cluster-slots.json | 22 +++++++++++++++++-- .../dashboards/betterdb-instance-vitals.json | 15 +++++++++++++ .../dashboards/betterdb-keyspace.json | 15 +++++++++++++ .../betterdb-monitor-internals.json | 15 +++++++++++++ .../dashboards/betterdb-persistence.json | 15 +++++++++++++ .../dashboards/betterdb-query-patterns.json | 15 +++++++++++++ .../dashboards/betterdb-security.json | 15 +++++++++++++ .../dashboards/betterdb-vector.json | 15 +++++++++++++ 10 files changed, 155 insertions(+), 2 deletions(-) diff --git a/deploy/observability/dashboards/betterdb-anomalies.json b/deploy/observability/dashboards/betterdb-anomalies.json index 2f1affbc0..a5dc64c06 100644 --- a/deploy/observability/dashboards/betterdb-anomalies.json +++ b/deploy/observability/dashboards/betterdb-anomalies.json @@ -540,5 +540,20 @@ "y": 40 } } + ], + "links": [ + { + "title": "BetterDB dashboards", + "type": "dashboards", + "tags": [ + "betterdb" + ], + "asDropdown": true, + "targetBlank": false, + "includeVars": true, + "keepTime": true, + "icon": "external link", + "keepTags": false + } ] } \ No newline at end of file diff --git a/deploy/observability/dashboards/betterdb-clients.json b/deploy/observability/dashboards/betterdb-clients.json index a5da87c57..7f69f9df0 100644 --- a/deploy/observability/dashboards/betterdb-clients.json +++ b/deploy/observability/dashboards/betterdb-clients.json @@ -375,5 +375,20 @@ "y": 24 } } + ], + "links": [ + { + "title": "BetterDB dashboards", + "type": "dashboards", + "tags": [ + "betterdb" + ], + "asDropdown": true, + "targetBlank": false, + "includeVars": true, + "keepTime": true, + "icon": "external link", + "keepTags": false + } ] } \ No newline at end of file diff --git a/deploy/observability/dashboards/betterdb-cluster-slots.json b/deploy/observability/dashboards/betterdb-cluster-slots.json index 8f78ae330..e460c961e 100644 --- a/deploy/observability/dashboards/betterdb-cluster-slots.json +++ b/deploy/observability/dashboards/betterdb-cluster-slots.json @@ -1,7 +1,10 @@ { "uid": "betterdb-cluster-slots", "title": "BetterDB \u00b7 Cluster Slots", - "tags": ["betterdb", "valkey"], + "tags": [ + "betterdb", + "valkey" + ], "schemaVersion": 39, "version": 1, "refresh": "30s", @@ -375,5 +378,20 @@ "y": 12 } } + ], + "links": [ + { + "title": "BetterDB dashboards", + "type": "dashboards", + "tags": [ + "betterdb" + ], + "asDropdown": true, + "targetBlank": false, + "includeVars": true, + "keepTime": true, + "icon": "external link", + "keepTags": false + } ] -} +} \ No newline at end of file diff --git a/deploy/observability/dashboards/betterdb-instance-vitals.json b/deploy/observability/dashboards/betterdb-instance-vitals.json index 89544ef49..5f0631819 100644 --- a/deploy/observability/dashboards/betterdb-instance-vitals.json +++ b/deploy/observability/dashboards/betterdb-instance-vitals.json @@ -671,5 +671,20 @@ "y": 56 } } + ], + "links": [ + { + "title": "BetterDB dashboards", + "type": "dashboards", + "tags": [ + "betterdb" + ], + "asDropdown": true, + "targetBlank": false, + "includeVars": true, + "keepTime": true, + "icon": "external link", + "keepTags": false + } ] } \ No newline at end of file diff --git a/deploy/observability/dashboards/betterdb-keyspace.json b/deploy/observability/dashboards/betterdb-keyspace.json index cdb48d235..e7484510d 100644 --- a/deploy/observability/dashboards/betterdb-keyspace.json +++ b/deploy/observability/dashboards/betterdb-keyspace.json @@ -294,5 +294,20 @@ "y": 16 } } + ], + "links": [ + { + "title": "BetterDB dashboards", + "type": "dashboards", + "tags": [ + "betterdb" + ], + "asDropdown": true, + "targetBlank": false, + "includeVars": true, + "keepTime": true, + "icon": "external link", + "keepTags": false + } ] } \ No newline at end of file diff --git a/deploy/observability/dashboards/betterdb-monitor-internals.json b/deploy/observability/dashboards/betterdb-monitor-internals.json index da3f40578..256165b73 100644 --- a/deploy/observability/dashboards/betterdb-monitor-internals.json +++ b/deploy/observability/dashboards/betterdb-monitor-internals.json @@ -816,5 +816,20 @@ "y": 56 } } + ], + "links": [ + { + "title": "BetterDB dashboards", + "type": "dashboards", + "tags": [ + "betterdb" + ], + "asDropdown": true, + "targetBlank": false, + "includeVars": true, + "keepTime": true, + "icon": "external link", + "keepTags": false + } ] } \ No newline at end of file diff --git a/deploy/observability/dashboards/betterdb-persistence.json b/deploy/observability/dashboards/betterdb-persistence.json index d5140fc00..86b5dbd4a 100644 --- a/deploy/observability/dashboards/betterdb-persistence.json +++ b/deploy/observability/dashboards/betterdb-persistence.json @@ -249,5 +249,20 @@ "y": 16 } } + ], + "links": [ + { + "title": "BetterDB dashboards", + "type": "dashboards", + "tags": [ + "betterdb" + ], + "asDropdown": true, + "targetBlank": false, + "includeVars": true, + "keepTime": true, + "icon": "external link", + "keepTags": false + } ] } \ No newline at end of file diff --git a/deploy/observability/dashboards/betterdb-query-patterns.json b/deploy/observability/dashboards/betterdb-query-patterns.json index 69b28db8b..3a953e410 100644 --- a/deploy/observability/dashboards/betterdb-query-patterns.json +++ b/deploy/observability/dashboards/betterdb-query-patterns.json @@ -523,5 +523,20 @@ "y": 48 } } + ], + "links": [ + { + "title": "BetterDB dashboards", + "type": "dashboards", + "tags": [ + "betterdb" + ], + "asDropdown": true, + "targetBlank": false, + "includeVars": true, + "keepTime": true, + "icon": "external link", + "keepTags": false + } ] } \ No newline at end of file diff --git a/deploy/observability/dashboards/betterdb-security.json b/deploy/observability/dashboards/betterdb-security.json index 99dccba83..9411508ad 100644 --- a/deploy/observability/dashboards/betterdb-security.json +++ b/deploy/observability/dashboards/betterdb-security.json @@ -288,5 +288,20 @@ "y": 16 } } + ], + "links": [ + { + "title": "BetterDB dashboards", + "type": "dashboards", + "tags": [ + "betterdb" + ], + "asDropdown": true, + "targetBlank": false, + "includeVars": true, + "keepTime": true, + "icon": "external link", + "keepTags": false + } ] } \ No newline at end of file diff --git a/deploy/observability/dashboards/betterdb-vector.json b/deploy/observability/dashboards/betterdb-vector.json index 5716fd305..9748a9cc4 100644 --- a/deploy/observability/dashboards/betterdb-vector.json +++ b/deploy/observability/dashboards/betterdb-vector.json @@ -222,5 +222,20 @@ "y": 8 } } + ], + "links": [ + { + "title": "BetterDB dashboards", + "type": "dashboards", + "tags": [ + "betterdb" + ], + "asDropdown": true, + "targetBlank": false, + "includeVars": true, + "keepTime": true, + "icon": "external link", + "keepTags": false + } ] } \ No newline at end of file From 94ef4e333a09907d442ffd48463d2e0262d211c7 Mon Sep 17 00:00:00 2001 From: Kristiyan Ivanov Date: Sat, 10 Oct 2026 10:09:17 +0300 Subject: [PATCH 06/13] fix(provisioning): address review findings - Split isAlreadyExistsError into isAlreadyExistsError/isConflictError so a genuine 409 Conflict on a create path is no longer swallowed as a benign AlreadyExists; the converge retry now matches Conflict specifically. - Two-tier backfillSecretKeys: missing/empty keys are written, control-plane config (RECONCILABLE_SECRET_KEYS) is reconciled to the current value on re-provision, generated secrets/tokens are never rotated. Treats empty as absent. Returns the written keys; ensureMetricsTokenInSecret now delegates. - Converge carries forward a small allowlist of control-plane env (COOKIE_DOMAIN) the desired spec may omit under env drift, and now also converges the image so a DEFAULT_IMAGE_TAG bump reaches re-provisioned tenants. - Converge retry re-reads inside the try with bounded attempts and backoff. - Generate the secretKeyRef env entries from CORE_SECRET_ENV_KEYS so the env list and the Secret key set can't drift. --- .../src/provisioning/provisioning.service.ts | 261 +++++++++--------- 1 file changed, 125 insertions(+), 136 deletions(-) diff --git a/proprietary/entitlement/src/provisioning/provisioning.service.ts b/proprietary/entitlement/src/provisioning/provisioning.service.ts index e606e8136..865c61cb7 100644 --- a/proprietary/entitlement/src/provisioning/provisioning.service.ts +++ b/proprietary/entitlement/src/provisioning/provisioning.service.ts @@ -26,6 +26,32 @@ const VALKEY_NAME_PATTERN = /^[a-z][a-z0-9-]{1,38}[a-z0-9]$/; const DEMO_OBSERVABILITY_NAMESPACE = 'demo-observability'; const DEMO_OBSERVABILITY_PROMETHEUS_PORT = 9090; +// db-credentials keys the Monitor Deployment references as non-optional +// secretKeyRefs. The Deployment env entries are generated from this list so the +// Secret key set and the pod env cannot drift (a key here with no Secret value +// wedges the pod in CreateContainerConfigError). +const CORE_SECRET_ENV_KEYS = [ + 'STORAGE_URL', + 'CLOUD_MODE', + 'AUTH_PUBLIC_KEY', + 'SESSION_SECRET', + 'OTEL_INGEST_TOKEN', + 'PROMETHEUS_METRICS_TOKEN', + 'ENTITLEMENT_API_URL', + 'ENTITLEMENT_API_KEY', +] as const; + +// Subset of db-credentials keys that are control-plane *config*, not per-tenant +// generated secrets: a re-provision reconciles these to the current desired +// value. Everything else (SESSION_SECRET and the *_TOKEN values) is generated +// once and never rotated out from under a running tenant. +const RECONCILABLE_SECRET_KEYS: ReadonlySet = new Set([ + 'STORAGE_URL', + 'CLOUD_MODE', + 'AUTH_PUBLIC_KEY', + 'ENTITLEMENT_API_URL', + 'ENTITLEMENT_API_KEY', +]); // The public SNI host is an opaque, stable label derived from the instance id // rather than its name: this keeps it globally unique on the shared wildcard @@ -1115,42 +1141,56 @@ export class ProvisioningService { if (!this.isAlreadyExistsError(error)) { throw error; } - // Re-provision: the Secret already exists. Backfill any keys it is - // missing. An older tenant's Secret can predate the current schema (e.g. - // lack CLOUD_MODE / AUTH_PUBLIC_KEY / SESSION_SECRET / ENTITLEMENT_API_*), - // and the Deployment references every key as a non-optional secretKeyRef — - // a missing one wedges the new pod in CreateContainerConfigError. Only ADD - // absent keys; never overwrite existing ones, so a running tenant's tokens - // and SESSION_SECRET are not rotated out from under it. (A non-demo - // tenant's stale PROMETHEUS_METRICS_PUBLIC_TOKEN is pruned later in - // createDeployment, once the spec no longer references it.) - this.logger.warn(`Secret db-credentials already exists in ${namespace}, backfilling missing keys...`); - await this.backfillSecretKeys(namespace, stringData); + // Re-provision: the Secret already exists. Reconcile it to the current + // schema. An older tenant's Secret can predate current keys (e.g. lack + // CLOUD_MODE / AUTH_PUBLIC_KEY / ENTITLEMENT_API_*), and the Deployment + // references every key as a non-optional secretKeyRef — a missing one + // wedges the new pod in CreateContainerConfigError. Missing/empty keys are + // written; control-plane config in RECONCILABLE_SECRET_KEYS is updated to + // the current value (so a migrated STORAGE_URL or rotated AUTH_PUBLIC_KEY + // is picked up); generated secrets/tokens are never overwritten. (A + // non-demo tenant's stale PROMETHEUS_METRICS_PUBLIC_TOKEN is pruned later + // in createDeployment, once the spec no longer references it.) + this.logger.warn(`Secret db-credentials already exists in ${namespace}, reconciling keys...`); + await this.backfillSecretKeys(namespace, stringData, RECONCILABLE_SECRET_KEYS); } } - // Adds any of `desired`'s keys that are absent from the db-credentials Secret, - // leaving existing values untouched. Brings an older tenant's Secret up to the - // current key set on re-provision without rotating live credentials. - private async backfillSecretKeys(namespace: string, desired: Record): Promise { + // Reconciles the db-credentials Secret toward `desired` and returns the keys + // it wrote. A key that is absent OR empty is always written (brings an older + // tenant up to the current key set and repairs a blank value). A key already + // present with a value is overwritten only when it is in `reconcileKeys` and + // its value changed — so control-plane config (STORAGE_URL, AUTH_PUBLIC_KEY, + // ...) is picked up on re-provision, while generated secrets (SESSION_SECRET + // and the *_TOKEN values) are never rotated out from under a running tenant. + private async backfillSecretKeys( + namespace: string, + desired: Record, + reconcileKeys: ReadonlySet = new Set(), + ): Promise { const secret = await this.coreApi.readNamespacedSecret({ name: 'db-credentials', namespace }); - const present = new Set(Object.keys(secret.data ?? {})); - const missing: Record = {}; + const data = secret.data ?? {}; + const current = (key: string): string | undefined => + data[key] === undefined ? undefined : Buffer.from(data[key], 'base64').toString('utf8'); + const patch: Record = {}; for (const [key, value] of Object.entries(desired)) { - if (!present.has(key)) { - missing[key] = value; + const existing = current(key); + if (!existing) { + patch[key] = value; // missing or empty -> write + } else if (reconcileKeys.has(key) && existing !== value) { + patch[key] = value; // config drifted -> reconcile to desired } } - if (Object.keys(missing).length === 0) { - return; + const written = Object.keys(patch); + if (written.length === 0) { + return []; } await this.coreApi.patchNamespacedSecret( - { name: 'db-credentials', namespace, body: { stringData: missing } }, + { name: 'db-credentials', namespace, body: { stringData: patch } }, k8s.setHeaderOptions('Content-Type', k8s.PatchStrategy.MergePatch), ); - this.logger.log( - `Backfilled missing db-credentials keys in ${namespace}: ${Object.keys(missing).join(', ')}`, - ); + this.logger.log(`Reconciled db-credentials keys in ${namespace}: ${written.join(', ')}`); + return written; } private async createResourceQuota( @@ -1291,80 +1331,14 @@ export class ProvisioningService { ] : []), ...(!isDemo && process.env.COOKIE_DOMAIN ? [{ name: 'COOKIE_DOMAIN', value: process.env.COOKIE_DOMAIN }] : []), - { - name: 'STORAGE_URL', - valueFrom: { - secretKeyRef: { - name: 'db-credentials', - key: 'STORAGE_URL', - }, - }, - }, - // Cloud auth env vars - { - name: 'CLOUD_MODE', - valueFrom: { - secretKeyRef: { - name: 'db-credentials', - key: 'CLOUD_MODE', - }, - }, - }, - { - name: 'AUTH_PUBLIC_KEY', - valueFrom: { - secretKeyRef: { - name: 'db-credentials', - key: 'AUTH_PUBLIC_KEY', - }, - }, - }, - { - name: 'SESSION_SECRET', - valueFrom: { - secretKeyRef: { - name: 'db-credentials', - key: 'SESSION_SECRET', - }, - }, - }, - { - name: 'OTEL_INGEST_TOKEN', - valueFrom: { - secretKeyRef: { - name: 'db-credentials', - key: 'OTEL_INGEST_TOKEN', - }, - }, - }, - { - name: 'PROMETHEUS_METRICS_TOKEN', - valueFrom: { - secretKeyRef: { - name: 'db-credentials', - key: 'PROMETHEUS_METRICS_TOKEN', - }, - }, - }, - // Entitlement API env vars (for workspace management) - { - name: 'ENTITLEMENT_API_URL', - valueFrom: { - secretKeyRef: { - name: 'db-credentials', - key: 'ENTITLEMENT_API_URL', - }, - }, - }, - { - name: 'ENTITLEMENT_API_KEY', - valueFrom: { - secretKeyRef: { - name: 'db-credentials', - key: 'ENTITLEMENT_API_KEY', - }, - }, - }, + // STORAGE_URL, cloud-auth secrets and Entitlement API + // config, all as non-optional secretKeyRefs generated from + // CORE_SECRET_ENV_KEYS so this list and the db-credentials + // key set cannot drift. + ...CORE_SECRET_ENV_KEYS.map((key) => ({ + name: key, + valueFrom: { secretKeyRef: { name: 'db-credentials', key } }, + })), ], // Rendered from the same appPodBudget the ResourceQuota // is sized from, so pod spec and quota cannot drift. @@ -1422,28 +1396,40 @@ export class ProvisioningService { // between `value` and `valueFrom` ends up carrying both, which the API // rejects with a 422. Replacing the container's env wholesale on the live // object avoids that, and naturally drops demo-only vars a now-non-demo - // tenant shouldn't keep (the desired env simply omits them). We touch - // only env, resources and replicas — the live image, pod-template - // annotations (e.g. the metrics-token restart marker) and probes are - // preserved. Retry on a 409 from a concurrent status write. + // tenant shouldn't keep (the desired env simply omits them) — except a + // small allowlist of control-plane-derived vars (COOKIE_DOMAIN) carried + // forward from the live pod when the desired env omits them, so env drift + // in this replica can't strip them. We converge env, resources, replicas + // AND the image (so a DEFAULT_IMAGE_TAG bump reaches re-provisioned + // tenants); pod-template annotations (e.g. the metrics-token restart + // marker) and probes are preserved. Retry only a 409 Conflict (stale + // resourceVersion from a concurrent status write), re-reading each attempt + // with a short backoff. this.logger.log(`Deployment already exists in ${namespace}, converging app container`); const desired = body.spec!.template!.spec!.containers![0]; - for (let attempt = 0; ; attempt++) { - const existing = await this.appsApi.readNamespacedDeployment({ name: 'betterdb', namespace }); - const c = existing.spec?.template?.spec?.containers?.find((x) => x.name === 'betterdb'); - if (!c) { - throw new Error(`betterdb container not found in ${namespace}`); - } - c.env = desired.env; - c.resources = desired.resources; - existing.spec!.replicas = body.spec!.replicas; + const PRESERVE_ENV = new Set(['COOKIE_DOMAIN']); + const MAX_ATTEMPTS = 3; + for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { try { + const existing = await this.appsApi.readNamespacedDeployment({ name: 'betterdb', namespace }); + const c = existing.spec?.template?.spec?.containers?.find((x) => x.name === 'betterdb'); + if (!c) { + throw new Error(`betterdb container not found in ${namespace}`); + } + const desiredEnv = desired.env ?? []; + const desiredNames = new Set(desiredEnv.map((e) => e.name)); + const carried = (c.env ?? []).filter( + (e) => PRESERVE_ENV.has(e.name) && !desiredNames.has(e.name), + ); + c.env = [...desiredEnv, ...carried]; + c.resources = desired.resources; + c.image = desired.image; + existing.spec!.replicas = body.spec!.replicas; await this.appsApi.replaceNamespacedDeployment({ name: 'betterdb', namespace, body: existing }); break; } catch (e: any) { - // isAlreadyExistsError also matches a 409 Conflict (stale - // resourceVersion) on replace; re-read and retry a few times. - if (this.isAlreadyExistsError(e) && attempt < 3) { + if (this.isConflictError(e) && attempt < MAX_ATTEMPTS) { + await this.sleep(100 * attempt); continue; } throw e; @@ -1814,23 +1800,12 @@ export class ProvisioningService { namespace: string, key: 'PROMETHEUS_METRICS_TOKEN' | 'PROMETHEUS_METRICS_PUBLIC_TOKEN' = 'PROMETHEUS_METRICS_TOKEN', ): Promise { - const secret = await this.coreApi.readNamespacedSecret({ - name: 'db-credentials', - namespace, - }); - if (secret.data?.[key]) { - return false; - } + // Add-only (the token key is not in RECONCILABLE_SECRET_KEYS): a missing or + // empty value is generated, an existing one is left in place so it is never + // rotated. Returns whether a fresh token was written. const token = crypto.randomBytes(32).toString('hex'); - await this.coreApi.patchNamespacedSecret( - { - name: 'db-credentials', - namespace, - body: { stringData: { [key]: token } }, - }, - k8s.setHeaderOptions('Content-Type', k8s.PatchStrategy.MergePatch), - ); - return true; + const written = await this.backfillSecretKeys(namespace, { [key]: token }); + return written.includes(key); } // Removes a key from the db-credentials Secret if present (JSON merge patch: @@ -2022,13 +1997,27 @@ export class ProvisioningService { return `demo.${this.appDomain}`; } + // A 409 is either AlreadyExists (expected when a create races a prior one) or + // Conflict (a stale-resourceVersion write, or a genuine fault). When the API + // server gives us the reason, trust it so a Conflict on a create path is NOT + // swallowed as a benign AlreadyExists; only fall back to a bare 409 match when + // no reason is available (older client-node embeds the code only in .message). private isAlreadyExistsError(error: any): boolean { - const code = error.statusCode ?? error.response?.statusCode ?? error.status; + const reason = error.body?.reason; + if (reason) return reason === 'AlreadyExists'; + return this.is409(error); + } + + private isConflictError(error: any): boolean { + const reason = error.body?.reason; + if (reason) return reason === 'Conflict'; + return this.is409(error); + } + + private is409(error: any): boolean { + const code = error.statusCode ?? error.response?.statusCode ?? error.status ?? error.body?.code; if (code === 409) return true; - if (error.body?.code === 409 || error.body?.reason === 'AlreadyExists') return true; - // k8s client-node v1.x embeds the status code only in the message string - if (typeof error.message === 'string' && error.message.startsWith('HTTP-Code: 409')) return true; - return false; + return typeof error.message === 'string' && error.message.startsWith('HTTP-Code: 409'); } private sleep(ms: number): Promise { From 0fb24380b2d33507591fb56195d239efbe5e1f61 Mon Sep 17 00:00:00 2001 From: Kristiyan Ivanov Date: Sat, 10 Oct 2026 10:09:17 +0300 Subject: [PATCH 07/13] fix(dashboards): correct metric types and demo-quality issues - ACL panels graph the gauges raw instead of rate() (acl_denied* are gauges, not counters, so rate() read ~0 during a real denial burst). - correlated_groups_total is a counter: show increase()[1h] split by severity, not the raw monotonic staircase under a {{connection}}-only legend. - Top command calls: instant topk in a table (per-step topk in a timeseries flickered and hid steady second-tier commands). - Persistence: an epoch-0 last-save timestamp no longer renders '63 years'; show 0 via multiplication by (ts > bool 0). - Security CVE panels coalesce absent series to 0 (no more 'No data'). - Vector percent-indexed axis capped at 0..100. - build-dashboards-configmap.sh: single python pass, nullglob + empty guard, skip-and-warn on a bad file (no all-or-nothing abort), ensure_ascii, and the Monitor-Internals dashboard defaults to betterdb-scrape (OTLP mirror skips histograms, so its GC/poll heatmaps need the pull path). - refresh 1m across the pack (was 30s, double-querying the 60s scrape path). --- .../dashboards/betterdb-anomalies.json | 10 ++-- .../dashboards/betterdb-clients.json | 4 +- .../dashboards/betterdb-cluster-slots.json | 4 +- .../dashboards/betterdb-instance-vitals.json | 4 +- .../dashboards/betterdb-keyspace.json | 4 +- .../betterdb-monitor-internals.json | 4 +- .../dashboards/betterdb-persistence.json | 6 +- .../dashboards/betterdb-query-patterns.json | 18 +++--- .../dashboards/betterdb-security.json | 34 +++++------ .../dashboards/betterdb-vector.json | 8 ++- .../build-dashboards-configmap.sh | 57 +++++++++++++------ 11 files changed, 89 insertions(+), 64 deletions(-) diff --git a/deploy/observability/dashboards/betterdb-anomalies.json b/deploy/observability/dashboards/betterdb-anomalies.json index a5dc64c06..d26c71bb4 100644 --- a/deploy/observability/dashboards/betterdb-anomalies.json +++ b/deploy/observability/dashboards/betterdb-anomalies.json @@ -1,13 +1,13 @@ { "uid": "betterdb-anomalies", - "title": "BetterDB \u00b7 Anomalies", + "title": "BetterDB · Anomalies", "tags": [ "betterdb", "valkey" ], "schemaVersion": 39, "version": 1, - "refresh": "30s", + "refresh": "1m", "time": { "from": "now-6h", "to": "now" @@ -471,7 +471,7 @@ { "id": 11, "type": "timeseries", - "title": "Correlated groups (total)", + "title": "Correlated groups (new, 1h)", "description": "Active groups of anomalies the correlator has linked together.", "datasource": { "type": "prometheus", @@ -493,8 +493,8 @@ "type": "prometheus", "uid": "${ds}" }, - "expr": "betterdb_correlated_groups_total{job=~\"$job\",connection=~\"$connection\"}", - "legendFormat": "{{connection}}" + "expr": "sum by (connection, severity) (increase(betterdb_correlated_groups_total{job=~\"$job\",connection=~\"$connection\"}[1h]))", + "legendFormat": "{{connection}} {{severity}}" } ], "gridPos": { diff --git a/deploy/observability/dashboards/betterdb-clients.json b/deploy/observability/dashboards/betterdb-clients.json index 7f69f9df0..071f17856 100644 --- a/deploy/observability/dashboards/betterdb-clients.json +++ b/deploy/observability/dashboards/betterdb-clients.json @@ -1,13 +1,13 @@ { "uid": "betterdb-clients", - "title": "BetterDB \u00b7 Clients & Pub/Sub", + "title": "BetterDB · Clients & Pub/Sub", "tags": [ "betterdb", "valkey" ], "schemaVersion": 39, "version": 1, - "refresh": "30s", + "refresh": "1m", "time": { "from": "now-6h", "to": "now" diff --git a/deploy/observability/dashboards/betterdb-cluster-slots.json b/deploy/observability/dashboards/betterdb-cluster-slots.json index e460c961e..b25df7890 100644 --- a/deploy/observability/dashboards/betterdb-cluster-slots.json +++ b/deploy/observability/dashboards/betterdb-cluster-slots.json @@ -1,13 +1,13 @@ { "uid": "betterdb-cluster-slots", - "title": "BetterDB \u00b7 Cluster Slots", + "title": "BetterDB · Cluster Slots", "tags": [ "betterdb", "valkey" ], "schemaVersion": 39, "version": 1, - "refresh": "30s", + "refresh": "1m", "time": { "from": "now-6h", "to": "now" diff --git a/deploy/observability/dashboards/betterdb-instance-vitals.json b/deploy/observability/dashboards/betterdb-instance-vitals.json index 5f0631819..0f3548e5f 100644 --- a/deploy/observability/dashboards/betterdb-instance-vitals.json +++ b/deploy/observability/dashboards/betterdb-instance-vitals.json @@ -1,13 +1,13 @@ { "uid": "betterdb-instance-vitals", - "title": "BetterDB \u00b7 Instance Vitals", + "title": "BetterDB · Instance Vitals", "tags": [ "betterdb", "valkey" ], "schemaVersion": 39, "version": 1, - "refresh": "30s", + "refresh": "1m", "time": { "from": "now-6h", "to": "now" diff --git a/deploy/observability/dashboards/betterdb-keyspace.json b/deploy/observability/dashboards/betterdb-keyspace.json index e7484510d..518eeabc8 100644 --- a/deploy/observability/dashboards/betterdb-keyspace.json +++ b/deploy/observability/dashboards/betterdb-keyspace.json @@ -1,13 +1,13 @@ { "uid": "betterdb-keyspace", - "title": "BetterDB \u00b7 Keyspace & Eviction", + "title": "BetterDB · Keyspace & Eviction", "tags": [ "betterdb", "valkey" ], "schemaVersion": 39, "version": 1, - "refresh": "30s", + "refresh": "1m", "time": { "from": "now-6h", "to": "now" diff --git a/deploy/observability/dashboards/betterdb-monitor-internals.json b/deploy/observability/dashboards/betterdb-monitor-internals.json index 256165b73..0d2f9f4f0 100644 --- a/deploy/observability/dashboards/betterdb-monitor-internals.json +++ b/deploy/observability/dashboards/betterdb-monitor-internals.json @@ -1,13 +1,13 @@ { "uid": "betterdb-monitor-internals", - "title": "BetterDB \u00b7 Monitor Internals", + "title": "BetterDB · Monitor Internals", "tags": [ "betterdb", "monitor" ], "schemaVersion": 39, "version": 1, - "refresh": "30s", + "refresh": "1m", "time": { "from": "now-6h", "to": "now" diff --git a/deploy/observability/dashboards/betterdb-persistence.json b/deploy/observability/dashboards/betterdb-persistence.json index 86b5dbd4a..891be6015 100644 --- a/deploy/observability/dashboards/betterdb-persistence.json +++ b/deploy/observability/dashboards/betterdb-persistence.json @@ -1,13 +1,13 @@ { "uid": "betterdb-persistence", - "title": "BetterDB \u00b7 Persistence", + "title": "BetterDB · Persistence", "tags": [ "betterdb", "valkey" ], "schemaVersion": 39, "version": 1, - "refresh": "30s", + "refresh": "1m", "time": { "from": "now-6h", "to": "now" @@ -139,7 +139,7 @@ "type": "prometheus", "uid": "${ds}" }, - "expr": "time() - betterdb_rdb_last_save_timestamp_seconds{job=~\"$job\",connection=~\"$connection\"}", + "expr": "(time() - betterdb_rdb_last_save_timestamp_seconds{job=~\"$job\",connection=~\"$connection\"}) * (betterdb_rdb_last_save_timestamp_seconds{job=~\"$job\",connection=~\"$connection\"} > bool 0)", "legendFormat": "{{connection}}" } ], diff --git a/deploy/observability/dashboards/betterdb-query-patterns.json b/deploy/observability/dashboards/betterdb-query-patterns.json index 3a953e410..91cba1c77 100644 --- a/deploy/observability/dashboards/betterdb-query-patterns.json +++ b/deploy/observability/dashboards/betterdb-query-patterns.json @@ -1,13 +1,13 @@ { "uid": "betterdb-query-patterns", - "title": "BetterDB \u00b7 Query Patterns", + "title": "BetterDB · Query Patterns", "tags": [ "betterdb", "valkey" ], "schemaVersion": 39, "version": 1, - "refresh": "30s", + "refresh": "1m", "time": { "from": "now-6h", "to": "now" @@ -160,7 +160,7 @@ }, "fieldConfig": { "defaults": { - "unit": "\u00b5s", + "unit": "µs", "custom": { "fillOpacity": 10 } @@ -339,7 +339,7 @@ }, "fieldConfig": { "defaults": { - "unit": "\u00b5s" + "unit": "µs" }, "overrides": [] }, @@ -384,8 +384,8 @@ }, { "id": 9, - "type": "timeseries", - "title": "Command calls by type", + "type": "table", + "title": "Top command calls/sec", "description": "Top command types by call rate.", "datasource": { "type": "prometheus", @@ -407,8 +407,10 @@ "type": "prometheus", "uid": "${ds}" }, - "expr": "topk(10, rate(betterdb_commandstats_calls_total{job=~\"$job\",connection=~\"$connection\"}[5m]))", - "legendFormat": "{{command}}" + "expr": "topk(15, sum by (command) (rate(betterdb_commandstats_calls_total{job=~\"$job\",connection=~\"$connection\"}[5m])))", + "format": "table", + "instant": true, + "range": false } ], "gridPos": { diff --git a/deploy/observability/dashboards/betterdb-security.json b/deploy/observability/dashboards/betterdb-security.json index 9411508ad..f59e04b43 100644 --- a/deploy/observability/dashboards/betterdb-security.json +++ b/deploy/observability/dashboards/betterdb-security.json @@ -1,13 +1,13 @@ { "uid": "betterdb-security", - "title": "BetterDB \u00b7 Security & ACL", + "title": "BetterDB · Security & ACL", "tags": [ "betterdb", "valkey" ], "schemaVersion": 39, "version": 1, - "refresh": "30s", + "refresh": "1m", "time": { "from": "now-6h", "to": "now" @@ -81,7 +81,7 @@ { "id": 1, "type": "timeseries", - "title": "ACL denials/sec", + "title": "ACL denied events captured", "description": "Commands rejected per second by ACL rules. Any sustained rate means a client is attempting unauthorized operations.", "datasource": { "type": "prometheus", @@ -89,7 +89,7 @@ }, "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "short", "custom": { "fillOpacity": 10 } @@ -103,7 +103,7 @@ "type": "prometheus", "uid": "${ds}" }, - "expr": "rate(betterdb_acl_denied{job=~\"$job\",connection=~\"$connection\"}[5m])", + "expr": "betterdb_acl_denied{job=~\"$job\",connection=~\"$connection\"}", "legendFormat": "{{connection}}" } ], @@ -117,7 +117,7 @@ { "id": 2, "type": "timeseries", - "title": "ACL denials by reason", + "title": "ACL denied by reason", "description": "Denial rate split by cause (command, key, channel, auth).", "datasource": { "type": "prometheus", @@ -125,7 +125,7 @@ }, "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "short", "custom": { "fillOpacity": 10 } @@ -139,7 +139,7 @@ "type": "prometheus", "uid": "${ds}" }, - "expr": "rate(betterdb_acl_denied_by_reason{job=~\"$job\",connection=~\"$connection\"}[5m])", + "expr": "betterdb_acl_denied_by_reason{job=~\"$job\",connection=~\"$connection\"}", "legendFormat": "{{reason}}" } ], @@ -153,7 +153,7 @@ { "id": 3, "type": "timeseries", - "title": "ACL denials by user", + "title": "ACL denied by user", "description": "Denial rate split by ACL user.", "datasource": { "type": "prometheus", @@ -161,7 +161,7 @@ }, "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "short", "custom": { "fillOpacity": 10 } @@ -175,7 +175,7 @@ "type": "prometheus", "uid": "${ds}" }, - "expr": "rate(betterdb_acl_denied_by_user{job=~\"$job\",connection=~\"$connection\"}[5m])", + "expr": "betterdb_acl_denied_by_user{job=~\"$job\",connection=~\"$connection\"}", "legendFormat": "{{username}}" } ], @@ -211,8 +211,8 @@ "type": "prometheus", "uid": "${ds}" }, - "expr": "betterdb_cve_findings{job=~\"$job\",connection=~\"$connection\"}", - "legendFormat": "{{connection}}" + "expr": "sum(betterdb_cve_findings{job=~\"$job\",connection=~\"$connection\"}) or vector(0)", + "legendFormat": "findings" } ], "gridPos": { @@ -244,8 +244,8 @@ "type": "prometheus", "uid": "${ds}" }, - "expr": "betterdb_cve_kev{job=~\"$job\",connection=~\"$connection\"}", - "legendFormat": "{{connection}}" + "expr": "sum(betterdb_cve_kev{job=~\"$job\",connection=~\"$connection\"}) or vector(0)", + "legendFormat": "kev" } ], "gridPos": { @@ -277,8 +277,8 @@ "type": "prometheus", "uid": "${ds}" }, - "expr": "betterdb_cve_dataset_stale{job=~\"$job\",connection=~\"$connection\"}", - "legendFormat": "{{connection}}" + "expr": "max(betterdb_cve_dataset_stale{job=~\"$job\",connection=~\"$connection\"}) or vector(0)", + "legendFormat": "dataset stale" } ], "gridPos": { diff --git a/deploy/observability/dashboards/betterdb-vector.json b/deploy/observability/dashboards/betterdb-vector.json index 9748a9cc4..68f010e86 100644 --- a/deploy/observability/dashboards/betterdb-vector.json +++ b/deploy/observability/dashboards/betterdb-vector.json @@ -1,13 +1,13 @@ { "uid": "betterdb-vector", - "title": "BetterDB \u00b7 Vector Search", + "title": "BetterDB · Vector Search", "tags": [ "betterdb", "valkey" ], "schemaVersion": 39, "version": 1, - "refresh": "30s", + "refresh": "1m", "time": { "from": "now-6h", "to": "now" @@ -164,7 +164,9 @@ "unit": "percent", "custom": { "fillOpacity": 10 - } + }, + "max": 100, + "min": 0 }, "overrides": [] }, diff --git a/proprietary/infra/k8s/demo-observability/build-dashboards-configmap.sh b/proprietary/infra/k8s/demo-observability/build-dashboards-configmap.sh index 2f5eec28a..ceff03888 100755 --- a/proprietary/infra/k8s/demo-observability/build-dashboards-configmap.sh +++ b/proprietary/infra/k8s/demo-observability/build-dashboards-configmap.sh @@ -1,32 +1,53 @@ #!/usr/bin/env bash # Build the grafana-dashboards ConfigMap from the shipped dashboard pack. # -# One demo-only override: the `job` template variable defaults to -# "betterdb-monitor" (the OTLP push path, ~2s fresh) so a visitor lands on a -# populated dashboard instead of the empty "betterdb-scrape" job they haven't -# selected yet. They can still flip the variable to compare pull vs push. +# Demo-only override: the `job` template variable is defaulted so a visitor +# lands on a populated dashboard. Most dashboards default to "betterdb-monitor" +# (the OTLP push path, ~2s fresh). The Monitor-Internals dashboard defaults to +# "betterdb-scrape" instead, because its GC/poll histograms exist only on the +# pull path (OTLP mirror mode skips histograms), so they would be empty under +# the push job. # -# The shipped JSON under deploy/observability/dashboards/ is NOT edited — -# external users import it with a generic default; the override is applied to -# throwaway copies here. Idempotent: safe for first install and pack updates. +# The shipped JSON under deploy/observability/dashboards/ is NOT edited — the +# override is applied to throwaway copies here. Idempotent; safe for first +# install and pack updates. set -euo pipefail +shopt -s nullglob SRC="$(cd "$(dirname "$0")/../../../../deploy/observability/dashboards" && pwd)" TMP="$(mktemp -d)" trap 'rm -rf "$TMP"' EXIT -for f in "$SRC"/*.json; do - python3 - "$f" "$TMP/$(basename "$f")" <<'PY' -import json, sys -src, dst = sys.argv[1], sys.argv[2] -d = json.load(open(src)) -dash = d["dashboard"] if isinstance(d, dict) and "dashboard" in d else d -for v in dash.get("templating", {}).get("list", []): - if v.get("name") == "job": - v["current"] = {"selected": True, "text": "betterdb-monitor", "value": "betterdb-monitor"} -json.dump(d, open(dst, "w"), indent=2) +files=("$SRC"/*.json) +if [ ${#files[@]} -eq 0 ]; then + echo "error: no dashboards found under $SRC" >&2 + exit 1 +fi + +# One python pass over every dashboard: set the job default, skip (warn on) any +# file that fails to parse instead of aborting the whole build. +python3 - "$TMP" "${files[@]}" <<'PY' +import json, os, sys +tmp, paths = sys.argv[1], sys.argv[2:] +for src in paths: + name = os.path.basename(src) + try: + d = json.load(open(src)) + except Exception as e: + print(f" skip {name}: {e}", file=sys.stderr) + continue + job = "betterdb-scrape" if d.get("uid") == "betterdb-monitor-internals" else "betterdb-monitor" + for v in d.get("templating", {}).get("list", []): + if v.get("name") == "job": + v["current"] = {"selected": True, "text": job, "value": job} + json.dump(d, open(os.path.join(tmp, name), "w"), indent=2, ensure_ascii=False) PY -done + +built=("$TMP"/*.json) +if [ ${#built[@]} -eq 0 ]; then + echo "error: no dashboards built (all failed to parse)" >&2 + exit 1 +fi kubectl create configmap grafana-dashboards -n demo-observability \ --from-file="$TMP" --dry-run=client -o yaml | kubectl apply -f - From 93e70d556645aeec34ca2cb18db34aeefe602c82 Mon Sep 17 00:00:00 2001 From: Kristiyan Ivanov Date: Sat, 10 Oct 2026 10:09:17 +0300 Subject: [PATCH 08/13] fix(infra): adopt vpc-cni as a managed addon + fix doc job name - Manage vpc-cni as an EKS addon (OVERWRITE adopts the bootstrapped self-managed CNI) carrying POD_MTU=1500; comment documents the bootstrap_self_managed_addons=false + before_compute ordering a clean cluster rebuild should use (not flipped on the live cluster). - README/prometheus.yaml header: the scraped copy is job betterdb-scrape. --- proprietary/infra/k8s/demo-observability/README.md | 6 +++--- proprietary/infra/k8s/demo-observability/prometheus.yaml | 2 +- proprietary/infra/terraform/eks.tf | 8 ++++++++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/proprietary/infra/k8s/demo-observability/README.md b/proprietary/infra/k8s/demo-observability/README.md index 60de58512..9b765fb04 100644 --- a/proprietary/infra/k8s/demo-observability/README.md +++ b/proprietary/infra/k8s/demo-observability/README.md @@ -208,8 +208,8 @@ them on the normal dependency cadence (Grafana CVEs are frequent). carries matching demo-only rules: ingress 3001 from this namespace (scrape) and egress 9090 to it (OTLP push). The policies in `namespace.yaml` are this side of the same pair. -- OTLP-pushed series land under the job derived from the pushed - `service.name` resource attribute; the scraped copy is job `betterdb`. The - dashboards' "Scrape job" variable switches between the two paths. +- OTLP-pushed series land under job `betterdb-monitor` (derived from the pushed + `service.name` resource attribute); the scraped copy is job `betterdb-scrape`. + The dashboards' "Scrape job" variable switches between the two paths. - Rate limiting is NOT active until the WAF ingress annotation from install step 6 is applied — which is why publishing the token is step 7. diff --git a/proprietary/infra/k8s/demo-observability/prometheus.yaml b/proprietary/infra/k8s/demo-observability/prometheus.yaml index f07bd6a85..737c0f73f 100644 --- a/proprietary/infra/k8s/demo-observability/prometheus.yaml +++ b/proprietary/infra/k8s/demo-observability/prometheus.yaml @@ -1,6 +1,6 @@ # Prometheus for the public demo. Cluster-internal only (no Ingress); Grafana # is the public face. Receives the demo tenant's metrics twice: -# - job "betterdb": direct scrape of /api/prometheus/metrics (bearer token) +# - job "betterdb-scrape": direct scrape of /api/prometheus/metrics (bearer token) # - OTLP push: the tenant's built-in mirror posts to the native OTLP # receiver (/api/v1/otlp/v1/metrics); series arrive under the job derived # from the pushed service.name resource attribute. diff --git a/proprietary/infra/terraform/eks.tf b/proprietary/infra/terraform/eks.tf index c29c7f7bb..d34bcb01c 100644 --- a/proprietary/infra/terraform/eks.tf +++ b/proprietary/infra/terraform/eks.tf @@ -32,6 +32,14 @@ module "eks" { # small requests succeed, then the big data packets are silently dropped until # timeout. Clamping only the pod MTU fixes it without touching node-level # jumbo traffic (node-to-node, EBS, VPC endpoints). + # + # NOTE on ordering for a FROM-SCRATCH cluster: EKS bootstraps a self-managed + # vpc-cni at MTU 9001 at creation, and this addon only overwrites it + # afterward, so early pods can briefly come up at 9001. A clean rebuild should + # additionally set `bootstrap_self_managed_addons = false` (cluster-level) and + # `before_compute = true` on this addon so the CNI config is in place before + # any node exists. We do NOT flip those on the live cluster: it already runs + # the CNI (with POD_MTU=1500 set out-of-band), so this addon just adopts it. cluster_addons = { vpc-cni = { addon_version = "v1.20.4-eksbuild.2" # matches the running CNI image From 4fb6b014d93f2f11774a075532a7c91585968024 Mon Sep 17 00:00:00 2001 From: Kristiyan Ivanov Date: Sat, 10 Oct 2026 10:46:59 +0300 Subject: [PATCH 09/13] chore(entitlement): release v1.14.3 Carries the provisioning review fixes (two-tier secret reconcile, env preserve + image converge, 409 split). Bumps the deployment image to v1.14.3. --- proprietary/entitlement/package.json | 2 +- proprietary/infra/k8s/entitlement-deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/proprietary/entitlement/package.json b/proprietary/entitlement/package.json index 8c0eecb27..3bc51a8e3 100644 --- a/proprietary/entitlement/package.json +++ b/proprietary/entitlement/package.json @@ -1,6 +1,6 @@ { "name": "@app/entitlement", - "version": "1.14.2", + "version": "1.14.3", "private": true, "scripts": { "dev": "nest start --watch", diff --git a/proprietary/infra/k8s/entitlement-deployment.yaml b/proprietary/infra/k8s/entitlement-deployment.yaml index 9d1ad58af..48e5e55ad 100644 --- a/proprietary/infra/k8s/entitlement-deployment.yaml +++ b/proprietary/infra/k8s/entitlement-deployment.yaml @@ -22,7 +22,7 @@ spec: serviceAccountName: entitlement containers: - name: entitlement - image: 811740411689.dkr.ecr.us-east-1.amazonaws.com/betterdb-entitlement:v1.14.2 + image: 811740411689.dkr.ecr.us-east-1.amazonaws.com/betterdb-entitlement:v1.14.3 imagePullPolicy: Always ports: - containerPort: 3002 From cb0f4135f333ba7f2614a09fb52c331441d00fbf Mon Sep 17 00:00:00 2001 From: Kristiyan Ivanov Date: Sun, 11 Oct 2026 12:32:24 +0300 Subject: [PATCH 10/13] fix(provisioning): address bot review; release v1.14.4 - Revert the secret reconcile to add-only (write missing/empty keys only). Overwriting a populated value does not reach a running pod without a restart (K8s injects Secret env only at container start), so reconcile was ineffective and risked mid-flight config changes. Missing/empty repair, which un-wedges older tenants, is kept. - Preserve the live image on converge instead of setting it: flipping the image here races waitForDeploymentReady (an old-image pod can report ready before the new template rolls). Fleet image rollouts use the version/rollout path, not re-provision. --- proprietary/entitlement/package.json | 2 +- .../src/provisioning/provisioning.service.ts | 81 ++++++++----------- .../infra/k8s/entitlement-deployment.yaml | 2 +- 3 files changed, 36 insertions(+), 49 deletions(-) diff --git a/proprietary/entitlement/package.json b/proprietary/entitlement/package.json index 3bc51a8e3..29c0e5b9a 100644 --- a/proprietary/entitlement/package.json +++ b/proprietary/entitlement/package.json @@ -1,6 +1,6 @@ { "name": "@app/entitlement", - "version": "1.14.3", + "version": "1.14.4", "private": true, "scripts": { "dev": "nest start --watch", diff --git a/proprietary/entitlement/src/provisioning/provisioning.service.ts b/proprietary/entitlement/src/provisioning/provisioning.service.ts index 865c61cb7..30ff5fdcb 100644 --- a/proprietary/entitlement/src/provisioning/provisioning.service.ts +++ b/proprietary/entitlement/src/provisioning/provisioning.service.ts @@ -41,18 +41,6 @@ const CORE_SECRET_ENV_KEYS = [ 'ENTITLEMENT_API_KEY', ] as const; -// Subset of db-credentials keys that are control-plane *config*, not per-tenant -// generated secrets: a re-provision reconciles these to the current desired -// value. Everything else (SESSION_SECRET and the *_TOKEN values) is generated -// once and never rotated out from under a running tenant. -const RECONCILABLE_SECRET_KEYS: ReadonlySet = new Set([ - 'STORAGE_URL', - 'CLOUD_MODE', - 'AUTH_PUBLIC_KEY', - 'ENTITLEMENT_API_URL', - 'ENTITLEMENT_API_KEY', -]); - // The public SNI host is an opaque, stable label derived from the instance id // rather than its name: this keeps it globally unique on the shared wildcard // endpoint (no cross-tenant collisions) without leaking the friendly name. @@ -1141,44 +1129,42 @@ export class ProvisioningService { if (!this.isAlreadyExistsError(error)) { throw error; } - // Re-provision: the Secret already exists. Reconcile it to the current - // schema. An older tenant's Secret can predate current keys (e.g. lack - // CLOUD_MODE / AUTH_PUBLIC_KEY / ENTITLEMENT_API_*), and the Deployment - // references every key as a non-optional secretKeyRef — a missing one - // wedges the new pod in CreateContainerConfigError. Missing/empty keys are - // written; control-plane config in RECONCILABLE_SECRET_KEYS is updated to - // the current value (so a migrated STORAGE_URL or rotated AUTH_PUBLIC_KEY - // is picked up); generated secrets/tokens are never overwritten. (A + // Re-provision: the Secret already exists. Backfill any key it is missing + // or has blank. An older tenant's Secret can predate current keys (e.g. + // lack CLOUD_MODE / AUTH_PUBLIC_KEY / ENTITLEMENT_API_*), and the + // Deployment references every key as a non-optional secretKeyRef — a + // missing one wedges the new pod in CreateContainerConfigError. Only ADD + // absent/empty keys; never overwrite a populated value, so a running + // tenant's secrets and tokens are not rotated out from under it (changing + // a live value also needs a pod restart, which this path does not do). (A // non-demo tenant's stale PROMETHEUS_METRICS_PUBLIC_TOKEN is pruned later // in createDeployment, once the spec no longer references it.) - this.logger.warn(`Secret db-credentials already exists in ${namespace}, reconciling keys...`); - await this.backfillSecretKeys(namespace, stringData, RECONCILABLE_SECRET_KEYS); + this.logger.warn(`Secret db-credentials already exists in ${namespace}, backfilling missing keys...`); + await this.backfillSecretKeys(namespace, stringData); } } - // Reconciles the db-credentials Secret toward `desired` and returns the keys - // it wrote. A key that is absent OR empty is always written (brings an older - // tenant up to the current key set and repairs a blank value). A key already - // present with a value is overwritten only when it is in `reconcileKeys` and - // its value changed — so control-plane config (STORAGE_URL, AUTH_PUBLIC_KEY, - // ...) is picked up on re-provision, while generated secrets (SESSION_SECRET - // and the *_TOKEN values) are never rotated out from under a running tenant. + // Writes any db-credentials key that is missing OR empty, and returns the + // keys written. Existing non-empty values are left untouched: re-provision + // brings an older tenant's key SET up to date (un-wedging a pod that lacks a + // required secretKeyRef — the kubelet recovers from CreateContainerConfigError + // on its own once the key exists) without rotating live secrets/tokens. It + // deliberately does NOT update a populated value: K8s injects Secret env only + // at container start, so changing one here wouldn't reach a running pod + // without a coordinated restart — that is handled by the dedicated rotation + // flow, not re-provision. private async backfillSecretKeys( namespace: string, desired: Record, - reconcileKeys: ReadonlySet = new Set(), ): Promise { const secret = await this.coreApi.readNamespacedSecret({ name: 'db-credentials', namespace }); const data = secret.data ?? {}; - const current = (key: string): string | undefined => - data[key] === undefined ? undefined : Buffer.from(data[key], 'base64').toString('utf8'); + const hasValue = (key: string): boolean => + data[key] !== undefined && Buffer.from(data[key], 'base64').toString('utf8') !== ''; const patch: Record = {}; for (const [key, value] of Object.entries(desired)) { - const existing = current(key); - if (!existing) { + if (!hasValue(key)) { patch[key] = value; // missing or empty -> write - } else if (reconcileKeys.has(key) && existing !== value) { - patch[key] = value; // config drifted -> reconcile to desired } } const written = Object.keys(patch); @@ -1189,7 +1175,7 @@ export class ProvisioningService { { name: 'db-credentials', namespace, body: { stringData: patch } }, k8s.setHeaderOptions('Content-Type', k8s.PatchStrategy.MergePatch), ); - this.logger.log(`Reconciled db-credentials keys in ${namespace}: ${written.join(', ')}`); + this.logger.log(`Backfilled missing db-credentials keys in ${namespace}: ${written.join(', ')}`); return written; } @@ -1399,12 +1385,14 @@ export class ProvisioningService { // tenant shouldn't keep (the desired env simply omits them) — except a // small allowlist of control-plane-derived vars (COOKIE_DOMAIN) carried // forward from the live pod when the desired env omits them, so env drift - // in this replica can't strip them. We converge env, resources, replicas - // AND the image (so a DEFAULT_IMAGE_TAG bump reaches re-provisioned - // tenants); pod-template annotations (e.g. the metrics-token restart - // marker) and probes are preserved. Retry only a 409 Conflict (stale - // resourceVersion from a concurrent status write), re-reading each attempt - // with a short backoff. + // in this replica can't strip them. We converge only env, resources and + // replicas; the live image is intentionally preserved — fleet image + // rollouts go through the version/rollout path, not re-provision, and + // flipping it here would race waitForDeploymentReady (an old-image pod can + // report ready before the new template rolls). Pod-template annotations + // (e.g. the metrics-token restart marker) and probes are preserved too. + // Retry only a 409 Conflict (stale resourceVersion from a concurrent + // status write), re-reading each attempt with a short backoff. this.logger.log(`Deployment already exists in ${namespace}, converging app container`); const desired = body.spec!.template!.spec!.containers![0]; const PRESERVE_ENV = new Set(['COOKIE_DOMAIN']); @@ -1423,7 +1411,6 @@ export class ProvisioningService { ); c.env = [...desiredEnv, ...carried]; c.resources = desired.resources; - c.image = desired.image; existing.spec!.replicas = body.spec!.replicas; await this.appsApi.replaceNamespacedDeployment({ name: 'betterdb', namespace, body: existing }); break; @@ -1800,9 +1787,9 @@ export class ProvisioningService { namespace: string, key: 'PROMETHEUS_METRICS_TOKEN' | 'PROMETHEUS_METRICS_PUBLIC_TOKEN' = 'PROMETHEUS_METRICS_TOKEN', ): Promise { - // Add-only (the token key is not in RECONCILABLE_SECRET_KEYS): a missing or - // empty value is generated, an existing one is left in place so it is never - // rotated. Returns whether a fresh token was written. + // backfillSecretKeys is add-only: a missing or empty token is generated, an + // existing one is left in place so it is never rotated. Returns whether a + // fresh token was written. const token = crypto.randomBytes(32).toString('hex'); const written = await this.backfillSecretKeys(namespace, { [key]: token }); return written.includes(key); diff --git a/proprietary/infra/k8s/entitlement-deployment.yaml b/proprietary/infra/k8s/entitlement-deployment.yaml index 48e5e55ad..7887bb76c 100644 --- a/proprietary/infra/k8s/entitlement-deployment.yaml +++ b/proprietary/infra/k8s/entitlement-deployment.yaml @@ -22,7 +22,7 @@ spec: serviceAccountName: entitlement containers: - name: entitlement - image: 811740411689.dkr.ecr.us-east-1.amazonaws.com/betterdb-entitlement:v1.14.3 + image: 811740411689.dkr.ecr.us-east-1.amazonaws.com/betterdb-entitlement:v1.14.4 imagePullPolicy: Always ports: - containerPort: 3002 From 9122716a7b7af7eb85602d48e312060f9e112ad6 Mon Sep 17 00:00:00 2001 From: Kristiyan Ivanov Date: Sun, 11 Oct 2026 12:32:24 +0300 Subject: [PATCH 11/13] fix(infra): close Grafana WAF Host:port rate-limit bypass Match the demo-grafana host with a regex allowing an optional :port, like the metrics rule, so a Host header such as demo-grafana.app.betterdb.com:443 can't skip the per-IP cap. --- proprietary/infra/terraform/demo-observability.tf | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/proprietary/infra/terraform/demo-observability.tf b/proprietary/infra/terraform/demo-observability.tf index a085cd86b..f6b4c4a6c 100644 --- a/proprietary/infra/terraform/demo-observability.tf +++ b/proprietary/infra/terraform/demo-observability.tf @@ -169,9 +169,10 @@ resource "aws_wafv2_web_acl" "tenant_alb" { aggregate_key_type = "IP" scope_down_statement { - byte_match_statement { - search_string = "demo-grafana.app.betterdb.com" - positional_constraint = "EXACTLY" + # Optional :port so a Host like demo-grafana.app.betterdb.com:443 + # can't bypass an exact match (same guard as the metrics rule above). + regex_match_statement { + regex_string = "^demo-grafana\\.app\\.betterdb\\.com(:[0-9]+)?$" field_to_match { single_header { name = "host" From 44e781fcdd37e573eba96469b43ec31ec328a9a6 Mon Sep 17 00:00:00 2001 From: Kristiyan Ivanov Date: Sun, 11 Oct 2026 12:32:24 +0300 Subject: [PATCH 12/13] fix(dashboards,docs): honest CVE staleness + document job-default override - CVE dataset-stale stat drops the 'or vector(0)' fallback: absent data now shows No data rather than a false 'fresh' (0). Findings/KEV keep the 0 (a genuine 'none found' is useful and not a false health claim). - docs/live-demo.md + deploy/observability/README.md: list all ten dashboards and state the demo-only job-default override (shipped files unchanged). --- deploy/observability/README.md | 4 +++- deploy/observability/dashboards/betterdb-security.json | 4 ++-- docs/live-demo.md | 10 +++++++--- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/deploy/observability/README.md b/deploy/observability/README.md index ca92139a3..a98f794d4 100644 --- a/deploy/observability/README.md +++ b/deploy/observability/README.md @@ -8,7 +8,9 @@ A hosted instance of this pack runs against the public demo workspace — read-only Grafana at `https://demo-grafana.app.betterdb.com` and a published scrape credential for `https://demo.app.betterdb.com/api/prometheus/metrics` (see [docs/live-demo.md](../../docs/live-demo.md)). The dashboards there are -these JSON files, unmodified. +these JSON files, with one demo-only override applied to the deployed copies: +the `job` template variable's default is pointed at the demo's push/scrape job +so panels populate on landing. The shipped files here are unchanged. ## What's here diff --git a/deploy/observability/dashboards/betterdb-security.json b/deploy/observability/dashboards/betterdb-security.json index f59e04b43..a01e6ce50 100644 --- a/deploy/observability/dashboards/betterdb-security.json +++ b/deploy/observability/dashboards/betterdb-security.json @@ -259,7 +259,7 @@ "id": 6, "type": "stat", "title": "CVE dataset stale", - "description": "1 when the local CVE dataset is older than the refresh window, so findings may be incomplete.", + "description": "1 when the local CVE dataset is older than the refresh window. No data means staleness could not be checked (CVE scanning not reporting) — not that the dataset is fresh.", "datasource": { "type": "prometheus", "uid": "${ds}" @@ -277,7 +277,7 @@ "type": "prometheus", "uid": "${ds}" }, - "expr": "max(betterdb_cve_dataset_stale{job=~\"$job\",connection=~\"$connection\"}) or vector(0)", + "expr": "max(betterdb_cve_dataset_stale{job=~\"$job\",connection=~\"$connection\"})", "legendFormat": "dataset stale" } ], diff --git a/docs/live-demo.md b/docs/live-demo.md index 80f166fd8..ebd23b0b5 100644 --- a/docs/live-demo.md +++ b/docs/live-demo.md @@ -11,9 +11,13 @@ data before wiring up your own instance. It renders the shipped dashboard pack ([`deploy/observability/dashboards/`](../deploy/observability/dashboards/)) -against the demo workspace: Instance Vitals, Query Patterns, Cluster Slots, -and Anomalies. These are the exact JSON files you can import into your own -Grafana — nothing demo-specific. +against the demo workspace — ten dashboards: Instance Vitals, Query Patterns, +Cluster Slots, Anomalies, Persistence, Keyspace & Eviction, Clients & Pub/Sub, +Security & ACL, Vector Search, and Monitor Internals. These are the exact JSON +files you can import into your own Grafana. The only demo-specific tweak is the +`job` variable's default (pointed at the demo's push/scrape job so panels +populate on landing), applied to the deployed copies — the shipped files +themselves are unchanged. ## Scrape the demo yourself From 12671c5b9c97df1e1bdd2d3195e88cef4147fb58 Mon Sep 17 00:00:00 2001 From: Kristiyan Ivanov Date: Mon, 12 Oct 2026 10:04:05 +0300 Subject: [PATCH 13/13] test(dashboards): validate the six new dashboards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit observability-assets.spec kept a hardcoded four-file list and asserted the directory matched it exactly, so api-tests failed and none of the new dashboards were checked. Add the five product dashboards (persistence, keyspace, clients, security, vector) to the contract list, and validate Monitor Internals under its own ds/job contract — it charts the Monitor's own runtime (no connection dimension), so the connection-scoped rules do not apply to it. --- .../prometheus/observability-assets.spec.ts | 81 ++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/apps/api/src/prometheus/observability-assets.spec.ts b/apps/api/src/prometheus/observability-assets.spec.ts index fdf74e80c..300ba69e4 100644 --- a/apps/api/src/prometheus/observability-assets.spec.ts +++ b/apps/api/src/prometheus/observability-assets.spec.ts @@ -156,12 +156,22 @@ function allPanels(dashboard: Dashboard): DashboardPanel[] { } describe('Grafana dashboard pack', () => { + // Product dashboards: every panel is scoped to a monitored connection, so + // they all carry the ds/job/connection contract. Monitor Internals is the + // Monitor's own runtime telemetry (no connection dimension) and is checked + // separately below. const files = [ 'betterdb-instance-vitals.json', 'betterdb-query-patterns.json', 'betterdb-cluster-slots.json', 'betterdb-anomalies.json', + 'betterdb-persistence.json', + 'betterdb-keyspace.json', + 'betterdb-clients.json', + 'betterdb-security.json', + 'betterdb-vector.json', ]; + const internalsFile = 'betterdb-monitor-internals.json'; it.each(files)('%s declares the shared contract', (file) => { const dashboard = loadDashboard(file); @@ -278,7 +288,76 @@ describe('Grafana dashboard pack', () => { it('provisions every dashboard file in the directory', () => { const onDisk = readdirSync(DASHBOARD_DIR).filter((file) => file.endsWith('.json')); - expect(onDisk.sort()).toEqual([...files].sort()); + expect(onDisk.sort()).toEqual([...files, internalsFile].sort()); + }); + + // Monitor Internals charts the Monitor process itself (Node.js runtime, poll + // loop, OTLP export). Those series have no `connection` label, so it carries + // only the ds/job contract — the connection-scoped rules above do not apply. + describe('Monitor Internals dashboard', () => { + it('declares the shared contract', () => { + const dashboard = loadDashboard(internalsFile); + + expect(dashboard.uid).toBe(internalsFile.replace('.json', '')); + expect(dashboard.title.startsWith('BetterDB · ')).toBe(true); + expect(dashboard.schemaVersion).toBeGreaterThanOrEqual(39); + }); + + it('exposes a datasource variable first, then a scrape job variable', () => { + const list = loadDashboard(internalsFile).templating.list; + + expect(list[0].name).toBe('ds'); + expect(list[0].type).toBe('datasource'); + expect(list.find((item) => item.name === 'job')?.type).toBe('query'); + }); + + it('points every target at the datasource variable and filters by job', () => { + const targets = allPanels(loadDashboard(internalsFile)).flatMap( + (panel) => panel.targets ?? [], + ); + + expect(targets.length).toBeGreaterThan(0); + for (const target of targets) { + expect(target.datasource).toEqual({ type: 'prometheus', uid: '${ds}' }); + expect(target.expr).toContain('job=~"$job"'); + } + }); + + it('leaves no literal datasource uid behind', () => { + expect(readFileSync(path.join(DASHBOARD_DIR, internalsFile), 'utf8')).not.toContain( + 'betterdb-prometheus', + ); + }); + + it('only queries metrics the service registers or collects by default', () => { + const registered = registeredMetricNames(); + // Node.js runtime and process metrics come from prom-client's + // collectDefaultMetrics (prefixed betterdb_), registered at startup rather + // than in the service constructor this registry snapshot is built from. + const isDefaultMetric = (name: string) => + name.startsWith('betterdb_nodejs_') || name.startsWith('betterdb_process_'); + const used = allPanels(loadDashboard(internalsFile)) + .flatMap((panel) => panel.targets ?? []) + .flatMap((target) => metricNamesIn(target.expr)); + + expect(used.length).toBeGreaterThan(0); + for (const name of used) { + const base = name.replace(/_(bucket|sum|count)$/, ''); + expect(registered.has(name) || registered.has(base) || isDefaultMetric(name)).toBe(true); + } + }); + + it('asks for table frames on every table panel', () => { + const tables = allPanels(loadDashboard(internalsFile)).filter( + (panel) => panel.type === 'table', + ); + + for (const panel of tables) { + for (const target of panel.targets ?? []) { + expect(target.format).toBe('table'); + } + } + }); }); });