From dab13c1877bceaa6ac0ba16ce19e2de74e99dba2 Mon Sep 17 00:00:00 2001 From: groundnuty Date: Sat, 1 Aug 2026 00:36:51 +0200 Subject: [PATCH 1/2] feat(prd): move Loki to IONOS S3 and restore a highly-available distributed topology Loki persisted ~470 days of logs but only ~2 hours were queryable. The `filesystem` object store needs one filesystem visible to every component; the ingester and querier each had their own ReadWriteOnce volume, so the querier searched an empty 200Gi disk while 127 GiB of chunks sat on the ingester's. Queries returned HTTP 200 with zero chunks, which Grafana renders as "No data". A shared volume is not possible here: the IONOS CSI driver refuses to mount one block device twice ("Resource busy"), even read-only, and no RWX storage class exists. Object storage removes the constraint - every component reads the same bucket - so the distributed topology is finally correct rather than broken. Storage: - object_store / shared_store: filesystem -> s3 (bucket loki-prd, eu-central-3) - schema (v11), store (boltdb-shipper) and index period are UNCHANGED, which is what lets the migrated historical chunks resolve - credentials via SealedSecret + `-config.expand-env=true`; never in git High availability: - replication_factor 1 -> 2, so a single ingester restart no longer stops ingestion cluster-wide (observed twice: 210 push-failures on 2026-07-13) - ingester/distributor/querier/query-frontend/gateway now run 2 replicas - PodDisruptionBudgets become meaningful now that replicas > 1 (see #2497) Also: - query-frontend was capped at 64Mi and OOMKilled every ~8 min (23 restarts), which users experienced as intermittent Loki errors; given real resources - memcached (index-queries, chunks, frontend) enabled - without caching every query refetches from S3 and browsing months of logs is unusably slow - serviceMonitor enabled; Loki previously exported no metrics at all, so failures of the logging stack itself were invisible - ingester PVC 200Gi -> 10Gi (WAL only; chunks live in S3), querier PVC removed The existing 1,892,200 objects / 122.28 GiB covering 2025-04-18 -> 2026-07-31 were migrated to loki-prd and verified: a Loki reading only from S3 answered queries at 1/7/30/90/180/300/400 days, matching pre-migration results. Retention is deliberately NOT enabled here and is tracked separately. Co-Authored-By: Claude Opus 5 (1M context) --- .../monitoring/loki-distributed.yaml | 9 + .../loki-s3-sealed-secret.yaml | 16 ++ ionos_prd/loki-distributed/values.yaml | 208 +++++++++++++++--- 3 files changed, 206 insertions(+), 27 deletions(-) create mode 100644 ionos_prd/loki-distributed/loki-s3-sealed-secret.yaml diff --git a/applications_prd/monitoring/loki-distributed.yaml b/applications_prd/monitoring/loki-distributed.yaml index 3a211577c..a0bed566e 100644 --- a/applications_prd/monitoring/loki-distributed.yaml +++ b/applications_prd/monitoring/loki-distributed.yaml @@ -18,6 +18,15 @@ spec: - repoURL: https://github.com/DOME-Marketplace/dome-gitops targetRevision: HEAD ref: values + # The `values` source above only supplies valueFiles; it does not deploy + # manifests. This third source deploys the S3 SealedSecret that every Loki + # component now needs, using the same pattern as applications_prd/argocd.yaml. + # `include` is required so values.yaml is not applied as a manifest. + - repoURL: https://github.com/DOME-Marketplace/dome-gitops + targetRevision: HEAD + path: ionos_prd/loki-distributed + directory: + include: 'loki-s3-sealed-secret.yaml' destination: namespace: loki-distributed server: https://kubernetes.default.svc diff --git a/ionos_prd/loki-distributed/loki-s3-sealed-secret.yaml b/ionos_prd/loki-distributed/loki-s3-sealed-secret.yaml new file mode 100644 index 000000000..7eb66bdf7 --- /dev/null +++ b/ionos_prd/loki-distributed/loki-s3-sealed-secret.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + creationTimestamp: null + name: loki-s3-credentials + namespace: loki-distributed +spec: + encryptedData: + AWS_ACCESS_KEY_ID: AgBlNMuBdtRvADeZ33VD/jtT7tV3gyoLNVX3xXarOZT17HQP/az+/q6XuqED+8zw6S5HYJ5p4+vOL2wSOKIWPGPpju5YMC6uRXx4EppjlhWoh/JUWDqkMydpgZD4OlyD+tHe0cLs8ELD8leTFpP8HMAjXNv3YTroWONHPH5iilWRwfzl3lXb4ITOM7/F45eXNnP4QEqVeKBtNab8RpaUOd6RaopJif/wmQj+QmfrExGFrR1YUUdoNfj8AhdlaYyHle5c0X1sNRFh611io3pyGpILoypyOYyjOrV1iVCIe98W6+RFORxz4UUex4R4o7o43xYQfVQhFtzYBqXwYbGla426LPM0h0ZeTizT0Q/raI24Zot2xaF4Buw/FXHgRLJiUtD0eH5pG1jqiNbI2eQp1nX6UFEbUJ7sNoCiYdKFhfaX4StCkpecADH8kUIcAeQsHw6ntqVVbVTPSlkbKEcLREHV8NioYStcXWhnvsNQZOI+X76kSitRqrvvi7hkoCvY4vqgphPyUuzZCe2Br3XrsULXU/QzWHhkWRhEURPJsTdqI+4Se3l6LGYeZtUVn+dcgjuomPZPRdnbP30bkPrhjgLyylXJXcaHX8aSJchHsUhDdjH53FL7sbxU2KjvCQGEejgNlL8LbAVFqZxQEYaj4Z2OF/OMPC3gic7TvrKBjBC8GsrqIKxks2MpWdsEdAJYRS4XM641w2RjZs46L+B7rDSWTKoK6HOPVXqg8htWUolvnsDNbuYjeakykfZyy/RFuXUnwEaSvi5zdKUYDSW+7G9xIwzz9KvUgCnQkU/LjhMB7hrBC2K0EsA/4Lg59Q== + AWS_SECRET_ACCESS_KEY: AgAYgAKbc2VQ0Rbt7LbYpsPDkuBy3il9gQU7t5dTAVU4CDVdp0VEQSSnUDbS6V4OKYX1AOLoJZqBvxU27H9lQjt39QGioaqWoiPtCvsp22Vu/mzDsXXUzQYKT7IVOivhY/qPnwtpWhByq59bvDoluGb2AMeNngVDXAshKyxxS+N+VJ0nWCBFoTZHb5mqI0Krz1Q5ZeTgvdl57xsDtfzjLjHJyMLxtPBZ96uU0UB3LgZRcZlIQwlne40XI6nHgv45skFGED/0aE1DFN9UYJ7550eLga1dgbNO6Re3f+0bBc9vmBvdtIRy4dhqFn0RVPyq+bZ0OOGxOCBB2L+8Ktjcvd4iuKSuIJbxq2kKFAStmT+yLY8FWrW/2Qsj3GzxzlzZZxl+5VjKpilOvQcb7IV5H3BBMTfuHUJjpDHXAT8qwwpeHv/taArEUoly+PAQpDfR1K9WM1Gzo5V3CogBjWb9+2d4rFHvNumqhchqfJL+vkKr/vkT1hCXYDcKOC5h56VP7yz4zuu7TB5gVZAuq2EokA1WKRnUzi3aiQJBmFmNkZ5pmdxT+YmvVBTUzUT40W90aAqjyI/i5Hy+eWiWcZIiSSEj5hmvYqucdOSYzMTNBGULS4VuCTaLWUixs4TmJw7p+yrmyat9LHY/OTeUz7XOsIueVtsYi2zKrqANsFJVpjd6Th9uvUdJhgpMXbzx+2U54IKwC8BbE6mHCcRjMkTJuSUrsJMEQhVSwd7oOX1z/oGxn7mbssPbj9uQHIWpX0f4i3AAkC4skF/wL1NVgtcQuY/M + template: + metadata: + creationTimestamp: null + name: loki-s3-credentials + namespace: loki-distributed diff --git a/ionos_prd/loki-distributed/values.yaml b/ionos_prd/loki-distributed/values.yaml index a9a1611cd..e2e399513 100644 --- a/ionos_prd/loki-distributed/values.yaml +++ b/ionos_prd/loki-distributed/values.yaml @@ -1,65 +1,219 @@ global: dnsService: "coredns" -compactor: - enabled: true - resources: - requests: - cpu: 10m - memory: 64Mi - limits: - cpu: 100m - memory: 128Mi +# --------------------------------------------------------------------------- +# Loki stores logs in IONOS Object Storage (bucket: loki-prd), not on local +# volumes. +# +# Why: the `filesystem` object store requires ONE filesystem visible to every +# component. Previously ingester and querier each had their own ReadWriteOnce +# volume, so the querier searched an empty disk while ~470 days of chunks sat on +# the ingester's. Queries returned HTTP 200 with zero chunks, which Grafana +# renders as "No data" — logs looked lost but were only unreachable. +# +# A shared volume is impossible here: the IONOS CSI driver refuses to mount one +# block device twice ("Resource busy"), even read-only, and no RWX storage class +# exists. Object storage removes the constraint entirely: every component reads +# the same bucket, so the distributed topology is finally correct. +# +# The existing 127 GiB (1,892,200 objects, covering 2025-04-18 -> 2026-07-31) +# has already been migrated into loki-prd and verified queryable end-to-end. +# --------------------------------------------------------------------------- +loki: + # Deep-merged over the chart's templated config. + structuredConfig: + schema_config: + configs: + # Unchanged from the previous deployment EXCEPT object_store, which + # moves filesystem -> s3. Schema, store and index period MUST stay the + # same or the migrated historical chunks will not resolve. + - from: "2020-09-07" + store: boltdb-shipper + object_store: s3 + schema: v11 + index: + prefix: loki_index_ + period: 24h + + storage_config: + boltdb_shipper: + shared_store: s3 + active_index_directory: /var/loki/index + cache_location: /var/loki/cache + cache_ttl: 168h + aws: + bucketnames: loki-prd + # eu-central-3 uses the DOT form; s3-eu-central-3... does not resolve. + endpoint: s3.eu-central-3.ionoscloud.com + region: eu-central-3 + # Expanded from env (see extraArgs / extraEnvFrom below) so credentials + # live in a SealedSecret and never appear in this file. + access_key_id: ${AWS_ACCESS_KEY_ID} + secret_access_key: ${AWS_SECRET_ACCESS_KEY} + s3forcepathstyle: true + + compactor: + shared_store: s3 + working_directory: /var/loki/compactor + # Retention deliberately NOT enabled here — tracked as separate work. + retention_enabled: false + + ingester: + lifecycler: + ring: + # HA: each stream is held by two ingesters, so losing one no longer + # stops ingestion cluster-wide. + replication_factor: 2 + +# Loki must expand ${...} in the config, and every component that touches object +# storage needs the credentials. Applied to each Loki binary; the gateway runs +# nginx and is excluded. ingester: + replicas: 2 + # Required by the chart whenever replicas > 1; also makes the PDB meaningful. + maxUnavailable: 1 + extraArgs: + - -config.expand-env=true + extraEnvFrom: + - secretRef: + name: loki-s3-credentials persistence: enabled: true claims: + # WAL only — chunks now live in S3. The previous 200Gi claim held the + # chunk store and is no longer needed at that size. - name: data - size: 200Gi + size: 10Gi resources: requests: - cpu: 20m + cpu: 100m memory: 512Mi limits: - cpu: 200m - memory: 768Mi + cpu: "1" + memory: 1Gi distributor: + replicas: 2 + # Required by the chart whenever replicas > 1; also makes the PDB meaningful. + maxUnavailable: 1 + extraArgs: + - -config.expand-env=true + extraEnvFrom: + - secretRef: + name: loki-s3-credentials resources: requests: - cpu: 20m + cpu: 50m memory: 192Mi limits: - cpu: 200m - memory: 256Mi + cpu: 500m + memory: 512Mi querier: + replicas: 2 + # Required by the chart whenever replicas > 1; also makes the PDB meaningful. + maxUnavailable: 1 + extraArgs: + - -config.expand-env=true + extraEnvFrom: + - secretRef: + name: loki-s3-credentials + # No chunk store on disk any more; the boltdb-shipper cache is ephemeral. persistence: - enabled: true - size: 200Gi + enabled: false resources: requests: - cpu: 10m - memory: 64Mi - limits: cpu: 100m - memory: 128Mi + memory: 512Mi + limits: + cpu: "1" + memory: 1Gi queryFrontend: + replicas: 2 + # Required by the chart whenever replicas > 1; also makes the PDB meaningful. + maxUnavailable: 1 + extraArgs: + - -config.expand-env=true + extraEnvFrom: + - secretRef: + name: loki-s3-credentials + # Previously limited to 64Mi and OOMKilled every ~8 minutes (23 restarts) — + # what users experienced as intermittent Loki errors. resources: requests: - cpu: 10m - memory: 32Mi + cpu: 50m + memory: 256Mi limits: - cpu: 100m - memory: 64Mi + cpu: 500m + memory: 512Mi + +compactor: + enabled: true + extraArgs: + - -config.expand-env=true + extraEnvFrom: + - secretRef: + name: loki-s3-credentials + resources: + requests: + cpu: 50m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi gateway: + replicas: 2 + # Required by the chart whenever replicas > 1; also makes the PDB meaningful. + maxUnavailable: 1 resources: requests: cpu: 10m memory: 32Mi limits: cpu: 100m - memory: 64Mi + memory: 128Mi + +# --------------------------------------------------------------------------- +# Caching. Without it every query re-fetches index and chunks from S3 over the +# network, which makes browsing months of logs in Grafana painfully slow. +# --------------------------------------------------------------------------- +memcachedIndexQueries: + enabled: true + replicas: 1 + resources: + requests: + cpu: 25m + memory: 128Mi + limits: + cpu: 250m + memory: 256Mi + +memcachedChunks: + enabled: true + replicas: 1 + resources: + requests: + cpu: 25m + memory: 256Mi + limits: + cpu: 250m + memory: 512Mi + +memcachedFrontend: + enabled: true + replicas: 1 + resources: + requests: + cpu: 25m + memory: 128Mi + limits: + cpu: 250m + memory: 256Mi + +# Loki previously exported no metrics to Prometheus at all, so failures of the +# logging stack itself were invisible. +serviceMonitor: + enabled: true From 0fcf01964e05bc198e23c3c6052e2e420ca5c1f8 Mon Sep 17 00:00:00 2001 From: groundnuty Date: Sat, 1 Aug 2026 00:47:16 +0200 Subject: [PATCH 2/2] feat(prd): collect logs from every node and add a developer-facing logs dashboard Completes the logging work: without these, Loki is highly available but still does not see all pods, and developers have no usable way to read what it holds. promtail: tolerate the ingress-node taint - The DaemonSet ran on 4 of 5 nodes because it did not tolerate `ingress-node=true:NoSchedule`, so nothing scheduled on ingress-pool-* was collected at all - including the ingress-nginx controller's HTTP access logs, which are exactly what is needed to debug 4xx/5xx coming from a component. - Setting `tolerations` replaces the chart defaults, so the master and control-plane entries are carried over deliberately. - Verified with `helm template`: all three tolerations reach the DaemonSet. Grafana: "Application Logs" dashboard - Namespace / Pod / Level / free-text Search variables, so a developer can go from "my component is broken" to its logs without writing LogQL. - Log volume by pod and errors-over-time, plus line/error/pod-count stats. - Time shortcuts out to 1y to make the retained history discoverable. - Deployed via the existing ionos_prd/grafana-dashboards source of the kube-prometheus-stack Application; no Application change needed. - Datasource is referenced through a `datasource` variable because the Loki datasource has no fixed UID in this cluster. Every dashboard query and both label_values() variable queries were executed against a Loki reading from the migrated S3 bucket and returned HTTP 200 with data. Dashboard confirmed provisioned by Grafana (uid dome-application-logs). Co-Authored-By: Claude Opus 5 (1M context) --- .../grafana-dashboards/application-logs.yaml | 502 ++++++++++++++++++ ionos_prd/promtail/values.yaml | 20 +- 2 files changed, 521 insertions(+), 1 deletion(-) create mode 100644 ionos_prd/grafana-dashboards/application-logs.yaml diff --git a/ionos_prd/grafana-dashboards/application-logs.yaml b/ionos_prd/grafana-dashboards/application-logs.yaml new file mode 100644 index 000000000..d915b8601 --- /dev/null +++ b/ionos_prd/grafana-dashboards/application-logs.yaml @@ -0,0 +1,502 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + grafana_dashboard: "1" + name: application-logs +data: + application-logs.json: |- + { + "title": "Application Logs", + "description": "Browse application logs from any pod in the cluster. Pick a namespace, optionally narrow to specific pods, filter by level, and search free-text. Backed by Loki; history goes back to 2025-04.", + "tags": [ + "dome", + "logs", + "loki" + ], + "uid": "dome-application-logs", + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "refresh": "", + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "1h" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d", + "90d", + "1y" + ] + }, + "templating": { + "list": [ + { + "name": "datasource", + "label": "Datasource", + "type": "datasource", + "query": "loki", + "current": {}, + "hide": 0, + "refresh": 1 + }, + { + "name": "namespace", + "label": "Namespace", + "type": "query", + "datasource": { + "type": "loki", + "uid": "${datasource}" + }, + "definition": "label_values(namespace)", + "query": { + "label": "namespace", + "refId": "namespace", + "type": 1 + }, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": [ + "marketplace" + ], + "value": [ + "marketplace" + ] + }, + "refresh": 2, + "sort": 1 + }, + { + "name": "pod", + "label": "Pod", + "type": "query", + "datasource": { + "type": "loki", + "uid": "${datasource}" + }, + "definition": "label_values({namespace=~\"$namespace\"}, pod)", + "query": { + "label": "pod", + "stream": "{namespace=~\"$namespace\"}", + "refId": "pod", + "type": 1 + }, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "refresh": 2, + "sort": 1 + }, + { + "name": "level", + "label": "Level", + "type": "custom", + "query": ". : All,(?i)(warn|error|fatal|panic|exception) : Warnings and errors,(?i)(error|fatal|panic|exception) : Errors only", + "options": [ + { + "selected": true, + "text": "All", + "value": "." + }, + { + "selected": false, + "text": "Warnings and errors", + "value": "(?i)(warn|error|fatal|panic|exception)" + }, + { + "selected": false, + "text": "Errors only", + "value": "(?i)(error|fatal|panic|exception)" + } + ], + "current": { + "selected": true, + "text": "All", + "value": "." + }, + "hide": 0 + }, + { + "name": "search", + "label": "Search", + "type": "textbox", + "query": "", + "current": { + "text": "", + "value": "" + }, + "description": "Free-text, case-insensitive. Leave empty to match everything.", + "hide": 0 + } + ] + }, + "panels": [ + { + "type": "row", + "title": "Overview", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "panels": [] + }, + { + "type": "stat", + "title": "Log lines (selected range)", + "description": "Total lines matching the current namespace / pod / level / search selection.", + "datasource": { + "type": "loki", + "uid": "${datasource}" + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(count_over_time({namespace=~\"$namespace\", pod=~\"$pod\"} |~ \"$level\" |~ \"(?i)$search\" [$__range]))", + "queryType": "instant", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "text", + "graphMode": "area" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "fixed", + "fixedColor": "text" + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "Error lines (selected range)", + "description": "Lines matching error|fatal|panic|exception, regardless of the Level selector.", + "datasource": { + "type": "loki", + "uid": "${datasource}" + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(count_over_time({namespace=~\"$namespace\", pod=~\"$pod\"} |~ \"(?i)(error|fatal|panic|exception)\" |~ \"(?i)$search\" [$__range]))", + "queryType": "instant", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "fixed", + "fixedColor": "text" + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "Pods reporting", + "description": "How many distinct pods produced matching lines. 0 usually means the selection is too narrow.", + "datasource": { + "type": "loki", + "uid": "${datasource}" + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(sum by (pod) (count_over_time({namespace=~\"$namespace\", pod=~\"$pod\"} |~ \"$level\" |~ \"(?i)$search\" [$__range])))", + "queryType": "instant", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "text", + "graphMode": "area" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "fixed", + "fixedColor": "text" + } + }, + "overrides": [] + } + }, + { + "type": "row", + "title": "Volume", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "Log volume by pod", + "description": "Stacked line counts per pod. A sudden spike or a pod dropping to zero is usually what you are looking for.", + "datasource": { + "type": "loki", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (pod) (count_over_time({namespace=~\"$namespace\", pod=~\"$pod\"} |~ \"$level\" |~ \"(?i)$search\" [$__interval]))", + "queryType": "range", + "refId": "A", + "legendFormat": "{{pod}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "sum" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "bars", + "fillOpacity": 30, + "lineWidth": 1, + "stacking": { + "mode": "normal", + "group": "A" + }, + "axisSoftMin": 0 + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Errors over time", + "description": "Error-like lines only. Independent of the Level selector so you always see errors.", + "datasource": { + "type": "loki", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (pod) (count_over_time({namespace=~\"$namespace\", pod=~\"$pod\"} |~ \"(?i)(error|fatal|panic|exception)\" |~ \"(?i)$search\" [$__interval]))", + "queryType": "range", + "refId": "A", + "legendFormat": "{{pod}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "sum" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "bars", + "fillOpacity": 30, + "lineWidth": 1, + "stacking": { + "mode": "normal", + "group": "A" + }, + "axisSoftMin": 0 + } + }, + "overrides": [] + } + }, + { + "type": "row", + "title": "Logs", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "panels": [] + }, + { + "type": "logs", + "title": "Logs", + "description": "Newest first. Click a line to expand labels. Use the Search variable for free-text; it is case-insensitive.", + "datasource": { + "type": "loki", + "uid": "${datasource}" + }, + "gridPos": { + "h": 20, + "w": 24, + "x": 0, + "y": 15 + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "{namespace=~\"$namespace\", pod=~\"$pod\"} |~ \"$level\" |~ \"(?i)$search\"", + "queryType": "range", + "refId": "A" + } + ], + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "dedupStrategy": "none", + "sortOrder": "Descending" + } + } + ] + } diff --git a/ionos_prd/promtail/values.yaml b/ionos_prd/promtail/values.yaml index c2068eeb1..8701297ab 100644 --- a/ionos_prd/promtail/values.yaml +++ b/ionos_prd/promtail/values.yaml @@ -9,4 +9,22 @@ resources: memory: 256Mi limits: cpu: 200m - memory: 384Mi \ No newline at end of file + memory: 384Mi + +# Promtail ran on 4 of 5 nodes: the DaemonSet did not tolerate the +# `ingress-node` taint, so NOTHING scheduled on ingress-pool-* was collected — +# including the ingress-nginx controller's HTTP access logs, which are exactly +# what developers need to debug 4xx/5xx from their components. +# +# Setting `tolerations` REPLACES the chart defaults, so the master and +# control-plane entries below are the chart's own and must be kept. +tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: ingress-node + operator: Exists + effect: NoSchedule