diff --git a/applications_prd/monitoring/loki-distributed.yaml b/applications_prd/monitoring/loki-distributed.yaml index 3a211577c..a0bed566e 100644 --- a/applications_prd/monitoring/loki-distributed.yaml +++ b/applications_prd/monitoring/loki-distributed.yaml @@ -18,6 +18,15 @@ spec: - repoURL: https://github.com/DOME-Marketplace/dome-gitops targetRevision: HEAD ref: values + # The `values` source above only supplies valueFiles; it does not deploy + # manifests. This third source deploys the S3 SealedSecret that every Loki + # component now needs, using the same pattern as applications_prd/argocd.yaml. + # `include` is required so values.yaml is not applied as a manifest. + - repoURL: https://github.com/DOME-Marketplace/dome-gitops + targetRevision: HEAD + path: ionos_prd/loki-distributed + directory: + include: 'loki-s3-sealed-secret.yaml' destination: namespace: loki-distributed server: https://kubernetes.default.svc diff --git a/ionos_prd/grafana-dashboards/application-logs.yaml b/ionos_prd/grafana-dashboards/application-logs.yaml new file mode 100644 index 000000000..25f86c330 --- /dev/null +++ b/ionos_prd/grafana-dashboards/application-logs.yaml @@ -0,0 +1,502 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + grafana_dashboard: "1" + name: application-logs +data: + application-logs.json: |- + { + "title": "Application Logs", + "description": "Browse application logs from any pod in the cluster. Pick a namespace, optionally narrow to specific pods, filter by level, and search free-text. Backed by Loki; history goes back to 2025-04.", + "tags": [ + "dome", + "logs", + "loki" + ], + "uid": "dome-application-logs", + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "refresh": "", + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "1h" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d", + "90d", + "1y" + ] + }, + "templating": { + "list": [ + { + "name": "datasource", + "label": "Datasource", + "type": "datasource", + "query": "loki", + "current": {}, + "hide": 0, + "refresh": 1 + }, + { + "name": "namespace", + "label": "Namespace", + "type": "query", + "datasource": { + "type": "loki", + "uid": "${datasource}" + }, + "definition": "label_values(namespace)", + "query": { + "label": "namespace", + "refId": "namespace", + "type": 1 + }, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": [ + "marketplace" + ], + "value": [ + "marketplace" + ] + }, + "refresh": 2, + "sort": 1 + }, + { + "name": "pod", + "label": "Pod", + "type": "query", + "datasource": { + "type": "loki", + "uid": "${datasource}" + }, + "definition": "label_values({namespace=~\"$namespace\"}, pod)", + "query": { + "label": "pod", + "stream": "{namespace=~\"$namespace\"}", + "refId": "pod", + "type": 1 + }, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "refresh": 2, + "sort": 1 + }, + { + "name": "level", + "label": "Level", + "type": "custom", + "query": ". : All,(?i)(warn|error|fatal|panic|exception) : Warnings and errors,(?i)(error|fatal|panic|exception) : Errors only", + "options": [ + { + "selected": true, + "text": "All", + "value": "." + }, + { + "selected": false, + "text": "Warnings and errors", + "value": "(?i)(warn|error|fatal|panic|exception)" + }, + { + "selected": false, + "text": "Errors only", + "value": "(?i)(error|fatal|panic|exception)" + } + ], + "current": { + "selected": true, + "text": "All", + "value": "." + }, + "hide": 0 + }, + { + "name": "search", + "label": "Search", + "type": "textbox", + "query": "", + "current": { + "text": "", + "value": "" + }, + "description": "Free-text, case-insensitive. Leave empty to match everything.", + "hide": 0 + } + ] + }, + "panels": [ + { + "type": "row", + "title": "Overview", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "panels": [] + }, + { + "type": "stat", + "title": "Log lines (selected range)", + "description": "Total lines matching the current namespace / pod / level / search selection.", + "datasource": { + "type": "loki", + "uid": "${datasource}" + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(count_over_time({namespace=~\"$namespace\", pod=~\"$pod\"} |~ \"$level\" |~ \"(?i)$search\" [$__range]))", + "queryType": "instant", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "text", + "graphMode": "area" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "fixed", + "fixedColor": "text" + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "Error lines (selected range)", + "description": "Lines matching error|fatal|panic|exception, regardless of the Level selector.", + "datasource": { + "type": "loki", + "uid": "${datasource}" + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(count_over_time({namespace=~\"$namespace\", pod=~\"$pod\"} |~ \"(?i)(error|fatal|panic|exception)\" |~ \"(?i)$search\" [$__range]))", + "queryType": "instant", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "fixed", + "fixedColor": "text" + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "title": "Pods reporting", + "description": "How many distinct pods produced matching lines. 0 usually means the selection is too narrow.", + "datasource": { + "type": "loki", + "uid": "${datasource}" + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 1 + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(sum by (pod) (count_over_time({namespace=~\"$namespace\", pod=~\"$pod\"} |~ \"$level\" |~ \"(?i)$search\" [$__range])))", + "queryType": "instant", + "refId": "A" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "text", + "graphMode": "area" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "fixed", + "fixedColor": "text" + } + }, + "overrides": [] + } + }, + { + "type": "row", + "title": "Volume", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "Log volume by pod", + "description": "Stacked line counts per pod. A sudden spike or a pod dropping to zero is usually what you are looking for.", + "datasource": { + "type": "loki", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (pod) (count_over_time({namespace=~\"$namespace\", pod=~\"$pod\"} |~ \"$level\" |~ \"(?i)$search\" [$__interval]))", + "queryType": "range", + "refId": "A", + "legendFormat": "{{pod}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "sum" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "bars", + "fillOpacity": 30, + "lineWidth": 1, + "stacking": { + "mode": "normal", + "group": "A" + }, + "axisSoftMin": 0 + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Errors over time", + "description": "Error-like lines only. Independent of the Level selector so you always see errors.", + "datasource": { + "type": "loki", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (pod) (count_over_time({namespace=~\"$namespace\", pod=~\"$pod\"} |~ \"(?i)(error|fatal|panic|exception)\" |~ \"(?i)$search\" [$__interval]))", + "queryType": "range", + "refId": "A", + "legendFormat": "{{pod}}" + } + ], + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "sum" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "bars", + "fillOpacity": 30, + "lineWidth": 1, + "stacking": { + "mode": "normal", + "group": "A" + }, + "axisSoftMin": 0 + } + }, + "overrides": [] + } + }, + { + "type": "row", + "title": "Logs", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "panels": [] + }, + { + "type": "logs", + "title": "Logs", + "description": "Newest first. Click a line to expand labels. Use the Search variable for free-text; it is case-insensitive.", + "datasource": { + "type": "loki", + "uid": "${datasource}" + }, + "gridPos": { + "h": 20, + "w": 24, + "x": 0, + "y": 15 + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "{namespace=~\"$namespace\", pod=~\"$pod\"} |~ \"$level\" |~ \"(?i)$search\"", + "queryType": "range", + "refId": "A" + } + ], + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "dedupStrategy": "none", + "sortOrder": "Descending" + } + } + ] + } diff --git a/ionos_prd/loki-distributed/loki-s3-sealed-secret.yaml b/ionos_prd/loki-distributed/loki-s3-sealed-secret.yaml new file mode 100644 index 000000000..7eb66bdf7 --- /dev/null +++ b/ionos_prd/loki-distributed/loki-s3-sealed-secret.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + creationTimestamp: null + name: loki-s3-credentials + namespace: loki-distributed +spec: + encryptedData: + AWS_ACCESS_KEY_ID: AgBlNMuBdtRvADeZ33VD/jtT7tV3gyoLNVX3xXarOZT17HQP/az+/q6XuqED+8zw6S5HYJ5p4+vOL2wSOKIWPGPpju5YMC6uRXx4EppjlhWoh/JUWDqkMydpgZD4OlyD+tHe0cLs8ELD8leTFpP8HMAjXNv3YTroWONHPH5iilWRwfzl3lXb4ITOM7/F45eXNnP4QEqVeKBtNab8RpaUOd6RaopJif/wmQj+QmfrExGFrR1YUUdoNfj8AhdlaYyHle5c0X1sNRFh611io3pyGpILoypyOYyjOrV1iVCIe98W6+RFORxz4UUex4R4o7o43xYQfVQhFtzYBqXwYbGla426LPM0h0ZeTizT0Q/raI24Zot2xaF4Buw/FXHgRLJiUtD0eH5pG1jqiNbI2eQp1nX6UFEbUJ7sNoCiYdKFhfaX4StCkpecADH8kUIcAeQsHw6ntqVVbVTPSlkbKEcLREHV8NioYStcXWhnvsNQZOI+X76kSitRqrvvi7hkoCvY4vqgphPyUuzZCe2Br3XrsULXU/QzWHhkWRhEURPJsTdqI+4Se3l6LGYeZtUVn+dcgjuomPZPRdnbP30bkPrhjgLyylXJXcaHX8aSJchHsUhDdjH53FL7sbxU2KjvCQGEejgNlL8LbAVFqZxQEYaj4Z2OF/OMPC3gic7TvrKBjBC8GsrqIKxks2MpWdsEdAJYRS4XM641w2RjZs46L+B7rDSWTKoK6HOPVXqg8htWUolvnsDNbuYjeakykfZyy/RFuXUnwEaSvi5zdKUYDSW+7G9xIwzz9KvUgCnQkU/LjhMB7hrBC2K0EsA/4Lg59Q== + AWS_SECRET_ACCESS_KEY: AgAYgAKbc2VQ0Rbt7LbYpsPDkuBy3il9gQU7t5dTAVU4CDVdp0VEQSSnUDbS6V4OKYX1AOLoJZqBvxU27H9lQjt39QGioaqWoiPtCvsp22Vu/mzDsXXUzQYKT7IVOivhY/qPnwtpWhByq59bvDoluGb2AMeNngVDXAshKyxxS+N+VJ0nWCBFoTZHb5mqI0Krz1Q5ZeTgvdl57xsDtfzjLjHJyMLxtPBZ96uU0UB3LgZRcZlIQwlne40XI6nHgv45skFGED/0aE1DFN9UYJ7550eLga1dgbNO6Re3f+0bBc9vmBvdtIRy4dhqFn0RVPyq+bZ0OOGxOCBB2L+8Ktjcvd4iuKSuIJbxq2kKFAStmT+yLY8FWrW/2Qsj3GzxzlzZZxl+5VjKpilOvQcb7IV5H3BBMTfuHUJjpDHXAT8qwwpeHv/taArEUoly+PAQpDfR1K9WM1Gzo5V3CogBjWb9+2d4rFHvNumqhchqfJL+vkKr/vkT1hCXYDcKOC5h56VP7yz4zuu7TB5gVZAuq2EokA1WKRnUzi3aiQJBmFmNkZ5pmdxT+YmvVBTUzUT40W90aAqjyI/i5Hy+eWiWcZIiSSEj5hmvYqucdOSYzMTNBGULS4VuCTaLWUixs4TmJw7p+yrmyat9LHY/OTeUz7XOsIueVtsYi2zKrqANsFJVpjd6Th9uvUdJhgpMXbzx+2U54IKwC8BbE6mHCcRjMkTJuSUrsJMEQhVSwd7oOX1z/oGxn7mbssPbj9uQHIWpX0f4i3AAkC4skF/wL1NVgtcQuY/M + template: + metadata: + creationTimestamp: null + name: loki-s3-credentials + namespace: loki-distributed diff --git a/ionos_prd/loki-distributed/values.yaml b/ionos_prd/loki-distributed/values.yaml index a9a1611cd..e2e399513 100644 --- a/ionos_prd/loki-distributed/values.yaml +++ b/ionos_prd/loki-distributed/values.yaml @@ -1,65 +1,219 @@ global: dnsService: "coredns" -compactor: - enabled: true - resources: - requests: - cpu: 10m - memory: 64Mi - limits: - cpu: 100m - memory: 128Mi +# --------------------------------------------------------------------------- +# Loki stores logs in IONOS Object Storage (bucket: loki-prd), not on local +# volumes. +# +# Why: the `filesystem` object store requires ONE filesystem visible to every +# component. Previously ingester and querier each had their own ReadWriteOnce +# volume, so the querier searched an empty disk while ~470 days of chunks sat on +# the ingester's. Queries returned HTTP 200 with zero chunks, which Grafana +# renders as "No data" — logs looked lost but were only unreachable. +# +# A shared volume is impossible here: the IONOS CSI driver refuses to mount one +# block device twice ("Resource busy"), even read-only, and no RWX storage class +# exists. Object storage removes the constraint entirely: every component reads +# the same bucket, so the distributed topology is finally correct. +# +# The existing 127 GiB (1,892,200 objects, covering 2025-04-18 -> 2026-07-31) +# has already been migrated into loki-prd and verified queryable end-to-end. +# --------------------------------------------------------------------------- +loki: + # Deep-merged over the chart's templated config. + structuredConfig: + schema_config: + configs: + # Unchanged from the previous deployment EXCEPT object_store, which + # moves filesystem -> s3. Schema, store and index period MUST stay the + # same or the migrated historical chunks will not resolve. + - from: "2020-09-07" + store: boltdb-shipper + object_store: s3 + schema: v11 + index: + prefix: loki_index_ + period: 24h + + storage_config: + boltdb_shipper: + shared_store: s3 + active_index_directory: /var/loki/index + cache_location: /var/loki/cache + cache_ttl: 168h + aws: + bucketnames: loki-prd + # eu-central-3 uses the DOT form; s3-eu-central-3... does not resolve. + endpoint: s3.eu-central-3.ionoscloud.com + region: eu-central-3 + # Expanded from env (see extraArgs / extraEnvFrom below) so credentials + # live in a SealedSecret and never appear in this file. + access_key_id: ${AWS_ACCESS_KEY_ID} + secret_access_key: ${AWS_SECRET_ACCESS_KEY} + s3forcepathstyle: true + + compactor: + shared_store: s3 + working_directory: /var/loki/compactor + # Retention deliberately NOT enabled here — tracked as separate work. + retention_enabled: false + + ingester: + lifecycler: + ring: + # HA: each stream is held by two ingesters, so losing one no longer + # stops ingestion cluster-wide. + replication_factor: 2 + +# Loki must expand ${...} in the config, and every component that touches object +# storage needs the credentials. Applied to each Loki binary; the gateway runs +# nginx and is excluded. ingester: + replicas: 2 + # Required by the chart whenever replicas > 1; also makes the PDB meaningful. + maxUnavailable: 1 + extraArgs: + - -config.expand-env=true + extraEnvFrom: + - secretRef: + name: loki-s3-credentials persistence: enabled: true claims: + # WAL only — chunks now live in S3. The previous 200Gi claim held the + # chunk store and is no longer needed at that size. - name: data - size: 200Gi + size: 10Gi resources: requests: - cpu: 20m + cpu: 100m memory: 512Mi limits: - cpu: 200m - memory: 768Mi + cpu: "1" + memory: 1Gi distributor: + replicas: 2 + # Required by the chart whenever replicas > 1; also makes the PDB meaningful. + maxUnavailable: 1 + extraArgs: + - -config.expand-env=true + extraEnvFrom: + - secretRef: + name: loki-s3-credentials resources: requests: - cpu: 20m + cpu: 50m memory: 192Mi limits: - cpu: 200m - memory: 256Mi + cpu: 500m + memory: 512Mi querier: + replicas: 2 + # Required by the chart whenever replicas > 1; also makes the PDB meaningful. + maxUnavailable: 1 + extraArgs: + - -config.expand-env=true + extraEnvFrom: + - secretRef: + name: loki-s3-credentials + # No chunk store on disk any more; the boltdb-shipper cache is ephemeral. persistence: - enabled: true - size: 200Gi + enabled: false resources: requests: - cpu: 10m - memory: 64Mi - limits: cpu: 100m - memory: 128Mi + memory: 512Mi + limits: + cpu: "1" + memory: 1Gi queryFrontend: + replicas: 2 + # Required by the chart whenever replicas > 1; also makes the PDB meaningful. + maxUnavailable: 1 + extraArgs: + - -config.expand-env=true + extraEnvFrom: + - secretRef: + name: loki-s3-credentials + # Previously limited to 64Mi and OOMKilled every ~8 minutes (23 restarts) — + # what users experienced as intermittent Loki errors. resources: requests: - cpu: 10m - memory: 32Mi + cpu: 50m + memory: 256Mi limits: - cpu: 100m - memory: 64Mi + cpu: 500m + memory: 512Mi + +compactor: + enabled: true + extraArgs: + - -config.expand-env=true + extraEnvFrom: + - secretRef: + name: loki-s3-credentials + resources: + requests: + cpu: 50m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi gateway: + replicas: 2 + # Required by the chart whenever replicas > 1; also makes the PDB meaningful. + maxUnavailable: 1 resources: requests: cpu: 10m memory: 32Mi limits: cpu: 100m - memory: 64Mi + memory: 128Mi + +# --------------------------------------------------------------------------- +# Caching. Without it every query re-fetches index and chunks from S3 over the +# network, which makes browsing months of logs in Grafana painfully slow. +# --------------------------------------------------------------------------- +memcachedIndexQueries: + enabled: true + replicas: 1 + resources: + requests: + cpu: 25m + memory: 128Mi + limits: + cpu: 250m + memory: 256Mi + +memcachedChunks: + enabled: true + replicas: 1 + resources: + requests: + cpu: 25m + memory: 256Mi + limits: + cpu: 250m + memory: 512Mi + +memcachedFrontend: + enabled: true + replicas: 1 + resources: + requests: + cpu: 25m + memory: 128Mi + limits: + cpu: 250m + memory: 256Mi + +# Loki previously exported no metrics to Prometheus at all, so failures of the +# logging stack itself were invisible. +serviceMonitor: + enabled: true diff --git a/ionos_prd/promtail/values.yaml b/ionos_prd/promtail/values.yaml index c2068eeb1..8701297ab 100644 --- a/ionos_prd/promtail/values.yaml +++ b/ionos_prd/promtail/values.yaml @@ -9,4 +9,22 @@ resources: memory: 256Mi limits: cpu: 200m - memory: 384Mi \ No newline at end of file + memory: 384Mi + +# Promtail ran on 4 of 5 nodes: the DaemonSet did not tolerate the +# `ingress-node` taint, so NOTHING scheduled on ingress-pool-* was collected — +# including the ingress-nginx controller's HTTP access logs, which are exactly +# what developers need to debug 4xx/5xx from their components. +# +# Setting `tolerations` REPLACES the chart defaults, so the master and +# control-plane entries below are the chart's own and must be kept. +tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: ingress-node + operator: Exists + effect: NoSchedule