From 6fe3b7a02ea9876b15220a46c2843d8402eeea83 Mon Sep 17 00:00:00 2001 From: Doug Oucharek Date: Fri, 9 Oct 2026 17:17:16 -0700 Subject: [PATCH] QFw: say which component wrote each log line The logs panels show a line's body and hide its labels, and a job's story has two writers, so the eight lines under a trace gave no hint which two came from the client. Each query now ends in a line_format that prefixes the line with its writer, "[client]" or "[qpm fake-iqm-20q]", on the QFw Trace panel, the warnings panel of QFw Jobs and the hop from a span to its logs in Explore. The labels stay in a line's details. Signed-off-by: Doug Oucharek --- telemetry/README.md | 5 +++++ telemetry/grafana/dashboards/qfw-jobs.json | 2 +- telemetry/grafana/dashboards/qfw-trace.json | 2 +- telemetry/grafana/provisioning/datasources/qfw.yaml | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/telemetry/README.md b/telemetry/README.md index f7464da..d499ab9 100644 --- a/telemetry/README.md +++ b/telemetry/README.md @@ -172,6 +172,11 @@ So a job's lines are one query away, from the client and the QPM alike: {service_name="qfw-qpm", qfw_device_name="fake-iqm-20q"} | severity_text=~"ERROR|CRITICAL" ``` +On the dashboards, and in the hop from a span to its logs, each line starts +with the component that wrote it, `[client]` or `[qpm fake-iqm-20q]`: the +panels show a line's body, not its labels, and a job's story has two +writers. The labels are still there in a line's details. + Three links tie the signals together: the QFw Trace dashboard shows a job's lines under its waterfall; a span's **Logs for this span** button in Explore runs the first query for its trace; and a log line's `trace_id` opens its diff --git a/telemetry/grafana/dashboards/qfw-jobs.json b/telemetry/grafana/dashboards/qfw-jobs.json index 4b80b8b..1f7d1d4 100644 --- a/telemetry/grafana/dashboards/qfw-jobs.json +++ b/telemetry/grafana/dashboards/qfw-jobs.json @@ -898,7 +898,7 @@ "uid": "qfw-loki" }, "refId": "A", - "expr": "{service_name=~\"qfw-.+\"} | severity_text=~\"WARN.*|ERROR|CRITICAL\"", + "expr": "{service_name=~\"qfw-.+\"} | severity_text=~\"WARN.*|ERROR|CRITICAL\" | line_format \"[{{.qfw_component_role}}{{if .qfw_device_name}} {{.qfw_device_name}}{{end}}] {{__line__}}\"", "queryType": "range" } ], diff --git a/telemetry/grafana/dashboards/qfw-trace.json b/telemetry/grafana/dashboards/qfw-trace.json index d9a5375..a6057e5 100644 --- a/telemetry/grafana/dashboards/qfw-trace.json +++ b/telemetry/grafana/dashboards/qfw-trace.json @@ -116,7 +116,7 @@ "uid": "qfw-loki" }, "refId": "A", - "expr": "{service_name=~\"qfw-.+\"} | trace_id=\"${traceId}\"", + "expr": "{service_name=~\"qfw-.+\"} | trace_id=\"${traceId}\" | line_format \"[{{.qfw_component_role}}{{if .qfw_device_name}} {{.qfw_device_name}}{{end}}] {{__line__}}\"", "queryType": "range" } ], diff --git a/telemetry/grafana/provisioning/datasources/qfw.yaml b/telemetry/grafana/provisioning/datasources/qfw.yaml index 134c416..8d9c879 100644 --- a/telemetry/grafana/provisioning/datasources/qfw.yaml +++ b/telemetry/grafana/provisioning/datasources/qfw.yaml @@ -38,7 +38,7 @@ datasources: customQuery: true # Grafana expands $VAR in provisioning files, so the span variable # the query needs at run time is written with a doubled dollar. - query: '{service_name=~"qfw-.+"} | trace_id="$${__span.traceId}"' + query: '{service_name=~"qfw-.+"} | trace_id="$${__span.traceId}" | line_format "[{{.qfw_component_role}}{{if .qfw_device_name}} {{.qfw_device_name}}{{end}}] {{__line__}}"' - name: Loki uid: qfw-loki