diff --git a/telemetry/README.md b/telemetry/README.md index f7464da..d499ab9 100644 --- a/telemetry/README.md +++ b/telemetry/README.md @@ -172,6 +172,11 @@ So a job's lines are one query away, from the client and the QPM alike: {service_name="qfw-qpm", qfw_device_name="fake-iqm-20q"} | severity_text=~"ERROR|CRITICAL" ``` +On the dashboards, and in the hop from a span to its logs, each line starts +with the component that wrote it, `[client]` or `[qpm fake-iqm-20q]`: the +panels show a line's body, not its labels, and a job's story has two +writers. The labels are still there in a line's details. + Three links tie the signals together: the QFw Trace dashboard shows a job's lines under its waterfall; a span's **Logs for this span** button in Explore runs the first query for its trace; and a log line's `trace_id` opens its diff --git a/telemetry/grafana/dashboards/qfw-jobs.json b/telemetry/grafana/dashboards/qfw-jobs.json index 4b80b8b..1f7d1d4 100644 --- a/telemetry/grafana/dashboards/qfw-jobs.json +++ b/telemetry/grafana/dashboards/qfw-jobs.json @@ -898,7 +898,7 @@ "uid": "qfw-loki" }, "refId": "A", - "expr": "{service_name=~\"qfw-.+\"} | severity_text=~\"WARN.*|ERROR|CRITICAL\"", + "expr": "{service_name=~\"qfw-.+\"} | severity_text=~\"WARN.*|ERROR|CRITICAL\" | line_format \"[{{.qfw_component_role}}{{if .qfw_device_name}} {{.qfw_device_name}}{{end}}] {{__line__}}\"", "queryType": "range" } ], diff --git a/telemetry/grafana/dashboards/qfw-trace.json b/telemetry/grafana/dashboards/qfw-trace.json index d9a5375..a6057e5 100644 --- a/telemetry/grafana/dashboards/qfw-trace.json +++ b/telemetry/grafana/dashboards/qfw-trace.json @@ -116,7 +116,7 @@ "uid": "qfw-loki" }, "refId": "A", - "expr": "{service_name=~\"qfw-.+\"} | trace_id=\"${traceId}\"", + "expr": "{service_name=~\"qfw-.+\"} | trace_id=\"${traceId}\" | line_format \"[{{.qfw_component_role}}{{if .qfw_device_name}} {{.qfw_device_name}}{{end}}] {{__line__}}\"", "queryType": "range" } ], diff --git a/telemetry/grafana/provisioning/datasources/qfw.yaml b/telemetry/grafana/provisioning/datasources/qfw.yaml index 134c416..8d9c879 100644 --- a/telemetry/grafana/provisioning/datasources/qfw.yaml +++ b/telemetry/grafana/provisioning/datasources/qfw.yaml @@ -38,7 +38,7 @@ datasources: customQuery: true # Grafana expands $VAR in provisioning files, so the span variable # the query needs at run time is written with a doubled dollar. - query: '{service_name=~"qfw-.+"} | trace_id="$${__span.traceId}"' + query: '{service_name=~"qfw-.+"} | trace_id="$${__span.traceId}" | line_format "[{{.qfw_component_role}}{{if .qfw_device_name}} {{.qfw_device_name}}{{end}}] {{__line__}}"' - name: Loki uid: qfw-loki