QFw: bring main's Loki logs and the dashboard's Grafana link into defw2-prototype

dashboard/docs/recipes/use-dashboard.md

@@ -10,7 +10,10 @@ qfw-dashboard status
 ```
 
 The Dashboard pane shows fixed health, node, service, allocation, experiment,
-topology, result, and alert widgets. Collapse a widget to reduce its footprint
+topology, result, and alert widgets. When the optional telemetry stack is up
+(see `telemetry/README.md` in the repository root), the health widget and
+the view controls in the top bar link to its Grafana, where QFw's own job
+metrics and traces live. Collapse a widget to reduce its footprint
 or select **Pop out** to mirror it in another browser window. The original
 widget remains in place.
 

dashboard/src/qfw_slurm_dashboard/assets/frontend.js

@@ -928,6 +928,7 @@
       return {
         health: state.health,
         observed_at: state.observed_at,
+        telemetry: telemetryLink(docker),
         sources: Object.values(state.sources || {}).map((source) => ({
           name: source.name,
           status: source.status,
@@ -980,10 +981,45 @@
     return {};
   }
 
+  function telemetryLink(docker) {
+    // The telemetry stack is optional and runs beside the cluster. When its
+    // Grafana container is up, the health widget links to it on the port it
+    // publishes, at the host this page was loaded from.
+    const grafana = (docker.records || []).find((item) =>
+      item.kind === "container" &&
+      (item.service === "grafana" || item.name === "grafana") &&
+      String(item.state || "").toLowerCase() === "running");
+    if (!grafana) return null;
+    const port = (grafana.ports || [])[0] || 3000;
+    return {
+      url: `${window.location.protocol}//${window.location.hostname}:${port}/`,
+      label: "Grafana: QFw jobs, traces and where the time goes",
+    };
+  }
+
+  function syncTelemetryAnchor(anchor) {
+    const telemetry = telemetryLink(selectedSource("docker"));
+    anchor.hidden = !telemetry;
+    if (telemetry) {
+      anchor.href = telemetry.url;
+      anchor.title = telemetry.label;
+    }
+  }
+
   function renderHealth(payload) {
     const root = element("div", "qfw-health");
     root.append(element("strong", `qfw-state qfw-${payload.health}`, payload.health));
     root.append(element("span", "qfw-freshness", payload.observed_at || "not observed"));
+    if (payload.telemetry) {
+      const line = element("span", "qfw-telemetry-link", "Telemetry: ");
+      const anchor = document.createElement("a");
+      anchor.href = payload.telemetry.url;
+      anchor.target = "_blank";
+      anchor.rel = "noopener";
+      anchor.textContent = payload.telemetry.label;
+      line.append(anchor);
+      root.append(line);
+    }
     root.append(table(payload.sources || [], [
       ["name", "Source"], ["status", "State"], ["observed_at", "Observed"],
     ]));
@@ -3768,6 +3804,17 @@
       element("span", "qfw-canvas-hint", "Wheel zoom · middle-drag pan"),
       zoomSlider, zoomLabel, reset, clear, clearStatus,
     );
+    // The same link the health widget carries, here where it is always in
+    // view: the health widget sits at the far end of the canvas. The header
+    // is built before the first state arrives, so the anchor is always
+    // present and the data refresh shows or hides it.
+    const telemetryAnchor = document.createElement("a");
+    telemetryAnchor.className = "qfw-telemetry-link qfw-telemetry-link-header";
+    telemetryAnchor.target = "_blank";
+    telemetryAnchor.rel = "noopener";
+    telemetryAnchor.textContent = "Grafana";
+    syncTelemetryAnchor(telemetryAnchor);
+    viewControls.append(telemetryAnchor);
     header.append(viewControls, identityLabel);
     root.append(header);
     const viewport = element("div", "qfw-canvas-viewport");
@@ -3803,6 +3850,7 @@
     const scrollPositions = captureWidgetScrollPositions(root);
     refreshPackagedExamples(root);
     refreshExperimentSubmissionStatus(root);
+    root.querySelectorAll(".qfw-telemetry-link-header").forEach(syncTelemetryAnchor);
     WIDGETS.forEach(([id]) => {
       const widget = root.querySelector(`.qfw-widget[data-widget="${id}"]`);
       if (!widget) return;

dashboard/src/qfw_slurm_dashboard/assets/qfw-slurm-cluster.css

@@ -545,6 +545,8 @@
 .qfw-stopped,
 .qfw-unavailable { color: var(--qfw-danger); }
 .qfw-freshness { color: var(--qfw-muted); font-size: 0.78em; }
+.qfw-telemetry-link { font-size: 0.85em; }
+.qfw-telemetry-link-header { margin-left: 0.75em; font-weight: 600; }
 .qfw-table-wrap { max-height: 22em; overflow: auto; }
 
 .qfw-table {

dashboard/src/qfw_slurm_dashboard/collectors.py

@@ -53,12 +53,29 @@ def docker_status(runner: CommandRunner) -> SourceState:
         "state": item.get("State", ""),
         "status": item.get("Status", ""),
         "health": item.get("Health", ""),
+        "ports": _published_ports(item),
     } for item in raw_records]
     running = all(str(item.get("state", "")).lower() == "running" for item in records)
     status = "ready" if records and running else "stopped" if not records else "degraded"
     return SourceState("docker", status, utc_now(), records)
 
 
+def _published_ports(item: dict[str, Any]) -> list[int]:
+    """The host ports a container publishes, so the browser can reach a
+    service beside the cluster, such as Grafana, without any configuration."""
+    ports: list[int] = []
+    for publisher in item.get("Publishers") or []:
+        if not isinstance(publisher, dict):
+            continue
+        try:
+            port = int(publisher.get("PublishedPort") or 0)
+        except (TypeError, ValueError):
+            continue
+        if port > 0 and port not in ports:
+            ports.append(port)
+    return ports
+
+
 def _pipe_records(result: CommandResult, fields: tuple[str, ...]) -> list[dict[str, str]]:
     records = []
     for line in result.stdout.splitlines():

dashboard/tests/test_collectors.py

@@ -65,6 +65,24 @@ def test_collectors_keep_partial_state_when_service_is_stopped() -> None:
     assert sources["service-plane"].status == "stopped"
 
 
+def test_docker_records_carry_published_ports() -> None:
+    runner = FakeRunner()
+    runner.host = lambda argv, timeout=None: CommandResult(
+        tuple(argv), 0,
+        '{"State":"running","Name":"grafana","Service":"grafana",'
+        '"Publishers":[{"URL":"0.0.0.0","TargetPort":3000,"PublishedPort":3000,'
+        '"Protocol":"tcp"},{"URL":"::","TargetPort":3000,"PublishedPort":3000,'
+        '"Protocol":"tcp"}]}\n'
+        '{"State":"running","Name":"c1","Publishers":[{"PublishedPort":0}]}\n',
+        "",
+    )
+    records = {item["name"]: item for item in docker_status(runner).records}
+    # One entry per host port, whatever the address families, and none for
+    # a container that publishes nothing.
+    assert records["grafana"]["ports"] == [3000]
+    assert records["c1"]["ports"] == []
+
+
 def test_malformed_docker_output_is_typed_unavailable() -> None:
     runner = FakeRunner()
     runner.host = lambda argv, timeout=None: CommandResult(tuple(argv), 0, "{", "")

do_telemetry.sh

@@ -1,13 +1,13 @@
 #!/bin/bash
 #
 # Start, stop or inspect the optional telemetry stack beside the running
-# cluster: the OpenTelemetry Collector, Prometheus, Tempo and Grafana from
+# cluster: the OpenTelemetry Collector, Prometheus, Tempo, Loki and Grafana from
 # docker-compose.telemetry.yml. The cluster's own containers are never
 # recreated by this script.
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 ENV_FILE="${SCRIPT_DIR}/qfw-install.env"
-SERVICES=(otel-collector prometheus tempo grafana)
+SERVICES=(otel-collector prometheus tempo loki grafana)
 
 usage() {
     cat <<USAGE
@@ -16,7 +16,7 @@ Usage: $(basename "$0") ACTION [compose arguments]
 Actions:
   up        Start the telemetry stack (pulls the pinned images on first use)
   down      Stop and remove its containers, keeping the data volumes
-  purge     down, then remove the Prometheus, Tempo and Grafana volumes
+  purge     down, then remove the Prometheus, Tempo, Loki and Grafana volumes
   status    Show its containers
   logs      Follow its logs (extra arguments go to docker compose logs)
   url       Print where Grafana is
@@ -63,7 +63,7 @@ case "${action}" in
     up)
         "${COMPOSE[@]}" up -d --no-recreate "${SERVICES[@]}" "$@"
         echo "Grafana: http://localhost:$(grafana_port)/ (anonymous viewers; admin password in QFW_GRAFANA_ADMIN_PASSWORD, default qfw-demo)"
-        echo "Prometheus: http://localhost:9090/  Tempo: http://localhost:3200/  OTLP/HTTP: http://localhost:4318/"
+        echo "Prometheus: http://localhost:9090/  Tempo: http://localhost:3200/  Loki: http://localhost:3100/  OTLP/HTTP: http://localhost:4318/"
         ;;
     down)
         "${COMPOSE[@]}" rm --stop --force "${SERVICES[@]}" "$@"
@@ -75,7 +75,7 @@ case "${action}" in
             echo "Could not determine the compose project name; volumes not removed." >&2
             exit 1
         fi
-        docker volume rm "${project}_prometheus_data" "${project}_tempo_data" "${project}_grafana_data"
+        docker volume rm "${project}_prometheus_data" "${project}_tempo_data" "${project}_loki_data" "${project}_grafana_data"
         ;;
     status)
         "${COMPOSE[@]}" ps "${SERVICES[@]}" "$@"

docker-compose.telemetry.yml

@@ -42,6 +42,12 @@ x-qfw-telemetry: &qfw-telemetry
     # up to the client's end to end. A few spans per job; off in QFw unless
     # asked for, on here because the dashboards are the point.
     QFW_TELEMETRY_TRANSPORT: ${QFW_TELEMETRY_TRANSPORT:-1}
+    # The logs tier: what QFw's processes log, stitched to the job's trace
+    # and kept in Loki. QFw writes a job's story at debug, so "debug" is
+    # the tier that puts lines under the waterfall; DEFw's transport
+    # internals come only with "all", and "error" is the production
+    # setting. Whatever a process logs at the chosen level leaves it.
+    QFW_TELEMETRY_LOGS: ${QFW_TELEMETRY_LOGS:-debug}
     OTEL_METRIC_EXPORT_INTERVAL: ${QFW_TELEMETRY_METRIC_INTERVAL_MS:-5000}
     # Deltas since the last export rather than totals since the process
     # started. A client process that lives for one job then exports exactly
@@ -97,6 +103,19 @@ services:
     networks:
       - slurm-network
 
+  loki:
+    image: grafana/loki:3.7.8
+    container_name: loki
+    hostname: loki
+    command: ["-config.file=/etc/loki.yaml"]
+    volumes:
+      - ./telemetry/loki.yaml:/etc/loki.yaml:ro
+      - loki_data:/loki
+    ports:
+      - "127.0.0.1:3100:3100"
+    networks:
+      - slurm-network
+
   grafana:
     image: grafana/grafana:13.2.3
     container_name: grafana
@@ -126,6 +145,7 @@ services:
     depends_on:
       - prometheus
       - tempo
+      - loki
     networks:
       - slurm-network
 
@@ -168,4 +188,5 @@ services:
 volumes:
   prometheus_data:
   tempo_data:
+  loki_data:
   grafana_data:

man/man8/qfw-site-services.8

@@ -125,6 +125,7 @@ The QFw telemetry variables
 .BR QFW_TELEMETRY_SAMPLE ,
 .BR QFW_TELEMETRY_DIR ,
 .BR QFW_TELEMETRY_TRANSPORT ,
+.BR QFW_TELEMETRY_LOGS ,
 .BR OTEL_METRIC_EXPORT_INTERVAL ,
 .BR OTEL_EXPORTER_OTLP_ENDPOINT
 and

telemetry/README.md

@@ -14,13 +14,14 @@ the collector is a deployment choice made here with stock components.
                  └ QFW_TELEMETRY=otlp
 ```
 
-Four containers on the cluster network, pinned in the overlay:
+Five containers on the cluster network, pinned in the overlay:
 
 | Container | Image | Role | Host port |
 | --- | --- | --- | --- |
 | `otel-collector` | `otel/opentelemetry-collector-contrib` | Receives OTLP from QFw, forwards traces to Tempo, exposes metrics for Prometheus | `127.0.0.1:4318` |
 | `prometheus` | `prom/prometheus` | Scrapes the collector every 5 s, keeps 15 days | `127.0.0.1:9090` |
 | `tempo` | `grafana/tempo` | Stores traces, 14 days | `127.0.0.1:3200` |
+| `loki` | `grafana/loki` | Stores log lines, 14 days, with the trace id each carries | `127.0.0.1:3100` |
 | `grafana` | `grafana/grafana` | The dashboards, provisioned from this directory | `${QFW_GRAFANA_PORT:-3000}` on every interface |
 
 Nothing in QFw changes when the stack is absent: with `QFW_TELEMETRY` unset,
@@ -64,6 +65,7 @@ The overlay sets these on every container that runs QFw code:
 | `QFW_TELEMETRY_ENDPOINT` | `http://otel-collector:4318` | The collector's OTLP/HTTP base URL |
 | `QFW_TELEMETRY_SAMPLE` | `always` (`QFW_TELEMETRY_SAMPLE` overrides) | Trace sampling. `always` for a demo or a benchmark; a ratio or `off` for production, where the metrics stay on regardless |
 | `QFW_TELEMETRY_TRANSPORT` | `1` (`QFW_TELEMETRY_TRANSPORT` overrides) | The transport extension: spans and a histogram for the client's run RPC, the QPM's completion-event push and the way back. Off in QFw by default; on here so the per-hop view adds up to the end to end |
+| `QFW_TELEMETRY_LOGS` | `debug` (`QFW_TELEMETRY_LOGS` overrides) | The logs tier: what QFw's processes log at that level or above, each line with the trace and span ids current when it was written, kept in Loki. `error` for production, `all` to include DEFw's transport internals. See [Logs](#logs) |
 | `OTEL_METRIC_EXPORT_INTERVAL` | `5000` (`QFW_TELEMETRY_METRIC_INTERVAL_MS` overrides) | Milliseconds between metric exports |
 | `OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE` | `delta` | Each export carries what happened since the last one, not totals since the process started. See the note on short-lived clients below |
 
@@ -106,15 +108,17 @@ on whatever the trace sampling:
   and the completion event back;
 - backend execute p50 by device and API path (`native`, `qrmi`, `qdmi`,
   `simulator`), and the QPM stages per job;
-- the latest job traces, as a list to open.
+- the latest job traces, as a list to open;
+- the recent warnings and errors from every QFw process, newest first.
 
 **QFw Traces** (`/d/qfw-traces`) is trace search over Tempo: recent jobs, the
 slowest provider interactions, and failed or cancelled jobs.
 
 **QFw Trace** (`/d/qfw-trace?var-traceId=<id>`) is one job as a waterfall:
 `qfw.app.job` from the client, `qfw.qpm.receive` in the QPM across the DEFw
-RPC, then queue, dispatch and the provider call. A trace ID in any of the
-tables above opens a small menu: **Open the trace** comes here, with the time
+RPC, then queue, dispatch and the provider call, with every log line any
+QFw process wrote while that trace was current underneath. A trace ID in any
+of the tables above opens a small menu: **Open the trace** comes here, with the time
 range and links back; **Trace: <id>**, which the Tempo data source adds on
 its own, opens the same trace in Explore. Explore is closed to the Viewer
 role unless `viewers_can_edit` is on, which the overlay sets
@@ -150,6 +154,53 @@ instance id and accumulates the deltas again
 in the first place. The result is one counter per client population and one
 per QPM device that behaves like a long-lived process's.
 
+## Logs
+
+QFw's logs tier (`QFW_TELEMETRY_LOGS` in QFw's `backends/qfw_telemetry`)
+puts an OpenTelemetry handler on each process's root logger, so what the
+process logs at the chosen level or above leaves as OTLP log records on the
+same resource as its spans, each stamped with the trace and span ids current
+when it was written. The collector forwards them to Loki, which keeps
+`service_name`, `qfw_component_role` and `qfw_device_name` as stream labels
+and the rest, `trace_id`, `span_id` and `severity_text` among them, as
+structured metadata on the line.
+
+So a job's lines are one query away, from the client and the QPM alike:
+
+```logql
+{service_name=~"qfw-.+"} | trace_id="<trace id>"
+{service_name="qfw-qpm", qfw_device_name="fake-iqm-20q"} | severity_text=~"ERROR|CRITICAL"
+```
+
+On the dashboards, and in the hop from a span to its logs, each line starts
+with the component that wrote it, `[client]` or `[qpm fake-iqm-20q]`: the
+panels show a line's body, not its labels, and a job's story has two
+writers. The labels are still there in a line's details.
+
+Three links tie the signals together: the QFw Trace dashboard shows a job's
+lines under its waterfall; a span's **Logs for this span** button in Explore
+runs the first query for its trace; and a log line's `trace_id` opens its
+trace in Tempo. Lines written outside any span, DEFw's transport internals
+for instance, carry no trace id and show only in a plain query.
+
+What arrives depends on the tier. QFw writes a job's story at `debug`: a
+QPM's device query, the circuit's qubit cap, a driver's progress. So
+`debug`, the overlay's default, is what puts a job's lines under its
+waterfall, about twenty per job on the fake IQM, and real errors go through
+at every tier. DEFw's own levels are categories, and its work-request and
+RPC chatter, hundreds of lines per job, stays local unless the tier is
+`all`. `error` is the production setting. Whatever a process logs at the
+chosen tier leaves it, so review a service's log before raising the tier on
+a shared collector. A process's own logging level still gates everything
+outside QFw's `qfw.*` loggers, which the tier opens to its level: a Qiskit
+client out of the box, root logger at Python's default `WARNING`, sends its
+story lines and the warnings and errors of everything else, nothing more.
+The site services report
+once the image carries a QFw with the tier and `qfw-site-services` has
+forwarded the variable to them.
+
+Loki keeps 14 days, like Tempo, in its own volume.
+
 ## Feeding the dashboards
 
 QFw's `examples/qfw_job_stream.sh` streams Qiskit jobs through one backend:
@@ -197,6 +248,14 @@ curl -s 'localhost:3200/api/search?q=%7B%20name%20%3D%20%22qfw.app.job%22%20%7D'
 ```
 
 The first lists the four metric families; the second returns the job's trace.
+For the logs tier:
+
+```bash
+curl -s -G 'localhost:3100/loki/api/v1/query_range' --data-urlencode 'query={service_name=~"qfw-.+"}' --data-urlencode 'limit=5'
+```
+
+returns the latest lines, each with its `trace_id` in the structured
+metadata when it was written inside a span.
 
 ## Notes for a demo
 
@@ -235,7 +294,7 @@ The output goes to `<QFW_CONTAINER_BASE>/qfw-fallback-<timestamp>` unless
 `/Applications`; `QFW_CHROME` names another binary. Without one the script
 makes the snapshots and the page without images.
 
-A third layer costs nothing: Prometheus keeps 15 days and Tempo 14, so any
+A third layer costs nothing: Prometheus keeps 15 days, Tempo and Loki 14, so any
 dashboard opened with an absolute time range over a rehearsal shows it, as
 long as the stack is up.
 
@@ -252,6 +311,11 @@ or set the time range to the rehearsal; if Grafana itself is down, open
 - **Metrics but no traces, or the reverse.** Prometheus and Tempo are fed by
   separate collector pipelines; `docker logs tempo` and the Prometheus targets
   page (`localhost:9090/targets`) show which side is unhappy.
+- **No log lines.** The process needs `QFW_TELEMETRY_LOGS` (the overlay
+  sets it; site services get it from `qfw-site-services`) and a QFw with the
+  logs tier; `docker logs loki` and `./do_telemetry.sh logs otel-collector`
+  show the hop. A line written outside any span has no trace id and does
+  not appear under a trace.
 - **Port 3000 is taken.** Set `QFW_GRAFANA_PORT` in `qfw-install.env`.
 - **Dashboard edits vanish.** They are provisioned from files; edit the JSON
   in `grafana/dashboards` instead.