From 33a7b2b4d0b711ac53451aa59b4f6f0503a62b60 Mon Sep 17 00:00:00 2001 From: "Dmitry V. Bulashev" Date: Sat, 29 Aug 2026 19:56:40 +0500 Subject: [PATCH 1/3] mcp io --- CHANGELOG.md | 5 + CHANGELOG.ru.md | 5 + backend/internal/mcpserver/client.go | 39 ++ backend/internal/mcpserver/e2e_test.go | 159 +++++ backend/internal/mcpserver/io.go | 642 ++++++++++++++++++ backend/internal/mcpserver/io_test.go | 630 +++++++++++++++++ .../internal/mcpserver/kb/en/pg-stat-io.md | 144 ++++ .../internal/mcpserver/kb/ru/pg-stat-io.md | 146 ++++ backend/internal/mcpserver/kb_sync_test.go | 50 +- backend/internal/mcpserver/prompts.go | 14 +- backend/internal/mcpserver/resources.go | 7 + backend/internal/mcpserver/resources_test.go | 2 +- backend/internal/mcpserver/tools.go | 150 +++- doc/en/mcp.md | 4 +- doc/ru/mcp.md | 4 +- 15 files changed, 1959 insertions(+), 42 deletions(-) create mode 100644 backend/internal/mcpserver/io.go create mode 100644 backend/internal/mcpserver/io_test.go create mode 100644 backend/internal/mcpserver/kb/en/pg-stat-io.md create mode 100644 backend/internal/mcpserver/kb/ru/pg-stat-io.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dc0ab23..f7aaa854 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## v1.7.2 + +### Features +- **Two MCP tools for I/O**, `io_summary` and `io_trend`: an assistant can ask who is doing the reading and writing — client load, autovacuum or the checkpointer — and when it started, with a knowledge-base page on how to read the counters. An empty answer names its cause and a period broken by a statistics reset carries the part that was measured; needs PostgreSQL 16 or newer. + ## v1.7.1 ### Features diff --git a/CHANGELOG.ru.md b/CHANGELOG.ru.md index 2668c45b..888f5522 100644 --- a/CHANGELOG.ru.md +++ b/CHANGELOG.ru.md @@ -1,5 +1,10 @@ # История изменений +## v1.7.2 + +### Фичи +- **Два MCP-инструмента по вводу-выводу** — `io_summary` и `io_trend`: ассистент может спросить, кто читает и пишет — клиентская нагрузка, автовакуум или чекпойнтер — и когда это началось, плюс страница базы знаний о том, как читать счётчики. Пустой ответ называет причину, а период, разорванный сбросом статистики, несёт измеренную часть; нужен PostgreSQL 16 или новее. + ## v1.7.1 ### Фичи diff --git a/backend/internal/mcpserver/client.go b/backend/internal/mcpserver/client.go index 5c208e1b..aa9c7b81 100644 --- a/backend/internal/mcpserver/client.go +++ b/backend/internal/mcpserver/client.go @@ -509,6 +509,45 @@ func (d *DashaClient) WaitEvents(ctx context.Context, cluster, instance string) return pick(r.JSON200, r.HTTPResponse, "wait_events") } +// IOHistory returns the pg_stat_io deltas of one host over a period. +func (d *DashaClient) IOHistory(ctx context.Context, params *apiclient.GetIOHistoryParams) (*apiclient.IOHistory, error) { + r, err := d.api.GetIOHistoryWithResponse(ctx, params, d.editor(ctx)) + if err != nil { + return nil, wrapErr("io_history", err) + } + + if r.JSON200 == nil && r.HTTPResponse != nil && r.HTTPResponse.StatusCode == http.StatusNotImplemented { + return nil, errors.New("dasha: I/O history needs snapshot storage, which this Dasha has not " + + "configured (501) — pg_stat_io history is unavailable on this deployment") + } + + if r.JSON200 == nil { + return nil, statusError("io_history", r.HTTPResponse) + } + + return r.JSON200, nil +} + +// IOSupported probes the live route, which answers 501 below PostgreSQL 16. +func (d *DashaClient) IOSupported(ctx context.Context, cluster, instance string) (bool, error) { + r, err := d.api.GetIOCurrentWithResponse(ctx, &apiclient.GetIOCurrentParams{ + ClusterName: cluster, Instance: instance, + }, d.editor(ctx)) + if err != nil { + return false, wrapErr("io_current", err) + } + + if r.HTTPResponse != nil && r.HTTPResponse.StatusCode == http.StatusNotImplemented { + return false, nil + } + + if r.JSON200 == nil { + return false, statusError("io_current", r.HTTPResponse) + } + + return true, nil +} + // SchemaLint returns the structural defects of one database's schema: code, // level, object and the numbers behind each finding. Reads the system catalog // only. Optional level filter ("error", "warning", "notice") narrows the list diff --git a/backend/internal/mcpserver/e2e_test.go b/backend/internal/mcpserver/e2e_test.go index 514007c3..373649fd 100644 --- a/backend/internal/mcpserver/e2e_test.go +++ b/backend/internal/mcpserver/e2e_test.go @@ -5,6 +5,8 @@ import ( "net/http" "net/http/httptest" "strings" + "sync" + "sync/atomic" "testing" "github.com/modelcontextprotocol/go-sdk/mcp" @@ -240,3 +242,160 @@ func firstText(res *mcp.CallToolResult) string { return "" } + +// TestE2E_IOTools drives both pg_stat_io tools over a real client session: the +// tools are advertised, arguments reach the history endpoint, and the shaped +// result comes back as readable JSON. +func TestE2E_IOTools(t *testing.T) { + t.Parallel() + + var ( + mu sync.Mutex + lastQuery string + ) + + query := func() string { + mu.Lock() + defer mu.Unlock() + + return lastQuery + } + + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/io/history" { + w.WriteHeader(http.StatusNotFound) + + return + } + + mu.Lock() + lastQuery = r.URL.RawQuery + mu.Unlock() + + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(ioHistoryJSON)) + })) + defer backend.Close() + + client, err := NewDashaClient(Config{DashaURL: backend.URL, Token: "t"}) //nolint:exhaustruct + if err != nil { + t.Fatalf("NewDashaClient: %v", err) + } + + ctx := context.Background() + st, ct := mcp.NewInMemoryTransports() + + ss, err := NewMCPServer(client, "test", "en").Connect(ctx, st, nil) + if err != nil { + t.Fatalf("server connect: %v", err) + } + defer ss.Close() + + c := mcp.NewClient(&mcp.Implementation{Name: "test-client", Version: "v0"}, nil) //nolint:exhaustruct + + cs, err := c.Connect(ctx, ct, nil) + if err != nil { + t.Fatalf("client connect: %v", err) + } + defer cs.Close() + + lt, err := cs.ListTools(ctx, nil) + if err != nil { + t.Fatalf("ListTools: %v", err) + } + + for _, want := range []string{"io_summary", "io_trend"} { + if !hasTool(lt.Tools, want) { + t.Errorf("tool %q not advertised", want) + } + } + + res, err := cs.CallTool(ctx, &mcp.CallToolParams{ //nolint:exhaustruct + Name: "io_summary", + Arguments: map[string]any{"cluster": "demo", "instance": "h1", "group_by": "full", "top": 5}, + }) + if err != nil { + t.Fatalf("CallTool(io_summary): %v", err) + } + + if res.IsError { + t.Fatalf("io_summary returned IsError: %s", firstText(res)) + } + + if got := firstText(res); !strings.Contains(got, `"ranked_by"`) || !strings.Contains(got, "vacuum") { + t.Errorf("result = %q, want a ranked table naming the vacuum context", got) + } + + if q := query(); !strings.Contains(q, "group_by=full") || !strings.Contains(q, "points=1") { + t.Errorf("history query = %q, want group_by=full and points=1", q) + } + + res, err = cs.CallTool(ctx, &mcp.CallToolParams{ //nolint:exhaustruct + Name: "io_trend", + Arguments: map[string]any{"cluster": "demo", "instance": "h1", "since": "6h"}, + }) + if err != nil { + t.Fatalf("CallTool(io_trend): %v", err) + } + + if res.IsError { + t.Fatalf("io_trend returned IsError: %s", firstText(res)) + } + + if q := query(); !strings.Contains(q, "points=24") || !strings.Contains(q, "group_by=context") { + t.Errorf("history query = %q, want the trend defaults", q) + } +} + +// TestE2E_IOToolsRejectBadArgs confirms local validation answers as a readable +// isError result without ever reaching Dasha. +func TestE2E_IOToolsRejectBadArgs(t *testing.T) { + t.Parallel() + + var called atomic.Bool + + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + called.Store(true) + + w.WriteHeader(http.StatusOK) + })) + defer backend.Close() + + client, err := NewDashaClient(Config{DashaURL: backend.URL, Token: "t"}) //nolint:exhaustruct + if err != nil { + t.Fatalf("NewDashaClient: %v", err) + } + + ctx := context.Background() + st, ct := mcp.NewInMemoryTransports() + + ss, err := NewMCPServer(client, "test", "en").Connect(ctx, st, nil) + if err != nil { + t.Fatalf("server connect: %v", err) + } + defer ss.Close() + + c := mcp.NewClient(&mcp.Implementation{Name: "test-client", Version: "v0"}, nil) //nolint:exhaustruct + + cs, err := c.Connect(ctx, ct, nil) + if err != nil { + t.Fatalf("client connect: %v", err) + } + defer cs.Close() + + res, err := cs.CallTool(ctx, &mcp.CallToolParams{ //nolint:exhaustruct + Name: "io_summary", + Arguments: map[string]any{"cluster": "demo", "instance": "h1", "group_by": "object"}, + }) + if err != nil { + t.Fatalf("CallTool: %v", err) + } + + if !res.IsError { + t.Errorf("an unknown group_by must be refused") + } + + if called.Load() { + t.Errorf("invalid arguments must not reach Dasha") + } +} diff --git a/backend/internal/mcpserver/io.go b/backend/internal/mcpserver/io.go new file mode 100644 index 00000000..9616debb --- /dev/null +++ b/backend/internal/mcpserver/io.go @@ -0,0 +1,642 @@ +package mcpserver + +import ( + "cmp" + "context" + "errors" + "math" + "slices" + "strings" + "time" + + "github.com/dbulashev/dasha/gen/apiclient" +) + +const ( + ioSummaryDefaultSince = time.Hour + ioTrendDefaultSince = 24 * time.Hour + ioSummaryDefaultTop = 20 + ioTrendDefaultPoints = 24 + ioMaxTop = 200 + ioMaxPoints = 200 + + // The history endpoint silently trims a longer window. + ioMaxWindow = 31 * 24 * time.Hour +) + +var ioRankMetrics = []string{"reads", "writes", "extends", "fsyncs"} + +const ioRankedBy = "reads+writes+extends+fsyncs" + +var ioTrendMetrics = []string{"reads", "read_bytes", "writes", "write_bytes", "extends"} + +var ioTrendTimeMetrics = []string{"read_time", "write_time"} + +// backend_type is absent on purpose: its set grows with every release. +var ( + ioContexts = []string{"normal", "vacuum", "bulkread", "bulkwrite", "init"} + ioObjects = []string{"relation", "temp relation", "wal"} +) + +type ioRange struct { + From time.Time `json:"from"` + To time.Time `json:"to"` +} + +type ioSummaryWindow struct { + From time.Time `json:"from"` + To time.Time `json:"to"` + DurationSeconds float64 `json:"duration_seconds"` + Complete bool `json:"complete"` +} + +type ioSummaryRow struct { + BackendType string `json:"backend_type,omitempty"` + Object string `json:"object,omitempty"` + Context string `json:"context,omitempty"` + IOOps int64 `json:"io_ops"` + SharePct float64 `json:"share_pct"` + OpsPerSecond float64 `json:"ops_per_second,omitempty"` + AvgReadMs *float64 `json:"avg_read_ms,omitempty"` + AvgWriteMs *float64 `json:"avg_write_ms,omitempty"` + Values map[string]int64 `json:"values"` +} + +type ioSummaryResult struct { + Requested ioRange `json:"requested"` + WindowCapped bool `json:"window_capped,omitempty"` + Window *ioSummaryWindow `json:"window,omitempty"` + Meta apiclient.IOHistoryMeta `json:"meta"` + GroupBy string `json:"group_by"` + RankedBy string `json:"ranked_by"` + RowsTotal int `json:"rows_total"` + RowsReturned int `json:"rows_returned"` + Rows []ioSummaryRow `json:"rows"` + Totals map[string]int64 `json:"totals,omitempty"` + EmptyReason string `json:"empty_reason,omitempty"` + EmptyDetail string `json:"empty_detail,omitempty"` +} + +// On an incomplete bucket the counters are real but cover only coverage_pct of +// its span. +type ioTrendPoint struct { + At time.Time `json:"at"` + DurationSeconds float64 `json:"duration_seconds"` + Complete bool `json:"complete"` + CoveragePct float64 `json:"coverage_pct,omitempty"` + Values map[string]int64 `json:"values,omitempty"` +} + +type ioTrendSeries struct { + Key apiclient.IOSeriesKey `json:"key"` + Points []ioTrendPoint `json:"points"` +} + +type ioTrendResult struct { + Requested ioRange `json:"requested"` + WindowCapped bool `json:"window_capped,omitempty"` + Window *ioRange `json:"window,omitempty"` + Meta apiclient.IOHistoryMeta `json:"meta"` + GroupBy string `json:"group_by"` + Points int `json:"points"` + Metrics []string `json:"metrics"` + IncompletePoints int `json:"incomplete_points"` + Series []ioTrendSeries `json:"series"` + EmptyReason string `json:"empty_reason,omitempty"` + EmptyDetail string `json:"empty_detail,omitempty"` +} + +type ioRequest struct { + Params *apiclient.GetIOHistoryParams + Top int + Capped bool + Filtered bool +} + +func ioWindow(since, from, to string, def time.Duration) (time.Time, time.Time, bool, string) { + start, end, msg := resolveWindow(since, from, to, def) + if msg != "" { + return start, end, false, msg + } + + if end.Sub(start) > ioMaxWindow { + return end.Add(-ioMaxWindow), end, true, "" + } + + return start, end, false, "" +} + +func ioFilterMsg(ioContext, object string) string { + if ioContext != "" && !slices.Contains(ioContexts, ioContext) { + return "context must be one of: " + strings.Join(ioContexts, ", ") + } + + if object != "" && !slices.Contains(ioObjects, object) { + return "object must be one of: " + strings.Join(ioObjects, ", ") + } + + return "" +} + +// points=1 asks for one bucket covering the whole window. +func ioSummaryParams(a ioSummaryArgs) (ioRequest, string) { + from, to, capped, msg := ioWindow(a.Since, a.From, a.To, ioSummaryDefaultSince) + if msg != "" { + return ioRequest{}, msg //nolint:exhaustruct + } + + if m := ioFilterMsg(a.Context, a.Object); m != "" { + return ioRequest{}, m //nolint:exhaustruct + } + + groupBy := apiclient.GetIOHistoryParamsGroupBy(cmp.Or(a.GroupBy, string(apiclient.Context))) + + switch groupBy { + case apiclient.Context, apiclient.BackendType, apiclient.Full: + default: + return ioRequest{}, "group_by must be 'context', 'backend_type' or 'full'" //nolint:exhaustruct + } + + top := a.Top + if top <= 0 { + top = ioSummaryDefaultTop + } + + if top > ioMaxTop { + return ioRequest{}, "top must be 200 or less" //nolint:exhaustruct + } + + one := 1 + + return ioRequest{ + Params: &apiclient.GetIOHistoryParams{ + ClusterName: a.Cluster, + Instance: a.Instance, + From: from, + To: to, + GroupBy: &groupBy, + Points: &one, + Context: opt(a.Context), + BackendType: opt(a.BackendType), + Object: opt(a.Object), + }, + Top: top, + Capped: capped, + Filtered: a.Context != "" || a.BackendType != "" || a.Object != "", + }, "" +} + +func ioSummary(ctx context.Context, c *DashaClient, a ioSummaryArgs) (any, error) { + req, msg := ioSummaryParams(a) + if msg != "" { + return nil, errors.New(msg) + } + + hist, err := c.IOHistory(ctx, req.Params) + if err != nil { + return nil, err + } + + out := ioSummaryResult{ //nolint:exhaustruct + Requested: ioRange{From: req.Params.From, To: req.Params.To}, + WindowCapped: req.Capped, + Meta: hist.Meta, + GroupBy: string(*req.Params.GroupBy), + RankedBy: ioRankedBy, + Rows: []ioSummaryRow{}, + } + + var ( + window ioSummaryWindow + rows []ioSummaryRow + totals = map[string]int64{} + seen int + ) + + window.Complete = true + + for _, s := range hist.Series { + seen += len(s.Points) + + values, span, duration, complete := ioFold(s.Points) + if span.From.IsZero() { + continue + } + + ioMergeWindow(&window, span, duration, complete) + + for k, v := range values { + totals[k] += v + } + + if ioIdle(values) { + continue + } + + rows = append(rows, ioRow(s.Key, values)) + } + + if !window.From.IsZero() { + out.Window = &window + } + + out.Totals = ioTrimZeros(totals) + + if len(rows) == 0 { + out.EmptyReason, out.EmptyDetail = ioEmptyReason(ctx, c, ioEmptyInput{ + Cluster: a.Cluster, + Instance: a.Instance, + Meta: hist.Meta, + Requested: out.Requested, + Seen: seen, + Measured: window.DurationSeconds > 0, + Filtered: req.Filtered, + }) + + return out, nil + } + + ioFinishRows(rows, window) + slices.SortStableFunc(rows, ioRowLess) + + out.RowsTotal = len(rows) + + if len(rows) > req.Top { + rows = rows[:req.Top] + } + + out.Rows = rows + out.RowsReturned = len(rows) + + return out, nil +} + +// duration counts only measurable spans — the denominator of any rate. +func ioFold(points []apiclient.IOPoint) (map[string]int64, ioRange, float64, bool) { + var ( + values = map[string]int64{} + span ioRange + complete = true + duration float64 + ) + + for _, p := range points { + if !p.Complete { + complete = false + } + + duration += p.DurationSeconds + + if span.From.IsZero() || p.From.Before(span.From) { + span.From = p.From + } + + if p.To.After(span.To) { + span.To = p.To + } + + for k, v := range p.Values { + values[k] += v + } + } + + return values, span, duration, complete +} + +// Series share the bucket grid: duration is the longest seen, not the sum. +func ioMergeWindow(w *ioSummaryWindow, span ioRange, duration float64, complete bool) { + if w.From.IsZero() || span.From.Before(w.From) { + w.From = span.From + } + + if span.To.After(w.To) { + w.To = span.To + } + + w.DurationSeconds = max(w.DurationSeconds, duration) + + if !complete { + w.Complete = false + } +} + +func ioIdle(values map[string]int64) bool { + for k, v := range values { + if k != "hits" && v != 0 { + return false + } + } + + return true +} + +func ioRow(key apiclient.IOSeriesKey, values map[string]int64) ioSummaryRow { + row := ioSummaryRow{ //nolint:exhaustruct + BackendType: deref(key.BackendType), + Object: deref(key.Object), + Context: deref(key.Context), + Values: ioTrimZeros(values), + } + + for _, m := range ioRankMetrics { + row.IOOps += values[m] + } + + return row +} + +func ioFinishRows(rows []ioSummaryRow, w ioSummaryWindow) { + var total int64 + for _, r := range rows { + total += r.IOOps + } + + for i := range rows { + r := &rows[i] + + if total > 0 { + r.SharePct = round2(float64(r.IOOps) / float64(total) * 100) + } + + if w.DurationSeconds > 0 { + r.OpsPerSecond = round2(float64(r.IOOps) / w.DurationSeconds) + } + + r.AvgReadMs = ioAvgMs(r.Values, "read_time", "reads") + r.AvgWriteMs = ioAvgMs(r.Values, "write_time", "writes") + } +} + +// nil, not zero: with track_io_timing off every time counter is zero. +func ioAvgMs(values map[string]int64, timeKey, opsKey string) *float64 { + t, ops := values[timeKey], values[opsKey] + if t <= 0 || ops <= 0 { + return nil + } + + v := round2(float64(t) / float64(ops)) + + return &v +} + +func ioRowLess(a, b ioSummaryRow) int { + if c := cmp.Compare(b.IOOps, a.IOOps); c != 0 { + return c + } + + return cmp.Or( + cmp.Compare(a.BackendType, b.BackendType), + cmp.Compare(a.Object, b.Object), + cmp.Compare(a.Context, b.Context), + ) +} + +func ioTrendParams(a ioTrendArgs) (ioRequest, string) { + from, to, capped, msg := ioWindow(a.Since, a.From, a.To, ioTrendDefaultSince) + if msg != "" { + return ioRequest{}, msg //nolint:exhaustruct + } + + if m := ioFilterMsg(a.Context, a.Object); m != "" { + return ioRequest{}, m //nolint:exhaustruct + } + + points := a.Points + if points <= 0 { + points = ioTrendDefaultPoints + } + + if points > ioMaxPoints { + return ioRequest{}, "points must be 200 or less" //nolint:exhaustruct + } + + groupBy := apiclient.Context + + return ioRequest{ //nolint:exhaustruct + Params: &apiclient.GetIOHistoryParams{ + ClusterName: a.Cluster, + Instance: a.Instance, + From: from, + To: to, + GroupBy: &groupBy, + Points: &points, + Context: opt(a.Context), + BackendType: opt(a.BackendType), + Object: opt(a.Object), + }, + Capped: capped, + Filtered: a.Context != "" || a.BackendType != "" || a.Object != "", + }, "" +} + +type ioTrendScan struct { + Window ioRange + Incomplete map[int64]bool + Measured bool + Seen int +} + +func ioTrend(ctx context.Context, c *DashaClient, a ioTrendArgs) (any, error) { + req, msg := ioTrendParams(a) + if msg != "" { + return nil, errors.New(msg) + } + + hist, err := c.IOHistory(ctx, req.Params) + if err != nil { + return nil, err + } + + metrics := slices.Clone(ioTrendMetrics) + if hist.Meta.TrackIoTiming || hist.Meta.TrackIoTimingChanged { + metrics = append(metrics, ioTrendTimeMetrics...) + } + + out := ioTrendResult{ //nolint:exhaustruct + Requested: ioRange{From: req.Params.From, To: req.Params.To}, + WindowCapped: req.Capped, + Meta: hist.Meta, + GroupBy: string(apiclient.Context), + Metrics: metrics, + Series: []ioTrendSeries{}, + } + + var series []ioTrendSeries + + scan := ioTrendScan{Incomplete: map[int64]bool{}} //nolint:exhaustruct + + for _, s := range hist.Series { + pts, active := ioTrendPoints(s.Points, metrics, &scan) + if !active { + continue + } + + series = append(series, ioTrendSeries{Key: s.Key, Points: pts}) + } + + out.IncompletePoints = len(scan.Incomplete) + + if !scan.Window.From.IsZero() { + out.Window = &scan.Window + } + + if len(series) == 0 { + out.EmptyReason, out.EmptyDetail = ioEmptyReason(ctx, c, ioEmptyInput{ + Cluster: a.Cluster, + Instance: a.Instance, + Meta: hist.Meta, + Requested: out.Requested, + Seen: scan.Seen, + Measured: scan.Measured, + Filtered: req.Filtered, + }) + + return out, nil + } + + slices.SortStableFunc(series, ioSeriesLess) + + out.Points = len(series[0].Points) + out.Series = series + + return out, nil +} + +func ioTrendPoints(points []apiclient.IOPoint, metrics []string, scan *ioTrendScan) ([]ioTrendPoint, bool) { + out := make([]ioTrendPoint, 0, len(points)) + active := false + + for _, p := range points { + scan.Seen++ + + if scan.Window.From.IsZero() || p.From.Before(scan.Window.From) { + scan.Window.From = p.From + } + + if p.To.After(scan.Window.To) { + scan.Window.To = p.To + } + + pt := ioTrendPoint{ //nolint:exhaustruct + At: p.To, + DurationSeconds: p.DurationSeconds, + Complete: p.Complete, + } + + if !p.Complete { + // The epoch break is instance-wide: count buckets, not series times buckets. + scan.Incomplete[p.To.UnixNano()] = true + pt.CoveragePct = ioCoverage(p) + } + + if p.DurationSeconds > 0 { + scan.Measured = true + + values := map[string]int64{} + + for _, m := range metrics { + if v := p.Values[m]; v != 0 { + values[m] = v + active = true + } + } + + if len(values) > 0 { + pt.Values = values + } + } + + out = append(out, pt) + } + + return out, active +} + +func ioCoverage(p apiclient.IOPoint) float64 { + span := p.To.Sub(p.From).Seconds() + if span <= 0 { + return 0 + } + + return round2(p.DurationSeconds / span * 100) +} + +func ioSeriesLess(a, b ioTrendSeries) int { + if c := cmp.Compare(ioSeriesWeight(b), ioSeriesWeight(a)); c != 0 { + return c + } + + return cmp.Compare(deref(a.Key.Context), deref(b.Key.Context)) +} + +func ioSeriesWeight(s ioTrendSeries) int64 { + var total int64 + + for _, p := range s.Points { + for _, m := range ioRankMetrics { + total += p.Values[m] + } + } + + return total +} + +type ioEmptyInput struct { + Cluster string + Instance string + Meta apiclient.IOHistoryMeta + Requested ioRange + Seen int + Measured bool + Filtered bool +} + +// The live probe runs only where no stored history exists at all. +func ioEmptyReason(ctx context.Context, c *DashaClient, in ioEmptyInput) (string, string) { + if in.Meta.EarliestAt == nil { + supported, err := c.IOSupported(ctx, in.Cluster, in.Instance) + + switch { + case err != nil: + return "support_unknown", err.Error() + case !supported: + return "unsupported_version", "" + default: + return "no_snapshots", "" + } + } + + // A filter is applied before the series are built: an empty response cannot + // tell a filter miss from a window with no captures. + switch { + case in.Meta.EarliestAt.After(in.Requested.To): + return "window_before_history", "" + case in.Meta.LatestAt != nil && in.Meta.LatestAt.Before(in.Requested.From): + return "window_after_history", "" + case in.Seen == 0 && in.Filtered: + return "no_io_matching_filter", "" + case in.Seen == 0: + return "no_snapshots_in_window", "" + case !in.Measured: + return "no_comparable_snapshots", "" + } + + return "no_io", "" +} + +// Absent means zero in a complete result. +func ioTrimZeros(values map[string]int64) map[string]int64 { + out := make(map[string]int64, len(values)) + + for k, v := range values { + if v != 0 { + out[k] = v + } + } + + return out +} + +func round2(v float64) float64 { + return math.Round(v*100) / 100 +} diff --git a/backend/internal/mcpserver/io_test.go b/backend/internal/mcpserver/io_test.go new file mode 100644 index 00000000..12c375c1 --- /dev/null +++ b/backend/internal/mcpserver/io_test.go @@ -0,0 +1,630 @@ +package mcpserver + +import ( + "context" + "net/http" + "net/http/httptest" + "slices" + "strings" + "testing" + "time" + + "github.com/dbulashev/dasha/gen/apiclient" +) + +func ioFakeAPI(t *testing.T, history string, ioStatus int) *DashaClient { + t.Helper() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/io/history": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(history)) + case "/api/io/current": + if ioStatus != http.StatusOK { + w.WriteHeader(ioStatus) + + return + } + + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"instance":"h1","captured_at":"2026-08-29T10:00:00Z",` + + `"version_num":170000,"track_io_timing":true,"rows":[]}`)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(srv.Close) + + c, err := NewDashaClient(Config{DashaURL: srv.URL, Token: "t"}) //nolint:exhaustruct + if err != nil { + t.Fatalf("NewDashaClient: %v", err) + } + + return c +} + +// A fixed latest_at would start reading as window_after_history a day later. +func ioLiveMeta(earliest string) string { + return `"meta":{"instance":"h1","earliest_at":"` + earliest + `","latest_at":"` + + time.Now().UTC().Format(time.RFC3339) + `","track_io_timing":true,` + + `"track_io_timing_changed":false,"version_changed":false}` +} + +const ioHistoryJSON = `{ + "meta": {"instance":"h1","earliest_at":"2026-08-01T00:00:00Z","latest_at":"2026-08-29T10:00:00Z", + "track_io_timing":true,"track_io_timing_changed":false,"version_changed":false}, + "series": [ + {"key":{"context":"normal"}, + "points":[{"from":"2026-08-29T09:00:00Z","to":"2026-08-29T10:00:00Z","duration_seconds":3600, + "complete":true,"values":{"reads":100,"writes":50,"hits":1000,"fsyncs":0}}]}, + {"key":{"context":"vacuum"}, + "points":[{"from":"2026-08-29T09:00:00Z","to":"2026-08-29T10:00:00Z","duration_seconds":3600, + "complete":true,"values":{"reads":900,"read_time":450,"hits":20}}]}, + {"key":{"context":"bulkwrite"}, + "points":[{"from":"2026-08-29T09:00:00Z","to":"2026-08-29T10:00:00Z","duration_seconds":3600, + "complete":true,"values":{"hits":500}}]} + ]}` + +func ioSummaryOf(t *testing.T, c *DashaClient, a ioSummaryArgs) ioSummaryResult { + t.Helper() + + got, err := ioSummary(context.Background(), c, a) + if err != nil { + t.Fatalf("ioSummary: %v", err) + } + + res, ok := got.(ioSummaryResult) + if !ok { + t.Fatalf("ioSummary returned %T, want ioSummaryResult", got) + } + + return res +} + +func TestIOSummary_RanksAndDropsCacheOnlyRows(t *testing.T) { + t.Parallel() + + c := ioFakeAPI(t, ioHistoryJSON, http.StatusOK) + res := ioSummaryOf(t, c, ioSummaryArgs{Cluster: "demo", Instance: "h1"}) //nolint:exhaustruct + + // bulkwrite did no physical I/O — no row. + if res.RowsTotal != 2 || res.RowsReturned != 2 { + t.Fatalf("rows total/returned = %d/%d, want 2/2", res.RowsTotal, res.RowsReturned) + } + + if res.Rows[0].Context != "vacuum" || res.Rows[1].Context != "normal" { + t.Fatalf("row order = %q, %q; want vacuum first (heaviest)", res.Rows[0].Context, res.Rows[1].Context) + } + + if res.Rows[0].IOOps != 900 || res.Rows[1].IOOps != 150 { + t.Errorf("io_ops = %d, %d; want 900, 150", res.Rows[0].IOOps, res.Rows[1].IOOps) + } + + if res.Rows[0].SharePct != 85.71 || res.Rows[1].SharePct != 14.29 { + t.Errorf("share_pct = %v, %v; want 85.71, 14.29", res.Rows[0].SharePct, res.Rows[1].SharePct) + } + + if res.Rows[0].OpsPerSecond != 0.25 { + t.Errorf("ops_per_second = %v, want 0.25", res.Rows[0].OpsPerSecond) + } + + // hits survive in the totals though the row is gone. + if res.Totals["hits"] != 1520 { + t.Errorf("totals[hits] = %d, want 1520 (every series, dropped ones included)", res.Totals["hits"]) + } + + // An explicit fsyncs:0 must not survive. + if _, ok := res.Rows[1].Values["fsyncs"]; ok { + t.Errorf("zero counters must be trimmed from values") + } +} + +func TestIOSummary_LatencyOnlyWhenMeasured(t *testing.T) { + t.Parallel() + + c := ioFakeAPI(t, ioHistoryJSON, http.StatusOK) + res := ioSummaryOf(t, c, ioSummaryArgs{Cluster: "demo", Instance: "h1"}) //nolint:exhaustruct + + if res.Rows[0].AvgReadMs == nil || *res.Rows[0].AvgReadMs != 0.5 { + t.Errorf("vacuum avg_read_ms = %v, want 0.5", res.Rows[0].AvgReadMs) + } + + // No read_time recorded: absent, never 0.00. + if res.Rows[1].AvgReadMs != nil { + t.Errorf("avg_read_ms = %v with no read_time, want absent", *res.Rows[1].AvgReadMs) + } +} + +func TestIOSummary_TopCutsTailAndReportsIt(t *testing.T) { + t.Parallel() + + c := ioFakeAPI(t, ioHistoryJSON, http.StatusOK) + res := ioSummaryOf(t, c, ioSummaryArgs{Cluster: "demo", Instance: "h1", Top: 1}) //nolint:exhaustruct + + if res.RowsReturned != 1 || res.RowsTotal != 2 { + t.Fatalf("returned/total = %d/%d, want 1/2 so the model can see the tail was cut", + res.RowsReturned, res.RowsTotal) + } + + if res.RankedBy != ioRankedBy { + t.Errorf("ranked_by = %q, want %q", res.RankedBy, ioRankedBy) + } +} + +func TestIOSummary_WindowComesFromTheData(t *testing.T) { + t.Parallel() + + c := ioFakeAPI(t, ioHistoryJSON, http.StatusOK) + res := ioSummaryOf(t, c, ioSummaryArgs{Cluster: "demo", Instance: "h1", Since: "24h"}) //nolint:exhaustruct + + if res.Window == nil { + t.Fatal("window must be reported") + } + + // Asked 24h, covered 1h: the window is the data's, not the request's. + if got := res.Window.To.Sub(res.Window.From); got != time.Hour { + t.Errorf("covered window = %v, want 1h", got) + } + + if res.Window.DurationSeconds != 3600 { + t.Errorf("duration_seconds = %v, want 3600 (longest series, not their sum)", res.Window.DurationSeconds) + } + + if req := res.Requested.To.Sub(res.Requested.From); req != 24*time.Hour { + t.Errorf("requested window = %v, want 24h", req) + } +} + +func TestIOSummary_EmptyReason(t *testing.T) { + t.Parallel() + + const noHistory = `{"meta":{"instance":"h1","earliest_at":null,"latest_at":null, + "track_io_timing":false,"track_io_timing_changed":false,"version_changed":false},"series":[]}` + + const laterHistory = `{"meta":{"instance":"h1","earliest_at":"2030-01-01T00:00:00Z", + "latest_at":"2030-01-02T00:00:00Z","track_io_timing":true, + "track_io_timing_changed":false,"version_changed":false},"series":[]}` + + // Every series is cache-only, so nothing survives the idle filter. + idleOnly := `{` + ioLiveMeta("2026-08-01T00:00:00Z") + `, + "series":[{"key":{"context":"normal"},"points":[{"from":"2026-08-29T09:00:00Z", + "to":"2026-08-29T10:00:00Z","duration_seconds":3600,"complete":true,"values":{"hits":10}}]}]}` + + // History stops days before the window starts: the collector is dead. + const staleHistory = `{"meta":{"instance":"h1","earliest_at":"2020-01-01T00:00:00Z", + "latest_at":"2020-01-02T00:00:00Z","track_io_timing":true, + "track_io_timing_changed":false,"version_changed":false},"series":[]}` + + // Snapshots exist on both sides of the window but none inside it. + const gapHistory = `{"meta":{"instance":"h1","earliest_at":"2020-01-01T00:00:00Z", + "latest_at":"2030-01-01T00:00:00Z","track_io_timing":true, + "track_io_timing_changed":false,"version_changed":false},"series":[]}` + + // Every interval spans a reset, so nothing in the window is comparable. + brokenEpoch := `{` + ioLiveMeta("2026-08-01T00:00:00Z") + `, + "series":[{"key":{"context":"normal"},"points":[{"from":"2026-08-29T09:00:00Z", + "to":"2026-08-29T10:00:00Z","duration_seconds":0,"complete":false,"values":{}}]}]}` + + tests := []struct { + name string + history string + ioStatus int + want string + }{ + {"pg 15 has no pg_stat_io", noHistory, http.StatusNotImplemented, "unsupported_version"}, + {"supported but not captured yet", noHistory, http.StatusOK, "no_snapshots"}, + {"the support probe was refused", noHistory, http.StatusForbidden, "support_unknown"}, + {"window ends before history starts", laterHistory, http.StatusOK, "window_before_history"}, + {"collector stopped before the window", staleHistory, http.StatusOK, "window_after_history"}, + {"no capture fell inside the window", gapHistory, http.StatusOK, "no_snapshots_in_window"}, + {"every interval spans a reset", brokenEpoch, http.StatusOK, "no_comparable_snapshots"}, + {"genuinely no physical I/O", idleOnly, http.StatusOK, "no_io"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + c := ioFakeAPI(t, tt.history, tt.ioStatus) + res := ioSummaryOf(t, c, ioSummaryArgs{Cluster: "demo", Instance: "h1"}) //nolint:exhaustruct + + if res.EmptyReason != tt.want { + t.Errorf("empty_reason = %q, want %q", res.EmptyReason, tt.want) + } + }) + } +} + +// A filter miss must not read as "the instance did no I/O". +func TestIOSummary_FilteredEmptyIsNotNoIO(t *testing.T) { + t.Parallel() + + nothingMatched := `{` + ioLiveMeta("2026-08-01T00:00:00Z") + `,"series":[]}` + + c := ioFakeAPI(t, nothingMatched, http.StatusOK) + + res := ioSummaryOf(t, c, ioSummaryArgs{ //nolint:exhaustruct + Cluster: "demo", Instance: "h1", BackendType: "autovacuum", + }) + + if res.EmptyReason != "no_io_matching_filter" { + t.Errorf("empty_reason = %q, want no_io_matching_filter", res.EmptyReason) + } +} + +// A cache-only instance is busy, not idle — the hits are the evidence. +func TestIOSummary_EmptyKeepsTotalsAndWindow(t *testing.T) { + t.Parallel() + + cachedOnly := `{` + ioLiveMeta("2026-08-01T00:00:00Z") + `, + "series":[{"key":{"context":"normal"},"points":[{"from":"2026-08-29T09:00:00Z", + "to":"2026-08-29T10:00:00Z","duration_seconds":3600,"complete":true, + "values":{"hits":50000000}}]}]}` + + c := ioFakeAPI(t, cachedOnly, http.StatusOK) + res := ioSummaryOf(t, c, ioSummaryArgs{Cluster: "demo", Instance: "h1"}) //nolint:exhaustruct + + if res.EmptyReason != "no_io" { + t.Fatalf("empty_reason = %q, want no_io", res.EmptyReason) + } + + if res.Totals["hits"] != 50000000 { + t.Errorf("totals[hits] = %d, want the counters of the dropped rows", res.Totals["hits"]) + } + + if res.Window == nil || res.Window.DurationSeconds != 3600 { + t.Errorf("an empty result must still report what the data covered") + } +} + +const ioTrendJSON = `{ + "meta": {"instance":"h1","earliest_at":"2026-08-01T00:00:00Z","latest_at":"2026-08-29T10:00:00Z", + "track_io_timing":true,"track_io_timing_changed":false,"version_changed":false}, + "series": [ + {"key":{"context":"vacuum"}, + "points":[ + {"from":"2026-08-29T07:00:00Z","to":"2026-08-29T08:00:00Z","duration_seconds":3600, + "complete":true,"values":{"reads":500,"read_time":250,"writes":0}}, + {"from":"2026-08-29T08:00:00Z","to":"2026-08-29T09:00:00Z","duration_seconds":0, + "complete":false,"values":{}}, + {"from":"2026-08-29T09:00:00Z","to":"2026-08-29T10:00:00Z","duration_seconds":3600, + "complete":true,"values":{"reads":700}}]}, + {"key":{"context":"bulkwrite"}, + "points":[ + {"from":"2026-08-29T07:00:00Z","to":"2026-08-29T08:00:00Z","duration_seconds":3600, + "complete":true,"values":{"hits":90}}]} + ]}` + +func TestIOTrend_GapsCarryNoValues(t *testing.T) { + t.Parallel() + + c := ioFakeAPI(t, ioTrendJSON, http.StatusOK) + + got, err := ioTrend(context.Background(), c, ioTrendArgs{Cluster: "demo", Instance: "h1"}) //nolint:exhaustruct + if err != nil { + t.Fatalf("ioTrend: %v", err) + } + + res, ok := got.(ioTrendResult) + if !ok { + t.Fatalf("ioTrend returned %T, want ioTrendResult", got) + } + + // The cache-only series carries no I/O and must not be plotted. + if len(res.Series) != 1 || deref(res.Series[0].Key.Context) != "vacuum" { + t.Fatalf("series = %d, want only vacuum", len(res.Series)) + } + + if res.IncompletePoints != 1 { + t.Errorf("incomplete_points = %d, want 1 (buckets, not series x buckets)", res.IncompletePoints) + } + + gap := res.Series[0].Points[1] + if gap.Complete { + t.Fatal("the middle bucket spans a stats reset and must be incomplete") + } + + // A zero here would read as a lull in the load. + if gap.Values != nil { + t.Errorf("a bucket that measured nothing must carry no values, got %v", gap.Values) + } + + if gap.CoveragePct != 0 { + t.Errorf("coverage_pct = %v, want 0 for a bucket that measured nothing", gap.CoveragePct) + } + + if !res.Series[0].Points[0].Complete || res.Series[0].Points[0].Values["reads"] != 500 { + t.Errorf("complete points must keep their counters") + } + + if _, ok := res.Series[0].Points[2].Values["writes"]; ok { + t.Errorf("zero counters must be trimmed inside a complete point") + } +} + +func TestIOTrend_TimeMetricsFollowTracking(t *testing.T) { + t.Parallel() + + c := ioFakeAPI(t, ioTrendJSON, http.StatusOK) + + got, _ := ioTrend(context.Background(), c, ioTrendArgs{Cluster: "demo", Instance: "h1"}) //nolint:exhaustruct + + res, ok := got.(ioTrendResult) + if !ok { + t.Fatalf("ioTrend returned %T", got) + } + + if !slices.Contains(res.Metrics, "read_time") { + t.Errorf("metrics = %v, must include read_time when track_io_timing is on", res.Metrics) + } + + off := `{"meta":{"instance":"h1","earliest_at":"2026-08-01T00:00:00Z","latest_at":"2026-08-29T10:00:00Z", + "track_io_timing":false,"track_io_timing_changed":false,"version_changed":false}, + "series":[{"key":{"context":"normal"},"points":[{"from":"2026-08-29T09:00:00Z", + "to":"2026-08-29T10:00:00Z","duration_seconds":3600,"complete":true,"values":{"reads":5}}]}]}` + + got2, _ := ioTrend(context.Background(), ioFakeAPI(t, off, http.StatusOK), + ioTrendArgs{Cluster: "demo", Instance: "h1"}) //nolint:exhaustruct + + res2, ok := got2.(ioTrendResult) + if !ok { + t.Fatalf("ioTrend returned %T", got2) + } + + if slices.Contains(res2.Metrics, "read_time") { + t.Errorf("metrics = %v, must omit time metrics that are zero by construction", res2.Metrics) + } +} + +func TestIOSummaryParams_Defaults(t *testing.T) { + t.Parallel() + + req, msg := ioSummaryParams(ioSummaryArgs{Cluster: "demo", Instance: "h1"}) //nolint:exhaustruct + if msg != "" { + t.Fatalf("ioSummaryParams(defaults) error = %q, want none", msg) + } + + p := req.Params + + if p.Points == nil || *p.Points != 1 { + t.Errorf("a summary must ask for exactly one bucket") + } + + if p.GroupBy == nil || *p.GroupBy != apiclient.Context { + t.Errorf("group_by = %v, want context", p.GroupBy) + } + + if req.Top != ioSummaryDefaultTop { + t.Errorf("top = %d, want %d", req.Top, ioSummaryDefaultTop) + } + + if got := p.To.Sub(p.From); got != ioSummaryDefaultSince { + t.Errorf("window = %v, want %v", got, ioSummaryDefaultSince) + } + + if p.Context != nil || p.BackendType != nil || p.Object != nil { + t.Errorf("unset filters must be omitted (nil)") + } + + if req.Capped || req.Filtered { + t.Errorf("a default call is neither capped nor filtered") + } +} + +func TestIOTrendParams_Defaults(t *testing.T) { + t.Parallel() + + req, msg := ioTrendParams(ioTrendArgs{Cluster: "demo", Instance: "h1"}) //nolint:exhaustruct + if msg != "" { + t.Fatalf("ioTrendParams(defaults) error = %q, want none", msg) + } + + if req.Params.Points == nil || *req.Params.Points != ioTrendDefaultPoints { + t.Errorf("points must default to %d", ioTrendDefaultPoints) + } + + if got := req.Params.To.Sub(req.Params.From); got != ioTrendDefaultSince { + t.Errorf("window = %v, want %v", got, ioTrendDefaultSince) + } + + // Filtering must not change the grouping. + req2, _ := ioTrendParams(ioTrendArgs{Cluster: "demo", Instance: "h1", Context: "vacuum"}) //nolint:exhaustruct + if req2.Params.GroupBy == nil || *req2.Params.GroupBy != apiclient.Context { + t.Errorf("group_by must always be context") + } + + if !req2.Filtered { + t.Errorf("a call carrying a dimension filter must be marked filtered") + } +} + +func TestIOParams_Errors(t *testing.T) { + t.Parallel() + + summary := []struct { + name string + args ioSummaryArgs + }{ + {"bad group_by", ioSummaryArgs{Cluster: "c", Instance: "h", GroupBy: "object"}}, //nolint:exhaustruct + {"top over cap", ioSummaryArgs{Cluster: "c", Instance: "h", Top: 500}}, //nolint:exhaustruct + {"bad since", ioSummaryArgs{Cluster: "c", Instance: "h", Since: "yesterday"}}, //nolint:exhaustruct + {"from without to", ioSummaryArgs{Cluster: "c", Instance: "h", From: "2026-07-10T12:00:00Z"}}, //nolint:exhaustruct + // An unmatched filter would otherwise come back as "no I/O". + {"bad context", ioSummaryArgs{Cluster: "c", Instance: "h", Context: "vacuuming"}}, //nolint:exhaustruct + {"bad object", ioSummaryArgs{Cluster: "c", Instance: "h", Object: "temp"}}, //nolint:exhaustruct + } + + for _, tt := range summary { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if _, msg := ioSummaryParams(tt.args); msg == "" { + t.Errorf("ioSummaryParams(%s) must return a validation error", tt.name) + } + }) + } + + if _, msg := ioTrendParams(ioTrendArgs{Cluster: "c", Instance: "h", Points: 5000}); msg == "" { //nolint:exhaustruct + t.Errorf("ioTrendParams must reject points over the cap") + } + + if _, msg := ioTrendParams(ioTrendArgs{Cluster: "c", Instance: "h", Object: "temp"}); msg == "" { //nolint:exhaustruct + t.Errorf("ioTrendParams must reject an object pg_stat_io cannot report") + } +} + +// Echoing the untrimmed request would overstate how far back it looked. +func TestIOParams_WindowCappedAtTheEndpointMaximum(t *testing.T) { + t.Parallel() + + req, msg := ioTrendParams(ioTrendArgs{Cluster: "c", Instance: "h", Since: "90d"}) //nolint:exhaustruct + if msg != "" { + t.Fatalf("ioTrendParams('90d') error = %q, want none", msg) + } + + if got := req.Params.To.Sub(req.Params.From); got != ioMaxWindow { + t.Errorf("window = %v, want it clamped to %v", got, ioMaxWindow) + } + + if !req.Capped { + t.Errorf("a clamped window must be flagged so the model does not over-read it") + } +} + +// The io_trend schema recommends '7d', which time.ParseDuration cannot read. +func TestParseSince_AcceptsDays(t *testing.T) { + t.Parallel() + + d, err := parseSince("7d") + if err != nil { + t.Fatalf("parseSince(7d): %v", err) + } + + if d != 7*24*time.Hour { + t.Errorf("parseSince(7d) = %v, want 168h", d) + } + + if _, err := parseSince("yesterday"); err == nil { + t.Errorf("parseSince must still reject prose") + } +} + +func TestResolveWindow_DefaultIsPerTool(t *testing.T) { + t.Parallel() + + from, to, msg := resolveWindow("", "", "", ioTrendDefaultSince) + if msg != "" { + t.Fatalf("resolveWindow error = %q, want none", msg) + } + + if got := to.Sub(from); got != ioTrendDefaultSince { + t.Errorf("window = %v, want the caller's default %v", got, ioTrendDefaultSince) + } +} + +// Every I/O call can hit this 501 on a Dasha without snapshot storage. +func TestIOHistory_StorageOffExplainsItself(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotImplemented) + })) + defer srv.Close() + + c, err := NewDashaClient(Config{DashaURL: srv.URL, Token: "t"}) //nolint:exhaustruct + if err != nil { + t.Fatalf("NewDashaClient: %v", err) + } + + _, err = ioSummary(context.Background(), c, ioSummaryArgs{Cluster: "demo", Instance: "h1"}) //nolint:exhaustruct + if err == nil { + t.Fatal("a 501 must surface as an error") + } + + if !strings.Contains(err.Error(), "snapshot storage") { + t.Errorf("error = %q, want it to name snapshot storage", err) + } +} + +const ioPartialJSON = `{ + "meta": {"instance":"h1","earliest_at":"2026-08-01T00:00:00Z","latest_at":"2026-08-29T10:00:00Z", + "track_io_timing":false,"track_io_timing_changed":false,"version_changed":false}, + "series": [ + {"key":{"context":"normal"}, + "points":[{"from":"2026-08-29T09:00:00Z","to":"2026-08-29T10:00:00Z","duration_seconds":3300, + "complete":false,"values":{"reads":5500,"writes":200}}]} + ]}` + +func ioTrendOf(t *testing.T, c *DashaClient, a ioTrendArgs) ioTrendResult { + t.Helper() + + got, err := ioTrend(context.Background(), c, a) + if err != nil { + t.Fatalf("ioTrend: %v", err) + } + + res, ok := got.(ioTrendResult) + if !ok { + t.Fatalf("ioTrend returned %T, want ioTrendResult", got) + } + + return res +} + +// Dropping a broken bucket's counters would hide 55 minutes of real I/O. +func TestIOTrend_PartialBucketKeepsWhatItMeasured(t *testing.T) { + t.Parallel() + + res := ioTrendOf(t, ioFakeAPI(t, ioPartialJSON, http.StatusOK), + ioTrendArgs{Cluster: "demo", Instance: "h1"}) //nolint:exhaustruct + + if len(res.Series) != 1 { + t.Fatalf("series = %d, want 1 — a partial bucket with counters is not an idle series", len(res.Series)) + } + + pt := res.Series[0].Points[0] + if pt.Complete { + t.Fatal("the bucket spans a reset and must stay flagged incomplete") + } + + if pt.Values["reads"] != 5500 { + t.Errorf("values[reads] = %d, want 5500", pt.Values["reads"]) + } + + // Without the coverage the counters would be compared against a full hour. + if pt.CoveragePct != 91.67 { + t.Errorf("coverage_pct = %v, want 91.67 (3300s of a 3600s bucket)", pt.CoveragePct) + } + + if res.IncompletePoints != 1 { + t.Errorf("incomplete_points = %d, want 1", res.IncompletePoints) + } + + if res.Points != 1 { + t.Errorf("points = %d, want the buckets actually returned", res.Points) + } +} + +// Every bucket unmeasurable is a broken record, not a quiet instance. +func TestIOTrend_AllIncompleteIsNotNoIO(t *testing.T) { + t.Parallel() + + allBroken := `{` + ioLiveMeta("2026-08-01T00:00:00Z") + `, + "series":[{"key":{"context":"normal"},"points":[ + {"from":"2026-08-29T08:00:00Z","to":"2026-08-29T09:00:00Z","duration_seconds":0, + "complete":false,"values":{}}, + {"from":"2026-08-29T09:00:00Z","to":"2026-08-29T10:00:00Z","duration_seconds":0, + "complete":false,"values":{}}]}]}` + + res := ioTrendOf(t, ioFakeAPI(t, allBroken, http.StatusOK), + ioTrendArgs{Cluster: "demo", Instance: "h1"}) //nolint:exhaustruct + + if res.EmptyReason != "no_comparable_snapshots" { + t.Errorf("empty_reason = %q, want no_comparable_snapshots", res.EmptyReason) + } + + if res.IncompletePoints != 2 { + t.Errorf("incomplete_points = %d, want 2", res.IncompletePoints) + } +} diff --git a/backend/internal/mcpserver/kb/en/pg-stat-io.md b/backend/internal/mcpserver/kb/en/pg-stat-io.md new file mode 100644 index 00000000..1c58973c --- /dev/null +++ b/backend/internal/mcpserver/kb/en/pg-stat-io.md @@ -0,0 +1,144 @@ +# Reading pg_stat_io + +How to read `io_summary` and `io_trend`. `pg_stat_io` splits physical I/O by +`backend_type` × `object` × `context`, which is the only place that answers +*whose* I/O it is. `wait_events` says backends are waiting on `DataFileRead` +but not who reads; `query_report` covers client backends in +`pg_stat_statements` only — autovacuum, the checkpointer, the WAL writer and +the background writer are invisible there. + +`pg_stat_io` is instance-wide. There is no per-database breakdown, and asking +for one is a category error. + +## The counters that mislead + +### hits are not I/O +A `hits` count is a page found in shared buffers — no disk was touched. It is +not part of the `io_ops` ranking and never answers "whose I/O". A row whose +only non-zero counter is `hits` is dropped from `io_summary` entirely; the +figure survives in `totals`, and `totals` is reported even when every row was +dropped — a large `hits` total with no rows means a busy, perfectly cached +instance, not an idle one. (The Dasha UI's "show idle" counts `hits` as +activity, so its row list can be longer than the tool's — the numbers agree, +the row filter does not.) + +`io_ops` is `reads + writes + extends + fsyncs`: the operations that touch a +file. Buffer management (`evictions`, `reuses`, `writebacks`) is not counted, +so a row whose only activity is eviction pressure is kept but ranks `io_ops: 0` +— read its `values`, not its rank. + +### evictions under bulkread are normal +`bulkread` and `bulkwrite` run through a small ring buffer that recycles its +own pages by design, so evictions there are the mechanism working, not cache +pressure. Evictions in the `normal` context are the ones worth reading as +pressure on shared buffers. + +### fsyncs on a client backend are an anomaly +Synchronising files is the checkpointer's job. `fsyncs` attributed to +`client backend` mean it could not keep up and backends are flushing for it — +look at `checkpoint_timeout`, `max_wal_size` and checkpoint frequency via +`settings_analyze`. They count towards `io_ops`, so such a row ranks on its own +rather than being cut off in the tail. + +### zero time is not fast +With `track_io_timing` off, `read_time`, `write_time`, `extend_time`, +`writeback_time` and `fsync_time` are zero by construction. Check +`meta.track_io_timing` before drawing any latency conclusion; +`meta.track_io_timing_changed` means the setting was toggled inside the window, +so the times cover only part of it. The tools omit `avg_read_ms` / +`avg_write_ms` rather than report `0.00`, so an absent latency means +"not measured", never "instant". + +## Contexts + +- **normal** — ordinary buffered access through shared buffers. The baseline. +- **vacuum** — autovacuum and manual VACUUM/ANALYZE. A large share at night is + expected; a large share during peak hours competes with the application. +- **bulkread** — sequential scans large enough to use a ring buffer instead of + polluting the cache. A high share means big scans: either the working set no + longer fits, or queries are missing indexes. +- **bulkwrite** — bulk writes (COPY, CREATE TABLE AS, some ALTER TABLE). +- **init** — relation forks being initialised. Normally negligible. + +## Objects + +- **relation** — tables and indexes. The bulk of the traffic. +- **temp relation** — spills to disk: sorts, hashes and CTEs that exceeded + `work_mem`. This is the instance-wide spill volume, which + `query_report`'s per-query temp counters cannot give you. +- **wal** — WAL I/O, reported from PostgreSQL 18 on. + +## Other counters + +- **extends** — file growth. A sustained extend rate is the honest measure of + how fast the database is growing, unlike a size delta that vacuum can mask. +- **reuses** — ring-buffer pages recycled during bulk operations. +- **writebacks** — pages handed to the kernel to flush. + +## Incomplete points + +A statistics reset, a restart or a major upgrade breaks the counter epoch. +`io_trend` marks the affected bucket `complete: false` and adds a +`coverage_pct`: the counters it carries are real, but they measure only that +share of the bucket's own span. So the values are usable — and throwing them +away would lose 55 minutes of real I/O in a bucket broken at minute five — but +they are **not comparable** with a complete bucket, and a smaller number there +is not a drop in load. `incomplete_points` counts such buckets. + +An incomplete bucket with no `values` at all measured nothing: that one is a +true gap in the record. + +## The window that was actually read + +`requested` is the window asked for, `window` the one the data covers; a +difference between them is snapshot coverage, not load. A request longer than +31 days is cut back to 31 and flagged `window_capped: true` — a conclusion of +the form "nothing changed in the last 90 days" cannot be drawn from a capped +window. + +## Empty results + +An empty result is not an answer until `empty_reason` says which one it is. +Exactly one of these values means the instance was idle; the rest mean the +question went unanswered: + +- **unsupported_version** — the host runs PostgreSQL 15 or older and has no + `pg_stat_io`. Nothing about its I/O can be read this way; fall back to + `wait_events` and `query_report`. +- **no_snapshots** — the server supports `pg_stat_io`, but no snapshot has been + captured yet. Nothing to do but wait for the collector. +- **support_unknown** — the probe that tells an unsupported server from an + un-captured one did not answer; `empty_detail` carries its error (a 403 means + the token may not read live I/O, a 404 a wrong cluster/instance name). +- **window_before_history** — the window ends before the stored history starts. + Ask for a later window; `meta.earliest_at` says from when. +- **window_after_history** — the window starts after the last stored capture: + the collector has not run since `meta.latest_at`. This says nothing about the + instance's I/O — say the history stops there, and do not report an all-clear. +- **no_snapshots_in_window** — history exists on both sides, but no capture + fell inside this window. Widen it. +- **no_comparable_snapshots** — captures exist in the window, but no two of + them are comparable: the counter epoch broke between every pair. Widen the + window past the resets, or read `meta.version_changed`. +- **no_io_matching_filter** — nothing matched `context` / `object` / + `backend_type`. The filter is applied before the series are built, so this + also covers a window that holds no captures at all. `backend_type` is not + validated, and `'autovacuum'` (the real value is `'autovacuum worker'`) + silently matches nothing. Re-run without the filter before concluding + anything. +- **no_io** — snapshots cover the window, they are comparable, and there + genuinely was no physical I/O. This is the only one that is a real answer, + and even here `totals` may show heavy cache activity. + +## Where to go next + +- `vacuum` dominates → `vacuum_danger`, `top_tables` — which tables keep + autovacuum busy. +- `bulkread` dominates → `top_queries` (by=time) and `list_indexes` (missing) — + large scans that an index would remove. +- `extends` growing → `top_tables` by size; check retention and bloat. +- `fsyncs` on backends → `settings_analyze` for checkpoint settings. +- `temp relation` significant → `top_queries` and `work_mem`: the spills come + from sorts and hashes that do not fit. +- a spike with an unclear owner → `io_summary` with `group_by=full` over the + window `io_trend` pointed at. diff --git a/backend/internal/mcpserver/kb/ru/pg-stat-io.md b/backend/internal/mcpserver/kb/ru/pg-stat-io.md new file mode 100644 index 00000000..6de4d3ff --- /dev/null +++ b/backend/internal/mcpserver/kb/ru/pg-stat-io.md @@ -0,0 +1,146 @@ +# Как читать pg_stat_io + +Как трактовать вывод `io_summary` и `io_trend`. `pg_stat_io` разбирает +физический I/O по `backend_type` × `object` × `context` — это единственный +источник, отвечающий на вопрос, *чей* это I/O. `wait_events` говорит, что +бэкенды ждут `DataFileRead`, но не говорит, кто читает; `query_report` +покрывает только клиентские бэкенды и только то, что попало в +`pg_stat_statements` — автовакуум, чекпойнтер, walwriter и bgwriter там не +видны вовсе. + +`pg_stat_io` инстанс-широкий. Разреза по базам данных нет, и параметра +`database` у этих инструментов не существует. + +## Счётчики, которые вводят в заблуждение + +### hits — это не I/O +`hits` — страница, найденная в разделяемых буферах; диска не было. В ранг +`io_ops` не входит и на вопрос «чей I/O» не отвечает. Строка, у которой +ненулевой только `hits`, из `io_summary` выбрасывается целиком; само число +остаётся в `totals`, и `totals` выводится даже когда выброшены все строки: +большой `hits` при пустых `rows` — это нагруженный инстанс с идеальным кэшем, +а не простаивающий. (В UI Dasha галка «показывать неактивные» считает `hits` +активностью, поэтому список строк там может быть длиннее — цифры совпадают, +фильтр строк отличается.) + +`io_ops` — это `reads + writes + extends + fsyncs`, операции, которые трогают +файл. Управление буферами (`evictions`, `reuses`, `writebacks`) в ранг не +входит, поэтому строка, у которой активность только в вытеснениях, остаётся, +но получает `io_ops: 0` — читать у неё `values`, а не ранг. + +### evictions в контексте bulkread — норма +`bulkread` и `bulkwrite` идут через маленький кольцевой буфер, который по +построению вытесняет собственные страницы. Вытеснения там — работа механизма, +а не давление на кэш. Читать как давление на разделяемые буферы стоит +`evictions` в контексте `normal`. + +### fsyncs у клиентского бэкенда — аномалия +Синхронизировать файлы — работа чекпойнтера. `fsyncs`, приписанные +`client backend`, означают, что тот не успевает и бэкенды сбрасывают за него. +Смотреть `checkpoint_timeout`, `max_wal_size` и частоту чекпойнтов через +`settings_analyze`. Они входят в `io_ops`, поэтому такая строка ранжируется +сама по себе, а не отсекается в хвосте. + +### ноль во времени — это не «быстро» +При выключенном `track_io_timing` `read_time`, `write_time`, `extend_time`, +`writeback_time` и `fsync_time` равны нулю по построению. Перед любым выводом +о латентности проверять `meta.track_io_timing`; +`meta.track_io_timing_changed` значит, что настройку переключали внутри окна и +времена покрывают только его часть. Инструменты не выводят `avg_read_ms` / +`avg_write_ms` вместо того, чтобы показать `0.00`: отсутствие латентности +значит «не измерялось», а не «мгновенно». + +## Контексты + +- **normal** — обычный буферизованный доступ через разделяемые буферы. База + отсчёта. +- **vacuum** — автовакуум и ручные VACUUM/ANALYZE. Большая доля ночью + ожидаема; большая доля в пике конкурирует с приложением. +- **bulkread** — последовательные чтения, достаточно крупные, чтобы идти + кольцевым буфером мимо кэша. Высокая доля значит крупные сканы: либо рабочий + набор перестал помещаться, либо запросам не хватает индексов. +- **bulkwrite** — массовая запись (COPY, CREATE TABLE AS, часть ALTER TABLE). +- **init** — инициализация слоёв отношения. Обычно пренебрежимо мало. + +## Объекты + +- **relation** — таблицы и индексы. Основной поток. +- **temp relation** — спиллы на диск: сортировки, хэши и CTE, не поместившиеся + в `work_mem`. Это объём спиллов по инстансу целиком, которого per-query + счётчики temp в `query_report` не дают. +- **wal** — I/O журнала; строки появляются с PostgreSQL 18. + +## Прочие счётчики + +- **extends** — рост файлов. Устойчивый темп extends — честная мера скорости + роста базы, в отличие от дельты размера, которую маскирует вакуум. +- **reuses** — переиспользованные страницы кольцевого буфера при массовых + операциях. +- **writebacks** — страницы, отданные ядру на сброс. + +## Неполные точки + +Сброс статистики, рестарт или мажорный апгрейд рвут эпоху счётчиков. +`io_trend` помечает такой бакет `complete: false` и добавляет `coverage_pct`: +счётчики в нём настоящие, но измеряют лишь эту долю собственного интервала +бакета. То есть значения пригодны — выбросить их значило бы потерять 55 минут +реального I/O в бакете, сломанном на пятой минуте, — но они **не сравнимы** с +полным бакетом, и меньшее число там не является спадом нагрузки. Количество +таких бакетов даёт `incomplete_points`. + +Неполный бакет вообще без `values` не измерил ничего: вот он и есть разрыв в +записи. + +## Какое окно прочитано на самом деле + +`requested` — запрошенное окно, `window` — то, которое покрывают данные; +расхождение между ними — это покрытие снимками, а не нагрузка. Запрос длиннее +31 дня обрезается до 31 и помечается `window_capped: true` — вывод вида «за +последние 90 дней ничего не менялось» из обрезанного окна не следует. + +## Пустые ответы + +Пустой результат не является ответом, пока `empty_reason` не скажет, какой это +из случаев. Ровно одно значение означает, что инстанс простаивал; остальные — +что на вопрос не ответили: + +- **unsupported_version** — на хосте PostgreSQL 15 или старше, `pg_stat_io` + там нет. Про его I/O этим путём ничего не узнать; остаются `wait_events` и + `query_report`. +- **no_snapshots** — сервер `pg_stat_io` поддерживает, но снимков ещё не + сделано. Остаётся дождаться сборщика. +- **support_unknown** — проба, отличающая неподдерживаемый сервер от ещё не + снятого, не ответила; её ошибка лежит в `empty_detail` (403 — токену не + разрешено читать живой I/O, 404 — неверное имя кластера/инстанса). +- **window_before_history** — окно заканчивается раньше начала хранимой + истории. Запросить более позднее окно; с какого момента она есть, говорит + `meta.earliest_at`. +- **window_after_history** — окно начинается позже последнего снимка: сборщик + не работал с `meta.latest_at`. Об I/O инстанса это не говорит ничего — + сообщать, что история обрывается там, а не выдавать «всё чисто». +- **no_snapshots_in_window** — история есть и до, и после, но в само окно не + попал ни один снимок. Расширить окно. +- **no_comparable_snapshots** — снимки в окне есть, но ни одна пара не + сравнима: эпоха счётчиков рвалась между всеми. Расширить окно за пределы + сбросов или посмотреть `meta.version_changed`. +- **no_io_matching_filter** — под `context` / `object` / `backend_type` ничего + не попало. Фильтр применяется до сборки серий, поэтому сюда же попадает окно, + в котором вообще нет снимков. `backend_type` не валидируется, и + `'autovacuum'` (настоящее значение — `'autovacuum worker'`) молча не + совпадает ни с чем. Прежде чем делать выводы, повторить запрос без фильтра. +- **no_io** — снимки окно покрывают, они сравнимы, физического I/O + действительно не было. Только это содержательный ответ, и даже здесь + `totals` может показывать активную работу с кэшем. + +## Куда идти дальше + +- доминирует `vacuum` → `vacuum_danger`, `top_tables` — какие таблицы держат + автовакуум занятым. +- доминирует `bulkread` → `top_queries` (by=time) и `list_indexes` (missing) — + крупные сканы, которые убрал бы индекс. +- растут `extends` → `top_tables` по размеру; проверить ретеншн и раздувание. +- `fsyncs` у бэкендов → `settings_analyze`, настройки чекпойнтера. +- заметный `temp relation` → `top_queries` и `work_mem`: спиллы дают + сортировки и хэши, которые не помещаются. +- всплеск с неясным виновником → `io_summary` с `group_by=full` по тому окну, + на которое указал `io_trend`. diff --git a/backend/internal/mcpserver/kb_sync_test.go b/backend/internal/mcpserver/kb_sync_test.go index 08554f4d..42962128 100644 --- a/backend/internal/mcpserver/kb_sync_test.go +++ b/backend/internal/mcpserver/kb_sync_test.go @@ -1,16 +1,14 @@ package mcpserver -// This file deliberately imports internal/health — the only place in mcpserver -// allowed to: the runtime boundary ("depend only on gen/apiclient") holds -// because a _test.go import is never linked into cmd/dasha-mcp. The import is -// the mechanism keeping the embedded knowledge base in lockstep with the rules -// engine: a rule added to (or removed from) health.Registry without a matching -// kb section fails CI. Threshold VALUES inside rule closures are not checkable -// this way — reviews changing severityFor(...) numbers must touch kb too. +// Importing internal/health is allowed only here: a _test.go import is never +// linked into cmd/dasha-mcp. Threshold values inside rule closures stay +// unchecked — changing them must touch kb too. import ( "io/fs" "regexp" + "slices" + "strings" "testing" "github.com/dbulashev/dasha/internal/health" @@ -134,4 +132,42 @@ func TestKB_SizeAndLanguageParity(t *testing.T) { } } } + + // Heading text cannot be compared across translations — the nesting can. + for _, name := range files["en"] { + want := headingLevels(t, "kb/en/"+name) + + for _, lang := range kbLangs[1:] { + if got := headingLevels(t, "kb/"+lang+"/"+name); !slices.Equal(got, want) { + t.Errorf("kb/%s/%s has heading structure %v, kb/en/%s has %v — a section is missing or extra", + lang, name, got, name, want) + } + } + } +} + +// headingLevels returns the depth of each markdown heading in order (2 for "##", +// 3 for "###"), which is a translation-independent fingerprint of a file's shape. +func headingLevels(t *testing.T, path string) []int { + t.Helper() + + b, err := kbFS.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + + var levels []int + + for _, line := range strings.Split(string(b), "\n") { + n := 0 + for n < len(line) && line[n] == '#' { + n++ + } + + if n > 0 && n < len(line) && line[n] == ' ' { + levels = append(levels, n) + } + } + + return levels } diff --git a/backend/internal/mcpserver/prompts.go b/backend/internal/mcpserver/prompts.go index e19ca8af..52890da2 100644 --- a/backend/internal/mcpserver/prompts.go +++ b/backend/internal/mcpserver/prompts.go @@ -55,7 +55,8 @@ Investigating: - query_compare needs snapshot IDs from list_snapshots. - search_logs works only on clusters with supports_logs=true (see list_clusters) and is rate-limited per user because every call reaches the Yandex Cloud API: combine all filters into one call, keep dedup on, and after a 429 wait ~30 seconds instead of retrying immediately. - schema_lint answers a different question from every other tool: what is wrong with the STRUCTURE, not what is happening now. Read its skipped list before concluding anything — a check that could not run says nothing about the schema, and reporting "clean" over a non-empty skipped list is a false all-clear. Two findings have a fix that is NOT the obvious one: sequence_exhaustion on an owned_column_type of 'integer' needs the column type changed (a table rewrite, needs a window), not just ALTER SEQUENCE; and no_primary_key on a table whose unique index is nullable cannot be answered with "you already have a unique index" — that index is no replica identity. Read dasha://kb/schema-checks before advising on a code you do not know. -- If unsure how to interpret a result or which tool to call next, read the resources first: dasha://kb/workflow (complaint-to-tool-chain playbooks), dasha://kb/health-rules (rule thresholds and first actions), dasha://kb/schema-checks (schema defect codes and their fixes), dasha://kb/wait-events (wait event glossary). +- "Who is doing all this I/O?" -> io_summary, and io_trend for when it started. These are the only tools that see non-client I/O: autovacuum, the checkpointer, the WAL writer. They need PostgreSQL 16+ (older hosts answer empty with empty_reason='unsupported_version', which does NOT mean no I/O) and snapshot storage. Never read a zero time metric as "fast" without checking meta.track_io_timing, and never read an incomplete io_trend point as a lull — its counters cover only coverage_pct of the bucket. An empty answer is never proof of an idle instance: only empty_reason='no_io' says that, and every other value means the question went unanswered. Read dasha://kb/pg-stat-io before interpreting the counters. +- If unsure how to interpret a result or which tool to call next, read the resources first: dasha://kb/workflow (complaint-to-tool-chain playbooks), dasha://kb/health-rules (rule thresholds and first actions), dasha://kb/schema-checks (schema defect codes and their fixes), dasha://kb/wait-events (wait event glossary), dasha://kb/pg-stat-io (how to read the I/O counters). If a result is refused as too large, narrow it (one database, a smaller range, or a more specific tool).`, @@ -99,6 +100,10 @@ If a result is refused as too large, narrow it (one database, a smaller range, o "3. blocked_queries — lock waits masquerade as slowness; if present, find the root blocker.\n" + "4. wait_events — the dominant event names the bottleneck class; interpret via the resource " + "dasha://kb/wait-events.\n" + + "5. Only if step 4 was dominated by an I/O event (DataFileRead, DataFileWrite, WALSync): io_summary — " + + "wait_events says backends wait on disk but never who reads, and pg_stat_statements covers only client " + + "backends. This is what separates client load from autovacuum, the checkpointer and bulk scans. " + + "PostgreSQL 16+ and snapshot storage only; an empty result is not proof of no I/O.\n" + "Report the heaviest statements, anything stuck or blocked, and next steps: EXPLAIN for plan problems, " + "caching/batching for frequency problems, terminating the blocker for lock problems.", @@ -129,7 +134,8 @@ If a result is refused as too large, narrow it (one database, a smaller range, o - query_compare требует ID снимков из list_snapshots. - search_logs работает только на кластерах с supports_logs=true (см. list_clusters) и лимитирован per-user, т.к. каждый вызов уходит в Yandex Cloud API: собирайте все фильтры в один вызов, держите dedup включённым, после 429 ждите ~30 секунд вместо немедленного повтора. - schema_lint отвечает не на тот вопрос, что остальные инструменты: что не так со СТРУКТУРОЙ, а не что происходит сейчас. Прежде чем делать выводы, прочитайте его список skipped — проверка, которая не выполнилась, не говорит о схеме ничего, и «всё чисто» при непустом skipped — ложное «отбой». У двух находок правильное лечение НЕ очевидное: sequence_exhaustion с owned_column_type = 'integer' требует смены типа колонки (переписывание таблицы, нужно окно), а не только ALTER SEQUENCE; а no_primary_key на таблице с nullable уникальным индексом нельзя закрывать фразой «у вас же есть unique» — такой индекс не годится в replica identity. Перед советами по незнакомому коду читайте dasha://kb/schema-checks. -- Если непонятно, как трактовать результат или какой инструмент звать дальше — сначала прочитайте ресурсы: dasha://kb/workflow (сценарии «жалоба -> цепочка»), dasha://kb/health-rules (пороги правил и первые действия), dasha://kb/schema-checks (коды дефектов схемы и их лечение), dasha://kb/wait-events (глоссарий wait events). +- «Кто делает весь этот I/O?» -> io_summary, а io_trend — когда он начался. Только эти инструменты видят неклиентский I/O: автовакуум, чекпойнтер, walwriter. Нужен PostgreSQL 16+ (на старых хостах ответ пустой с empty_reason='unsupported_version', и это НЕ значит «I/O нет») и хранилище снимков. Никогда не читайте нулевое время как «быстро», не проверив meta.track_io_timing, и никогда не читайте неполную точку io_trend как затишье — её счётчики покрывают лишь coverage_pct бакета. Пустой ответ не доказывает простой: это говорит только empty_reason='no_io', любое другое значение значит, что на вопрос не ответили. Перед трактовкой счётчиков читайте dasha://kb/pg-stat-io. +- Если непонятно, как трактовать результат или какой инструмент звать дальше — сначала прочитайте ресурсы: dasha://kb/workflow (сценарии «жалоба -> цепочка»), dasha://kb/health-rules (пороги правил и первые действия), dasha://kb/schema-checks (коды дефектов схемы и их лечение), dasha://kb/wait-events (глоссарий wait events), dasha://kb/pg-stat-io (как читать счётчики I/O). Если результат отклонён как слишком большой — сузьте запрос (одна база, меньший диапазон или более специфичный инструмент).`, @@ -172,6 +178,10 @@ If a result is refused as too large, narrow it (one database, a smaller range, o "2. running_queries — запросы, работающие минутами, и idle-in-transaction сессии.\n" + "3. blocked_queries — ожидания блокировок маскируются под медленность; если есть — найди корневого блокировщика.\n" + "4. wait_events — доминирующее событие называет класс узкого места; трактуй через ресурс dasha://kb/wait-events.\n" + + "5. Только если на шаге 4 доминировало I/O-событие (DataFileRead, DataFileWrite, WALSync): io_summary — " + + "wait_events говорит, что бэкенды ждут диск, но не говорит, кто читает, а pg_stat_statements видит только " + + "клиентские бэкенды. Это отделяет клиентскую нагрузку от автовакуума, чекпойнтера и массовых сканов. " + + "Нужен PostgreSQL 16+ и хранилище снимков; пустой ответ не доказывает отсутствие I/O.\n" + "Доложи самые тяжёлые запросы, всё застрявшее или заблокированное, и следующие шаги: EXPLAIN для проблем " + "плана, кэширование/батчинг для проблем частоты, завершение блокировщика для проблем блокировок.", diff --git a/backend/internal/mcpserver/resources.go b/backend/internal/mcpserver/resources.go index f2ba1c3e..abe1170b 100644 --- a/backend/internal/mcpserver/resources.go +++ b/backend/internal/mcpserver/resources.go @@ -68,6 +68,13 @@ func registerResources(s *mcp.Server, lang string) { "which params it fills, what the defect leads to and what the first action " + "is — including the two cases where the obvious fix is the wrong one.", }, + { + "pg-stat-io", "Reading pg_stat_io", + "Read before interpreting io_summary / io_trend results: which counters are not I/O " + + "(hits), which look alarming but are normal (evictions under bulkread), which are " + + "genuinely anomalous (fsyncs on a client backend), what each context and object means, " + + "and what an empty or incomplete result actually says.", + }, { "workflow", "Diagnostic workflows", "Read when unsure which tool to call next: complaint-to-tool-chain playbooks " + diff --git a/backend/internal/mcpserver/resources_test.go b/backend/internal/mcpserver/resources_test.go index 7bd970dd..a1ffe097 100644 --- a/backend/internal/mcpserver/resources_test.go +++ b/backend/internal/mcpserver/resources_test.go @@ -7,7 +7,7 @@ import ( "testing" ) -var kbResourceNames = []string{"health-rules", "schema-checks", "wait-events", "workflow"} +var kbResourceNames = []string{"health-rules", "pg-stat-io", "schema-checks", "wait-events", "workflow"} func TestKBHandler_ServesMarkdown(t *testing.T) { t.Parallel() diff --git a/backend/internal/mcpserver/tools.go b/backend/internal/mcpserver/tools.go index c6ad9146..264d46a0 100644 --- a/backend/internal/mcpserver/tools.go +++ b/backend/internal/mcpserver/tools.go @@ -239,6 +239,33 @@ type searchLogsArgs struct { PageToken string `json:"page_token,omitempty" jsonschema:"Cursor from a previous dedup=false result to fetch the next page"` } +// pg_stat_io is instance-wide, so neither I/O tool takes a database: there is +// nothing to narrow it to. +type ioSummaryArgs struct { + Cluster string `json:"cluster" jsonschema:"Dasha cluster name"` + Instance string `json:"instance" jsonschema:"Dasha instance / host name"` + Since string `json:"since,omitempty" jsonschema:"Look-back window ending now, e.g. '15m', '1h', '24h', '7d' (default '1h'); ignored when from/to are set"` + From string `json:"from,omitempty" jsonschema:"Window start, RFC3339; set together with to"` + To string `json:"to,omitempty" jsonschema:"Window end, RFC3339; set together with from"` + GroupBy string `json:"group_by,omitempty" jsonschema:"Dimensions to keep: 'context' (default, cheapest answer to whose I/O), 'backend_type' or 'full' (backend_type x object x context)"` + Object string `json:"object,omitempty" jsonschema:"Keep only this object: 'relation', 'temp relation' (work_mem spills) or 'wal'"` + BackendType string `json:"backend_type,omitempty" jsonschema:"Keep only this backend type, e.g. 'client backend', 'autovacuum worker', 'checkpointer'"` + Context string `json:"context,omitempty" jsonschema:"Keep only this context: 'normal', 'vacuum', 'bulkread', 'bulkwrite' or 'init'"` + Top int `json:"top,omitempty" jsonschema:"Max rows to return, heaviest first (default 20, max 200); rows_total says how many were dropped"` +} + +type ioTrendArgs struct { + Cluster string `json:"cluster" jsonschema:"Dasha cluster name"` + Instance string `json:"instance" jsonschema:"Dasha instance / host name"` + Since string `json:"since,omitempty" jsonschema:"Look-back window ending now, e.g. '6h', '24h', '7d' (default '24h'); ignored when from/to are set"` + From string `json:"from,omitempty" jsonschema:"Window start, RFC3339; set together with to"` + To string `json:"to,omitempty" jsonschema:"Window end, RFC3339; set together with from"` + Points int `json:"points,omitempty" jsonschema:"Buckets per series (default 24 — a day by the hour over the default window, max 200); raise only when the shape matters more than the breakdown"` + Context string `json:"context,omitempty" jsonschema:"Keep only this context: 'normal', 'vacuum', 'bulkread', 'bulkwrite' or 'init'"` + BackendType string `json:"backend_type,omitempty" jsonschema:"Keep only this backend type before grouping by context"` + Object string `json:"object,omitempty" jsonschema:"Keep only this object: 'relation', 'temp relation' or 'wal'"` +} + func registerTools(s *mcp.Server, c *DashaClient) { addTool(s, &mcp.Tool{ Name: "list_clusters", @@ -658,6 +685,46 @@ func registerTools(s *mcp.Server, c *DashaClient) { return jsonResult(c.SearchLogs(ctx, params)) }) + + addTool(s, &mcp.Tool{ + Name: "io_summary", + Description: "Break instance-wide physical I/O down over a time window: who read, wrote and extended, " + + "from the stored pg_stat_io snapshots. Answers what wait_events (who waits, not who causes it) and " + + "query_report (client backends only, pg_stat_statements only) cannot — autovacuum vs client load, " + + "the bulkread share (sequential scans bypassing the cache), extends (real file growth), fsyncs on a " + + "regular backend (the checkpointer is falling behind), 'temp relation' (instance-wide work_mem " + + "spills). group_by=context (default) is the cheapest answer to 'whose I/O'; 'full' breaks it down " + + "by backend_type x object x context and needs top. Requires PostgreSQL 16 or newer: older hosts have " + + "no pg_stat_io at all and come back empty with empty_reason='unsupported_version', which does NOT " + + "mean 'no I/O'. Every empty answer carries an empty_reason, and only 'no_io' means the instance was " + + "idle: 'no_snapshots_in_window', 'no_comparable_snapshots', 'window_after_history' and " + + "'no_io_matching_filter' all mean the question went unanswered — check totals and meta before " + + "reporting an all-clear. With track_io_timing off every time metric is zero by construction — a " + + "missing measurement, not a missing load; meta.track_io_timing says which, and avg_read_ms/" + + "avg_write_ms are absent rather than 0. A window longer than 31 days is cut back to it and flagged " + + "window_capped. pg_stat_io is instance-wide: there is no database parameter. A counter absent " + + "from values is zero. Needs snapshot storage (501 otherwise). " + + "Read dasha://kb/pg-stat-io before interpreting the numbers.", + }, func(ctx context.Context, _ *mcp.CallToolRequest, a ioSummaryArgs) (*mcp.CallToolResult, any, error) { + return jsonResult(ioSummary(ctx, c, a)) + }) + + addTool(s, &mcp.Tool{ + Name: "io_trend", + Description: "Coarse time series of physical I/O per pg_stat_io context — when the load started and " + + "whether it lines up with the autovacuum window. Defaults to the last 24 hours in 24 buckets, " + + "grouped by context; use io_summary on the window this narrows down to find out who is behind one. " + + "A point covering a statistics reset, a restart or a major upgrade carries complete=false and a " + + "coverage_pct: its counters are real but measure only that share of the bucket's span, so it is not " + + "comparable with a complete point and a lower number there is not a drop in load (incomplete_points " + + "counts such buckets). An incomplete point with no values at all measured nothing. In a complete " + + "point an absent metric is zero. Same preconditions as io_summary, including empty_reason on an " + + "empty answer: PostgreSQL 16+, time metrics are zero unless track_io_timing is on, instance-wide " + + "(no database), snapshot storage required (501 otherwise). " + + "Read dasha://kb/pg-stat-io before interpreting the numbers.", + }, func(ctx context.Context, _ *mcp.CallToolRequest, a ioTrendArgs) (*mcp.CallToolResult, any, error) { + return jsonResult(ioTrend(ctx, c, a)) + }) } // closedWorld marks the tools as not interacting with an open world of external @@ -706,6 +773,58 @@ func trendWindow(rng string) (span time.Duration, step int) { } } +// resolveWindow maps the since / from+to argument pair every windowed tool +// accepts onto an absolute window, defaulting to the last def. Returns a +// non-empty message instead of a window when the arguments are invalid. +func resolveWindow(since, from, to string, def time.Duration) (time.Time, time.Time, string) { + end := time.Now() + start := end.Add(-def) + + switch { + case from != "" || to != "": + if from == "" || to == "" { + return time.Time{}, time.Time{}, "from and to must be set together (RFC3339)" + } + + var err error + if start, err = time.Parse(time.RFC3339, from); err != nil { + return time.Time{}, time.Time{}, "from must be RFC3339 (e.g. 2026-07-10T12:00:00Z)" + } + + if end, err = time.Parse(time.RFC3339, to); err != nil { + return time.Time{}, time.Time{}, "to must be RFC3339 (e.g. 2026-07-10T13:00:00Z)" + } + + if !start.Before(end) { + return time.Time{}, time.Time{}, "from must be before to" + } + case since != "": + d, err := parseSince(since) + if err != nil || d <= 0 { + return time.Time{}, time.Time{}, "since must be a positive duration like '15m', '1h', '24h' or '7d'" + } + + start = end.Add(-d) + } + + return start, end, "" +} + +// time.ParseDuration has no day unit; models write '7d'. +func parseSince(since string) (time.Duration, error) { + days, ok := strings.CutSuffix(since, "d") + if !ok { + return time.ParseDuration(since) + } + + n, err := strconv.Atoi(days) + if err != nil { + return 0, err + } + + return time.Duration(n) * 24 * time.Hour, nil +} + // logsDefaultSince is the default look-back window for search_logs; a short // window keeps the upstream Yandex API scan (and the result) small. const logsDefaultSince = time.Hour @@ -724,34 +843,9 @@ func logsParams(a searchLogsArgs) (*apiclient.GetLogsParams, string) { return nil, "service_type must be 'postgresql' or 'pooler'" } - to := time.Now() - from := to.Add(-logsDefaultSince) - - switch { - case a.From != "" || a.To != "": - if a.From == "" || a.To == "" { - return nil, "from and to must be set together (RFC3339)" - } - - var err error - if from, err = time.Parse(time.RFC3339, a.From); err != nil { - return nil, "from must be RFC3339 (e.g. 2026-07-10T12:00:00Z)" - } - - if to, err = time.Parse(time.RFC3339, a.To); err != nil { - return nil, "to must be RFC3339 (e.g. 2026-07-10T13:00:00Z)" - } - - if !from.Before(to) { - return nil, "from must be before to" - } - case a.Since != "": - d, err := time.ParseDuration(a.Since) - if err != nil || d <= 0 { - return nil, "since must be a positive duration like '15m', '1h' or '24h'" - } - - from = to.Add(-d) + from, to, msg := resolveWindow(a.Since, a.From, a.To, logsDefaultSince) + if msg != "" { + return nil, msg } // Dedup defaults to on: grouped results are far smaller and usually enough. diff --git a/doc/en/mcp.md b/doc/en/mcp.md index 89e093e6..7106cbdf 100644 --- a/doc/en/mcp.md +++ b/doc/en/mcp.md @@ -4,9 +4,9 @@ `dasha-mcp` is a separate, **read-only** [MCP](https://modelcontextprotocol.io) server over the Dasha API. It lets AI assistants query the fleet's PostgreSQL diagnostics as tools/prompts, forwarding each caller's token to Dasha so its RBAC is preserved. Any MCP-compatible client works — Claude Desktop, Claude Code, Cursor, Continue, **opencode**, etc. -- **Tools (28):** `list_clusters`, `fleet_health`, `get_instance_info`, `get_health_score`, `get_health_recommendations`, `health_details` (turns a recommendation into a target: pass its `rule_id` as `detail` to get the offending tables, databases or sessions — the per-table drill-downs also take a `database`, the instance-wide ones do not), `health_trend`, `health_databases`, `top_queries` (by time/WAL), `query_report`, `list_snapshots`, `query_compare`, `running_queries`, `blocked_queries`, `list_indexes` (missing/unused/usage), `unused_index_report` (cluster-wide verdict on whether an index is safe to DROP: weighs the scan counter against every host of the cluster and against the statistics window behind it, because `idx_scan` is not replicated and a counter without its window means nothing), `top_tables`, `schema_lint` / `schema_lint_summary` (structural defects of a schema from the system catalog: sequences running out of values, tables without a primary key, unlogged relations, schemas PUBLIC may create in — with a `skipped` list naming the checks that could not run, so a missing check is never mistaken for a clean result), `hot_tables` / `hot_indexes` (top hot objects per metric class — reads/writes/io — from the daily delta snapshots, summed over every cluster host, with a coverage ratio that says how representative the top is; needs snapshot storage), `describe_table`, `get_replication`, `settings_analyze`, `wait_events`, `connections`, `vacuum_danger`, `search_logs` (Yandex Cloud PostgreSQL/pooler logs; Yandex-MDB-discovered clusters only, rate-limited per user). All are annotated **read-only** and closed-world so compatible clients can surface (and auto-approve) them as safe. The server also ships usage **instructions** that prime the model on which tool/prompt to reach for. +- **Tools (30):** `list_clusters`, `fleet_health`, `get_instance_info`, `get_health_score`, `get_health_recommendations`, `health_details` (turns a recommendation into a target: pass its `rule_id` as `detail` to get the offending tables, databases or sessions — the per-table drill-downs also take a `database`, the instance-wide ones do not), `health_trend`, `health_databases`, `top_queries` (by time/WAL), `query_report`, `list_snapshots`, `query_compare`, `running_queries`, `blocked_queries`, `list_indexes` (missing/unused/usage), `unused_index_report` (cluster-wide verdict on whether an index is safe to DROP: weighs the scan counter against every host of the cluster and against the statistics window behind it, because `idx_scan` is not replicated and a counter without its window means nothing), `top_tables`, `schema_lint` / `schema_lint_summary` (structural defects of a schema from the system catalog: sequences running out of values, tables without a primary key, unlogged relations, schemas PUBLIC may create in — with a `skipped` list naming the checks that could not run, so a missing check is never mistaken for a clean result), `hot_tables` / `hot_indexes` (top hot objects per metric class — reads/writes/io — from the daily delta snapshots, summed over every cluster host, with a coverage ratio that says how representative the top is; needs snapshot storage), `describe_table`, `get_replication`, `settings_analyze`, `wait_events`, `connections`, `vacuum_danger`, `search_logs` (Yandex Cloud PostgreSQL/pooler logs; Yandex-MDB-discovered clusters only, rate-limited per user), `io_summary` / `io_trend` (physical I/O from `pg_stat_io`, broken down by backend type, object and context, and its shape over time — the only tools that see I/O done by autovacuum, the checkpointer and the WAL writer rather than by client backends; PostgreSQL 16+ and snapshot storage). All are annotated **read-only** and closed-world so compatible clients can surface (and auto-approve) them as safe. The server also ships usage **instructions** that prime the model on which tool/prompt to reach for. - **Prompts (5):** `diagnose_cluster`, `explain_health_score`, `find_index_opportunities`, `investigate_slow_queries`, `fleet_overview` — linear playbooks: numbered steps, one tool per step, with an interpretation criterion on each (built for models without deep PostgreSQL expertise; strong models simply move faster through them). -- **Resources (4):** an embedded knowledge base the model can read on demand — `dasha://kb/health-rules` (every health rule with LOW/MED/HIGH thresholds and first actions), `dasha://kb/schema-checks` (every schema-check code, the params it fills and its first action), `dasha://kb/wait-events` (wait event glossary), `dasha://kb/workflow` (complaint-to-tool-chain playbooks and API care rules). +- **Resources (5):** an embedded knowledge base the model can read on demand — `dasha://kb/health-rules` (every health rule with LOW/MED/HIGH thresholds and first actions), `dasha://kb/schema-checks` (every schema-check code, the params it fills and its first action), `dasha://kb/wait-events` (wait event glossary), `dasha://kb/pg-stat-io` (how to read the `pg_stat_io` counters), `dasha://kb/workflow` (complaint-to-tool-chain playbooks and API care rules). - **Language:** `--lang en|ru` (or `DASHA_MCP_LANG`) selects the language of the knowledge base, playbooks and instructions; tool names, schemas and results stay English. **Prerequisite:** a Dasha API token — a [personal access token](auth.md#personal-access-tokens-optional) (`dasha_pat_…`) or a static config token. It determines the role (`viewer` is enough). diff --git a/doc/ru/mcp.md b/doc/ru/mcp.md index 9ea201a3..e3f0d123 100644 --- a/doc/ru/mcp.md +++ b/doc/ru/mcp.md @@ -4,9 +4,9 @@ `dasha-mcp` — отдельный **read-only** [MCP](https://modelcontextprotocol.io)-сервер поверх Dasha API. Позволяет AI-ассистентам запрашивать диагностику флота PostgreSQL как tools/prompts, прокидывая токен каждого вызывающего в Dasha (RBAC сохраняется). Подходит любой MCP-совместимый клиент — Claude Desktop, Claude Code, Cursor, Continue, **opencode** и т.д. -- **Tools (28):** `list_clusters`, `fleet_health`, `get_instance_info`, `get_health_score`, `get_health_recommendations`, `health_details` (превращает рекомендацию в цель: передайте её `rule_id` как `detail` — вернутся сами таблицы, базы или сессии; потабличным drill-down нужна ещё `database`, инстанс-уровневым — нет), `health_trend`, `health_databases`, `top_queries` (по времени/WAL), `query_report`, `list_snapshots`, `query_compare`, `running_queries`, `blocked_queries`, `list_indexes` (missing/unused/usage), `unused_index_report` (вердикт по всему кластеру: можно ли удалить индекс — счётчик сканов взвешивается по всем хостам и по окну статистики, т.к. `idx_scan` не реплицируется, а счётчик без окна ничего не значит), `top_tables`, `schema_lint` / `schema_lint_summary` (дефекты структуры схемы по системному каталогу: кончающиеся последовательности, таблицы без первичного ключа, unlogged-объекты, схемы, где может создавать объекты PUBLIC — со списком `skipped`, называющим невыполнившиеся проверки, чтобы пропуск не приняли за чистый результат), `hot_tables` / `hot_indexes` (топ горячих объектов по классам метрик — чтения/записи/io — из суточных дельта-снимков, просуммированных по всем хостам кластера, с coverage-долей репрезентативности топа; требует snapshot-хранилище), `describe_table`, `get_replication`, `settings_analyze`, `wait_events`, `connections`, `vacuum_danger`, `search_logs` (логи PostgreSQL/пулера из Yandex Cloud; только для кластеров из Yandex MDB discovery, с per-user rate limit). Все помечены **read-only** и closed-world, чтобы совместимые клиенты показывали (и авто-аппрувили) их как безопасные. Сервер также отдаёт **инструкции** по использованию, которые подсказывают модели, какой tool/prompt выбрать. +- **Tools (30):** `list_clusters`, `fleet_health`, `get_instance_info`, `get_health_score`, `get_health_recommendations`, `health_details` (превращает рекомендацию в цель: передайте её `rule_id` как `detail` — вернутся сами таблицы, базы или сессии; потабличным drill-down нужна ещё `database`, инстанс-уровневым — нет), `health_trend`, `health_databases`, `top_queries` (по времени/WAL), `query_report`, `list_snapshots`, `query_compare`, `running_queries`, `blocked_queries`, `list_indexes` (missing/unused/usage), `unused_index_report` (вердикт по всему кластеру: можно ли удалить индекс — счётчик сканов взвешивается по всем хостам и по окну статистики, т.к. `idx_scan` не реплицируется, а счётчик без окна ничего не значит), `top_tables`, `schema_lint` / `schema_lint_summary` (дефекты структуры схемы по системному каталогу: кончающиеся последовательности, таблицы без первичного ключа, unlogged-объекты, схемы, где может создавать объекты PUBLIC — со списком `skipped`, называющим невыполнившиеся проверки, чтобы пропуск не приняли за чистый результат), `hot_tables` / `hot_indexes` (топ горячих объектов по классам метрик — чтения/записи/io — из суточных дельта-снимков, просуммированных по всем хостам кластера, с coverage-долей репрезентативности топа; требует snapshot-хранилище), `describe_table`, `get_replication`, `settings_analyze`, `wait_events`, `connections`, `vacuum_danger`, `search_logs` (логи PostgreSQL/пулера из Yandex Cloud; только для кластеров из Yandex MDB discovery, с per-user rate limit), `io_summary` / `io_trend` (физический I/O по `pg_stat_io` в разрезе типа бэкенда, объекта и контекста и его динамика — единственные инструменты, которые видят I/O автовакуума, чекпойнтера и walwriter, а не только клиентских бэкендов; нужен PostgreSQL 16+ и snapshot-хранилище). Все помечены **read-only** и closed-world, чтобы совместимые клиенты показывали (и авто-аппрувили) их как безопасные. Сервер также отдаёт **инструкции** по использованию, которые подсказывают модели, какой tool/prompt выбрать. - **Prompts (5):** `diagnose_cluster`, `explain_health_score`, `find_index_opportunities`, `investigate_slow_queries`, `fleet_overview` — линейные плейбуки: нумерованные шаги, один tool на шаг, с критерием трактовки на каждом (рассчитаны на модели без глубокой экспертизы PostgreSQL; сильные модели просто проходят их быстрее). -- **Resources (4):** встроенная база знаний, которую модель читает по запросу — `dasha://kb/health-rules` (каждое правило health score с порогами LOW/MED/HIGH и первыми действиями), `dasha://kb/schema-checks` (каждый код проверки схемы, его params и первое действие), `dasha://kb/wait-events` (глоссарий wait events), `dasha://kb/workflow` (сценарии «жалоба → цепочка инструментов» и правила бережности к API). +- **Resources (5):** встроенная база знаний, которую модель читает по запросу — `dasha://kb/health-rules` (каждое правило health score с порогами LOW/MED/HIGH и первыми действиями), `dasha://kb/schema-checks` (каждый код проверки схемы, его params и первое действие), `dasha://kb/wait-events` (глоссарий wait events), `dasha://kb/pg-stat-io` (как читать счётчики `pg_stat_io`), `dasha://kb/workflow` (сценарии «жалоба → цепочка инструментов» и правила бережности к API). - **Язык:** `--lang en|ru` (или `DASHA_MCP_LANG`) выбирает язык базы знаний, плейбуков и инструкций; имена tools, схемы и результаты остаются английскими. **Предусловие:** токен Dasha API — [персональный токен](auth.md#персональные-api-токены-опционально) (`dasha_pat_…`) или статический config-токен. Он определяет роль (`viewer` достаточно). From 8c889c31a7ea269fa7e78557e2397bad8d4e05fd Mon Sep 17 00:00:00 2001 From: "Dmitry V. Bulashev" Date: Sun, 30 Aug 2026 09:25:58 +0500 Subject: [PATCH 2/3] review fixes --- CHANGELOG.md | 2 +- CHANGELOG.ru.md | 2 +- backend/gen/apiclient/client.gen.go | 9 + backend/gen/serverhttp/api.gen.go | 689 +++++++++--------- backend/internal/http/v1_io.go | 20 +- backend/internal/http/v1_io_test.go | 27 + backend/internal/mcpserver/io.go | 15 +- backend/internal/mcpserver/io_test.go | 80 +- .../internal/mcpserver/kb/en/pg-stat-io.md | 50 +- .../internal/mcpserver/kb/ru/pg-stat-io.md | 54 +- backend/internal/mcpserver/prompts.go | 4 +- backend/internal/mcpserver/tools.go | 35 +- .../statio/snapshot/180000/snapshot.tmpl.sql | 5 +- .../sql/statio/snapshot/snapshot.tmpl.sql | 5 +- backend/internal/repository/statio.go | 3 +- .../repository/statio_integration_test.go | 2 +- backend/internal/statio/statio.go | 25 +- backend/internal/storage/migrate.go | 22 +- backend/internal/storage/statio.go | 17 +- .../storage/statio_integration_test.go | 12 +- doc/en/features.md | 2 +- doc/ru/features.md | 2 +- doc/swagger.yaml | 16 + frontend/src/api/models/iOHistoryMeta.ts | 4 + frontend/src/api/models/iOSnapshot.ts | 2 + frontend/src/components/io/IOModeBar.vue | 14 +- frontend/src/components/io/useIoLive.ts | 1 + frontend/src/components/io/useIoRows.ts | 8 + frontend/src/locales/de_DE.json | 4 +- frontend/src/locales/en_US.json | 4 +- frontend/src/locales/ru_RU.json | 4 +- frontend/src/views/IOView.vue | 4 + 32 files changed, 704 insertions(+), 439 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7aaa854..9536a3e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## v1.7.2 ### Features -- **Two MCP tools for I/O**, `io_summary` and `io_trend`: an assistant can ask who is doing the reading and writing — client load, autovacuum or the checkpointer — and when it started, with a knowledge-base page on how to read the counters. An empty answer names its cause and a period broken by a statistics reset carries the part that was measured; needs PostgreSQL 16 or newer. +- **Two MCP tools for I/O**, `io_summary` and `io_trend`: an assistant can ask who is doing the reading and writing — client load, autovacuum or the checkpointer — and when it started, with a knowledge-base page on how to read the counters. An empty answer names its cause and a period broken by a statistics reset carries the part that was measured. Write-ahead log I/O times follow `track_wal_io_timing`, so they are read even where `track_io_timing` is off; needs PostgreSQL 16 or newer. ## v1.7.1 diff --git a/CHANGELOG.ru.md b/CHANGELOG.ru.md index 888f5522..d80f5273 100644 --- a/CHANGELOG.ru.md +++ b/CHANGELOG.ru.md @@ -3,7 +3,7 @@ ## v1.7.2 ### Фичи -- **Два MCP-инструмента по вводу-выводу** — `io_summary` и `io_trend`: ассистент может спросить, кто читает и пишет — клиентская нагрузка, автовакуум или чекпойнтер — и когда это началось, плюс страница базы знаний о том, как читать счётчики. Пустой ответ называет причину, а период, разорванный сбросом статистики, несёт измеренную часть; нужен PostgreSQL 16 или новее. +- **Два MCP-инструмента по вводу-выводу** — `io_summary` и `io_trend`: ассистент может спросить, кто читает и пишет — клиентская нагрузка, автовакуум или чекпойнтер — и когда это началось, плюс страница базы знаний о том, как читать счётчики. Пустой ответ называет причину, а период, разорванный сбросом статистики, несёт измеренную часть. Времена ввода-вывода журнала предзаписи берутся по `track_wal_io_timing` и читаются даже при выключенном `track_io_timing`; нужен PostgreSQL 16 или новее. ## v1.7.1 diff --git a/backend/gen/apiclient/client.gen.go b/backend/gen/apiclient/client.gen.go index a8389bf5..3b191a5b 100644 --- a/backend/gen/apiclient/client.gen.go +++ b/backend/gen/apiclient/client.gen.go @@ -801,6 +801,12 @@ type IOHistoryMeta struct { // TrackIoTimingChanged The setting was toggled inside the period, so I/O times cover only part of it. TrackIoTimingChanged bool `json:"track_io_timing_changed"` + // TrackWalIoTiming Value at the newest capture in the period. Governs the time counters of WAL rows (object 'wal', PostgreSQL 18 and newer) independently of track_io_timing. + TrackWalIoTiming bool `json:"track_wal_io_timing"` + + // TrackWalIoTimingChanged The setting was toggled inside the period, so WAL times cover only part of it. + TrackWalIoTimingChanged bool `json:"track_wal_io_timing_changed"` + // VersionChanged The server was upgraded inside the period; the intervals spanning the upgrade are incomplete. VersionChanged bool `json:"version_changed"` } @@ -856,6 +862,9 @@ type IOSnapshot struct { // TrackIoTiming When false the server collects no I/O times and the Time metrics are absent. TrackIoTiming bool `json:"track_io_timing"` + // TrackWalIoTiming Governs the time counters of WAL rows (object 'wal', PostgreSQL 18 and newer) independently of track_io_timing. + TrackWalIoTiming bool `json:"track_wal_io_timing"` + // VersionNum server_version_num at capture time — it decides how the counters are read. VersionNum int `json:"version_num"` } diff --git a/backend/gen/serverhttp/api.gen.go b/backend/gen/serverhttp/api.gen.go index 6b9abfb1..e249c3f9 100644 --- a/backend/gen/serverhttp/api.gen.go +++ b/backend/gen/serverhttp/api.gen.go @@ -806,6 +806,12 @@ type IOHistoryMeta struct { // TrackIoTimingChanged The setting was toggled inside the period, so I/O times cover only part of it. TrackIoTimingChanged bool `json:"track_io_timing_changed"` + // TrackWalIoTiming Value at the newest capture in the period. Governs the time counters of WAL rows (object 'wal', PostgreSQL 18 and newer) independently of track_io_timing. + TrackWalIoTiming bool `json:"track_wal_io_timing"` + + // TrackWalIoTimingChanged The setting was toggled inside the period, so WAL times cover only part of it. + TrackWalIoTimingChanged bool `json:"track_wal_io_timing_changed"` + // VersionChanged The server was upgraded inside the period; the intervals spanning the upgrade are incomplete. VersionChanged bool `json:"version_changed"` } @@ -861,6 +867,9 @@ type IOSnapshot struct { // TrackIoTiming When false the server collects no I/O times and the Time metrics are absent. TrackIoTiming bool `json:"track_io_timing"` + // TrackWalIoTiming Governs the time counters of WAL rows (object 'wal', PostgreSQL 18 and newer) independently of track_io_timing. + TrackWalIoTiming bool `json:"track_wal_io_timing"` + // VersionNum server_version_num at capture time — it decides how the counters are read. VersionNum int `json:"version_num"` } @@ -13578,345 +13587,347 @@ var swaggerSpec = []string{ "XxG8KTX5MFnkB1THmuDegs5OQJCOtNWVPEAJ9ZLRz7tKILWZf1i7C7kGPQcSil2ls/NeGSD3V2vb1qvh", "dJ3HI4o8G8xUzs6vaMAuaQE3U02/9SRxlgBcZwKMjdYoPM/SxuvkAVy9UChxOsJCMkFB1FfioerkhnHL", "eJbdv7bBFhmcXPXVvu8pUmqe3EyFmlqRR6/V91RD11ZVyY2twBD0CdBCpfGHujO/T5VO40URDVjrtNAV", - "d1r5YkFtLiq9k9bB0PSzR+fE3qhbATUncLfZoYZ8cpt78SXid+1B32LzC8PKgoL3N/fwzJsRyBZlmCm4", - "lKGquR/mG2sEcXsSz+jv0aK6aOkH5Oax4legL9xliHLFSWWSXn+aQ5a5wyu6BLMyuQH7jKWarxhnC16M", - "qcw4+wm0iqMiVBeamjqcoFPGELhxvMzblyqzGdkJa9BXVdBBKozd8NkQXK6Zk4om7L9BKzqHVNXAQLu0", - "eSod38DUgMd3P/1vuOrm45rMx9U4tiiEG7hoKfF+M3Giuozp682YnV0O/80CnP0m3I8Clo6k5O0fY6oz", - "XXMVIVkuskx4kHgbQ/sKOilByQk7Jkkh2FKo3rjTdKRvieFZClXMN4wb9vbd69eTnRUNW4Bs2BgD/HYg", - "56p6cNv4uYH10Mf3O7hP0GZgNrtebreRavJtZ/gO1nFmnYqc+gx5/c5Ht98AFIY6Q6hQs5SbwDvwwXBI", - "IhFvs2bxpyHgn+MHbOhHH0FT2SouqKK2K3R4PZbbc6v7mm/iJ6g7PMzW7EIZu9Bw9dfX7MkfHz35UyjF", - "7cSeJ//BlBy7K+IJv0Pvbs36dqRCQ2Kz9UA3hheeB9LdZUtKrpXQtvEtUo25rnSIT9wzxpkv6hIq/LW7", - "vAh5yzOBwnNLFKg/X0J+f6lvp1SG6ZPYV6oJ8URlGRo7ZFNCCl13rkUOvmql6dB/v6wkyfre0XpxuWnj", - "GycgJg21M9hTU0iwyusSm1dAm0lim7PdJq2mcbml4DT3GBOZkHiifEWmcHec3gqj9AmXqeiLD8jKPFZN", - "+ztYM/8jtmHTKeinDH4seUaV0iHF3mGGFGpUL7H8eq6w+jqQpaj+0H+BE2Gqqp/cS1+dr+tGLpkD4ZrN", - "NPAbenP8DPtpqShDQzr1nWKGX7gmHGkOiriMJW6lWbQsqG/CZMrFgswlKFYCVpSmjjbU3YPRuNraT+68", - "oMS7CSF92mRUGsjyd3L54vj6BcNGQuzk/O3Ju8vLF2+vX/+dGto0htOcqGBQ9iyuSDWXfH6wQmdsCndk", - "DDx/y87fvv47kTvxBLxvS5VhkKFUVBrUS6kSE3FTbD9HsyjprQvVNliiZG2gdJNxa3myBOO7bkkISbu2", - "1HJjX7SJqkZ3xo2lp7BRcHOhkAqdcorXdeyEksL8mDEDTtxx3OvpFsD5xmC6lEE3avdnwl5PE3bKzZL7", - "nnpwB0npwHh6+jrediLjUoKmmIqYdvbSszutygWmYNOTb6GYsONQpxyjuROuNbmqE34L3D71FrlmaB/D", - "sbpMbKl5Nmbu3ty2Ms0b9Uk9B0245ZlaUFexHsN0uKQRs5ySJLY6XYYH25ODJTGgcXWffyxVSLWmP1iF", - "/zGuss/dbn549eLyBbuB9UrpdMwgL+za+wSLjAtPYRN2LNnZFYqcDRaScYdnfLKbjbGEDLznKbnN0EJY", - "GcXcwzWda57gHcHrY9TcHqWATapoqOP7cIftNwzjGfI8qSzlgMu08rsLySS1mkR6zoAm1O5pmGd8Qfnn", - "N4CB/TlHnROvDJET5S9WDQk7KOu8Lc2U2t4IBBtCLjsCQk/cgJP9XxK4uyyESFNUrd3CFfUXFy9EfeFb", - "W/fdDh3DaPAy9zDxRUPccBtJ1SrewYUiI4Ls1D6Nhsxirtq4JQ+jbSPsyAQy6p5rMqwE7oprKeQiWgKY", - "m0paryg/lBngbMZTJlLgLAVTCAtVYMIKyIq297v0A+0lapzF8MBpEevRUTuRsDEmz9Y/QeqZGOLMCTmm", - "cAgTskt8ndNRQ80JO0ObMzVO+AmavacyyMfs7fm1gzleUXf9F+4OOzqQinnmiA1dsD+EdpirSYE6FM7w", - "2UyjRsf+vHstLOwHUhqxWfbAB8NYH4oc5KcmVyRBoAX7TfmjRcHVDht0tfla7BTzmuJJRNfyBXQGEPdc", - "yAXoQntzWscJwU1tqqqFG//OONGC+IOPsUkUyrb+iaHFrRffNm41WvH7/B5VNA8VGfVPY9gAdT/i9Lrh", - "a1kRJzUjbnLOUpI5taolRYVO6MEUWcZWStsliUXuagYLHFEitQbE+EnPxmb+vfJhJ/iOKNvQF5HQ8UGi", - "/sW+06xbzzFPo/aTZn8soyEzb+s4thowTo0fM8OlsO6XKNyp5YxIp7N1FUI4OGd9U9cMrUk7KPJdfVAg", - "dGIeuhZEB3Vj99yTRk5NdMPejJc6jW/ojNoqFFzjW4+eCBXa/OJziY+sylIgezoP8hP6zyQAJstVXVTZ", - "QtyCpD2tHAlQCGC2RqFqIR07c6h2fBsKlgkMgPSSfAH6qCoRRJ06yTSAQ1BgwwMK721Dugj9UtW82YoV", - "z0XHma1ZoQwiIRoLWkEm0tOBBO7NOvRarTxQGubtUgqL8jEhFfsVo/dW36Le/8dv3T8TYajNK/svfsuv", - "cLUJe0MdiDm16qZKC1XwZ/d6Jk55Sxnm0vDMK+9USXs+B6cPYLVZ4/t8NnqDI9JuYE2dE7FvngNPxrGp", - "18bTudtr2XoY9w1MqCEfuTtt/jmu+iW33wNkx4Hj7WLtb5W94NqQyjA4XJKq5U0T3yiqK6o72RWpsdmv", - "dqX0TaZ8F1+rS0lM7iDJBFapmq29jbgqcuzbVYmf4HDMrFJTJzqOnWBlYIoVPcaslL6GNKRTs5aW340d", - "+ZfzuUjcnZwWWtyKDBYQhHy8atLrKiScoD3aPmWlvJFqJaeYPYaNNczaWMirP7AD7v9UqTMo6+VKCqvw", - "ojkC8i4cyRr9bkkGfNb+2CqVUYMyVqgMa2k1dAss1GW8O2jBiyAs4aaNMIdjxvOZWJSqNBjKyw64ZKVE", - "U8pcQIricHUnvDQSGtRa9YxpmKPM09RiFiUYg5xNOJ4Tu+l429BQ1uiM39oNSS/jCqL1v2t0ObIaE1im", - "DruoplR63JQikFM8E/qDEUv4TAZVD9lIR3upXlRfjIv5QtCH7uZPG0ajun9K869S0cK4q+rvE3ah3Uuf", - "iVuoYq+I6e+uaNW8LCGgdte17IsuqkSPCGd2j6kyyH4xdnvsQHAr0AMfBPKODL4ZsjLYYlVZ/mLmquAU", - "y4fKhVLZaVHxoa65tvZOtxhJoVVaJhhsigry2FtIqcUQgX3CziUwkNY9WqBrOqkMVTgG/RvBeiaryj8t", - "tYRvRLhW2OjaUAyZbN3lhTsb5q3lRvcKP8VnCFmS+ycZnTjLuF74TREXqB++cO/Jh4yGPeI0lN2ARjeH", - "KR8fuFAqdZgv4F4Yrl+GmFOgLmQ+dL5Q+xz9uTbWOKeiKVO1r+cLIXlHTGmQTSm1k/fwsu4j4ZPcHmFt", - "reAnNIOwa4xuQoLzxjhpsQ9HEyU1NbVwTJio+pDP1g3TC3e0kiy9kmpZDhqydejG3EtRTzvqBkZOBJUD", - "2T/Kgyht+gvSIlWv3459YjZZliAzVUCKBlMo6dSa/fWHjZCqilu1rnhNPzEUBvpo85FdHLO3tH6wPzQN", - "9D2G9BD/L1MSMtI41fmnf1oJMXG/WJAQKiUEsSQkMrOgOLSsOGnpdb/axI10golMQG5B3hMrlKgs44WB", - "dLrQqiwix3wnBd0Ex0Cpiz/1jpp7e3uDsGZr1hA1e8DgDQ/uJdlhB+qzAbVujFMOymQ5YVfN+3hLd20l", - "ksAVURejV57shO5q4FXytxzvKvFAk6gCiElWX1XmHsNvu+2MtyRp78tdqmcKt+mVNofJupzofooFBSJ6", - "IzZ1cd0ayBlKVqMNgaz6kkzhaEXZ5H7I63xzYrIt14iTSrPGVX26Ubu2LPDerJbcIqUK9HCYdg6Ne9CC", - "/4Xs1WTtQEkuKApuhyYY+1SBiipSkUgz2A9kNc+ZbtFmioUx7YJevcFcQ54MYsRkZOiQRjg71vqtNdNa", - "eCUdniMsgl5P1q2yZtcDouE6RxpvcsEIx4jAK3LLY/wv3I84ke5i3cHKHNFAY/olWjSnTrZdV8J+RafV", - "rSO7Z8PZOeea5Qr5kUUvKJcVkW6YoNumYdQzqieefnH8OnEYziszhbDIUsyEZWo19eWVww4DJ2t6xpyO", - "x3KunZCT1UJ64GGT2ocwRR9lOxmz673tuDK9m3EGnuFQP8iGI7I60Onpa7ZQCCx0D1aOWYzr4g2fKnlg", - "ya8q2fH19fHJKxSrq42QXNH1yhZOlAjB87i9Z+5XnptJw2/iJXGM/qCcsxB2XVny6foFY5NmRuUVyLzF", - "PBxLaHLoV5wE/9HWFVdaURnmPJwpOPvmInOs3GF5yW+BzbBRpXszVVndau8jXIkUSGus4+QbWDY3gOZp", - "opQQnYAU4z69gTUto7J0wnJu0QQVJqosD8gycm5BC2+Idd89rfyVly9eXr64esU0eMr3nEOD95oTTirH", - "ubDBcRhGtpzUnpm4RUspfiyh6T+kFb3DUvIqkhhFSVxGq5WXcIKnVumGxl354IDMG1RSkwk7YYbK0zfg", - "2a//U10yWtHvJtBC7RcWjhsrfM2IBJRmM1gKmTL6q8fIMwy5CTsiC2UblWOqK47nJTuqu/GqerTcXFz6", - "sJ0qmIkzCauwcUdkHkWOCSkJDuUyWDu6+dbhoMG+7e9M+LhS9A2FJOS+YHkpM3HjlAmrsDDb0u3GWGYp", - "tDo8C81m4w2mitUYAv8ajUdtJjQKEWF1RkRN/1QG1hEmJsw1MIm/1OeM1neRod9CJ4ugSkJDoQpF1SrL", - "TaVAHn8CXYdQcMZxxXkStEyicISBk04XQhT4KNQqPAanJUQ33tzhcgfh6QNrIMbKEe+EQsPJ4fY/YT+g", - "wIH2bfe8+NQpRp1h0WPiRj+rnRgkfnk3UhUDalix1Bx5MTo4dgfgJn3t6CPe0IjNx6nHXEjr+w80Y37w", - "zHRB6H127/CYZKcgoJUG9JSaeU/YVcJlI7QYbz0G3nhugfcCH1CSflckPm/KAJvhrhS8kQ40c4n0bmrc", - "ZoZ+Lw3o4dMb+HGv6X15o/ukylY7q2cZV8Bo7qR56F5qeJ4pHrF24p/3aTF0CnMhRacvR6NLLXKi3l/2", - "WemC/LE9Pco+fmWyM89EGyBp7bp19np7/RC3GuBcHiPHinT/7QPUJzta705POBZfj9SVFPbSx4oN4Kaf", - "4Ujjaou9h2ucoVP7bujJOlvZsZwy9l28UdZ9KmaUBnt/A/igB7JXN0rC+g40fRUyUtibHfYH6ONE/bV5", - "Gqs1q/KgPEvJnm5cLYDhgRKVu6fcvTSlpMy4IXV66oj5Kbq/dhbfCGUxv6KgBifDJIatAC0xZPczYJ+2", - "Nmp9FHtm2IwnN+hZV8bmHG1QZUGhqWRvqHvOeI0Y++fgZHHcNBaKVQRdm+Ad8Vsl5wOXVQ2q8EyjEcTH", - "cA2BXn/4ersgSpN42tvtUkIMH70X5IzCkM/1W2Uvnegda9kojdVc9BQExml6m2WdmWrWTaCfme/d4vEf", - "B7KiFgfyXbTCvPXy4+YpeoHxps4Wb4PghaHS0Omlj5Ic8mBSxYXz+bXIweD870xH9qiIoifHo9GP8ePz", - "7PaheoHi26c9iTyaT96hpoyf7ZJEmt/2U8sTB6IzuYvkvh605Nc7Fvt60GKf4rXcAEZ7t+Ne0MbOHgVb", - "9Hi7cPz1fg0a8aevP0OPPFp312m+iVHsMFr9VVHplcgLCo8Zsp/703Q/NffvYBxBySei8WtVfPd8He/s", - "ukVW7ms6vm8z2P7K7PjLwIbjjW83hW/fdLzeWS8s3skyGgG3Q1+82kN03UqQHwd0Q5WSZm/dxjl2QKcv", - "IikY9PZKe/sedCoSGzOd3TekY7N2DYVvHDOp5FEjzmaudALBanxLG3HC8/uRD1Z7P3rakmXJcoQpW1Vs", - "Nhm3g4VIpJR/E+rodR2wPkMjODmDhQ1ttvilz0n8gFCLgIYYAPsxG1cFf8viXaDMOJ3HAz28bZ+dXp5f", - "kPpLjcFiGU4hqkNUHm6riqMMbiFz32FgGql8jXFfGZYsRdbN0JyXBoMmUM3GpOvgnBPWsFdnb6/JzG1C", - "BZqwgPsEVXNyuXNMshBF0/Gp5nP24vsXl3+vdxFPPKzPtr3TrU/p2gTG00p5NFR/xmDQeyOlE48VfB7u", - "LtZ1THyYub/GlSNkVpqm+xFLGpDlfKmyCmGZuKXIPZWlja/NUq0cYH4Crfy+mq5WVugSTdIrpW8w6pAC", - "dFN011deq8rRv1qu6QgapI+tcOj9V5kuIPVcQ2jWTkdrJkVWDtIYJ1yh+2grxNCM0AeyeGiP052bxpbh", - "nL22MEV4O4VkxiM823gUEp0cZoxujRAZomZohU9D9WRHx8ipqVbNC7nIhFmi6WbsnZ8yRQ4cotFDIG5I", - "D3X/JmeNT8F4d+b7H6kELUQmJB1j7NRKMgPSgkwg5LoqU+VbRC/I1lD5N2jbhLoNveOV7haE4NXrKhwV", - "nY3e00XFkdr7bzrzYjF1Uk3hVqRAxhT3ok+VDE11pw5k/s+1OQWDspfu1R+P5rCqrC1IwPgv2Uoe2zi0", - "VD0l9m6Bh0eSgtXJ1xRe45Coi1UkKDh/XOOSgta80xWh9L7Jht6PApi6aZkCsrT1xAZ4NQ1EeLhRm7H9", - "Y0DVvhYy9hGCKKvzgsbtU+W7DdTX7slBW1tRVaEmWYS4Eaar4wtF/qxmElLGRT4wO9X2yu239ctZ9fXQ", - "qphWrviavozlGTTJsZWxkXKsSeelsd1NdTZSJ4PDuYnCTv3WsNmKJ407YfptdHY44i7pwTHBuFl/Pwv2", - "B1hL23Pt2nCLBqMyDhVLwuAF96m/YsG9jFfLv83Qci43eS5x2KFuZnLao1CfQu10bsPTUXgd2tfzONI1", - "qCryNN51bGUPqBOwFleLP4tb9ZG4AtJaYoMv7xdVkAs53WqYp1KXmGO/8gKIVBVXFcZnIja31GX1A10c", - "GKFd+266DeIdbJsJ8qRsVQUfw+LVozKQ+fhXqw8DiLsgKwQx1oJkIDGszSqUBN1D2sKKCZ35I4/ifqkT", - "3fvfW9S0B4FXGPxvbMsL5GUd31igoWI2HkwQi+XAWOZoKS3POM7kXMXMIJcNz+CmhPo9lRKK0rD/7WWZ", - "Zdt+f0uVknYEHTQ20hoZZ2/oyWnbCdvn6jUyXgIG5iaQblFE64/6bVmfQI/1NsSNLW5uKAaV1yq5ec6T", - "mwW20LkAfhNJmNijZBrWqtkWXd2tytf6fuzW6ttms7hbR9RRyc0RlpGTqO4sNC+WoYJWigVmFlRXRzJe", - "WnUUSo/Fq9URLKaFB8a2Sx4BXwwIHXcphqQllvnvQqPVKpFwybVZAuXrzKAn5YQOF7+A/d3tw9QfVOY2", - "53fTFRd2ePNdPMg01HXbBMlfFAu5PT4zvAqXxKEmamqgX+IJyfuUusM6Fs8JGQNymDzku3iO0+2ip/lT", - "D22kkJYFSUFHjbxD30u/yrT1hIJx+vHKnNtaXpF09gGVF+ZllrGcAplzMBj1kfOCHbjnMwVsFQB3BZfp", - "YayaAGaZTg2A3AaAvnp/0VIevd16M/4xl2o2/twUz3wVy04XZl+oI8AJgT/20IvWGwqF5odz3dKA3v10", - "1BP3kOoVJm9fgikzG2tXlpZFnN/s18akuhSxBB24s9OCL2Bq1U0MaSFeFa+Ir0PsFIy0LLzUhkHP4aJg", - "SkUj7SVur4yxJMfkyJbkRHRI2RHTCBnzyOcltEtSR211wSay+yEkyI09lOt9xVD1pm6id1z1736pAX6C", - "N/zuOGbC7/9u30DPnpkG77M3L/OSclePKXMnIgwfvz1+/ff/fuFehASMAdNoPRISX0U7foodhPjfQquF", - "BmOmPjPosKti9LQP97v6Hvce2dT3xyfv3r35wD0RZIZuiZycpyV4UJ07DhYvtGZYwQ3Z+p3g48/OrBaL", - "hS/0vu+iz5Vdbl9tpuwyLGH2nZ4APfxIvnl9ONFBCjw9wpps2KwKo5LRilpm/JDyNdQmQHwFwsEbvY4n", - "rOOfsdgmhaNTTSqBZspB80fd2W2o9OC/i6H2TjfoeLxx3XZc4LgmeAp8H6fga26sX/D+0idOUuPuwyci", - "Evqweb7/0DnELewByE+gRzZOsQGaNuY2UTCuCaFxlB0UdV2XOD1LT7lckAzTJrB79fJvzGxew9x+lKb+", - "G5PuOB7B8sKz+M2TXSz7xPMLsZGBMYRZXGBEJ00b29sFaOMk/ePEvVTXQbzqqCVYnne/MulwVwgN5oNU", - "yp5G+yi9oxnuQybvVQ4KDXMRc+Jjfw76Fd2/Eg2DIgVpxTwUcDoIL4mBRIM97HH33aqbav+RuuPk8nfr", - "UV0I/DxUhMdwgQy7WrlPJszxerSeorUS5V8hsTrs1A/10/h+Wfevmq5V1urJfitghY4wnuZigBcGqVGS", - "ccrD2U86blLZQEo9wRGRdkOe9oTssaBS+36esUzMAWtMCMkwXv4g53fsmz/+4TH7M3vymK2Ba3P4jKlc", - "WAupkx4e+1o7UjFcZz1hx+7whAyvXEgGdwngKo2MMV+qNuFYyoxb9s1jXJXc0HUqtjCm9L3wVGlpNreS", - "9414PAZCUBJlHPcPTFA8bJwKO/zlRad4F5Xt9vFJWN6T1dd1o8JLyIu8pTZFwSOIVcuIlnJ+R93BHeCi", - "zcKbRZyi3YRflTmXtYc74zPIqpWJzEdb6DF2U91vz+pEE4cQCEWmswz0VwY/mbBTmHOnwjGrGBF0002+", - "L4nj+fYi4PSL57gDeOWHs4rxqEe7v14CQ+MSsdQJuwRbaglpqGiUwBi7I2C51GgI0h5siDaxmx0trqiV", - "1h4eg8aIzd9UqXs8vu+ksLtltxA27Nfww6qJo2fwckhD+u5ksZDaerIU2Rb/xSnvt7S9uLNXlltzovKi", - "HJ6jGoZVKtWQ4MIPlp3GoyvsZPg8uzFXtZFmiOhdjdtnxwjSHmqJynEB0s2h4eCbm4gdpwvaCIbGEbRv", - "o57n3MBzntyQHbCTm4t/v7IaeD4YmDTooVEfjuNjVPkC7hmWWoebmz0P3hg5/PRbBP42KMddbEQWjO9+", - "G/ZPKLI6wjvoh/7o+BOV51yme7OUV8CL/W9oGLUPWbkx11jJf/+1aNwPVMJj4DiE1SWVYjkJrqAHIv+P", - "xIsCUmsSbNFBDKYxeHXxtYn1GLS20WlFhxvFA5Ibc6okDGVOOGAfKvLc9LVKbkAPR8i2K7A9+4lW2udQ", - "fsReLLcKmdxjnXrQQ3P3beQ9HhHx7XEQGvCBTDp+g5o5YOEOtfDTRnCEvtpEOm7SeHvnrYNvYmcDydvu", - "V23r7BiDt5DxKfAUd/A8Hq36BnKl1w3nCQZhzda+G2ll078+O6X2wRN28Rf25E//HvVhIGJ298N72OfF", - "jSHQDV6KYshwzD6PxBt+NxDemciFrbuYfyI4N/cTKx9QLzsr53OsmlXwBLskSWqwcIDB9XltXZ2ulL6Z", - "5pAf4gbfl48ff5OwJ3/CbT4Le054gaVLczptaINwsAGgw3ue7G2Z7zoZExbyynQI2oTGgph9QUYKX0zP", - "+7KSdZIBO8Dwc+Oj27GLRWhJhl8V3JjDceXNQn/TV4ZhICblnjjIVegL/SNlmU/dr1O3ralIh4ZafkGC", - "R2CVu+WGjWsXuVJd8uwiNcYJWwFDcUED0tNGCE63Xob3BPuUxU4cGDuo/ki1nKHR/Cr0R+J57VGOW57D", - "NnxcVRR3nW/emHsqQn6eNz6ppW+d4aTiB1QdhvpmfBePfPG/C7nYffzGRx92fiEX2wEg5GJPCAi56D2h", - "FwbO9SU4LRZBdSarhSggoV9qTM8s5LGeYr6+JLaKqIqPJjc+hxH7pQhr2PnZabN9pPutbocQCiwGCn3q", - "i+dpMCq7rfoOtvoA+rWqMAm8BNQ82CcHYInk9zJG7Fgi+ny++/QdPtMARYtKxxu3uE1yHQrdvG3tO9Em", - "gA5ye3c/FMcRau+QZC8TO8GCRhBooVeg62FglFEldNXGz6pnWEmPnCMhbHG2Zge099Mx87PGmVZw1/Is", - "O5+Pnv7PgMBNSkR/Q614Rz//o3tdw1GjV8HvKhJntlnYuWrgFLoXDW9eFDvqJZbT/Mhn7ZB3OF4AQS8d", - "1Nn8HfeZBMYXCw0LbM4TgQq10sAmnEbIRVbf4Ak7rmAmfGqPrx27Enbp2ET1rS8Oq9UKC4QHaalSAZDI", - "IjTEahteK+loyTWYsAz5uLxHlErAV2W9KnaD1YPdT3+u99TMEPaVzZGvrYQBYkQdQxhPlkAl3IS651Ny", - "0ts9b7c4imMv+hpN7V66KK9FDvd+Bf34+29gCMMRgeRqljNhL7Ccw0b5NOoYjE5TLARL3b6qzxy1JSoP", - "zal41ZKcVJ+06joC+qiaklvq+OLuOE4KKTv4l1EydPBmTw57HqkXd5B8EIDP1EcYfn/0vOF3H3yEN/zu", - "IuPyw+YALj98I8Dlh+9EfISNiA/fxwdP8GW+kH0Bcbs5oRt6f0LHPiWp0xxPhWPu6ceYyhu8P2Aqm6Zw", - "26a3jZj1A4NfTeEOEuzRcBgyBbc/41qtDJZZyEWWCQOJkmlPhcaBG23TZO9Gi4zLz7XRa8jRPnBPIgvD", - "749SNF180MWtZrj/JpwS0lNqvUpVa9aZUVkomu7b0PuWNuH2HzQQ6FHag8fD7fnOPTuvkkV+4Fl/nazd", - "6AvD7w+5H3j2shD3X/6SslTuNcGHSfpBhdj05f4uwn7QZfpdxvtdxvtCZbzfxamd4tRHEnZ+F0U+hijy", - "u2BwX8Gg5+WnjK946B7I9BpHxTwNWKHnOE11pHYXVe85u2A8TTUYE9pqU8h6oqQETJqhwi/YPeqOGZXc", - "QFTn2+qv2eWn2eSZg90t/QrwlnDdK8u1hfR4j9BtNPXHY3/7HD0/cGFf3EKsXECQq0OL9AmWZYBbrOoI", - "baNY8KX4DuLuS3QvYm3+hnxP1YL+c/LH2ParvQRiGbwf3977U23KE3Zvmwo3Mykt6Fg1LCTfUpsorEfU", - "2axP7GBE4mPmWBslaaTs2B9wd+dFcvgQ2itiatB5vf0udJuYb13CQK1NAmxdDk9M49bN7tUIMCbY/U8Z", - "0QeOm20wNxPcX2Cjtp4yJJc+x6OnKU51r9pIwz0dNQoewZ0FiRYj39yqYinCN9L17sSnET0TLb7FotDK", - "dwqs6cqwC60m7Jgc6wasr4wHsmp3is1KjeVZBth6k7yYvj4ZWyltl+EDR7ZL0NRE7Q6jsaNGsp0ZC8eN", - "cgUBvA1Ybg3zR9hdq+LJ4+drR64R/a4oe5+1j2WPf9a95b32+Mk2M3lPlsF2eS2iENVH3oyuVMOB8RkN", - "oYRWXcpkt0e7NgTU0TMVRFvwa8AmQGI8qv7QWHMXpf1AcXD7O48/Pu18qUjaIZNGhcDvVVYOiZWKYbwe", - "3lh5D6SeLHmsuHtl2RleV6meDoMMIjVfLveu1bR7zg2V89Mt8IpStj7N5I7tf+zZg674KeduZEN81Omd", - "itl8Hz7m5G296+PN3LmudIe6Z/HXoEtYG7QQI+0OSjex0DrbgIt//3Cgj8/RfwEP9AB+7E4RA/yl7yAh", - "lDxRci4i2vnVWiZLraQqzYnKc9HT4Kb+6oo68r0NdpQdhUB6Bo4jC+84wVWmIm8Gqko9OoDTTzK1+BBj", - "zem2SndXfA4/8Cx0qbnH9O5M/Vm+mbK9VhO3bqVU7cBBWKQx5TgAbhfQ+xS3ovqkd/9tw85mB6usNMvX", - "fHFFTtaB15BGGbktWDxiYYYi4+vXfLGPkFQN2m+HfljPFq9A2t7ftlJDr2nH3aSthp/evBnsh7z3AWlU", - "9AxR60SXVhr2imrnUTJUGZwsuVzANdXfijwaQpOy3iwQQD0pp1aF0spYYJ2KLFs1pZ9H49FM2WW0fMAW", - "s0PngLUGXe8kfpRV6AizxQhy1WlD1Y3X7rTDavo1qal8lTTU+8UPwi7RIhP/qtm2pn+1lyLL5jyxrSqw", - "rbyVLHsubM6L/jku+ALemY79pnX5qPnGFpg4kealsGcyfNv/2QXoC5+IHWHCZX58u/hBpFR6LuL+UNzY", - "k9BYYLj81h4Xlzi5sddLDWapsp79Y6bIK2z43w/OH0SW4WQDiLampRYqO1QUIZk2qDZ3toH5CEFtnLm5", - "9xZZjLsXo43KLonE6GETdbHLScW4Xgtpw2vfW0Vya+Hbbl3kBnrmXGTRlkVNgXWzmj+3jGfZhJ1Zwxpl", - "QbGlwAwSXqKYa5fYH4gbKsED6dhPQh8IS+V7gPe0/JHKir47Zm5EUcR2fuKWMq1+YkyXkjKIhGnUo5RO", - "9BW3IoMFjFkpTVn41gg+pHXMHHhKDYcT9t/uaHMhU+HE+0Ztbeyrhgen8kHY/FkYxiUrpT83o4sXWiTh", - "kbH4IvO7TQWSVp0ah0JxKA0d+q5RI7bUB3ErA1RziaVeYA/zL3gRb9yw4lq2y7P0xdWkdf4HUU89uMJL", - "jYSKjrZT8UsCX6wqdKwr0BUm9zVyCKtedw5oE3YtbAZmzOCuyLjk1N2Gy5TNxR2ap3WZ+D9iadxmDyTf", - "+8DNdnxxxhKutQAseVVoZSCqkmF3suZ73guZ2NvtAbKZaoB/x5148kLqmanSTtiLH0ueGU8+Plbf/afv", - "lIaAMM/I1yOrGG1oNVbTPrWJanARdTn+SVZ+VrdN6tny1HpRv2oTBD+WvmFNmHs0rtvPVLsYjUdJsy2n", - "6DS4b1UjHtAtaIOQ6mZBuA9Ip1qVNkJJLzVWyU8Z/o75EVQvGavn39mqTTedeExKu5KwTUPvbVDUrZxO", - "mVBEPW2gNqDmyWPQBRrUoMYT01cGO4g5shraq+ZTNakhsotwbCLH2A2oGpQQQmrQ4ReB0KLoodUq4u1m", - "KCcZd8wA50K2gpt3Q8YMJosJS5Zc88SCZrdcr4VcHHzzx8OelZq9NbpnC7/tf7563uiyPZ0Yz3xnxT0X", - "w9nibI8bO0WyihyvSuF2T59nz4E/DEyxzvld7/Qgsqo3Y6cunwltJcNyTK0kZU5hQ1L7beBzgRl+/eTb", - "P337H9/88ds/MbgFWVvmqhmENZDNqUHjTCyEtE+bDFPdgp5namUYlvIfeD5MnZr2sf9rKqSpZMqWPJtT", - "RlnAHFru0FrHMRXM94ikRtaEsnHrTzewHnumH5qgUIo8+oPro7gJb2DNsItMD/N30JzOIvWoccJJUt/a", - "Cn5zd/FbwQ4k4xnLZepkJtiy1NYLe13dU8SzAw59P2HHiG0HbZK9qHglXYA7wmXYq5sEm+kBiUr/Ko1l", - "x6+vX1yyqxd/fffi7ckLz/icfMcc8/JZwQCIEO7bEPUeQ8faQOHjjb+2Gi4V5SwTyZTK8E0rUXR7f9EI", - "269rmDCt0GVfFo0Ol1qtJuzC9w+oaR4z6WHuZA8noc9VKUMbzma3QDWn9rwO589abMUHPvnAAK2Upa/i", - "omdNkFGaondwQhKf3wXlUqOUTbdCaRALSUReHaTmkpRtGYcfJWBOa8Nn7B6GYhI+W5PqTtAVcXK+Hxuo", - "qdU0TU19w4vpDawn7KpMlnVfZF/3lIzs3LDLFxevz06O2dnpi7fXZ9d/xyzQ96O1KhnP3Fu7ZkvuPm3t", - "5P3ItyycAVtphRi4FQnEdSesHlzEOA46Urq8mml3K6rlHVDLHNpt28K3U7hb8tKEQpT36rRVyzR9DbVD", - "Y5xp3tPwFa9opQ4YZpW6GciSgx4Xu0sLhE34YuykJ2MZydvE9tm5TilIxhC9+mwPn2trBvdK29SMIpaY", - "QYpuoVVaOmYvG0pq0LOxOm+tw+tSTtgVVneWTvZFPgH1MOro3dVdMVQNKzDnwlh+AzKEOlbK7P7HvroR", - "RfTMtZlj4ER+QGjJF1EBwvEwOT90x5uLDDub+4xX4n0LIJtF2dMV2OpSJqGqbadXhy82vHTvHmX7sIQX", - "VTPq2oDQaqJCDW4bhhT3AK1As5njypPd0XYVQTd18gDFAJPmzset+7Vd60AsDdTZr8N93DDAUERaeFim", - "7r0wVQmmJWQFaHJPshWZNmYGGVMG/BaafacrGJaSnrseoXx7M0w6qKkZSOOGZErdMB7UQGEdA2eGr+Pk", - "QI2fIqCAZClFwjOWc33j24548eLqr6+vro+vXziYvMcORaq070cT9havWDBHeGUOTW6+zzI0agc4qqI7", - "iUYLLx2GxsycOnKQ7oYhg94A4msLeN0O53XIHIeepliq24FvSyPnpj2g1bW2EmOoB39lUwtp4o7yvLGr", - "si3FWycTddcTRtqnae71qlxJYRU6tLXKvAbqqUt482ONTvo1pxvPQ0SuEYmZsCv/2PUcanfl5Vbj3F03", - "q+ZzPdzK3Qp8e8aV9Q8DyusSDdSdqa6I5fjXpu69xQS8zco63GA4wBbWgACVcDZvVd1XYJPDfAfxyPTa", - "/DG4j1tnB529u4WqaaP79QUSequZ+x/2iYg/a4fol6VId9LWGdYurtbattXXwvTEs9xjr6fcLPm29qYD", - "in4hbwqFJqhZQx3C7BQUdUs17mo+6cV7i6ErSaODbtqpJBbKlPQUoAh7MP3bw77IpCOhFR61juaWnzXy", - "O9plL/ZrXvyKm9cquTE9UfvIojhVnmpBK9LWE1lb+CiqBAyisfHov4ySm9ht+kQXBiPlzSUY+IDi+5fV", - "29HtCLLepA9fnv4pez/KuSx59n5EzyUvrXpKpQ99/ys0GuBf4P2oL/3E3CvyfgvR1h3NGhFW7bD7wwl7", - "GzqahHkMI/G5kjmF8VMnSqeQkjBoVyo0wgxBW6mYow5tszVqjPWE+IZXPzODBzUNv1FecI2qihNIZ2u0", - "CeBd2YGy7fynwxjahLSNOd0r66I3XyC6UkWt8XyA8Nch/K/bnUz0tL/AYo0nHLWLzRUpufy+JdfP0rsP", - "m+B+HaUUN/bD1sUpPnD323tU9aLiFG/5LBbYgg1K3oBdqnh1dtSta9v9HpEVzYUbTbcjL4GPYrjv1OhL", - "iUzbDGwYGlD28lOfFZ0ScM+pqbB5ZFZqcmT6hETSF8/nO9qWP1/fb1utGWLR7P037kr8BA2QRH/f0k3d", - "/doJqen+2g63aj+EoadhnBH0lxB3v/aGfdZdFfZtwve2Ueq2CgFt3M/W3DXSmydtQqwJnzaku3ejAY02", - "vdR3s6bcCE/o3psOUe3kS88zxW00glWru5cawBf5Gxj9WI+rQtMGXP3OqAsNtqfPM32JoVX7VOVuDHs9", - "uE9Ee9CwTe0HrarU8T5nqQYNP0lzyJZzVJ/tdwqkp+GbCZ/3biTWiPU1ZlJ0hkbIYQPVPWiMIm0DIR1g", - "RwEZAdvmNegj8XHkou28sidVCENXcK3DNDey7gdU/1FZtiX9n7q39fzW0GQiv78NpaxaO5sj5xkgvfU+", - "Bk6real5cs9pG264SIa0VbodE9t4ZuLvT7xTmH9Naug2Vq7BWi/YBucAYmjGfXQ7BM+FFP1oGVQN3h+j", - "MdfOPfU0gdmxnTNzQd7LOELOzDv0Q/b9+j3PRE8ufH/WQDtiuP3D8GSInRBrHq5xknrbzRX9tnZCuRIW", - "9miNV415cVdoqngfP/92sfHjw6ySxkKfgchG7wOkroT9MS5IqAIwsNtzyOpojBl2nRraaqdbf68Ou11n", - "vezLJcHlttATtqgzx7eLXuS3vtgnjYgG7iP/0IgdO9l3Gxdce4EndmdA2quBgZatr6t52+fc3GTrVFFo", - "dv7Yj8ZrVXz3fB3As9Edg8cP+dLk/S2wehTEN1z03Juz9G4g3Ldpz7vVxH57Uc8vfRop/rIPvXyfDxRc", - "Rx4crTXCVsJmmxoeQpUQgsu0FMTnXrgJYItSQTuvZquY2JcJtSXPrk8s6mZa1/OMR42smDBBdOdkSK/q", - "LUU7HfYnVN7HpVR7jHb6DV5orfQbMMaff+eAgY6PM+83ih7qvLSJ6sOFN2TH19m5PQ/tE4o+7/dktoZv", - "IGuYLLxhs28iswGC9rT18WPUQo2EqnS9DpFTr91W8tg+irIZ3CXMXAmZQN2Xbdgg0PZ7nuy7vdfc2EYX", - "4fv5v3CS0ir+sSa63Tz73vN8/6FziFvYC3FvVIqIi4FzC9vHbe6Hts5VaLW7amx7c0sb1LW5/niT0KPk", - "Fbs/rdJ2HVa7hzjWmmZ3zbrtfGJrEbae5qM/j0cGklILu0axKxgwxXewPi7pndvI6bIiwZyr0IbcSTKj", - "JSZrhp7lT0d/Ozq+ODuiwIxgWMdpMecYuAYdX+C/frimmckpen52esIKrW4FTY+WfNRRcY569qW1xejn", - "nzGvwss9wmIts3cGtNvwaDwKMUxPR08mjyePMei7AMkLMXo6+mbyePLNaDwquF0iHB7xQjzipV0+wobw", - "j3Bb+MsCIqFpr4XxETy+jT7jaAan42AkNkU/UCT5AU4qjNXcKk3ZOYcTdoUd5Ks0IA0+Qkz7jvLPKLyC", - "M7NU2jLqDU8RTz7PjsJxJuwSbtUNpGF5DBrOhbWQslJmbmNCJlmZwlT7LwWWk5uMECoUXegeSDzZcZZd", - "+HNdExx82hdY0Ka3DVD9yaMzWs1vC7sAaTCFkj6w4+vHjykuEQNG8FGt8/sf/csHHtRZW4M8PscOyGHj", - "5JbA7UfK3WAoYIvWS/zeffrt428iAbw8yyiOGNM2JWthlGEkHqY2aKwrCaFRnlvr53EPeT36X5H+TGtl", - "EMuFI/gxjIF0hPSViRNcH4G9k5m48bExkM2PDOhbkQADmRaK8pwo3QbjPsFxG2z66MOMEzz0V8Yt7klr", - "k2Boj8dy3SKZTYpB1uHuW804UACr+Rq9Wf2peptE9G0fyNJPgEg34bd9NFht7NFbZV+qUqY9mC8NAmQn", - "X9FCJqLgmQ8axxwDIxaS8o65ZRlwY5mSCYwpSRKb3ImqI57QLHNfuEFHoo9Eovf/He7xwa4sFgL9ku5o", - "eFU8itoA+gs4aW7pNc0PAtFWyIQ1tgGi8aaPnv7PP9qnGPqEuY/da4LB5u07H3/dvtyH65f4an3og+V+", - "KpSJYPiN4/C858UI0fwx7FNU27zMMmYI0xiSTrhl529PXlQVGgJ8KGvC1HSgBdxSws2CC0kz1g3uMPDb", - "p1rBXQKQth8c/N1NmZLjx+2a3QpYuc25c4Vc2SxTK0iJ3KhDWvSKU8i1b6HW94RKK46SJRdSyMXhhCFv", - "qgiTKvDCXSF01Rvwm8csdX89MAD+JzMVcur+GGGtpNh3n0kPlOcqXX805hGhKVqcyKX95P68QeVPPv1G", - "0hhNVz85Bv84li59yzORBjrqfQgu23RGBGY2KQzTXPKamvDZ8BRV5SO4C8JyCm/pPhT7iXFKVkHQ+/Pc", - "HrnrCxS67i8jqRAl+yghu1f/63UB+sh/hEkWmhq6ypTBfA5096kwC2KSlLEwQFC9CAd2dsBvlcCCCRV/", - "wkSOanIKLM7ca+kYUI/IdNzY/UnY/INIUKVVV+2Fzz089pSp/kD3vmsDQDu0D02mQjewG2+P/tdR2887", - "hKguxAbRr69W+SEU/FGltH7o75Bg97one6NnPCrKCOAvygcG/Md/3LbA/Ey6Qw964yJc7F2Rbn1+MAaB", - "agxQBtUDoDF+y6oyrIMuF339ULeAVvuorGYgLdfn/MRE1zjiA5HZx6AaUyVyDKEan/bxQFTjV9uu58RP", - "Vadl7hQUqkEI15DDfeDEAA2WC6fdLIWxSq8PKwXJL8Asn20+/V2gVVncn/7h96zvqrP0w7/5Pq/rCLvS", - "DKavphPb9Lw/P/oWMv4B8mic+odoS02v+HjlvaT3GNrMXbvPeKd5tsYNy2Xq2Yz6aFNlIscK7vVsaQhg", - "/cPjmI+uB7LzuYGeeWLTPIhhpRUm8SDXIqK1bNyBB1UOgni3nz2pOo7KcyUfhQTeo67peuNsIU4k2I/3", - "s741Yx0coe001vlgiI9HTTsSgj+mPL8B5SXwzC6PTKI0bIPxK/zuCj/7xUB4G402D/RwIK6o2gwEdp2X", - "/muDen2yBwY/pT5vyqUdKza1gWb1CJbxGWRHVXGgVPCFVJgO/pSsPTxZkunvwC3GHrFCqQw0e8SWythD", - "prGOVSgoowzIKu4glOTz9VcMYHEXAxkkFiu3pmwZ6tL4H3EjkLIDuOOJzda+GGkBiYX0cMIu0XJuGFCd", - "9z/PeWagLoqW0wGPsM9gynKVYpFHNZ9H5c0O5ggklbT+6yPN1gEfkkKxVJB5tBSL5VEKPD1CPBxhGbF+", - "cj2TmZDAaHRdM0ksltOc303dRFOcCGsI5DlI0vWeMqsKqmpn2GzNcEVbFhkw/HwXLbwSi+Up8BSbolMS", - "5sORw+5vqxDSe0iiT/7wC5dEt6LpM8oZNY0rLX5S8gj7jgq5OPJus73JnOaZZnwxvROp2aDxMC8zVLx9", - "tmYqS8HY0GV1epcLyQ6wDF7hC6y/+f7kJEx9uPMe0HfP/UmuwkEe8i78luk7Cv4vgcYztTpaKntUFqQn", - "34eNZ2o1XSo7pTl6+HgBmmZnr86v/XLEwwPVc5MA1gzbRcyv1eqVsmQw/J2nf6k0v4GlL4Hcib6PeBV4", - "fqTm8z3JneaYroRdTuuJpmo+3xReSHDBQLTGp22RV0imIVOUlLOL9one68D58/n8d+L/8ogf0XQZsPol", - "UP6dSI9Wmhdcu1FHMUV/CPHfiXRaTzPVwtxEWX1V385qLg3HLiZHZ6eMLzC+pCrjyRduicD52UHBF6gd", - "BqI7WokUdl2Kv4n0h2pPn8Ek8dum9Sj0PyfJe2fVLttJMC3gIGY1SIpv+p+5VvmYWfUPL7WIHIzleYEh", - "NTzLaMQYf0y4hYXSa/obxdpk3IJM1j4AmTODNQN5xg6WqtRHan60Arg5ZA5OeMcopM+iZYSlojATdkke", - "VNO0hTSNPQffPv6WysuvhIHd8r8Hyee+Et7j0x8t8cEeoA+fukOkPC8y6mQBhXurqdWFYQf+ErJvHj9G", - "+Me25AZN/YiYh+zj3drBmjYRwgNex/bzMNSyfNkZ9ZCU2y3tCQ7rdkzxtLp0Ip1HBobyVjlP9atHxfa5", - "hveyihA+OFFSgu/n9Yg1OsGGOXUo5w4pU6U9pAKXMbpqNDf7PCFeWzD1gKS1ArFYWtOOMm1T1Sn+vbHd", - "H/yYDyKoBwJt2OuuSPcB9+nXeOy++KdPcuyPHz21uU3SlofFUH02yO8IQ1/VU3SubqhxfbQrkygwY59N", - "9Cvw3rRO9GkZ5Gbo12a0BX5Zh2V9geaDB1Fi2nB4MI0lSAGPfJXr7agKX1/5j79grfabX3pwVBfan4Mm", - "LLdHmKYl7HogZVhuj8OIL5g8PmXsXM80pQF938BI4zvpfzoBe1+abOL581DmYGZF3/7ygt2GY+JzMIcV", - "F3ZARHG90aoEyq8NF3Vtl4fDQiiT/UiQqHuUtEvO94uz+Hm70PZvVebaAMZDITCYTLwFYUjELilGv05k", - "bcNR5/yfGiHFwhhsXmOONBiwZJvdgp92M59rsuT+xnDUOf+nxpHxJUF3XRlfg/Q3emHw9J8cFXvclN/C", - "LRnWsbVzXR7mzZnfmEeFMkbMMjiSZZZtlRNe3pgL//Fb/Pa3i64NUHwOhPmW9ANRduW//h1pDWA8INrc", - "Oke5MJh3sB1n1+sC3oQvf8v4agHigXC1VHZnhMY51skqCuDawc4wH/PsRmFZrAKdpTdSrbBDvbgVacmz", - "RtNNDFoSht1AYRk3jFMK80LznPIIjFjIse8UTsVAmSpt1TTTLbHkhknlHbtVXstBaYBVJymo74rIgAp5", - "WcOSUlMPQ8sxqKknOEPZc1z2gUIzdkbm3Qi5vQZOaJZsfWVybOkf6Xzca08LZe2H18roMw5S8dY9Z+pp", - "o421qmY8uWFWYd/sSa/PfW32sHt/Ul9pl3gepKpK6CJq+lJ+G23vR6cOrPXdosvsyzSGe+wms6oYtXiD", - "qJsG9MQlpnDH5hm/VaVmak53kSJt3QQT5svhYw/yNQVtUPYaTylASyiWZNwYMMgvLpSxCw1Xf33NbgCK", - "6tLjTthKC+s724OOx+f6BV9hD9vPe4vxXHEqHSEARuPqIod/C9Vzi9uAf3HHExu6+DZratb9Za2i9rIH", - "nKogUm258PMk5RbM4bNW+TpMIHB/t/V3PReQ24fI8P+SnVg7eMIlJnB+gczg2r2myjJ/t4ksGpg/UJpx", - "thC3IB0//socshQyy+ve0C0WUT+6vVziROVFaX38JPGgrwzLxC34mQ/CMx1uNsuFLE1TzOAyWSo9xlon", - "oWv9UhkbAvqxpqK7E6iRMyhUsjxEBlNo5VY0KA8E+UFzC0zJOM1/ZUhkqaSUCbv2VYjxBI6JLZU9oqOY", - "Zq1i7u5dmSz9YS3XC7BVg/E+4eOihuHvkscnljw+FqPGp6ifY39i7tKgmC+Pw5z427xUVoLBSxfkcjWn", - "GpN1xf+KkexId7tEiaHBEZo3sHr0DlIusjVec8cdiL+YwDV89RPMY3jGMBfeyTxHb7GeIkV2kzTCiqx0", - "ikqbD1QazZuT7/+9/qsbdtOUXBrN30sDhxN2zW8ARZkQdvWUhbCCmueF/uraB6NCijpR61HnmaO/tTsQ", - "lpb1PNDxORKXeH32mQZ+k/qCnQyk1Wsqcptg7PyiSvlD/zDVDCiTZRAkrLJY4tNvMyhiPLGk3eEscRmM", - "EqN+qSLYjpv9uyz2uyz2sWUxn5v4AaKYl+Ye8fRWGKX7i81xbYJwogpHqMVi6jgA/g/k7rx1ZeEqhl5J", - "X6AWOYsnb89OxwzurOZJqFZCza6pxEk1qQ+mN2P2LyWkGWOiMTGuhVZlYdhsPQ7yWqHcHmdWA1RiKspW", - "UjG4wxa0C/qhYoieG7EXVEY3NE1jjiDDs1GdT1i2UmWGtVNuYex7EZgl1nafM7iDpMRcAHd5aFdUO8DQ", - "GddMl5IpOXkvzxzyNph7BKbC1Jw5VCLfZPa8ATE6MVUrL6UJdX4Laj/qzshmZCBbLVVGOjVbKX2TKZ66", - "c/AwOx2BMyPkIoOwB7MC7eGAdYZ5je4VZJmHPOVhcYaTirAhw1e1bu/A1jmHe+RPLl8cX79gZ29PX/yt", - "u3kEvGnSlHtaV0uRLB1kVsu1r4Xt8egX4fImCP8NAnTKhNsdmhYRFpP3kqq5mzLD0vCcLaHU+CYjLKSy", - "WJhnCRqrMLvvfIX3IuNSgn4a/mOaLCHxZf0pSbi6DNX2AoCTTDjM5aWxiN1CgyFUgo/Fxk3KqvYOMwVA", - "elQW9DKbcrGgsuBpmtG+6TrwOuuzNO70CknQAe2UmyX3aCHKBcNOT19PmBOZ3Damhbv0Ke6xlNpdTMdv", - "pkTPM5grAkKSlZj4iYTXoAaJtcmRwBHhT5t3CT9OkIrciWfAfDuyFFFG0k0ajMZzoY0dB81tqQwQ6la8", - "LszPq68pWcsDVzLIC9u821jxegZoJwZhl43OGizJwN0PkWXuJlCYyeS9DIWOvn38bV3eCI819ZyTzYHj", - "Q+7YnzCYHB72E/g4PkNjJz4tkRK5rLdHhZsILHS5ZrgCca1SGIQ9sfr3o0CHVgXcvx9tM2sde/b+kHLV", - "hjKf8yMD7mvsgeGjOgM6G6Th8J/BHI35VVl5RxxGmEqUNjwHlkOy5FIYev1QmAiALA2YcV3q1LcH0iqD", - "FgEZvjbVreYztyIuV4sNXxlCtPsAKdrBGe6KTKUweoo3Oy5ewR219yh9zcD9y/KNR8ausfuXk81GvyHh", - "C8nWE+2nkMJa0tRZJSo0n4bqRfQXjOgyF8a9ho3GRA3CncFSyBQlmE0JaxZa7/aHGOKHocPrr6IexC8+", - "mwCRQih5IN9qRS9OjD1S8ohWG0A4bsC5PMbPf8vBqQ5jTVg8MOISjsUbB2DsxH/5+2X/gi57QMoDU81S", - "2CPt23XvIJtXwl5SWstv+o4HMDwwokLigNJHUtkjNCMMQJqPmT/Xb5W9xDG/cfRtAuSBEelFuQG4e+O/", - "/I1jLIDhgfHkQzifDEDUVfj0N46pCg6fB1VfD0fV17+jysPh86Dqm+Go+uZ3VHk4PDCqrCqObo5m66Nd", - "CTweX9eq+O75+tebxTMcYw1IPDDOSlkaSAcg6x19+LsKeN/yADzLyCMRn8kbaP1sM6Uy4HJL/bmlBrNU", - "WfolqqaeVj4LKR+RXb3XQewb8kdGjr1/LrSm0HzFTMJlFcEyJqcxOrrQCptTa2XfxXslZKpW5D9dgQbG", - "k6TMywzdCOjTQz+PH7HF3ew+096Rw9kt6FQktjbkaizxKORiwuLeWZHeTXHj3icbftnml5Xe3yzSDLrO", - "zJyv2cz7JtB91uORlV2PbBUO3QxMYoVI0AHldviv0lgxXzPOTi/PLybsmCBCTjdqTj3DdbC9x1pAlpoA", - "kT+XkpIyWn4q78N8PyKkvh+R59GPGVMoAkGCgrvJb49Gc6R/SBmfWwgdawP4l9ywGYCjBoz2rIKX1kxy", - "rdWKrdymm226HcZUSaj2HwX4wa1IAVFSLaDRdq/kNvcYXSzvaPh8PrLvAApCbohe8MQpKuT0xfL4n0ex", - "kMtUq2JauT9H45F/cYzlGUwDyHBiU87nIhEg7TTllrtviRgGRTRF9o+eNqINt1PmuKK7qHQoU85otjFL", - "uAF3o0AaYcUtZOu+o4YI0j0SRfo2tveW2EsMaii4tsLNDYHccSy1yyFHUsE1SDuuI38dFgoERClFLx4p", - "LPYesai/Vidg62p+Oh/gSZOTBtahJFstAZkgRiV5Vm6Y4XNgViFONz19peGLIXL6O/zud8nvS5KwECUP", - "JWCpRz6BoVeqwrBbEnao77uP+Q39uLipwhswBqZq/pWPKRQ5PJ4SIMWsS+bAqgsNKFzlTysftk9+GDOr", - "eXIzFWpqRY58sLHCLWgjnEAiU4zE8EFAjRjmJ/+BP6hiOltbMBP21gdXCMNS0OIWUoyeeuabz98C9QWj", - "MGQSj+BWqNIwk4kEQgzNTKuV8bKeWaoVfZuK+Rw0vvgzsCsnR9iVYlYkN2ZShdMJ5VYXzXL0KGVI1QhW", - "QjnC+GATswIdFxjOfZT6r6Sm5flVHRL66aNcr300JAUmNqhGZWmQNJ/8kVrR+cTdCoU9yQJOm2jgmWjG", - "pwu4pUbN67YrZ/mUwv4rUqqzBkLkrGnOjYqLlHSvuGW5O9o/MafH/JPNyuQGLLY8c9vGAKP0Xzxxu8ZL", - "eMszLEGOa6KcnKYoaByGeFRKD0q4xqxlJE0Mo2cZyIWtQu7DbExYH9CKIAxvF0r91TcrblDWzsDCUx84", - "6oRrrI7NUVzm2jKUc3L+L6VZWSw0T4ESA2gbnlWECFc+n1NYIm0YhfUidARiQob12iqF08J+Aq0m7Pm6", - "kZ6JoAgRkBoWXKcZGAR8hwv1ciBUH9w0geNg2FzNkxwd+qYYObda3FGOlmjkZKHk/U+MMZ7O1n+el1n2", - "Tzyvx++fn/yzh0F8jjr93X6+iMF5qGkuVOpUQK9IZ0ouAg6+eUK50+74WmCyi88hCL/4ph7csn9a9c8+", - "sfWX1xPgVOROrldVxDlVBsBn6Jm3BxiLRNTIHew7f6CTnpQR5NV3tpE0Uv8ltIfzDbgdoQ1Stt7wO5GX", - "OZNlPgNMoibCxKwdOs2zGMNpnMgqNu9XRWi6+Im+ftwj7vXpXXi5Gnzan58dSIe9bMyoFdKYzcrsRgNP", - "6b8wp2bMhBS2tz9DDcp7aYS4M48E5sb1rdNB1P0X8+n0BxoyHxFsIS9Y/c8Vz3pPW2Uvfp6eATV7u0+f", - "8b2SXK7oRjZSTYQKiXlk9vO8rX7iM7XozwS8Aq6TJfs7Ry3uJFNlldnH3EB2UD8ij6it7iG7FZwy906f", - "s+OLsyjLf+1W/VBuH815pajlQHG7k2sLOoD5MXP7wSPskWL7S2DiVeA1TRCH2i1oKqP8yUKuUbLciwdc", - "BbuST33AXA+eZWxWCzsitE02Tg1lB10D1Jgdvz09bMWf94PBz/KxoNB5PbUqsGeYToMtLey6MqdxufYy", - "iIHarGY2j3U4Zug7WGgo2NHthL2QFq++nyoYd9+P3pePH3+T/Bv+P3g/YkXGE1iS8oDR2b5fddN9QLv8", - "yrCcmxv36EFeZNwC6oOYXOkD9assIJZCWhaUX8bMkhcwFOZ+pk9JeYO6xmyppH2fcQiPj+JeK/gCpoY8", - "07GWao/3sCLhXFbdgPxsr+FrtaBX5RIzxu7TcsOHE7LG6/HR1G181dwba8rCq2NViqZ/+Wjs15tjy8JY", - "DTwP7+XxxRkDrZUf8e2WEY57q9I2TV05d+iUTl951GjcOdcAP8FRzu+OdphM39QT1I0zX+LwN/zu+CHt", - "qA9ijdx93gcyUvZgbrNDSTfdq5RkLPE5yt7Sma0ZZGIhZr7YWj2lU64f+b/4VMCmRPaVYWolnZa6cNqr", - "45FlxseNnsR1A1a2VFI5idIrcmrFwFiRI9c3od+xV5oWU0z2n2jIsCm97/8nboH5f4f8WlrD/9F7UZci", - "Baz3MMvgsLKH1DpZA3ys0MqhAwzTpZSUi0xuXG/L2lrvJUoRv+r+L3vfj6oLzKe/CrvaHzX297AdkH7R", - "8T1tnL3EEAI2WzcdyBvSYy1akvh3uM1tPN3ZSOShuTt1k3p4Xt7s6SvSo5RLh4VhFH1djz1LT2nkbzfS", - "cAdgPgNu/RtdaLXQbt5hWP0eR12EQb8jdBMmD4TLgLhHXg7a2kfBf3vsP/11icHd0z00AhzVYY/pshiC", - "hOfcwHP6+teJh8YBHxoVQWEdgAYPy18pDk5qzf1BEUChaQPAf+Zj2H6NwKezPTTo6UEfAnt6sH6lwPeH", - "eyDoO/1BUOGO5GZ7Lsdf6dPn/ssvVdnb8e1VooqHQqmD2DrA64ER6gZwDQMQeuK//GUjtM+76L2xUz7M", - "M1eWIh3u76tmn432nG173Sgs3qXmjfpRVjHvd2EHlGmAf0ML3nN0ux9+topND3eXPKmeWfjEDNIN+M/d", - "A84kXTM+yyD4/bdEBvQFBGxcXkSZ7zTn/rc0u+7xGvtGXdG3vzlD6QYEPjFz3UgT6+OtD5Ntcx+EnCNc", - "eLYfi/0AtlXVb9UBJp+tvly3TwdtkDJjqgqj3m0RCAKDzP8Pe1e3m0buxV/FivTXv5WA9Gt7sXtFs63U", - "LdrNNq16UVXBzBiwMGM6HkLRKtI+xD7hPsnK59geExgwJGMIyV1DPTO2z4eP7XN+PyjG4gVgbZoMS4PE", - "xxWRxZDlM66YDyALaXcInm0wTFPGJkTwDGhcliBNBSB5Z4AD3scyQQRKzJmS4goKMgySX4u0FbwAC3kK", - "iSkW+RXkyr5+pf9MOOSQ8oz8Rq/oBYy6KukK/uTp/g+OwZptMUzcqAnJCsHx6hdNpFph3udSlfatDHfh", - "fXG6r1atSw6RHcZPFM5ofyrEHKf15fJT7wwUqI8DuoABejcCwUvEEFdrWh5qGHsjsZVnkNiKcJ6pmTI9", - "h4jrqiqMdMyzS9u64vYq5NbprPp+CWta4d4aO1ewH0WLfFaMXPzZITMu0oTmqSJP/tcgl0+J9uI84YWY", - "t8hbQHk1GqEMqK1+3Vqfc4ltFoYzpj86kP4P+TLPAmJoc5EGdTau8+bVkETfFXzEuqYGx1QxNkg3k8Ul", - "/o9LkdJuN6THl/pjFdnQ+pVeKrT5034sKPm5vBpENgYv/T+X7qqwO1VM/6tbmU1rl+OqCX79qrFPR2/s", - "NrKnt1u30794er2WmdbWDi05FpjrCQXibVvZmt52m1lf+HZca/et91gLCnAqZDJSIWrQgYYRdKHelL5k", - "FLUmbnuxBAlDHfbOKg5htFVMroodDka2FFdjTUxc7SkPMgRrCyUdMw0l2gN4XEmTXPaYIuZcnCimoI7q", - "aSWkAe5ThXEPS1uoMmt4WSue3x3Zvun/Wc70jnyl7O0QE9tmj0Z+uvl0y9m6O9yqzS3aTwUcIi2PqZCT", - "58+avflGqnuzX/mk27+ZHyrd/UEHGP7cRY4bnZhnVIRL+QsVj0LeTchfoHt7kHEypEFnvNDPM2j8KOJw", - "yeKM1SRJi1CmO4hL0TpBfixbn2Hjo8C9WB5WhNlWYkPs7vXqQkQN4aO4rhvDi+W6FkSwMabyOxn5yjC6", - "EEwsF0cM2IGm4Gvgjjyy1rkq2JgktKBCDrB4/t+//zFkX8BlltKCwm8+rmPK+iyx9GL5NANazQkdlBU7", - "aihngHvCvk8RKhBZn5AZzVBOAdumatjKHgc5SB1444jNLfaIkAOeUEE8RYMyI0QrlFmD0IxMMyEHA5a6", - "Mn+SyZ5M5yRnYzbusVzhrgMurCjB2SLnn9903p8BUiTuSYilruVZi7QJcMzd5FLT4/AhWHhGumrEJxOW", - "dhHXhAPiC1VyEeSxx3g2cNRbDdJjCZ0qvQP0vpPy8iuOsA4QKwHQBQnUkEGvRd5xgA8BEkIHmmdI+aQQ", - "LCXTiUUcyaVltGyRtpjRuXKYkPrxXMtLFQ3LJMhyB+xCSUKT4eqyqQuYyA7P7sveuwSL6LvJ65sbWFPZ", - "bkn/xsiJWHAhED3HMiOaJ6tOvQW7YmIlVCRWeDZOZjQ3N0aZLHjCtoR/XOw5UwwVSFVDeOxYNB3UCZ4Z", - "c9oZ9rGi246TfCcskL5nG9aq77Z/AVgh25eAbVWjfUhwkqUnqIvXN+ABDPJdgfQC3kmRT5NimlPhVjGD", - "NWbrQv+vjCKfrFpVTzfV5OIEuJdDghzPPMbOBvhy+MmBJhsmcZPLYSGVXTGvYgKRvxyQXiqZIdM0TLF6", - "0UCUMguXXCItCClHCtAfWuQLFSPrwMozMo9LuH/jNd6yjyybUwt/BA7OeMmZNjhwnnq9LzgV2oT0Ymjz", - "WMpqXfstrAHWsUOzp8XL0l98YtNl7lJkH1VkTPORXpUo12ubv7b6gGhDCoDPmXQ+YMPSFb3MN87xvBuf", - "XQ5dAW/d12O3s9RzljedMhgRWvXryxxslru5dKbKCh3dBVV1XZi2x1nVZUf3uyx437wr2jbEScEDMzC/", - "rZOIV2VuWx9ZkcXADCy6JCaDIAm4Dsa92nxoOMjR9QABQkJoMz9By0fWzMNSGJBKZNZMozP44h7brDS/", - "2pb3SWvc/rI6VSYUfc9wJoS/qM6dGIjESeSuq1NevNz8wB+wM+648rNdIsIqZWxuZPpeVMn7R/j9IPTS", - "kH4fm3KWp7DhGnpePvOoprd60Zo44MVRxAFLKlN7MeLeLSqXs6aFlwu3qY9y9tY+9GhU+/f9vjziBK1N", - "e/pgS6rCFAdBEA64sOqBKY4vj3oVZ8iLZh7kZCw1/sPF8IJ5sNMQd1O6TYR16JFVPFlFChiWpTXQ/hdA", - "XZv0inIBzmXtMaR7oO3aH6XwJrkefMHx6bY/Oct43543/uq1LfMGzGV0vdI0KrXZ8C5Mw+O2uk3ZE7WK", - "AoDRAySB7Y4/lFnJRoJ1xE/ed95/eBuKZPv9jjM6fqp1y7hPHQxmzkdVPHTi/F2SdfZ6GhCRf/+6caJY", - "MgXiFy2vN4zmLG9Pi+HJz1+/6alrT/gHNne/fLv+LwAA///cJDrEu0sCAA==", + "d1r5YkFtLiq9k9bB0PSzR+fE3qhbATUncLfZoYZ8cn17WfHs45yX/cUtHcqpihwqdcpt4ofj19Qp4cAL", + "HV+tePbVmF0oYxcarv76mj35D3Itwgr0IWuUunGCyZx1QDf4TB8Lxu4Ee8PYl+HftQd9iw1GDCsLSpDY", + "3MMzb6ohe59hpuBShsrxfphvXhJUmkm8akKPptol/X5ijZPOduBvgiLOmvrCkIYovZxUWen12jlkmQOY", + "IuY0K5MbsM9YqvmKcbbgxZjKv7OfQKs4+kLVp6mpwzw65SWBG/fGeLtfZc4k+22Nrqo6PUiFMTU+S4XL", + "NXPS6oT9N2hF55CqGhjuGG2eSvo3sDtAKNpPLx+uUvt4M/NxNcEtivoGLlrGFb+ZOFFdxuwozViqXYEY", + "m4VR+03rHwUsHQnWM9Ix1f+uub2QLBdZJjxIvO2nfW2d9KbkhB2TBFcxZawD7zRQ6VuVeDZEnQwM44a9", + "fff69WRnpckWIBu23wC/Hci5qgShNn5uYD1UKPoO7hNMG5jNLonKbaSafNsZvoN1nMGnIqf+T17v9lkH", + "NwCFoY4dKtSS5SbwDnxkHJJI9N6sJf1pCPjn+AEbeutH0CC3inGqqO09HV6PZRDd6r4Wn/gJ6s4bs3VL", + "mvjjoyd/CiXSnTj65D+YkmN3RTzhd+jdrVnfjlRoSGy2Huhe8krNQLq7bGkvtXGgbRSNVMmuK1DiE/eM", + "ceaL7YTKi+3uO0Le8kygUtMSH+rPl5DfXxrfKS1jWiv2+2pCPFFZhkYo2ZRcQzekayc6+lzDDv3fU4b9", + "EgTTIP1Ict90zCYIl2njGydxJw27RTDIp5BgmeAldj+BNjfHPnm7baJN70RLQ27usV8e7Ep9SPtRtihT", + "uDtOb4VR+oTLVPSFnWRlHivS/h2smf8Ru/vpFPRTBj+WPKMC/JBiSzpDdhq0WmBV/1xhUX8gA2T9of8C", + "J8IMaD+5Fx47X9f9gTIH2DWbaeA39GT6GfYzfqDaAOnUNyAazi+acKQ5KJA3lg+YZtFqs763lykXC7LC", + "oVQMWKicGiVR0xhG42onEnmJg23ITQjp0+bl0EAG5ZPLF8fXLxj2p2In529P3l1evnh7/frv1CepMZzm", + "RJ2KkrJxRSrl5dPOFfr4U7gjG/P5W3b+9vXf6RIQS8MbuVQZxq5KRRVnvZAtMb87xa6GNIuS3mhVbYMl", + "StZ2bzcZt5YnSzC+mZuEkAtuy8A8GvuiTVSl3zNuLL3kjTquC4VUyJ2O4S7x2MlUhfkxYwactOaY79Mt", + "gPP95nQpgzrYbvuFLcQm7JSbJfetGuEOktKB8fT0dbybScalBE2hOjGF9KXn1lqVC8zsJ4nFQjFhx6H8", + "PSYJJFxrioBI+C1w+9QbepsRowzH6jKxpebZmLl7c9sqYNAoe+sfgIRbnqkFNavr8XeESxqx9ipJUrdj", + "zTyYNB0siQGNq/v8Y6lCBj/9wSr8j3FV1MDt5odXLy5fsBtYr5ROxwzywq69q7nIuPAUNmHHkp1docTc", + "YCEZd3hGiaPZb03IwHuekjcWDc+VrdW9u9O55gneEbw+Rs3tUQrY+4yGutcA7rCri2E8Q54nlaXSAjKt", + "wjmEZJI6mCI9Z0ATavdgzDO+oLIGN4D5IjlHlRmvDJETpcVWfS47KOu8OM1M7d7AFhsieTvyTU84ilNd", + "XhK4uyyESFNUHQPDFfUXFy9EfeFbW/dNNB3DaPAy9zDxRUNachtJ1SreGIgCboLo1z6NhsxiCuS4Jc6j", + "OSfsyAQy6p5rMqyy8oprKeQiWlmam0rZqCg/VK/gbMZTJlLgLAVTCAtVvMsKyDi797v0A+0lavPHqNNp", + "EWv9Uvsmsd8qz9Y/QeqZGOLMiT6mcAgTskt8ndNRn9YJO0NXBvXj+AmaLc0yyMfs7fm1gzleUXf9F+4O", + "OzqQinnmiH2CsO2IdpirSYEaX87w2Uyjtuz+cg5aWNgPpDRis5qGj7GyPsI9yE9NrkiCQAv2m/JHi4Kr", + "HTboavO12CnmNcWTiKro6zINIO65kAvQhfbWwI5vi5va0lYLN/6dcaIF8QcfupUolHj9E0OL27akXt9q", + "dA71udOqIDGqXeufxrABaqrF6XXD17IiTupx3eScpSQLclWijOrn0IMpsoytlLZLEovc1QwGRKJE6jiJ", + "Ybmejc38e+WjmfAdUbah7iKh44NEbbF9A2O3nmOeRu0nzf5YRiOx3tbhkTVgLNzZMTNcCut+icKdOhmJ", + "dDpbV5Gpg0shbKrKoeNtB0W+WRQKhE7MQ4+V6KBu7J57MihQb+awN+OlTuP7hKOyDQXX+Najg0uF7tH4", + "XOIjq7IUyIXAg/yEblkJgDmYVXNethC3IGlPK0cCFFmarVGoWkjHzhyqHd+GgmUC42q9JF+APqoqT1ED", + "WLJs4BAU2PCAwjtxkS5CG141b3b4xXPRcWZrViiDSIiGGFeQibQKIYF7s72BVisPlIZ1vpTConxMSMU2", + "2BgUoG/RbPHHb90/E2GoezD7L37Lr3C1CXtDja05dYCnAh5VTHH3eiZOeUsZpmjxzNseqED7fA5OH8Ai", + "xsa3j220nEek3cCaGnJiO0YHnoxjr7iNp3O3M7z1MO4b71JDPnJ32vxzXLXhbr8HyI4Dx9vF2t8qe8G1", + "IZVhcBQuFWGcJr7/WFdUd7IrUmOzDfJK6ZtM+ebQVpeSmNxBkgksfjZbe5NLVTvbd0ETP8HhmFmlpk50", + "HDvBysAUC8WMWSl9aXJIp2YtLb8bO/Iv53ORuDs5LbS4FRksIAj5eNWk11VIOEFzun3KSnkj1UpOMSkR", + "+7WYtbGQV39gB9z/qVJnUNbLlRRW4UVzBOQ9UJI12iiTDPis/bFVKqO+d6xQGZZoa+gWWP/NeG/WghdB", + "WMJNG2EOx4znM7EoVWkwQpwdcMlKiaaUuYAUxeHqTnhpJPQ9tuoZ0zBHmaepxSxKMAY5m3A8J3bT8bah", + "nY9Ku08Lbpet3ZD0Mq4gWv+7RpcjqzGBZeqwi2pKpcdNKbA9xTNhmAFiCZ/JoOohG+loL9WL6mu8MV9f", + "/NDd/GnDaFS35Wn+VSpaGHdV/X3CLrR76TNxC1VIHzH93YXSmpclxGnvupZ9QWuV6BHhzO4xVQbZL6YE", + "jB0IbgUGdgSBvCODb0ZCDbZYVZa/mLkq+PTyoXKhVHZaVHyoa22uHfItRlJolZYJxjCjgjz2dlPqXEVg", + "n7BzCQykdY8W6JpOKkMVjkH3TLCeyaqgVEst4RuB0xU2ujYUQ4Zcd3nhzoZ5a7nRvcJP8RlCluT+SUYn", + "zjKuF35TxAXqhy/ce3KBo2GPOA0lzaDRzWHKh50ulEod5gu4F4brlyHm06jr4w+dL5TUR3e0jfVjqmjK", + "hFbYBV8IyTtiSoNsSqmdvIeXdR8Jn+T2CGtrxdShGYRdY9AcEpw3xkmL7V2aKKmpqYVjwkTV3n62bphe", + "uKOVZOmVVMty0JCtQ5PvXop62lE3MFgkqBzI/lEeRGnTX5AWqXr9duzz/cmyBJmp4n40mEJJp9bsrz9s", + "ROpV3Kp1xWv6iaEw0Eebj+zimL0dG4L9oWmg7zGkh7QSmZKQkcapzj/900qIibv1goRQKSGIJSGRmQXF", + "oWXFSUuv+9UmbqQTzI8D8mrynhC0RGUZLwyk04VWZRE55jsp6CY4BsoymFvfkmzu7e0NwpqtWUPU7AGD", + "Nzy4l2SHHajPBtS6MU45KJPlhF017+Mt3bWVSAJXRF2MXnmyE7qrgVfJ33K8q8QDTaIKICZZfVWZewy/", + "7XbJ3pL7vy93qZ4p3KZX2hwm6yq1+ykWFN/qjdjUHHhrfHCohI42BLLqSzKFoxVlk/shr/M9r8m2XCNO", + "Ks0aV/XpRknkssB7s1pyi5Qq0MNh2qlZ7kEL/heyV5O1AyW5oCi4HZpg7FMFKqpIRSLNYD+Q1TxnukWb", + "KRbGtOvE9caiDXkyiBGTkaFDGuHsWEK61kxr4ZV0eI6wCHo9WbfKml0PCADsHGm8yQUjHCMCr8gtj/G/", + "cD/iRLqLdQcrc0QDjemXaNGcOtl2XQn7FZ1Wt47sng1n55xrlivkRxa9oFxWRLphgm6bhlHPqJ54+sXx", + "68RhOK/MFMIiSzETlqnV1FftDjsMnKzpGXM6Hsu5dkJOVgvpgYdNah/CFH2U7Rzfrve248r0bsYZeIZD", + "bUYbjsjqQKenr9lCIbDQPVg5ZjEsjTd8quSBJb+qZMfX18cnr1CsrjZCckXXK1s4USLkZOD2nrlfeW4m", + "Db+Jl8QxeIVSGUM0f2XJp+sXjE2aGZVXIPMW83AsocmhX3ES/EdbV1xpRdW983Cm4Oybi8yxcoflJb8F", + "NsP+p+7NVGV1q72PcCVSIK2xTr9oYNncAJqniVJCdAJSjPv0Bta0jMrSCcu5RRNUmKiyPCDLyLkFLbwh", + "1n33tPJXXr54efni6hXT4Cnfcw4N3mtOOKkc58IGx2EY2XJSe2biFi2l+LGEpv+QVvQOS8mr4GkUJXEZ", + "rVZewgmeWqUbGnflgwMyb1ClVibshBnqetCAZ7/+T+XuaEW/m0ALtV9YOG6s8DUjElCazWApZMrorx4j", + "zzAQJ+yILJRtVI6pXD2el+yo7sar6tFyc3Hpg3mqWCzOJKzCxh2ReRQ5JqQkOJTLYO3opvGHgwb7tr8z", + "4eNK0TcUkpD7OvilzMSNUyaswnp/S7cbY5mlaPLwLDR72DeYKhb5CPxrNB61mdAoBLTViTY1/VN1YUeY", + "mIfZwCT+Up8zWjZIhjYeneSUKrcRhSoUVavkSZUCefwJdB1CwRnHFedJ0DKJwhHGfTpdCFHgg2ir8Bic", + "lhDdeHOHyx2Epw8srRmrcr0TCg0nh9v/hP2AAgfat93z4jPyGAW7ocfEjX5WOzFI/PJupCqE1bBiqTny", + "YnRw7I4fxsd759Nf+U+7Nh+nHnMhrW9r0Yz5wTPTBaH32b3DY5KdgoBWGtBT6hE/YVcJl43IaLz1GHjj", + "uQXeC3xASfpdkfi8KQNsRutS8EY60Mwl0rupcZsZ+r00oIdPb+DHvab3VbPuk4Fd7ayeZVwBo7mT5qF7", + "qeF5pnjE2ol/3qdz1SnMhRSddi+N5sfIiXp/2WelC/LH9rS++/gF7848E22ApLXr1tnr7fVD3GqAc3mM", + "HCvSVLoPUJ/saL07PeFY0z9SrlTYSx8rNoCbfoYjjast9h6ucYZOScWhJ+tsZcdyyth38f5r9ynEUhps", + "KQ/ggx7IXt2oNOwbG/UVXklhb3bYn1+AE/WXfGqs1iz2hPIs5RC7cbUAhgdKVO6ecvfSlJKSAYeUf6oD", + "/qfo/tpZ0yVUW/2KghqcDJMYtgK0xJDdz4B92tqo9UH4mWEzntygZ10Zm3O0QZUFhaaSvaFuZeQ1YmzL", + "hJPFcdNYKFZodm2Cd8RvlZwPXFalzcIzjUYQH8M1BHr9Qe3tOjtN4mlvt0sJMXz0XpAzCkM+12+VvXSi", + "d6wTqDRWc9FTZxqn6e3BdmaqWTeBfma+d4vHfxzIilocyDdnC/PWy4+bp+gFxpu6CEEbBC8MVRxPL32U", + "5JAHkwp5nM+vRQ4G539nOrJHRRQ9KSqNNp8fn2e3D9ULFN+V70nk0XzyDjVl/GyXJNL8tp9anjgQncld", + "JPf1oCW/3rHY14MW+xSv5QYw2rsd94I2dvYo2KLH24Xjr/fr+4k/ff0ZWi/SurtO802MYofR6q+KSq9E", + "XlB4zJD93J+m+6m5fwfjCEo+EY1fq+K75+t4w+AtsnJfL/t9ewz3F/zHXwb2sW98uyl8+1729c56YfFO", + "ltEIuB364tUeoutWgvw4oBuqlDRbNjfOsQM6fRFJwaC3V9rb96BTkdiY6ey+IR2bJZEofOOYSSWPGnE2", + "c6UTCFbjW9qIE57fj3yw2vvR05YsS5YjTNmqYrPJuB0sRCKl/JtQnrHrgPUZGsHJGSxsaLPFL31O4geE", + "WgQ0xADYj9m4KvhbFu8CZcbpPB7o4W377PTy/ILUX+o3F8twClEdovJwW1UcZXALmfsOA9NI5WuM+8qw", + "ZCmybobmvDQYNIFqNuaMB+ecsIa9Ont7TWZuEwr9hAXcJ6iak8udY5KFKJqOTzWfsxffv7j8e72LeOJh", + "fbbtDZR9StcmMJ5WyqOhkjsGg94bKZ14rODzcHexLsPiw8z9Na4cIbPSNN2PWJGBLOdLlVUIy8QtRe6p", + "LG18bZZq5QDzE2jl99V0tbJCl2iSXil9g1GHFKCboru+8lpVjv7Vck1H0CB9bIVD77/KdAGp5xpCs3Y6", + "WjMpsnKQxjjhCt1HWyGGZoQ+kMVDe5zu3DS2DOfstYUpwtspJDMe4dnGo5Do5DBjdGuEyBA1Qyt8Gopy", + "OzpGTk2ldl7IRSbMEk03Y+/8lCly4BCNHgJxQ3qo+zc5a3wKxrsz31ZLJWghMiHpGGOnVpIZkBZkAiHX", + "VZkq3yJ6QbaGyr9B2yYcVUE5jle6WxCCV6+rcFR0NnpPF9WDau+/6cyLxdRJNYVbkQIZU9yLPlUy9Gqe", + "OpD5P9fmFAzKXrpXfzyaw6qytiAB479kK3ls49BS9VRuvAUeHkkKVidfU3iNQ6IuFsGg4PxxjUsKWvNO", + "V4TS+yYbej8KYOqmZQrI0tYTG+DVNBDh4UZtxvaPAcUgW8jYRwiirM4LGrdP8fg2UF+7JwdtbUVV3Jxk", + "EeJGmK6OLxT5s5pJSBkX+cDsVNsrt9/WL2fVLkarYlq54mv6MpZn0CTHVsZGyrHUoZfGdvdq2kidDA7n", + "Jgo7ZYHDZiueNO6E6bfR2eGIu6QHxwTjZv39LNgfYC1tz7Vrwy0ajMo4VOsJgxfcp/6KBfcyXi3/NkPL", + "udzkucRhh7qZyWmPQn0KtdO5DU9H4XVoX8/jSNegKijUeNfdjtyG3VotrhZ/FrfqI3EFpLXEBl/eL6og", + "F3K61TBPFVQxx37lBRCpKq4qjM9EbG6py+oHujgwQrv23XTCfhG2zQR5UraqOqJh8epRGch8/KvVhwHE", + "XZAVghhrQTKQGNZmFUqC7iFtYcX4chGxR3G/1Inu/e+tlduDwCsM/je25QXyso7vV9FQMRsPJojFcmAs", + "c7QSmGccZ3KuYmaQy4ZncFNC/Z4KDEVp2P/2ssyybb+/pfpJO4IOGhtpjYyzN/TktO2E7XP1GhkvAQNz", + "E0i3KKL1R/22rE+gx3ob4sYWNzcUg8prldw858nNAjszXQC/iSRM7FHxDWvVbIuu7hYVbH0/dmv1bbNZ", + "m64j6qjk5gir4ElUdxaaF8tQVyvFAjMLqqsjGS+tOgqV0+LF9ggW08IDY9slj4AvBoSOuxRD0hLL/Heh", + "f2+VSLjk2iyB8nVm0JNyQoeLX0DMhY1SWZj6g6on5/xuuuLCDu/pjAeZhrJ0sdJtLOT2+MzwKlwSh5qo", + "qYF+iSck71OpD+tYPCdkDMhh8pDv4jlOt4uenmI9tJFCWhYkBR018g5zbpMlpFWmrScUjNOPFxbd1kmN", + "pLMPqLwwL7OM5RTInIPBqI+cF+zAPZ8pYAcKuCu4TA9j1QQwy3RqAOQ2APSVK4yW8uhtAp3xj7lUs5/s", + "pnjmi3B2mnv7Qh0BTgj8sYdetN5Q6F8wnOuWBvTup6OeuIdUrzB5+xJMmdlYF7y0LOL8Zr/uONWliCXo", + "wJ2dFnwBU6tuYkgL8ap4RXwZZadgpGXhpTYMeg4XBVMqGmkvcXtljCU5Jke2JCeiQ8qOmEbImEc+L6Fd", + "hTtqqws2kd0PIUFu7KFc7yuGqjd1b8bjqi38Sw3wE7zhd8cxE37/d/sGevbMNHifvXmZl5S7ekyZOxFh", + "+Pjt8eu///cL9yIkYAyYRkebkPgq2vFT7CDE/xZaLTQYM/WZQYddFaOnK73f1fe498imvj8+effuzQfu", + "iSAzdEvk5DwtwYPq3HGweKE1wwpuyNbvBB9/dma1WCx8bft9F32u7HL7ajNll2EJs+/0BOjhRyLQVSc6", + "SIGnR1iTDXugYVQyWlHLjB9SvobaBIivQDh4o9fxhHX8MxbbpHB0qkkl0Ew5aP6oO7sNlR78dzHU3ukG", + "HY83rtuOCxzXBE+B7+MUfM2N9QveX/rESWrcffhEREIfNs/3HzqHuIU9APkJ9MjGKTZA08bcJgrGNSE0", + "jrKDoq7rEqdn6SmXC5Jh2gR2j4OOR42ZzWuY2/s8cxvw2Zh0x/EIlheexW+e7GLZJ55fiI0MjCHM4gIj", + "Omna2N4uQBsn6R8n7qW6DuJVRy3B8rz7VXmHu0JoMB+kUraiXDvSO5rhPmTyXuWg0DAXMSc+thehX9H9", + "K9EwKFKQVsxDAaeD8JIYSDTYwx533626qfYfKZtOLn+3HtWFwM9DQXsMF8iwWZr7ZMIcr0frKVorUf4V", + "EqvDTv1QP41vw3b/ou9aZa1W/7cCVugI42kuBnhhkBolGac8nP2k4yaVDaTUExwR6WLlaU/IHgvqeUH6", + "LcvEHLDGhJAM4+UPcn7HvvnjHx6zP7Mnj9kauDaHz5jKhbWQOunhsa+1IxXDddYTduwOT8jwyoVkcJcA", + "rtLIGPOlahOOpcy4Zd88xlXJDV2nYgtjSt9iUZWWZnMred+Ix2MgBCVRxnH/wATFw8apsHFkXnSKd1HZ", + "bh+fhOU9WX1dNyq8hLzIW+oGFTyCWLWMaCnnd9R03gEu2oO+WcQp2qT6VZlzWXu4Mz6DrFqZyHy0hR5j", + "N9X99qxONHEIgVBkOstAf2Xwkwk7hTl3KhyzihFBN93k+5I4nm8vAk6/eI47gFd+OKsYj3q0++slMDQu", + "EUudsEuwpZaQhopGCYyxZwKWS42GIO3BhmgTu9nR4oq6h+3hMWiM2PxNlbrH4/tOCrtbdgthw34NP6ya", + "OHoGL4c0pO9OFguprSdLkW3xX5zyfkvbizt7Zbk1JyovyuE5qmFYpVINCS78YNlpPLrCBpnPsxtzVRtp", + "hoje1bh9dowg7aGWqBwXIN0cGg6+uYnYcbqgjWBoHEH7Nup5zg0858kN2QE7ubn49yurgeeDgUmDHhr1", + "4Tg+RpUv4J5hqXW4udnz4I2Rw0+/ReBvg3LcxUZkwfjut2H/hCKrI7yDfuiPjj9Rec5lujdLeQW82P+G", + "hlH7kJUbc42V/Pdfi8b9QCU8Bo5DWF1SKZaT4Ap6IPL/SLwoILUmwRYdxGAag1cXX5tYj0FrG51WdLhR", + "PCC5MadKwlDmhAP2oSLPTV+r5Ab0cIRsuwLbs59opX0O5UfsxXKrkMk91qkHPTR330be4xER3x4HoQEf", + "yKTjN6iZAxbuUAs/bQRH6KtNpOMmjbd33jr4JnY2kLztftW2zo4xeAsZnwJPcQfP49GqbyBXet1wnmAQ", + "1mztm6lWNv3rs1PqSj1hF39hT/7071EfBiJmdzu/h31e3BgC3eClKIYMx+zzSLzhdwPhnYlc2Lo5/ieC", + "c3M/sfIB9bKzcj7HqlkFT7BLkvRN+zC4Pq+tq9OV0jfTHPJD3OD78vHjbxL25E+4zWdhzwkvsHRpTqcN", + "bRAONgB0eM+TvS3zXSdjwkJemQ5Bm9AXEbMvyEjhi+l5X1ayTjJgBxh+bnx0O3axCC3J8KuCG3M4rrxZ", + "6G/6yjAMxKTcEwe5Cn2h/aUs86n7deq2NRXp0FDLL0jwCKxyt9ywce0iV6pLnl2kxjhhK2AoLmhAetoI", + "wenWy/CeYJ+y2IkDYwfVH6mWMzSaX4X+SDyvPcpxy3PYho+riuKu880bc09FyM/zxie19K0znFT8gKrD", + "UN+M7+KRL/53IRe7j9/46MPOL+RiOwCEXOwJASEXvSf0wsC5vgSnxSKozmS1EAUk9EuN6ZmFPNZTzNeX", + "xFYRVfHR5MbnMGK/FGENOz87bbaPdL/V7RBCgcVAoU998TwNRmW3Vd/BVh9Av1YVJoGXgHof++QALJH8", + "XsaIHUtEn893n77DZxqgaFHpeOMWt0muQ6Gbt619J9oE0EFu7+6H4jhC7R2S7GViJ1jQCAIt9Ap0PQyM", + "MqqErtr4WfUMK+mRcySELc7W7ID2fjpmftY40wruWp5l5/PR0/8ZELhJiehvqJPw6Od/dK9rOGr0Kvhd", + "ReLMNgs7Vw2cQvei4c2LYke9xHKaH/msHfIOxwsg6KWDOpu/4z6TwPhioWGBzXkiUKFWGtiE0wi5yOob", + "PGHHFcyET+3xtWNXwi4dm6i+9cVhtVphgfAgLVUqABJZhIZYbcNrJR0tuQYTliEfl/eIUgn4qqxXxW6w", + "erD76c/1npoZwr6yOfK1lTBAjKhjCOPJEqiEm1D3fEpOervn7RZHcexFX6Op3UsX5bXI4d6voB9//w0M", + "YTgikFzNcibsBZZz2CifRh2D0WmKhWCp21f1maO2ROWhORWvOqqT6pNWXUdAH1VTcksdX9wdx0khZQf/", + "MkqGvt7syWHPI/XiDpIPAvCZ+gjD74+eN/zug4/wht9dZFx+2BzA5YdvBLj88J2Ij7AR8eH7+OAJvswX", + "si8gbjcndEPvT+jYpyR1muOpcMw9/RhTeYP3B0xl0xRu2/S2EbN+YPCrKdxBgj0aDkOm4PZnXKuVwTIL", + "ucgyYSBRMu2p0Dhwo22a7N1okXH5uTZ6DTnaB+5JZGH4/VGKposPurjVDPffhFNCekqtV6lqzTozKgtF", + "030bet/SJtz+gwYCPUp78Hi4Pd+5Z+dVssgPPOuvk7UbfWH4/SH3A89eFuL+y19Slsq9JvgwST+oEJu+", + "3N9F2A+6TL/LeL/LeF+ojPe7OLVTnPpIws7vosjHEEV+FwzuKxj0vPyU8RUP3QOZXuOomKcBK/Qcp6mO", + "1O6i6j1nF4ynqQZjQlttCllPlJSASTNU+AW7R90xo5IbiOp8W/01u/w0mzxzsLulXwHeEq57Zbm2kB7v", + "EbqNpv547G+fo+cHLuyLW4iVCwhydWiRPsGyDHCLVR2hbRQLvhTfQdx9ie5FrM3fkO+pWtB/Tv4Y2361", + "l0Asg/fj23t/qk15wu5tU+FmJqUFHauGheRbahOF9Yg6m/WJHYxIfMwca6MkjZQd+wPu7rxIDh9Ce0VM", + "DTqvt9+FbhPzrUsYqLVJgK3L4Ylp3LrZvRoBxgS7/ykj+sBxsw3mZoL7C2zU1lOG5NLnePQ0xanuVRtp", + "uKejRsEjuLMg0WLkm1tVLEX4Rrrenfg0omeixbdYFFr5ToE1XRl2odWEHZNj3YD1lfFAVu1OsVmpsTzL", + "AFtvkhfT1ydjK6XtMnzgyHYJmpqo3WE0dtRItjNj4bhRriCAtwHLrWH+CLtrVTx5/HztyDWi3xVl77P2", + "sezxz7q3vNceP9lmJu/JMtgur0UUovrIm9GVajgwPqMhlNCqS5ns9mjXhoA6eqaCaAt+DdgESIxH1R8a", + "a+6itB8oDm5/5/HHp50vFUk7ZNKoEPi9ysohsVIxjNfDGyvvgdSTJY8Vd68sO8PrKtXTYZBBpObL5d61", + "mnbPuaFyfroFXlHK1qeZ3LH9jz170BU/5dyNbIiPOr1TMZvvw8ecvK13fbyZO9eV7lD3LP4adAlrgxZi", + "pN1B6SYWWmcbcPHvHw708Tn6L+CBHsCP3SligL/0HSSEkidKzkVEO79ay2SplVSlOVF5Lnoa3NRfXVFH", + "vrfBjrKjEEjPwHFk4R0nuMpU5M1AValHB3D6SaYWH2KsOd1W6e6Kz+EHnoUuNfeY3p2pP8s3U7bXauLW", + "rZSqHTgIizSmHAfA7QJ6n+JWVJ/07r9t2NnsYJWVZvmaL67IyTrwGtIoI7cFi0cszFBkfP2aL/YRkqpB", + "++3QD+vZ4hVI2/vbVmroNe24m7TV8NObN4P9kPc+II2KniFqnejSSsNeUe08SoYqg5Mllwu4pvpbkUdD", + "aFLWmwUCqCfl1KpQWhkLrFORZaum9PNoPJopu4yWD9hidugcsNag653Ej7IKHWG2GEGuOm2ouvHanXZY", + "Tb8mNZWvkoZ6v/hB2CVaZOJfNdvW9K/2UmTZnCe2VQW2lbeSZc+FzXnRP8cFX8A707HftC4fNd/YAhMn", + "0rwU9kyGb/s/uwB94ROxI0y4zI9vFz+IlErPRdwfiht7EhoLDJff2uPiEic39nqpwSxV1rN/zBR5hQ3/", + "+8H5g8gynGwA0da01EJlh4oiJNMG1ebONjAfIaiNMzf33iKLcfditFHZJZEYPWyiLnY5qRjXayFteO17", + "q0huLXzbrYvcQM+ciyzasqgpsG5W8+eW8SybsDNrWKMsKLYUmEHCSxRz7RL7A3FDJXggHftJ6ANhqXwP", + "8J6WP1JZ0XfHzI0oitjOT9xSptVPjOlSUgaRMI16lNKJvuJWZLCAMSulKQvfGsGHtI6ZA0+p4XDC/tsd", + "bS5kKpx436itjX3V8OBUPgibPwvDuGSl9OdmdPFCiyQ8MhZfZH63qUDSqlPjUCgOpaFD3zVqxJb6IG5l", + "gGousdQL7GH+BS/ijRtWXMt2eZa+uJq0zv8g6qkHV3ipkVDR0XYqfkngi1WFjnUFusLkvkYOYdXrzgFt", + "wq6FzcCMGdwVGZecuttwmbK5uEPztC4T/0csjdvsgeR7H7jZji/OWMK1FoAlrwqtDERVMuxO1nzPeyET", + "e7s9QDZTDfDvuBNPXkg9M1XaCXvxY8kz48nHx+q7//Sd0hAQ5hn5emQVow2txmrapzZRDS6iLsc/ycrP", + "6rZJPVueWi/qV22C4MfSN6wJc4/GdfuZahej8ShptuUUnQb3rWrEA7oFbRBS3SwI9wHpVKvSRijppcYq", + "+SnD3zE/guolY/X8O1u16aYTj0lpVxK2aei9DYq6ldMpE4qopw3UBtQ8eQy6QIMa1Hhi+spgBzFHVkN7", + "1XyqJjVEdhGOTeQYuwFVgxJCSA06/CIQWhQ9tFpFvN0M5STjjhngXMhWcPNuyJjBZDFhyZJrnljQ7Jbr", + "tZCLg2/+eNizUrO3Rvds4bf9z1fPG122pxPjme+suOdiOFuc7XFjp0hWkeNVKdzu6fPsOfCHgSnWOb/r", + "nR5EVvVm7NTlM6GtZFiOqZWkzClsSGq/DXwuMMOvn3z7p2//45s/fvsnBrcga8tcNYOwBrI5NWiciYWQ", + "9mmTYapb0PNMrQzDUv4Dz4epU9M+9n9NhTSVTNmSZ3PKKAuYQ8sdWus4poL5HpHUyJpQNm796QbWY8/0", + "QxMUSpFHf3B9FDfhDawZdpHpYf4OmtNZpB41TjhJ6ltbwW/uLn4r2IFkPGO5TJ3MBFuW2nphr6t7inh2", + "wKHvJ+wYse2gTbIXFa+kC3BHuAx7dZNgMz0gUelfpbHs+PX1i0t29eKv7168PXnhGZ+T75hjXj4rGAAR", + "wn0bot5j6FgbKHy88ddWw6WinGUimVIZvmklim7vLxph+3UNE6YVuuzLotHhUqvVhF34/gE1zWMmPcyd", + "7OEk9LkqZWjD2ewWqObUntfh/FmLrfjAJx8YoJWy9FVc9KwJMkpT9A5OSOLzu6BcapSy6VYoDWIhicir", + "g9RckrIt4/CjBMxpbfiM3cNQTMJna1LdCboiTs73YwM1tZqmqalveDG9gfWEXZXJsu6L7OuekpGdG3b5", + "4uL12ckxOzt98fb67PrvmAX6frRWJeOZe2vXbMndp62dvB/5loUzYCutEAO3IoG47oTVg4sYx0FHSpdX", + "M+1uRbW8A2qZQ7ttW/h2CndLXppQiPJenbZqmaavoXZojDPNexq+4hWt1AHDrFI3A1ly0ONid2mBsAlf", + "jJ30ZCwjeZvYPjvXKQXJGKJXn+3hc23N4F5pm5pRxBIzSNEttEpLx+xlQ0kNejZW5611eF3KCbvC6s7S", + "yb7IJ6AeRh29u7orhqphBeZcGMtvQIZQx0qZ3f/YVzeiiJ65NnMMnMgPCC35IipAOB4m54fueHORYWdz", + "n/FKvG8BZLMoe7oCW13KJFS17fTq8MWGl+7do2wflvCiakZdGxBaTVSowW3DkOIeoBVoNnNcebI72q4i", + "6KZOHqAYYNLc+bh1v7ZrHYilgTr7dbiPGwYYikgLD8vUvRemKsG0hKwATe5JtiLTxswgY8qA30Kz73QF", + "w1LSc9cjlG9vhkkHNTUDadyQTKkbxoMaKKxj4MzwdZwcqPFTBBSQLKVIeMZyrm982xEvXlz99fXV9fH1", + "CweT99ihSJX2/WjC3uIVC+YIr8yhyc33WYZG7QBHVXQn0WjhpcPQmJlTRw7S3TBk0BtAfG0Br9vhvA6Z", + "49DTFEt1O/BtaeTctAe0utZWYgz14K9saiFN3FGeN3ZVtqV462Si7nrCSPs0zb1elSsprEKHtlaZ10A9", + "dQlvfqzRSb/mdON5iMg1IjETduUfu55D7a683Gqcu+tm1Xyuh1u5W4Fvz7iy/mFAeV2igboz1RWxHP/a", + "1L23mIC3WVmHGwwH2MIaEKASzuatqvsKbHKY7yAemV6bPwb3cevsoLN3t1A1bXS/vkBCbzVz/8M+EfFn", + "7RD9shTpTto6w9rF1VrbtvpamJ54lnvs9ZSbJd/W3nRA0S/kTaHQBDVrqEOYnYKibqnGXc0nvXhvMXQl", + "aXTQTTuVxEKZkp4CFGEPpn972BeZdCS0wqPW0dzys0Z+R7vsxX7Ni19x81olN6Ynah9ZFKfKUy1oRdp6", + "ImsLH0WVgEE0Nh79l1FyE7tNn+jCYKS8uQQDH1B8/7J6O7odQdab9OHL0z9l70c5lyXP3o/oueSlVU+p", + "9KHvf4VGA/wLvB/1pZ+Ye0XebyHauqNZI8KqHXZ/OGFvQ0eTMI9hJD5XMqcwfupE6RRSEgbtSoVGmCFo", + "KxVz1KFttkaNsZ4Q3/DqZ2bwoKbhN8oLrlFVcQLpbI02AbwrO1C2nf90GEObkLYxp3tlXfTmC0RXqqg1", + "ng8Q/jqE/3W7k4me9hdYrPGEo3axuSIll9+35PpZevdhE9yvo5Tixn7YujjFB+5+e4+qXlSc4i2fxQJb", + "sEHJG7BLFa/Ojrp1bbvfI7KiuXCj6XbkJfBRDPedGn0pkWmbgQ1DA8pefuqzolMC7jk1FTaPzEpNjkyf", + "kEj64vl8R9vy5+v7bas1Qyyavf/GXYmfoAGS6O9buqm7XzshNd1f2+FW7Ycw9DSMM4L+EuLu196wz7qr", + "wr5N+N42St1WIaCN+9mau0Z686RNiDXh04Z09240oNGml/pu1pQb4Qnde9Mhqp186XmmuI1GsGp191ID", + "+CJ/A6Mf63FVaNqAq98ZdaHB9vR5pi8xtGqfqtyNYa8H94loDxq2qf2gVZU63ucs1aDhJ2kO2XKO6rP9", + "ToH0NHwz4fPejcQasb7GTIrO0Ag5bKC6B41RpG0gpAPsKCAjYNu8Bn0kPo5ctJ1X9qQKYegKrnWY5kbW", + "/YDqPyrLtqT/U/e2nt8amkzk97ehlFVrZ3PkPAOkt97HwGk1LzVP7jltww0XyZC2SrdjYhvPTPz9iXcK", + "869JDd3GyjVY6wXb4BxADM24j26H4LmQoh8tg6rB+2M05tq5p54mMDu2c2YuyHsZR8iZeYd+yL5fv+eZ", + "6MmF788aaEcMt38YngyxE2LNwzVOUm+7uaLf1k4oV8LCHq3xqjEv7gpNFe/j598uNn58mFXSWOgzENno", + "fYDUlbA/xgUJVQAGdnsOWR2NMcOuU0Nb7XTr79Vht+usl325JLjcFnrCFnXm+HbRi/zWF/ukEdHAfeQf", + "GrFjJ/tu44JrL/DE7gxIezUw0LL1dTVv+5ybm2ydKgrNzh/70Xitiu+erwN4Nrpj8PghX5q8vwVWj4L4", + "houee3OW3g2E+zbtebea2G8v6vmlTyPFX/ahl+/zgYLryIOjtUbYSthsU8NDqBJCcJmWgvjcCzcBbFEq", + "aOfVbBUT+zKhtuTZ9YlF3Uzrep7xqJEVEyaI7pwM6VW9pWinw/6Eyvu4lGqP0U6/wQutlX4Dxvjz7xww", + "0PFx5v1G0UOdlzZRfbjwhuz4Oju356F9QtHn/Z7M1vANZA2ThTds9k1kNkDQnrY+foxaqJFQla7XIXLq", + "tdtKHttHUTaDu4SZKyETqPuyDRsE2n7Pk32395ob2+gifD//F05SWsU/1kS3m2ffe57vP3QOcQt7Ie6N", + "ShFxMXBuYfu4zf3Q1rkKrXZXjW1vbmmDujbXH28SepS8YvenVdquw2r3EMda0+yuWbedT2wtwtbTfPTn", + "8chAUmph1yh2BQOm+A7WxyW9cxs5XVYkmHMV2pA7SWa0xGTN0LP86ehvR8cXZ0cUmBEM6zgt5hwD16Dj", + "C/zXD9c0MzlFz89OT1ih1a2g6dGSjzoqzlHPvrS2GP38M+ZVeLlHWKxl9s6AdhsejUchhunp6Mnk8eQx", + "Bn0XIHkhRk9H30weT74ZjUcFt0uEwyNeiEe8tMtH2BD+EW4Lf1lAJDTttTA+gse30WcczeB0HIzEpugH", + "iiQ/wEmFsZpbpSk753DCrrCDfJUGpMFHiGnfUf4ZhVdwZpZKW0a94SniyefZUTjOhF3CrbqBNCyPQcO5", + "sBZSVsrMbUzIJCtTmGr/pcBycpMRQoWiC90DiSc7zrILf65rgoNP+wIL2vS2Aao/eXRGq/ltYRcgDaZQ", + "0gd2fP34McUlYsAIPqp1fv+jf/nAgzpra5DH59gBOWyc3BK4/Ui5GwwFbNF6id+7T799/E0kgJdnGcUR", + "Y9qmZC2MMozEw9QGjXUlITTKc2v9PO4hr0f/K9Kfaa0MYrlwBD+GMZCOkL4ycYLrI7B3MhM3PjYGsvmR", + "AX0rEmAg00JRnhOl22DcJzhug00ffZhxgof+yrjFPWltEgzt8ViuWySzSTHIOtx9qxkHCmA1X6M3qz9V", + "b5OIvu0DWfoJEOkm/LaPBquNPXqr7EtVyrQH86VBgOzkK1rIRBQ880HjmGNgxEJS3jG3LANuLFMygTEl", + "SWKTO1F1xBOaZe4LN+hI9JFI9P6/wz0+2JXFQqBf0h0Nr4pHURtAfwEnzS29pvlBINoKmbDGNkA03vTR", + "0//5R/sUQ58w97F7TTDYvH3n46/bl/tw/RJfrQ99sNxPhTIRDL9xHJ73vBghmj+GfYpqm5dZxgxhGkPS", + "Cbfs/O3Ji6pCQ4APZU2Ymg60gFtKuFlwIWnGusEdBn77VCu4SwDS9oODv7spU3L8uF2zWwErtzl3rpAr", + "m2VqBSmRG3VIi15xCrn2LdT6nlBpxVGy5EIKuTicMORNFWFSBV64K4SuegN+85il7q8HBsD/ZKZCTt0f", + "I6yVFPvuM+mB8lyl64/GPCI0RYsTubSf3J83qPzJp99IGqPp6ifH4B/H0qVveSbSQEe9D8Flm86IwMwm", + "hWGaS15TEz4bnqKqfAR3QVhO4S3dh2I/MU7JKgh6f57bI3d9gULX/WUkFaJkHyVk9+p/vS5AH/mPMMlC", + "U0NXmTKYz4HuPhVmQUySMhYGCKoX4cDODvitElgwoeJPmMhRTU6BxZl7LR0D6hGZjhu7PwmbfxAJqrTq", + "qr3wuYfHnjLVH+jed20AaIf2oclU6AZ24+3R/zpq+3mHENWF2CD69dUqP4SCP6qU1g/9HRLsXvdkb/SM", + "R0UZAfxF+cCA//iP2xaYn0l36EFvXISLvSvSrc8PxiBQjQHKoHoANMZvWVWGddDloq8f6hbQah+V1Qyk", + "5fqcn5joGkd8IDL7GFRjqkSOIVTj0z4eiGr8atv1nPip6rTMnYJCNQjhGnK4D5wYoMFy4bSbpTBW6fVh", + "pSD5BZjls82nvwu0Kov70z/8nvVddZZ++Dff53UdYVeawfTVdGKbnvfnR99Cxj9AHo1T/xBtqekVH6+8", + "l/QeQ5u5a/cZ7zTP1rhhuUw9m1EfbapM5FjBvZ4tDQGsf3gc89H1QHY+N9AzT2yaBzGstMIkHuRaRLSW", + "jTvwoMpBEO/2sydVx1F5ruSjkMB71DVdb5wtxIkE+/F+1rdmrIMjtJ3GOh8M8fGoaUdC8MeU5zegvASe", + "2eWRSZSGbTB+hd9d4We/GAhvo9HmgR4OxBVVm4HArvPSf21Qr0/2wOCn1OdNubRjxaY20KwewTI+g+yo", + "Kg6UCr6QCtPBn5K1hydLMv0duMXYI1YolYFmj9hSGXvINNaxCgVllAFZxR2Ekny+/ooBLO5iIIPEYuXW", + "lC1DXRr/I24EUnYAdzyx2doXIy0gsZAeTtglWs4NA6rz/uc5zwzURdFyOuAR9hlMWa5SLPKo5vOovNnB", + "HIGkktZ/faTZOuBDUiiWCjKPlmKxPEqBp0eIhyMsI9ZPrmcyExIYja5rJonFcprzu6mbaIoTYQ2BPAdJ", + "ut5TZlVBVe0Mm60ZrmjLIgOGn++ihVdisTwFnmJTdErCfDhy2P1tFUJ6D0n0yR9+4ZLoVjR9RjmjpnGl", + "xU9KHmHfUSEXR95ttjeZ0zzTjC+mdyI1GzQe5mWGirfP1kxlKRgbuqxO73Ih2QGWwSt8gfU335+chKkP", + "d94D+u65P8lVOMhD3oXfMn1Hwf8l0HimVkdLZY/KgvTk+7DxTK2mS2WnNEcPHy9A0+zs1fm1X454eKB6", + "bhLAmmG7iPm1Wr1SlgyGv/P0L5XmN7D0JZA70fcRrwLPj9R8vie50xzTlbDLaT3RVM3nm8ILCS4YiNb4", + "tC3yCsk0ZIqScnbRPtF7HTh/Pp//TvxfHvEjmi4DVr8Eyr8T6dFK84JrN+oopugPIf47kU7raaZamJso", + "q6/q21nNpeHYxeTo7JTxBcaXVGU8+cItETg/Oyj4ArXDQHRHK5HCrkvxN5H+UO3pM5gkftu0HoX+5yR5", + "76zaZTsJpgUcxKwGSfFN/zPXKh8zq/7hpRaRg7E8LzCkhmcZjRjjjwm3sFB6TX+jWJuMW5DJ2gcgc2aw", + "ZiDP2MFSlfpIzY9WADeHzMEJ7xiF9Fm0jLBUFGbCLsmDapq2kKax5+Dbx99SefmVMLBb/vcg+dxXwnt8", + "+qMlPtgD9OFTd4iU50VGnSygcG81tbow7MBfQvbN48cI/9iW3KCpHxHzkH28WztY0yZCeMDr2H4ehlqW", + "LzujHpJyu6U9wWHdjimeVpdOpPPIwFDeKuepfvWo2D7X8F5WEcIHJ0pK8P28HrFGJ9gwpw7l3CFlqrSH", + "VOAyRleN5mafJ8RrC6YekLRWIBZLa9pRpm2qOsW/N7b7gx/zQQT1QKANe90V6T7gPv0aj90X//RJjv3x", + "o6c2t0na8rAYqs8G+R1h6Kt6is7VDTWuj3ZlEgVm7LOJfgXem9aJPi2D3Az92oy2wC/rsKwv0HzwIEpM", + "Gw4PprEEKeCRr3K9HVXh6yv/8Res1X7zSw+O6kL7c9CE5fYI07SEXQ+kDMvtcRjxBZPHp4yd65mmNKDv", + "GxhpfCf9Tydg70uTTTx/HsoczKzo219esNtwTHwO5rDiwg6IKK43WpVA+bXhoq7t8nBYCGWyHwkSdY+S", + "dsn5fnEWP28X2v6tylwbwHgoBAaTibcgDInYJcXo14msbTjqnP9TI6RYGIPNa8yRBgOWbLNb8NNu5nNN", + "ltzfGI465//UODK+JOiuK+NrkP5GLwye/pOjYo+b8lu4JcM6tnauy8O8OfMb86hQxohZBkeyzLKtcsLL", + "G3PhP36L3/520bUBis+BMN+SfiDKrvzXvyOtAYwHRJtb5ygXBvMOtuPsel3Am/DlbxlfLUA8EK6Wyu6M", + "0DjHOllFAVw72BnmY57dKCyLVaCz9EaqFXaoF7ciLXnWaLqJQUvCsBsoLOOGcUphXmieUx6BEQs59p3C", + "qRgoU6Wtmma6JZbcMKm8Y7fKazkoDbDqJAX1XREZUCEva1hSauphaDkGNfUEZyh7jss+UGjGzsi8GyG3", + "18AJzZKtr0yOLf0jnY977WmhrP3wWhl9xkEq3rrnTD1ttLFW1YwnN8wq7Js96fW5r80edu9P6ivtEs+D", + "VFUJXURNX8pvo+396NSBtb5bdJl9mcZwj91kVhWjFm8QddOAnrjEFO7YPOO3qtRMzekuUqStm2DCfDl8", + "7EG+pqANyl7jKQVoCcWSjBsDBvnFhTJ2oeHqr6/ZDUBRXXrcCVtpYX1ne9Dx+Fy/4CvsYft5bzGeK06l", + "IwTAaFxd5PBvoXpucRvwL+54YkMX32ZNzbq/rFXUXvaAUxVEqi0Xfp6k3II5fNYqX4cJBO7vtv6u5wJy", + "+xAZ/l+yE2sHT7jEBM4vkBlcu9dUWebvNpFFA/MHSjPOFuIWpOPHX5lDlkJmed0busUi6ke3l0ucqLwo", + "rY+fJB70lWGZuAU/80F4psPNZrmQpWmKGVwmS6XHWOskdK1fKmNDQD/WVHR3AjVyBoVKlofIYAqt3IoG", + "5YEgP2hugSkZp/mvDIkslZQyYde+CjGewDGxpbJHdBTTrFXM3b0rk6U/rOV6AbZqMN4nfFzUMPxd8vjE", + "ksfHYtT4FPVz7E/MXRoU8+VxmBN/m5fKSjB46YJcruZUY7Ku+F8xkh3pbpcoMTQ4QvMGVo/eQcpFtsZr", + "7rgD8RcTuIavfoJ5DM8Y5sI7mefoLdZTpMhukkZYkZVOUWnzgUqjeXPy/b/Xf3XDbpqSS6P5e2ngcMKu", + "+Q2gKBPCrp6yEFZQ87zQX137YFRIUSdqPeo8c/S3dgfC0rKeBzo+R+ISr88+08BvUl+wk4G0ek1FbhOM", + "nV9UKX/oH6aaAWWyDIKEVRZLfPptBkWMJ5a0O5wlLoNRYtQvVQTbcbN/l8V+l8U+tizmcxM/QBTz0twj", + "nt4Ko3R/sTmuTRBOVOEItVhMHQfA/4HcnbeuLFzF0CvpC9QiZ/Hk7dnpmMGd1TwJ1Uqo2TWVOKkm9cH0", + "Zsz+pYQ0Y0w0Jsa10KosDJutx0FeK5Tb48xqgEpMRdlKKgZ32IJ2QT9UDNFzI/aCyuiGpmnMEWR4Nqrz", + "CctWqsywdsotjH0vArPE2u5zBneQlJgL4C4P7YpqBxg645rpUjIlJ+/lmUPeBnOPwFSYmjOHSuSbzJ43", + "IEYnpmrlpTShzm9B7UfdGdmMDGSrpcpIp2YrpW8yxVN3Dh5mpyNwZoRcZBD2YFagPRywzjCv0b2CLPOQ", + "pzwsznBSETZk+KrW7R3YOudwj/zJ5Yvj6xfs7O3pi791N4+AN02ack/raimSpYPMarn2tbA9Hv0iXN4E", + "4b9BgE6ZcLtD0yLCYvJeUjV3U2ZYGp6zJZQa32SEhVQWC/MsQWMVZvedr/BeZFxK0E/Df0yTJSS+rD8l", + "CVeXodpeAHCSCYe5vDQWsVtoMIRK8LHYuElZ1d5hpgBIj8qCXmZTLhZUFjxNM9o3XQdeZ32Wxp1eIQk6", + "oJ1ys+QeLUS5YNjp6esJcyKT28a0cJc+xT2WUruL6fjNlOh5BnNFQEiyEhM/kfAa1CCxNjkSOCL8afMu", + "4ccJUpE78QyYb0eWIspIukmD0XgutLHjoLktlQFC3YrXhfl59TUla3ngSgZ5YZt3GytezwDtxCDsstFZ", + "gyUZuPshsszdBAozmbyXodDRt4+/rcsb4bGmnnOyOXB8yB37EwaTw8N+Ah/HZ2jsxKclUiKX9faocBOB", + "hS7XDFcgrlUKg7AnVv9+FOjQqoD796NtZq1jz94fUq7aUOZzfmTAfY09MHxUZ0BngzQc/jOYozG/Kivv", + "iMMIU4nShufAckiWXApDrx8KEwGQpQEzrkud+vZAWmXQIiDD16a61XzmVsTlarHhK0OIdh8gRTs4w12R", + "qRRGT/Fmx8UruKP2HqWvGbh/Wb7xyNg1dv9ystnoNyR8Idl6ov0UUlhLmjqrRIXm01C9iP6CEV3mwrjX", + "sNGYqEG4M1gKmaIEsylhzULr3f4QQ/wwdHj9VdSD+MVnEyBSCCUP5Fut6MWJsUdKHtFqAwjHDTiXx/j5", + "bzk41WGsCYsHRlzCsXjjAIyd+C9/v+xf0GUPSHlgqlkKe6R9u+4dZPNK2EtKa/lN3/EAhgdGVEgcUPpI", + "KnuEZoQBSPMx8+f6rbKXOOY3jr5NgDwwIr0oNwB3b/yXv3GMBTA8MJ58COeTAYi6Cp/+xjFVweHzoOrr", + "4aj6+ndUeTh8HlR9MxxV3/yOKg+HB0aVVcXRzdFsfbQrgcfj61oV3z1f/3qzeIZjrAGJB8ZZKUsD6QBk", + "vaMPf1cB71segGcZeSTiM3kDrZ9tplQGXG6pP7fUYJYqS79E1dTTymch5SOyq/c6iH1D/sjIsffPhdYU", + "mq+YSbisIljG5DRGRxdaYXNqrey7eK+ETNWK/Kcr0MB4kpR5maEbAX166OfxI7a4m91n2jtyOLsFnYrE", + "1oZcjSUehVxMWNw7K9K7KW7c+2TDL9v8stL7m0WaQdeZmfM1m3nfBLrPejyysuuRrcKhm4FJrBAJOqDc", + "Dv9VGivma8bZ6eX5xYQdE0TI6UbNqWe4Drb3WAvIUhMg8udSUlJGy0/lfZjvR4TU9yPyPPoxYwpFIEhQ", + "cDf57dFojvQPKeNzC6FjbQD/khs2A3DUgNGeVfDSmkmutVqxldt0s023w5gqCdX+owA/uBUpIEqqBTTa", + "7pXc5h6ji+UdDZ/PR/YdQEHIDdELnjhFhZy+WB7/8ygWcplqVUwr9+doPPIvjrE8g2kAGU5syvlcJAKk", + "nabccvctEcOgiKbI/tHTRrThdsocV3QXlQ5lyhnNNmYJN+BuFEgjrLiFbN131BBBukeiSN/G9t4Se4lB", + "DQXXVri5IZA7jqV2OeRIKrgGacd15K/DQoGAKKXoxSOFxd4jFvXX6gRsXc1P5wM8aXLSwDqUZKslIBPE", + "qCTPyg0zfA7MKsTppqevNHwxRE5/h9/9Lvl9SRIWouShBCz1yCcw9EpVGHZLwg71ffcxv6EfFzdVeAPG", + "wFTNv/IxhSKHx1MCpJh1yRxYdaEBhav8aeXD9skPY2Y1T26mQk2tyJEPNla4BW2EE0hkipEYPgioEcP8", + "5D/wB1VMZ2sLZsLe+uAKYVgKWtxCitFTz3zz+VugvmAUhkziEdwKVRpmMpFAiKGZabUyXtYzS7Wib1Mx", + "n4PGF38GduXkCLtSzIrkxkyqcDqh3OqiWY4epQypGsFKKEcYH2xiVqDjAsO5j1L/ldS0PL+qQ0I/fZTr", + "tY+GpMDEBtWoLA2S5pM/Uis6n7hbobAnWcBpEw08E834dAG31Kh53XblLJ9S2H9FSnXWQIicNc25UXGR", + "ku4Vtyx3R/sn5vSYf7JZmdyAxZZnbtsYYJT+iydu13gJb3mGJchxTZST0xQFjcMQj0rpQQnXmLWMpIlh", + "9CwDubBVyH2YjQnrA1oRhOHtQqm/+mbFDcraGVh46gNHnXCN1bE5istcW4ZyTs7/pTQri4XmKVBiAG3D", + "s4oQ4crncwpLpA2jsF6EjkBMyLBeW6VwWthPoNWEPV830jMRFCECUsOC6zQDg4DvcKFeDoTqg5smcBwM", + "m6t5kqND3xQj51aLO8rREo2cLJS8/4kxxtPZ+s/zMsv+ief1+P3zk3/2MIjPUae/288XMTgPNc2FSp0K", + "6BXpTMlFwME3Tyh32h1fC0x28TkE4Rff1INb9k+r/tkntv7yegKcitzJ9aqKOKfKAPgMPfP2AGORiBq5", + "g33nD3TSkzKCvPrONpJG6r+E9nC+AbcjtEHK1ht+J/IyZ7LMZ4BJ1ESYmLVDp3kWYziNE1nF5v2qCE0X", + "P9HXj3vEvT69Cy9Xg0/787MD6bCXjRm1QhqzWZndaOAp/Rfm1IyZkML29meoQXkvjRB35pHA3Li+dTqI", + "uv9iPp3+QEPmI4It5AWr/7niWe9pq+zFz9MzoGZv9+kzvleSyxXdyEaqiVAhMY/Mfp631U98phb9mYBX", + "wHWyZH/nqMWdZKqsMvuYG8gO6kfkEbXVPWS3glPm3ulzdnxxFmX5r92qH8rtozmvFLUcKG53cm1BBzA/", + "Zm4/eIQ9Umx/CUy8CrymCeJQuwVNZZQ/Wcg1SpZ78YCrYFfyqQ+Y68GzjM1qYUeEtsnGqaHsoGuAGrPj", + "t6eHrfjzfjD4WT4WFDqvp1YF9gzTabClhV1X5jQu114GMVCb1czmsQ7HDH0HCw0FO7qdsBfS4tX3UwXj", + "7vvR+/Lx42+Sf8P/B+9HrMh4AktSHjA62/erbroPaJdfGZZzc+MePciLjFtAfRCTK32gfpUFxFJIy4Ly", + "y5hZ8gKGwtzP9Ckpb1DXmC2VtO8zDuHxUdxrBV/A1JBnOtZS7fEeViScy6obkJ/tNXytFvSqXGLG2H1a", + "bvhwQtZ4PT6auo2vmntjTVl4daxK0fQvH439enNsWRirgefhvTy+OGOgtfIjvt0ywnFvVdqmqSvnDp3S", + "6SuPGo075xrgJzjK+d3RDpPpm3qCunHmSxz+ht8dP6Qd9UGskbvP+0BGyh7MbXYo6aZ7lZKMJT5H2Vs6", + "szWDTCzEzBdbq6d0yvUj/xefCtiUyL4yTK2k01IXTnt1PLLM+LjRk7huwMqWSionUXpFTq0YGCty5Pom", + "9Dv2StNiisn+Ew0ZNqX3/f/ELTD/75BfS2v4P3ov6lKkgPUeZhkcVvaQWidrgI8VWjl0gGG6lJJykcmN", + "621ZW+u9RCniV93/Ze/7UXWB+fRXYVf7o8b+HrYD0i86vqeNs5cYQsBm66YDeUN6rEVLEv8Ot7mNpzsb", + "iTw0d6duUg/Py5s9fUV6lHLpsDCMoq/rsWfpKY387UYa7gDMZ8Ctf6MLrRbazTsMq9/jqIsw6HeEbsLk", + "gXAZEPfIy0Fb+yj4b4/9p78uMbh7uodGgKM67DFdFkOQ8JwbeE5f/zrx0DjgQ6MiKKwD0OBh+SvFwUmt", + "uT8oAig0bQD4z3wM268R+HS2hwY9PehDYE8P1q8U+P5wDwR9pz8IKtyR3GzP5fgrffrcf/mlKns7vr1K", + "VPFQKHUQWwd4PTBC3QCuYQBCT/yXv2yE9nkXvTd2yod55spSpMP9fdXss9Ges22vG4XFu9S8UT/Kqv/D", + "3tXutm1z4VshArx4W8B2+rUC2345WQtkNdasbtEfRRHTEm0TpkVXlOMaQ4FdxK5wVzLwHJKiI8uWnYh2", + "nPwZFpeSSJ4PHpLnPA8x9y7kCVYawG9wgncG1+5P94bYFM6WjKpeZKxmB6kf+HnzAxcJmhntC2bv/ddk", + "BpQlBBSMF0RmmOb0f2dqkx0vgDeqi20f3EFpYQZqdq6FMrEy3xqm2mYXgbyHeaFiOxd7C7fl8FtTOyd7", + "w5e7ydOBHcTKGIcwaq4trEJAkjkUY/EMsDZNhqVB4uOKyGzE0jlXzAeQhbQ7BM82GKYxY1MieAI0LgVI", + "UwFI3gnggA+wTBCBElOmpLiGggyD5NcibQUvwEKeTGKKRXoNubKvX+k/Iw45pDwhv9Nr2oVRlyVdwZ88", + "3v/BMVizLYYJGzUhWSE4Xv2iqVQrzPtSqty+leEuvC9O99WqdckhssP4icIZHcyEWOC0viw+9dZAgfo4", + "oEsYoHcjELxErOJqTctDDWNvJLbyBBJbEc4zNlOm5xBxXVWJkU54cmVbl9xeVbl1Oi+/X8KaVri3xs5l", + "7HvWIp8UI90/O2TORRzRNFbkyf8a5Oop0V6cRzwTixZ5AyivRiOUAbXVr1vrc66wzdJwJvR7B9L/IV/m", + "WYUY2lykQZ2N67x5NSTR9wQfs56pwTFVjA3SS2R2hf/iUqS0263S4yv9sZJsaP1KLxXa/Gk/Vin5Ob8a", + "RDYGL/0/le6qsDdTTP9frzSb1i7HZRP8+lVjn47e2G1gT2+3bqd/8fjHWmZaWztUcCww11MKxNu2sjW+", + "7TazvvDtuNbuW++xlhTgVMhorKqoQQcaBtCFelP6onHQmrjtxVJJGOqwd1ZhCKOtYnKV7XAwsqW4Gmti", + "4nJPeZAhWFso6ZhpKNEewONKmqayzxQx5+JEMQV1VE9LIQ1wnyqMeyhsofKs4aJWPL87sn3T//OU6R35", + "StnbIUa2zR6N/HTz6ZazdXe4VZtbtJ+qcIhUHFMmp8+fNfuLjVT3Zr/yUbc/Wxwq3f1BBxj+3AWOG52Y", + "51RUl/JnKh6FvJuQP0P39iDjaEQrnfFCP8+h8aOIq0sWZ6wmSVqEMt1BXIrWCfJD3vocGx8F7kVxWAFm", + "W4kNsbvXq64IGsIHcV03hhfKdS2JYGNM5Xcy8JVhcCGYWC6MGLADTcHXwB15ZK0LlbEJiWhGhRxi8fy/", + "f/9jyL6AyyymGYXffFzHmA1YZOnF0lkCtJpTOswrdtRIzgH3hH2bIVQgsj4hM5qhnAK2TdWwlT0OcpA6", + "8MYxW1jsESGHPKKCeIoGZUaIViiTBqEJmSVCDocsdmX+JJF9GS9IyiZs0mepwl0HXFhRgrNFLj+ddS7O", + "ASkS9yTEUtfypEXaBDjmbnKp6XH4ECw8IT015tMpi3uIa8IB8YUquQzy2Gc8GTrqrQbps4jOlN4Bet+J", + "ef4VR1gHiJUA6IIEasig1yJvOcCHAAmhA80zpHxSCBaT2dQijqTSMlq2SFvM6UI5TEj9eKrlpbKGZRJk", + "qQN2oSSi0Wh12VQXJrLDk/uy987BIgZu8gbmBtZUtlvSvwlyImZcCETPscyI5smyU2/BrplYCRWJFZ6N", + "kzlNzY1RIjMesS3hH5d7zhRDBVLlEB47Fk1X6gRPjDntDPtY0m3HSb4TFsjAsw1r1XfbvwpYIduXgG1V", + "o31IcJK5J6iL17fCAxjkuwLpJbyTLJ1F2Sylwq1iBmvM1oX+XxlFPlm1qp5uqsnFCXAvhwQ5nniMnQ3w", + "5fCTA002TOIml8NCKrtiXsUEIn85IL1YMkOmaZhi9aKBKGUWLjlHWhBSjhWgP7TIZyrG1oHlZ2Qel/Dg", + "xmu8ZR9ZNmcW/ggcnPGSc21w4Dz1ep9xKrQJ6cXQ5rHk1br2W1gDrGOHZl+Ll8W/+sSmRe5SZB9VZELT", + "sV6VKNdrm7+2+oBoIwqAz4l0PmDD0hW8zDfM8bwbn10OXQFv3ddjt7PUS5Y2nTIYEVr1G8gUbJa7uXSm", + "yjId3VWq6uqatsdZ1WVH94fM+MC8K9g2xEnBAzMwv62TiFdlblsfWZHF0AwsuCSmw0oScB0Me7X50HCQ", + "g+sBAoRUoc38CC0fWTMPS2FAKoFZM43O4Iv7bLPS/GZb3ietcfvL8lSZquh7hjOh+ovq3ImBSJxE7ro6", + "5cXLzQ+8h51xx5Wf7RIRliljcyPT97JK3j/C7wehl4b0+9iUMz+Fra6hl/kzj2p6qxetiQNeHEUcUFCZ", + "2osR925RqZw3LbxcdZv6IOdv7EOPRrV/3+/LI0zQ2rSnD7akqpriIAjCARdWPTDF8eVRr+KMeNZMKzkZ", + "S43/cDG8YB7sNITdlG4TYR16ZBVOVoEChqK0htr/Aqhrk15TLsC5rD2GdA+0XfujFN401YPPOD7d9ien", + "iPfteeMvXts8b8BcRtcrTaNSmw2vaxoet9Vtyp6oVRQAjF5BEtju+EOZlWwkWEf85KJz8e5NVSTbb3ec", + "0fFTrVvGfepgZeZ8VMVDJ87fJVlnr6cBAfn3fzROFItmQPyi5XXGaMrS9iwbnfzy5aueuvaUv2ML98vX", + "H/8FAAD//6LdI3wSTgIA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/backend/internal/http/v1_io.go b/backend/internal/http/v1_io.go index d2c0f218..484aa7d7 100644 --- a/backend/internal/http/v1_io.go +++ b/backend/internal/http/v1_io.go @@ -170,12 +170,17 @@ func ioHistoryMeta(instance string, earliest, latest *time.Time, metas []statio. } meta.TrackIoTiming = metas[len(metas)-1].TrackIOTiming + meta.TrackWalIoTiming = metas[len(metas)-1].TrackWALIOTiming for _, m := range metas[1:] { if m.TrackIOTiming != metas[0].TrackIOTiming { meta.TrackIoTimingChanged = true } + if m.TrackWALIOTiming != metas[0].TrackWALIOTiming { + meta.TrackWalIoTimingChanged = true + } + if m.VersionNum/10000 != metas[0].VersionNum/10000 { meta.VersionChanged = true } @@ -186,13 +191,14 @@ func ioHistoryMeta(instance string, earliest, latest *time.Time, metas []statio. func ioSnapshotToAPI(instance string, snap statio.Snapshot) serverhttp.IOSnapshot { out := serverhttp.IOSnapshot{ - Instance: instance, - CapturedAt: snap.CapturedAt, - VersionNum: snap.VersionNum, - OpBytes: snap.OpBytes, - TrackIoTiming: snap.TrackIOTiming, - StatsReset: snap.StatsReset, - Rows: make([]serverhttp.IORow, 0, len(snap.Rows)), + Instance: instance, + CapturedAt: snap.CapturedAt, + VersionNum: snap.VersionNum, + OpBytes: snap.OpBytes, + TrackIoTiming: snap.TrackIOTiming, + TrackWalIoTiming: snap.TrackWALIOTiming, + StatsReset: snap.StatsReset, + Rows: make([]serverhttp.IORow, 0, len(snap.Rows)), } for _, r := range snap.Rows { diff --git a/backend/internal/http/v1_io_test.go b/backend/internal/http/v1_io_test.go index 14a7b02a..8be3f7ec 100644 --- a/backend/internal/http/v1_io_test.go +++ b/backend/internal/http/v1_io_test.go @@ -147,6 +147,33 @@ func TestIOHistoryMetaReportsEpochShifts(t *testing.T) { } } +// The two timing GUCs are reported apart: WAL times survive track_io_timing off. +func TestIOHistoryMetaSeparatesWALTiming(t *testing.T) { + t.Parallel() + + at := time.Date(2026, 8, 22, 3, 0, 0, 0, time.UTC) + earliest, latest := at.Add(-24*time.Hour), at + + metas := []statio.Meta{ + {CapturedAt: at.Add(-2 * time.Minute), VersionNum: 180001, TrackWALIOTiming: true}, + {CapturedAt: at.Add(-time.Minute), VersionNum: 180001, TrackWALIOTiming: true}, + } + + meta := ioHistoryMeta("h1", &earliest, &latest, metas) + + if meta.TrackIoTiming || meta.TrackIoTimingChanged { + t.Error("relation timing was off the whole period") + } + + if !meta.TrackWalIoTiming { + t.Error("track_wal_io_timing must reflect the newest capture") + } + + if meta.TrackWalIoTimingChanged { + t.Error("track_wal_io_timing held steady") + } +} + func TestIOHistoryMetaWithoutSnapshots(t *testing.T) { t.Parallel() diff --git a/backend/internal/mcpserver/io.go b/backend/internal/mcpserver/io.go index 9616debb..f4a4a1f9 100644 --- a/backend/internal/mcpserver/io.go +++ b/backend/internal/mcpserver/io.go @@ -250,6 +250,7 @@ func ioSummary(ctx context.Context, c *DashaClient, a ioSummaryArgs) (any, error Requested: out.Requested, Seen: seen, Measured: window.DurationSeconds > 0, + Complete: window.Complete, Filtered: req.Filtered, }) @@ -429,6 +430,14 @@ func ioTrendParams(a ioTrendArgs) (ioRequest, string) { }, "" } +// Relation times answer to track_io_timing, the 'wal' object's to +// track_wal_io_timing; the series are grouped by context, so either setting +// alone can put real times in a bucket. +func ioTimingMeasured(m apiclient.IOHistoryMeta) bool { + return m.TrackIoTiming || m.TrackIoTimingChanged || + m.TrackWalIoTiming || m.TrackWalIoTimingChanged +} + type ioTrendScan struct { Window ioRange Incomplete map[int64]bool @@ -448,7 +457,7 @@ func ioTrend(ctx context.Context, c *DashaClient, a ioTrendArgs) (any, error) { } metrics := slices.Clone(ioTrendMetrics) - if hist.Meta.TrackIoTiming || hist.Meta.TrackIoTimingChanged { + if ioTimingMeasured(hist.Meta) { metrics = append(metrics, ioTrendTimeMetrics...) } @@ -488,6 +497,7 @@ func ioTrend(ctx context.Context, c *DashaClient, a ioTrendArgs) (any, error) { Requested: out.Requested, Seen: scan.Seen, Measured: scan.Measured, + Complete: len(scan.Incomplete) == 0, Filtered: req.Filtered, }) @@ -588,6 +598,7 @@ type ioEmptyInput struct { Requested ioRange Seen int Measured bool + Complete bool Filtered bool } @@ -619,6 +630,8 @@ func ioEmptyReason(ctx context.Context, c *DashaClient, in ioEmptyInput) (string return "no_snapshots_in_window", "" case !in.Measured: return "no_comparable_snapshots", "" + case !in.Complete: + return "no_io_in_measured_part", "" } return "no_io", "" diff --git a/backend/internal/mcpserver/io_test.go b/backend/internal/mcpserver/io_test.go index 12c375c1..660eb5f7 100644 --- a/backend/internal/mcpserver/io_test.go +++ b/backend/internal/mcpserver/io_test.go @@ -48,12 +48,14 @@ func ioFakeAPI(t *testing.T, history string, ioStatus int) *DashaClient { func ioLiveMeta(earliest string) string { return `"meta":{"instance":"h1","earliest_at":"` + earliest + `","latest_at":"` + time.Now().UTC().Format(time.RFC3339) + `","track_io_timing":true,` + - `"track_io_timing_changed":false,"version_changed":false}` + `"track_io_timing_changed":false,"track_wal_io_timing":true,` + + `"track_wal_io_timing_changed":false,"version_changed":false}` } const ioHistoryJSON = `{ "meta": {"instance":"h1","earliest_at":"2026-08-01T00:00:00Z","latest_at":"2026-08-29T10:00:00Z", - "track_io_timing":true,"track_io_timing_changed":false,"version_changed":false}, + "track_io_timing":true,"track_io_timing_changed":false, + "track_wal_io_timing":true,"track_wal_io_timing_changed":false,"version_changed":false}, "series": [ {"key":{"context":"normal"}, "points":[{"from":"2026-08-29T09:00:00Z","to":"2026-08-29T10:00:00Z","duration_seconds":3600, @@ -206,6 +208,12 @@ func TestIOSummary_EmptyReason(t *testing.T) { "series":[{"key":{"context":"normal"},"points":[{"from":"2026-08-29T09:00:00Z", "to":"2026-08-29T10:00:00Z","duration_seconds":0,"complete":false,"values":{}}]}]}` + // A reset cut the window in half; the half that was measured saw no I/O. + partialIdle := `{` + ioLiveMeta("2026-08-01T00:00:00Z") + `, + "series":[{"key":{"context":"normal"},"points":[{"from":"2026-08-29T09:00:00Z", + "to":"2026-08-29T10:00:00Z","duration_seconds":1800,"complete":false, + "values":{"hits":10}}]}]}` + tests := []struct { name string history string @@ -220,6 +228,7 @@ func TestIOSummary_EmptyReason(t *testing.T) { {"no capture fell inside the window", gapHistory, http.StatusOK, "no_snapshots_in_window"}, {"every interval spans a reset", brokenEpoch, http.StatusOK, "no_comparable_snapshots"}, {"genuinely no physical I/O", idleOnly, http.StatusOK, "no_io"}, + {"quiet only where it was measured", partialIdle, http.StatusOK, "no_io_in_measured_part"}, } for _, tt := range tests { @@ -280,7 +289,8 @@ func TestIOSummary_EmptyKeepsTotalsAndWindow(t *testing.T) { const ioTrendJSON = `{ "meta": {"instance":"h1","earliest_at":"2026-08-01T00:00:00Z","latest_at":"2026-08-29T10:00:00Z", - "track_io_timing":true,"track_io_timing_changed":false,"version_changed":false}, + "track_io_timing":true,"track_io_timing_changed":false, + "track_wal_io_timing":true,"track_wal_io_timing_changed":false,"version_changed":false}, "series": [ {"key":{"context":"vacuum"}, "points":[ @@ -377,6 +387,35 @@ func TestIOTrend_TimeMetricsFollowTracking(t *testing.T) { } } +// WAL times answer to their own GUC: track_io_timing off does not zero them. +func TestIOTrend_WALTimingAloneKeepsTimeMetrics(t *testing.T) { + t.Parallel() + + walOnly := `{"meta":{"instance":"h1","earliest_at":"2026-08-01T00:00:00Z", + "latest_at":"2026-08-29T10:00:00Z","track_io_timing":false, + "track_io_timing_changed":false,"track_wal_io_timing":true, + "track_wal_io_timing_changed":false,"version_changed":false}, + "series":[{"key":{"context":"normal"},"points":[{"from":"2026-08-29T09:00:00Z", + "to":"2026-08-29T10:00:00Z","duration_seconds":3600,"complete":true, + "values":{"writes":40,"write_time":120}}]}]}` + + got, _ := ioTrend(context.Background(), ioFakeAPI(t, walOnly, http.StatusOK), + ioTrendArgs{Cluster: "demo", Instance: "h1"}) //nolint:exhaustruct + + res, ok := got.(ioTrendResult) + if !ok { + t.Fatalf("ioTrend returned %T", got) + } + + if !slices.Contains(res.Metrics, "write_time") { + t.Errorf("metrics = %v, must keep time metrics when track_wal_io_timing is on", res.Metrics) + } + + if len(res.Series) != 1 || res.Series[0].Points[0].Values["write_time"] != 120 { + t.Errorf("the WAL write_time must survive into the series: %+v", res.Series) + } +} + func TestIOSummaryParams_Defaults(t *testing.T) { t.Parallel() @@ -510,6 +549,22 @@ func TestParseSince_AcceptsDays(t *testing.T) { } } +// A day count that overflows time.Duration must not wrap into a short window +// that then slips past the 31-day cap. +func TestParseSince_RejectsOverflowingDays(t *testing.T) { + t.Parallel() + + for _, since := range []string{"106752d", "9223372036854775807d"} { + if d, err := parseSince(since); err == nil { + t.Errorf("parseSince(%s) = %v, want an error", since, d) + } + } + + if _, _, _, msg := ioWindow("106752d", "", "", ioTrendDefaultSince); msg == "" { + t.Errorf("ioWindow must reject a day count it cannot represent") + } +} + func TestResolveWindow_DefaultIsPerTool(t *testing.T) { t.Parallel() @@ -628,3 +683,22 @@ func TestIOTrend_AllIncompleteIsNotNoIO(t *testing.T) { t.Errorf("incomplete_points = %d, want 2", res.IncompletePoints) } } + +// A gap in the record cannot carry an all-clear for the window as asked. +func TestIOTrend_PartialWindowIsNotNoIO(t *testing.T) { + t.Parallel() + + partial := `{` + ioLiveMeta("2026-08-01T00:00:00Z") + `, + "series":[{"key":{"context":"normal"},"points":[ + {"from":"2026-08-29T08:00:00Z","to":"2026-08-29T09:00:00Z","duration_seconds":3600, + "complete":true,"values":{"hits":10}}, + {"from":"2026-08-29T09:00:00Z","to":"2026-08-29T10:00:00Z","duration_seconds":1800, + "complete":false,"values":{"hits":10}}]}]}` + + res := ioTrendOf(t, ioFakeAPI(t, partial, http.StatusOK), + ioTrendArgs{Cluster: "demo", Instance: "h1"}) //nolint:exhaustruct + + if res.EmptyReason != "no_io_in_measured_part" { + t.Errorf("empty_reason = %q, want no_io_in_measured_part", res.EmptyReason) + } +} diff --git a/backend/internal/mcpserver/kb/en/pg-stat-io.md b/backend/internal/mcpserver/kb/en/pg-stat-io.md index 1c58973c..37d8a7a1 100644 --- a/backend/internal/mcpserver/kb/en/pg-stat-io.md +++ b/backend/internal/mcpserver/kb/en/pg-stat-io.md @@ -41,14 +41,22 @@ look at `checkpoint_timeout`, `max_wal_size` and checkpoint frequency via rather than being cut off in the tail. ### zero time is not fast -With `track_io_timing` off, `read_time`, `write_time`, `extend_time`, -`writeback_time` and `fsync_time` are zero by construction. Check -`meta.track_io_timing` before drawing any latency conclusion; -`meta.track_io_timing_changed` means the setting was toggled inside the window, -so the times cover only part of it. The tools omit `avg_read_ms` / +Two settings govern the time counters and move independently: +`track_io_timing` covers relation and temp-relation I/O, `track_wal_io_timing` +covers the `wal` object alone (PostgreSQL 18 and newer). Under whichever is +off, `read_time`, `write_time`, `extend_time`, `writeback_time` and +`fsync_time` are zero by construction. Check `meta.track_io_timing` before any +conclusion about relation latency and `meta.track_wal_io_timing` before any +about WAL; the matching `*_changed` flag means that setting was toggled inside +the window, so its times cover only part of it. The tools omit `avg_read_ms` / `avg_write_ms` rather than report `0.00`, so an absent latency means "not measured", never "instant". +`io_trend` groups by context, which puts WAL and relation rows in the same +series, so it carries the time metrics when either setting is on: with only +`track_wal_io_timing` on, the times in a bucket are WAL's alone. Separate them +with `io_summary` at `group_by=full`, where `wal` is a row of its own. + ## Contexts - **normal** — ordinary buffered access through shared buffers. The baseline. @@ -90,17 +98,19 @@ true gap in the record. ## The window that was actually read -`requested` is the window asked for, `window` the one the data covers; a -difference between them is snapshot coverage, not load. A request longer than -31 days is cut back to 31 and flagged `window_capped: true` — a conclusion of -the form "nothing changed in the last 90 days" cannot be drawn from a capped -window. +`requested` is the window the tools read and `window` the one the data covers; +a difference between them is snapshot coverage, not load. A request longer than +31 days is cut back to 31 and flagged `window_capped: true`; `requested` then +holds the capped range, not the range asked for, and the original ask is +nowhere in the result. A conclusion of the form "nothing changed in the last 90 +days" cannot be drawn from a capped window. ## Empty results An empty result is not an answer until `empty_reason` says which one it is. -Exactly one of these values means the instance was idle; the rest mean the -question went unanswered: +Only `no_io` means the instance was idle across the whole window, +`no_io_in_measured_part` answers for part of it, and the rest mean the question +went unanswered: - **unsupported_version** — the host runs PostgreSQL 15 or older and has no `pg_stat_io`. Nothing about its I/O can be read this way; fall back to @@ -126,9 +136,19 @@ question went unanswered: validated, and `'autovacuum'` (the real value is `'autovacuum worker'`) silently matches nothing. Re-run without the filter before concluding anything. -- **no_io** — snapshots cover the window, they are comparable, and there - genuinely was no physical I/O. This is the only one that is a real answer, - and even here `totals` may show heavy cache activity. +- **no_io_in_measured_part** — comparable captures exist and show no I/O, but a + counter epoch broke inside the window, so part of it was never measured. The + quiet covers the measured part alone: `io_summary` reports it as + `window.complete: false`, `io_trend` as `incomplete_points` with a + `coverage_pct` per bucket. Not an all-clear for the window as asked. +- **no_io** — snapshots cover the window, they are comparable, and none of the + counters the tool classifies moved. For `io_summary` that is every counter + except `hits`. `io_trend` classifies `reads`, `writes`, `extends` and their + byte and time counters only, so `fsyncs`, `evictions`, `reuses` and + `writebacks` may be non-zero behind an `io_trend` `no_io` — read `io_summary` + over the same window before calling the instance quiet. This is the only + value that is a full answer, and even here `totals` may show heavy cache + activity. ## Where to go next diff --git a/backend/internal/mcpserver/kb/ru/pg-stat-io.md b/backend/internal/mcpserver/kb/ru/pg-stat-io.md index 6de4d3ff..8284870b 100644 --- a/backend/internal/mcpserver/kb/ru/pg-stat-io.md +++ b/backend/internal/mcpserver/kb/ru/pg-stat-io.md @@ -42,13 +42,22 @@ сама по себе, а не отсекается в хвосте. ### ноль во времени — это не «быстро» -При выключенном `track_io_timing` `read_time`, `write_time`, `extend_time`, -`writeback_time` и `fsync_time` равны нулю по построению. Перед любым выводом -о латентности проверять `meta.track_io_timing`; -`meta.track_io_timing_changed` значит, что настройку переключали внутри окна и -времена покрывают только его часть. Инструменты не выводят `avg_read_ms` / -`avg_write_ms` вместо того, чтобы показать `0.00`: отсутствие латентности -значит «не измерялось», а не «мгновенно». +Временными счётчиками управляют две настройки, и они меняются независимо: +`track_io_timing` отвечает за I/O отношений и временных отношений, +`track_wal_io_timing` — только за объект `wal` (PostgreSQL 18 и новее). При +выключенной любой из них соответствующие `read_time`, `write_time`, +`extend_time`, `writeback_time` и `fsync_time` равны нулю по построению. Перед +выводом о латентности отношений проверять `meta.track_io_timing`, о WAL — +`meta.track_wal_io_timing`; парный флаг `*_changed` значит, что настройку +переключали внутри окна и её времена покрывают только его часть. Инструменты +не выводят `avg_read_ms` / `avg_write_ms` вместо того, чтобы показать `0.00`: +отсутствие латентности значит «не измерялось», а не «мгновенно». + +`io_trend` группирует по контексту, и строки WAL и отношений попадают в одну +серию, поэтому временные метрики он несёт, если включена хотя бы одна из +настроек: при одной только `track_wal_io_timing` времена в бакете — это +времена WAL. Разделить их можно через `io_summary` с `group_by=full`, где +`wal` — отдельная строка. ## Контексты @@ -93,16 +102,19 @@ ## Какое окно прочитано на самом деле -`requested` — запрошенное окно, `window` — то, которое покрывают данные; -расхождение между ними — это покрытие снимками, а не нагрузка. Запрос длиннее -31 дня обрезается до 31 и помечается `window_capped: true` — вывод вида «за -последние 90 дней ничего не менялось» из обрезанного окна не следует. +`requested` — окно, которое прочитали инструменты, `window` — то, которое +покрывают данные; расхождение между ними — это покрытие снимками, а не +нагрузка. Запрос длиннее 31 дня обрезается до 31 и помечается +`window_capped: true`; тогда в `requested` лежит обрезанный диапазон, а не +запрошенный, и исходного запроса в ответе нет нигде. Вывод вида «за последние +90 дней ничего не менялось» из обрезанного окна не следует. ## Пустые ответы Пустой результат не является ответом, пока `empty_reason` не скажет, какой это -из случаев. Ровно одно значение означает, что инстанс простаивал; остальные — -что на вопрос не ответили: +из случаев. Только `no_io` означает, что инстанс простаивал всё окно, +`no_io_in_measured_part` отвечает за его часть, остальные — что на вопрос не +ответили: - **unsupported_version** — на хосте PostgreSQL 15 или старше, `pg_stat_io` там нет. Про его I/O этим путём ничего не узнать; остаются `wait_events` и @@ -128,9 +140,19 @@ в котором вообще нет снимков. `backend_type` не валидируется, и `'autovacuum'` (настоящее значение — `'autovacuum worker'`) молча не совпадает ни с чем. Прежде чем делать выводы, повторить запрос без фильтра. -- **no_io** — снимки окно покрывают, они сравнимы, физического I/O - действительно не было. Только это содержательный ответ, и даже здесь - `totals` может показывать активную работу с кэшем. +- **no_io_in_measured_part** — сравнимые снимки есть и I/O не показывают, но + внутри окна оборвалась эпоха счётчиков, и часть его не измерялась. Тишина + относится только к измеренной части: `io_summary` сообщает об этом через + `window.complete: false`, `io_trend` — через `incomplete_points` и + `coverage_pct` у бакета. Это не «всё чисто» по запрошенному окну. +- **no_io** — снимки окно покрывают, они сравнимы, и ни один из счётчиков, + которые инструмент классифицирует, не сдвинулся. У `io_summary` это все + счётчики, кроме `hits`. `io_trend` классифицирует только `reads`, `writes`, + `extends` и их байтовые и временные счётчики, поэтому за `no_io` от + `io_trend` могут стоять ненулевые `fsyncs`, `evictions`, `reuses` и + `writebacks` — прежде чем называть инстанс спокойным, прочитать `io_summary` + за то же окно. Только это значение — полный ответ, и даже здесь `totals` + может показывать активную работу с кэшем. ## Куда идти дальше diff --git a/backend/internal/mcpserver/prompts.go b/backend/internal/mcpserver/prompts.go index 52890da2..4e2ce33a 100644 --- a/backend/internal/mcpserver/prompts.go +++ b/backend/internal/mcpserver/prompts.go @@ -55,7 +55,7 @@ Investigating: - query_compare needs snapshot IDs from list_snapshots. - search_logs works only on clusters with supports_logs=true (see list_clusters) and is rate-limited per user because every call reaches the Yandex Cloud API: combine all filters into one call, keep dedup on, and after a 429 wait ~30 seconds instead of retrying immediately. - schema_lint answers a different question from every other tool: what is wrong with the STRUCTURE, not what is happening now. Read its skipped list before concluding anything — a check that could not run says nothing about the schema, and reporting "clean" over a non-empty skipped list is a false all-clear. Two findings have a fix that is NOT the obvious one: sequence_exhaustion on an owned_column_type of 'integer' needs the column type changed (a table rewrite, needs a window), not just ALTER SEQUENCE; and no_primary_key on a table whose unique index is nullable cannot be answered with "you already have a unique index" — that index is no replica identity. Read dasha://kb/schema-checks before advising on a code you do not know. -- "Who is doing all this I/O?" -> io_summary, and io_trend for when it started. These are the only tools that see non-client I/O: autovacuum, the checkpointer, the WAL writer. They need PostgreSQL 16+ (older hosts answer empty with empty_reason='unsupported_version', which does NOT mean no I/O) and snapshot storage. Never read a zero time metric as "fast" without checking meta.track_io_timing, and never read an incomplete io_trend point as a lull — its counters cover only coverage_pct of the bucket. An empty answer is never proof of an idle instance: only empty_reason='no_io' says that, and every other value means the question went unanswered. Read dasha://kb/pg-stat-io before interpreting the counters. +- "Who is doing all this I/O?" -> io_summary, and io_trend for when it started. These are the only tools that see non-client I/O: autovacuum, the checkpointer, the WAL writer. They need PostgreSQL 16+ (older hosts answer empty with empty_reason='unsupported_version', which does NOT mean no I/O) and snapshot storage. Never read a zero time metric as "fast" without checking meta.track_io_timing, and never read an incomplete io_trend point as a lull — its counters cover only coverage_pct of the bucket. An empty answer is never proof of an idle instance: only empty_reason='no_io' says that, 'no_io_in_measured_part' answers for the measured part of the window alone, and every other value means the question went unanswered. Read dasha://kb/pg-stat-io before interpreting the counters. - If unsure how to interpret a result or which tool to call next, read the resources first: dasha://kb/workflow (complaint-to-tool-chain playbooks), dasha://kb/health-rules (rule thresholds and first actions), dasha://kb/schema-checks (schema defect codes and their fixes), dasha://kb/wait-events (wait event glossary), dasha://kb/pg-stat-io (how to read the I/O counters). If a result is refused as too large, narrow it (one database, a smaller range, or a more specific tool).`, @@ -134,7 +134,7 @@ If a result is refused as too large, narrow it (one database, a smaller range, o - query_compare требует ID снимков из list_snapshots. - search_logs работает только на кластерах с supports_logs=true (см. list_clusters) и лимитирован per-user, т.к. каждый вызов уходит в Yandex Cloud API: собирайте все фильтры в один вызов, держите dedup включённым, после 429 ждите ~30 секунд вместо немедленного повтора. - schema_lint отвечает не на тот вопрос, что остальные инструменты: что не так со СТРУКТУРОЙ, а не что происходит сейчас. Прежде чем делать выводы, прочитайте его список skipped — проверка, которая не выполнилась, не говорит о схеме ничего, и «всё чисто» при непустом skipped — ложное «отбой». У двух находок правильное лечение НЕ очевидное: sequence_exhaustion с owned_column_type = 'integer' требует смены типа колонки (переписывание таблицы, нужно окно), а не только ALTER SEQUENCE; а no_primary_key на таблице с nullable уникальным индексом нельзя закрывать фразой «у вас же есть unique» — такой индекс не годится в replica identity. Перед советами по незнакомому коду читайте dasha://kb/schema-checks. -- «Кто делает весь этот I/O?» -> io_summary, а io_trend — когда он начался. Только эти инструменты видят неклиентский I/O: автовакуум, чекпойнтер, walwriter. Нужен PostgreSQL 16+ (на старых хостах ответ пустой с empty_reason='unsupported_version', и это НЕ значит «I/O нет») и хранилище снимков. Никогда не читайте нулевое время как «быстро», не проверив meta.track_io_timing, и никогда не читайте неполную точку io_trend как затишье — её счётчики покрывают лишь coverage_pct бакета. Пустой ответ не доказывает простой: это говорит только empty_reason='no_io', любое другое значение значит, что на вопрос не ответили. Перед трактовкой счётчиков читайте dasha://kb/pg-stat-io. +- «Кто делает весь этот I/O?» -> io_summary, а io_trend — когда он начался. Только эти инструменты видят неклиентский I/O: автовакуум, чекпойнтер, walwriter. Нужен PostgreSQL 16+ (на старых хостах ответ пустой с empty_reason='unsupported_version', и это НЕ значит «I/O нет») и хранилище снимков. Никогда не читайте нулевое время как «быстро», не проверив meta.track_io_timing, и никогда не читайте неполную точку io_trend как затишье — её счётчики покрывают лишь coverage_pct бакета. Пустой ответ не доказывает простой: это говорит только empty_reason='no_io', 'no_io_in_measured_part' отвечает лишь за измеренную часть окна, любое другое значение значит, что на вопрос не ответили. Перед трактовкой счётчиков читайте dasha://kb/pg-stat-io. - Если непонятно, как трактовать результат или какой инструмент звать дальше — сначала прочитайте ресурсы: dasha://kb/workflow (сценарии «жалоба -> цепочка»), dasha://kb/health-rules (пороги правил и первые действия), dasha://kb/schema-checks (коды дефектов схемы и их лечение), dasha://kb/wait-events (глоссарий wait events), dasha://kb/pg-stat-io (как читать счётчики I/O). Если результат отклонён как слишком большой — сузьте запрос (одна база, меньший диапазон или более специфичный инструмент).`, diff --git a/backend/internal/mcpserver/tools.go b/backend/internal/mcpserver/tools.go index 264d46a0..eaf5762d 100644 --- a/backend/internal/mcpserver/tools.go +++ b/backend/internal/mcpserver/tools.go @@ -3,6 +3,8 @@ package mcpserver import ( "cmp" "context" + "errors" + "math" "strconv" "strings" "time" @@ -697,11 +699,15 @@ func registerTools(s *mcp.Server, c *DashaClient) { "by backend_type x object x context and needs top. Requires PostgreSQL 16 or newer: older hosts have " + "no pg_stat_io at all and come back empty with empty_reason='unsupported_version', which does NOT " + "mean 'no I/O'. Every empty answer carries an empty_reason, and only 'no_io' means the instance was " + - "idle: 'no_snapshots_in_window', 'no_comparable_snapshots', 'window_after_history' and " + - "'no_io_matching_filter' all mean the question went unanswered — check totals and meta before " + - "reporting an all-clear. With track_io_timing off every time metric is zero by construction — a " + - "missing measurement, not a missing load; meta.track_io_timing says which, and avg_read_ms/" + - "avg_write_ms are absent rather than 0. A window longer than 31 days is cut back to it and flagged " + + "idle across the whole window: 'no_io_in_measured_part' means a counter epoch broke inside it and " + + "the quiet covers the measured part alone, while 'no_snapshots_in_window', " + + "'no_comparable_snapshots', 'window_after_history' and 'no_io_matching_filter' all mean the " + + "question went unanswered — check totals and meta before reporting an all-clear. Time counters " + + "answer to two settings: track_io_timing for relation and temp relation rows, " + + "track_wal_io_timing for the 'wal' object (PostgreSQL 18+). Under whichever is off every time " + + "metric is zero by construction — a missing measurement, not a missing load; meta.track_io_timing " + + "and meta.track_wal_io_timing say which, and avg_read_ms/avg_write_ms are absent rather than 0. " + + "A window longer than 31 days is cut back to it and flagged " + "window_capped. pg_stat_io is instance-wide: there is no database parameter. A counter absent " + "from values is zero. Needs snapshot storage (501 otherwise). " + "Read dasha://kb/pg-stat-io before interpreting the numbers.", @@ -718,9 +724,14 @@ func registerTools(s *mcp.Server, c *DashaClient) { "coverage_pct: its counters are real but measure only that share of the bucket's span, so it is not " + "comparable with a complete point and a lower number there is not a drop in load (incomplete_points " + "counts such buckets). An incomplete point with no values at all measured nothing. In a complete " + - "point an absent metric is zero. Same preconditions as io_summary, including empty_reason on an " + - "empty answer: PostgreSQL 16+, time metrics are zero unless track_io_timing is on, instance-wide " + - "(no database), snapshot storage required (501 otherwise). " + + "point an absent metric is zero. This series carries reads, writes, extends and their byte and " + + "time counters only, so its empty_reason='no_io' does not rule out fsyncs, evictions, reuses or " + + "writebacks — confirm with io_summary over the same window. Same preconditions as io_summary, " + + "including empty_reason on an empty answer: PostgreSQL 16+, instance-wide (no database), snapshot " + + "storage required (501 otherwise). Time metrics are carried when track_io_timing or " + + "track_wal_io_timing is on; grouping is by context, which merges WAL and relation rows, so with " + + "only track_wal_io_timing on a bucket's times are WAL's alone — read meta.track_io_timing and " + + "meta.track_wal_io_timing before attributing them, and split them with io_summary group_by=full. " + "Read dasha://kb/pg-stat-io before interpreting the numbers.", }, func(ctx context.Context, _ *mcp.CallToolRequest, a ioTrendArgs) (*mcp.CallToolResult, any, error) { return jsonResult(ioTrend(ctx, c, a)) @@ -810,6 +821,8 @@ func resolveWindow(since, from, to string, def time.Duration) (time.Time, time.T return start, end, "" } +const maxSinceDays = int64(math.MaxInt64 / (24 * time.Hour)) + // time.ParseDuration has no day unit; models write '7d'. func parseSince(since string) (time.Duration, error) { days, ok := strings.CutSuffix(since, "d") @@ -817,11 +830,15 @@ func parseSince(since string) (time.Duration, error) { return time.ParseDuration(since) } - n, err := strconv.Atoi(days) + n, err := strconv.ParseInt(days, 10, 64) if err != nil { return 0, err } + if n > maxSinceDays { + return 0, errors.New("day count out of range") + } + return time.Duration(n) * 24 * time.Hour, nil } diff --git a/backend/internal/query/sql/statio/snapshot/180000/snapshot.tmpl.sql b/backend/internal/query/sql/statio/snapshot/180000/snapshot.tmpl.sql index 330c6c63..46a7289f 100644 --- a/backend/internal/query/sql/statio/snapshot/180000/snapshot.tmpl.sql +++ b/backend/internal/query/sql/statio/snapshot/180000/snapshot.tmpl.sql @@ -3,8 +3,9 @@ SELECT s.object, s.context, s.stats_reset, - current_setting('track_io_timing')::boolean AS track_io_timing, - (SELECT max(op_bytes)::int FROM pg_stat_io) AS op_bytes, + current_setting('track_io_timing')::boolean AS track_io_timing, + coalesce(current_setting('track_wal_io_timing', true)::boolean, false) AS track_wal_io_timing, + (SELECT max(op_bytes)::int FROM pg_stat_io) AS op_bytes, jsonb_strip_nulls(jsonb_build_object( 'reads', s.reads, 'read_time', round(s.read_time)::bigint, diff --git a/backend/internal/query/sql/statio/snapshot/snapshot.tmpl.sql b/backend/internal/query/sql/statio/snapshot/snapshot.tmpl.sql index 8bd206c6..42f79fea 100644 --- a/backend/internal/query/sql/statio/snapshot/snapshot.tmpl.sql +++ b/backend/internal/query/sql/statio/snapshot/snapshot.tmpl.sql @@ -3,8 +3,9 @@ SELECT s.object, s.context, s.stats_reset, - current_setting('track_io_timing')::boolean AS track_io_timing, - NULL::int AS op_bytes, + current_setting('track_io_timing')::boolean AS track_io_timing, + coalesce(current_setting('track_wal_io_timing', true)::boolean, false) AS track_wal_io_timing, + NULL::int AS op_bytes, jsonb_strip_nulls(jsonb_build_object( 'reads', s.reads, 'read_bytes', s.read_bytes, diff --git a/backend/internal/repository/statio.go b/backend/internal/repository/statio.go index 201b1b55..0f0df74f 100644 --- a/backend/internal/repository/statio.go +++ b/backend/internal/repository/statio.go @@ -55,7 +55,8 @@ func (p *PgxPool) GetIOSample(ctx context.Context, clusterName, instanceName str counters []byte ) - if err := rows.Scan(&r.BackendType, &r.Object, &r.Context, &reset, &snap.TrackIOTiming, &opBytes, &counters); err != nil { + if err := rows.Scan(&r.BackendType, &r.Object, &r.Context, &reset, + &snap.TrackIOTiming, &snap.TrackWALIOTiming, &opBytes, &counters); err != nil { return nil, fmt.Errorf("GetIOSample scan | %w", err) } diff --git a/backend/internal/repository/statio_integration_test.go b/backend/internal/repository/statio_integration_test.go index b4074622..b3c6ab1f 100644 --- a/backend/internal/repository/statio_integration_test.go +++ b/backend/internal/repository/statio_integration_test.go @@ -52,7 +52,7 @@ func TestStatioSnapshotTemplate(t *testing.T) { ) require.NoError(t, rows.Scan(&r.BackendType, &r.Object, &r.Context, &reset, - &snap.TrackIOTiming, &opBytes, &counters)) + &snap.TrackIOTiming, &snap.TrackWALIOTiming, &opBytes, &counters)) if opBytes.Valid { v := int(opBytes.Int32) diff --git a/backend/internal/statio/statio.go b/backend/internal/statio/statio.go index c191836b..8969e193 100644 --- a/backend/internal/statio/statio.go +++ b/backend/internal/statio/statio.go @@ -40,25 +40,30 @@ type Snapshot struct { // byte counters instead and leave this nil. OpBytes *int TrackIOTiming bool - StatsReset *time.Time - Rows []Row + // The 'wal' object rows (PostgreSQL 18 and newer) time themselves by this + // setting, not by TrackIOTiming. + TrackWALIOTiming bool + StatsReset *time.Time + Rows []Row } // Meta is one capture's header: what a read plan needs before any matrix body // is fetched. type Meta struct { - CapturedAt time.Time - VersionNum int - TrackIOTiming bool - StatsReset *time.Time + CapturedAt time.Time + VersionNum int + TrackIOTiming bool + TrackWALIOTiming bool + StatsReset *time.Time } func (s Snapshot) Meta() Meta { return Meta{ - CapturedAt: s.CapturedAt, - VersionNum: s.VersionNum, - TrackIOTiming: s.TrackIOTiming, - StatsReset: s.StatsReset, + CapturedAt: s.CapturedAt, + VersionNum: s.VersionNum, + TrackIOTiming: s.TrackIOTiming, + TrackWALIOTiming: s.TrackWALIOTiming, + StatsReset: s.StatsReset, } } diff --git a/backend/internal/storage/migrate.go b/backend/internal/storage/migrate.go index 492675ae..a97cb9e9 100644 --- a/backend/internal/storage/migrate.go +++ b/backend/internal/storage/migrate.go @@ -273,17 +273,22 @@ ALTER TABLE autosnapshot_config_global DROP COLUMN IF EXISTS hot_interval` // decide which bodies a request has to touch at all. createIOSnapshotSQL = ` CREATE TABLE IF NOT EXISTS io_snapshot ( - cluster_name text NOT NULL, - instance text NOT NULL, - captured_at timestamptz NOT NULL DEFAULT now(), - version_num int NOT NULL, - op_bytes int, - track_io_timing boolean NOT NULL, - stats_reset timestamptz, - rows jsonb NOT NULL, + cluster_name text NOT NULL, + instance text NOT NULL, + captured_at timestamptz NOT NULL DEFAULT now(), + version_num int NOT NULL, + op_bytes int, + track_io_timing boolean NOT NULL, + track_wal_io_timing boolean NOT NULL DEFAULT false, + stats_reset timestamptz, + rows jsonb NOT NULL, CONSTRAINT io_snapshot_pkey PRIMARY KEY (cluster_name, instance, captured_at) ) PARTITION BY RANGE (captured_at)` + addIOSnapshotWALTimingSQL = ` +ALTER TABLE io_snapshot + ADD COLUMN IF NOT EXISTS track_wal_io_timing boolean NOT NULL DEFAULT false` + addAutosnapshotIOConfigSQL = ` ALTER TABLE autosnapshot_config_global ADD COLUMN IF NOT EXISTS io_enabled boolean NOT NULL DEFAULT true, @@ -369,6 +374,7 @@ func (s *Storage) migrate(ctx context.Context, logger *zap.Logger) error { dropAutosnapshotHotIntervalSQL, addSnapshotDatabasesSQL, createIOSnapshotSQL, + addIOSnapshotWALTimingSQL, addAutosnapshotIOConfigSQL, } { if _, err := s.ddlPool.Exec(ctx, ddl); err != nil { diff --git a/backend/internal/storage/statio.go b/backend/internal/storage/statio.go index ce3aa971..b6774d00 100644 --- a/backend/internal/storage/statio.go +++ b/backend/internal/storage/statio.go @@ -42,10 +42,10 @@ func (s *Storage) InsertIOSnapshot( _, err = s.pool.Exec(ctx, ` INSERT INTO io_snapshot (cluster_name, instance, captured_at, version_num, op_bytes, - track_io_timing, stats_reset, rows) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb)`, + track_io_timing, track_wal_io_timing, stats_reset, rows) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb)`, clusterName, instance, snap.CapturedAt, snap.VersionNum, snap.OpBytes, - snap.TrackIOTiming, snap.StatsReset, jsonbArg(body), + snap.TrackIOTiming, snap.TrackWALIOTiming, snap.StatsReset, jsonbArg(body), ) if err != nil { return fmt.Errorf("storage: insert io snapshot: %w", err) @@ -96,7 +96,7 @@ func (s *Storage) GetIOSnapshotMetas( ) ([]statio.Meta, error) { rows, err := s.pool.Query(ctx, ` ( - SELECT captured_at, version_num, track_io_timing, stats_reset + SELECT captured_at, version_num, track_io_timing, track_wal_io_timing, stats_reset FROM io_snapshot WHERE cluster_name = $1 AND instance = $2 AND captured_at < $3 ORDER BY captured_at DESC @@ -104,7 +104,7 @@ func (s *Storage) GetIOSnapshotMetas( ) UNION ALL ( - SELECT captured_at, version_num, track_io_timing, stats_reset + SELECT captured_at, version_num, track_io_timing, track_wal_io_timing, stats_reset FROM io_snapshot WHERE cluster_name = $1 AND instance = $2 AND captured_at >= $3 AND captured_at <= $4 ) @@ -120,7 +120,8 @@ func (s *Storage) GetIOSnapshotMetas( for rows.Next() { var m statio.Meta - if err := rows.Scan(&m.CapturedAt, &m.VersionNum, &m.TrackIOTiming, &m.StatsReset); err != nil { + if err := rows.Scan(&m.CapturedAt, &m.VersionNum, &m.TrackIOTiming, + &m.TrackWALIOTiming, &m.StatsReset); err != nil { return nil, fmt.Errorf("storage: scan io snapshot meta: %w", err) } @@ -142,7 +143,7 @@ func (s *Storage) GetIOSnapshotsAt( } rows, err := s.pool.Query(ctx, ` - SELECT captured_at, version_num, op_bytes, track_io_timing, stats_reset, rows + SELECT captured_at, version_num, op_bytes, track_io_timing, track_wal_io_timing, stats_reset, rows FROM io_snapshot WHERE cluster_name = $1 AND instance = $2 AND captured_at >= $3 AND captured_at <= $4 @@ -163,7 +164,7 @@ func (s *Storage) GetIOSnapshotsAt( ) if err := rows.Scan(&snap.CapturedAt, &snap.VersionNum, &snap.OpBytes, - &snap.TrackIOTiming, &snap.StatsReset, &body); err != nil { + &snap.TrackIOTiming, &snap.TrackWALIOTiming, &snap.StatsReset, &body); err != nil { return nil, fmt.Errorf("storage: scan io snapshot: %w", err) } diff --git a/backend/internal/storage/statio_integration_test.go b/backend/internal/storage/statio_integration_test.go index edccd0bb..721dca91 100644 --- a/backend/internal/storage/statio_integration_test.go +++ b/backend/internal/storage/statio_integration_test.go @@ -36,11 +36,12 @@ func ioTestSnapshot(at time.Time, reset time.Time, reads int64) statio.Snapshot opBytes := 8192 return statio.Snapshot{ - CapturedAt: at, - VersionNum: 170004, - OpBytes: &opBytes, - TrackIOTiming: true, - StatsReset: &reset, + CapturedAt: at, + VersionNum: 170004, + OpBytes: &opBytes, + TrackIOTiming: true, + TrackWALIOTiming: true, + StatsReset: &reset, Rows: []statio.Row{ { Key: statio.Key{BackendType: "client backend", Object: "relation", Context: "normal"}, @@ -70,6 +71,7 @@ func TestIOSnapshotRoundTrip(t *testing.T) { assert.Equal(t, 170004, metas[0].VersionNum) assert.True(t, metas[0].TrackIOTiming) + assert.True(t, metas[0].TrackWALIOTiming) require.NotNil(t, metas[0].StatsReset) assert.WithinDuration(t, reset, *metas[0].StatsReset, time.Second) diff --git a/doc/en/features.md b/doc/en/features.md index 5870993d..8d4a73f1 100644 --- a/doc/en/features.md +++ b/doc/en/features.md @@ -51,7 +51,7 @@ - Server I/O split by backend type, object and context (`normal` / `vacuum` / `bulkread` / `bulkwrite`) - History from scheduled snapshots plus a live mode polling every 3–30 s, or one snapshot per click, that needs no snapshot storage - shared_buffers efficiency, vacuum cost and bulk-operation cost as summary cards -- Count and Time metrics; the Time mode requires `track_io_timing = on`, and the WAL rows additionally require `track_wal_io_timing = on` +- Count and Time metrics; the Time mode needs `track_io_timing = on` for relation rows and `track_wal_io_timing = on` for the WAL rows, and is available whenever either is on - Broken series (statistics reset, restart, major upgrade) are drawn as gaps, never as zeros - Write-ahead log I/O (PostgreSQL 18) in both the chart and the breakdown diff --git a/doc/ru/features.md b/doc/ru/features.md index 626f40c5..764b8469 100644 --- a/doc/ru/features.md +++ b/doc/ru/features.md @@ -51,7 +51,7 @@ - Ввод-вывод сервера в разрезе типа процесса, объекта и контекста (`normal` / `vacuum` / `bulkread` / `bulkwrite`) - История по снимкам плюс живой режим с интервалом опроса 3–30 с или снимком по кнопке, не требующий хранилища снимков - Карточки: эффективность shared_buffers, цена очистки, цена массовых операций -- Метрики «Счётчики» и «Время»; режим времени требует `track_io_timing = on`, а строкам журнала предзаписи нужен ещё и `track_wal_io_timing = on` +- Метрики «Счётчики» и «Время»; режиму времени нужен `track_io_timing = on` для строк отношений и `track_wal_io_timing = on` для строк журнала предзаписи, и он доступен, если включена хотя бы одна из настроек - Разрывы ряда (сброс статистики, рестарт, мажорный апгрейд) рисуются разрывом, а не нулём - Ввод-вывод журнала предзаписи (PostgreSQL 18) — в графике и детализации diff --git a/doc/swagger.yaml b/doc/swagger.yaml index 238a2924..9490c518 100644 --- a/doc/swagger.yaml +++ b/doc/swagger.yaml @@ -4429,6 +4429,7 @@ components: - captured_at - version_num - track_io_timing + - track_wal_io_timing - rows properties: instance: @@ -4446,6 +4447,11 @@ components: track_io_timing: type: boolean description: When false the server collects no I/O times and the Time metrics are absent. + track_wal_io_timing: + type: boolean + description: >- + Governs the time counters of WAL rows (object 'wal', PostgreSQL 18 and newer) + independently of track_io_timing. stats_reset: type: string format: date-time @@ -4514,6 +4520,8 @@ components: - instance - track_io_timing - track_io_timing_changed + - track_wal_io_timing + - track_wal_io_timing_changed - version_changed properties: instance: @@ -4533,6 +4541,14 @@ components: track_io_timing_changed: type: boolean description: The setting was toggled inside the period, so I/O times cover only part of it. + track_wal_io_timing: + type: boolean + description: >- + Value at the newest capture in the period. Governs the time counters of WAL rows + (object 'wal', PostgreSQL 18 and newer) independently of track_io_timing. + track_wal_io_timing_changed: + type: boolean + description: The setting was toggled inside the period, so WAL times cover only part of it. version_changed: type: boolean description: The server was upgraded inside the period; the intervals spanning the upgrade are incomplete. diff --git a/frontend/src/api/models/iOHistoryMeta.ts b/frontend/src/api/models/iOHistoryMeta.ts index 6512ce1d..f593d0e5 100644 --- a/frontend/src/api/models/iOHistoryMeta.ts +++ b/frontend/src/api/models/iOHistoryMeta.ts @@ -18,6 +18,10 @@ export interface IOHistoryMeta { track_io_timing: boolean /** The setting was toggled inside the period, so I/O times cover only part of it. */ track_io_timing_changed: boolean + /** Value at the newest capture in the period. Governs the time counters of WAL rows (object 'wal', PostgreSQL 18 and newer) independently of track_io_timing. */ + track_wal_io_timing: boolean + /** The setting was toggled inside the period, so WAL times cover only part of it. */ + track_wal_io_timing_changed: boolean /** The server was upgraded inside the period; the intervals spanning the upgrade are incomplete. */ version_changed: boolean } diff --git a/frontend/src/api/models/iOSnapshot.ts b/frontend/src/api/models/iOSnapshot.ts index f37913e7..ebba397f 100644 --- a/frontend/src/api/models/iOSnapshot.ts +++ b/frontend/src/api/models/iOSnapshot.ts @@ -18,6 +18,8 @@ export interface IOSnapshot { op_bytes?: number | null /** When false the server collects no I/O times and the Time metrics are absent. */ track_io_timing: boolean + /** Governs the time counters of WAL rows (object 'wal', PostgreSQL 18 and newer) independently of track_io_timing. */ + track_wal_io_timing: boolean /** * The pg_stat_io epoch; a change between two snapshots invalidates the interval between them. * @nullable diff --git a/frontend/src/components/io/IOModeBar.vue b/frontend/src/components/io/IOModeBar.vue index 198509a7..ed1d56d3 100644 --- a/frontend/src/components/io/IOModeBar.vue +++ b/frontend/src/components/io/IOModeBar.vue @@ -11,6 +11,7 @@ const props = defineProps<{ backendTypes: string[] ranges: string[] trackIoTiming: boolean + trackWalIoTiming: boolean liveActive: boolean liveRemaining: number liveLoading: boolean @@ -41,6 +42,13 @@ const backendTypeOptions = computed(() => [ ...props.backendTypes.map((b) => ({ value: b, title: b })), ]) +const timingHint = computed(() => { + if (!props.trackIoTiming && !props.trackWalIoTiming) return t('io.trackIoTimingOff') + if (!props.trackIoTiming) return t('io.trackIoTimingWalOnly') + if (!props.trackWalIoTiming) return t('io.trackWalIoTimingOff') + return null +}) + const windowLabel = computed(() => { if (props.windowSeconds <= 0) return null if (props.windowSeconds < 90) return t('io.window.seconds', { n: Math.round(props.windowSeconds) }) @@ -72,10 +80,10 @@ const windowLabel = computed(() => { {{ t('io.metric.count') }} - + {{ t('io.metric.time') }} - - {{ t('io.trackIoTimingOff') }} + + {{ timingHint }} diff --git a/frontend/src/components/io/useIoLive.ts b/frontend/src/components/io/useIoLive.ts index 70609e1e..db38e261 100644 --- a/frontend/src/components/io/useIoLive.ts +++ b/frontend/src/components/io/useIoLive.ts @@ -144,5 +144,6 @@ export function useIoLive(source: { waiting: computed(() => current.value !== null && previous.value === null), lastAt: computed(() => current.value?.captured_at ?? null), trackIoTiming: computed(() => current.value?.track_io_timing ?? true), + trackWalIoTiming: computed(() => current.value?.track_wal_io_timing ?? false), } } diff --git a/frontend/src/components/io/useIoRows.ts b/frontend/src/components/io/useIoRows.ts index a68fa8c9..d9d47024 100644 --- a/frontend/src/components/io/useIoRows.ts +++ b/frontend/src/components/io/useIoRows.ts @@ -9,6 +9,7 @@ export function useIoRows(source: { liveRows: Ref liveWindowSeconds: Ref liveTrackIoTiming: Ref + liveTrackWalIoTiming: Ref history: Ref matrix: Ref backendType: Ref @@ -66,6 +67,12 @@ export function useIoRows(source: { return source.history.value.meta.track_io_timing }) + const trackWalIoTiming = computed(() => { + if (source.live.value) return source.liveTrackWalIoTiming.value + if (!source.history.value?.series?.length) return false + return source.history.value.meta.track_wal_io_timing + }) + const partial = computed( () => !source.live.value && @@ -81,6 +88,7 @@ export function useIoRows(source: { backendTypes, windowSeconds, trackIoTiming, + trackWalIoTiming, partial, noData, } diff --git a/frontend/src/locales/de_DE.json b/frontend/src/locales/de_DE.json index 6d834b83..9559fa22 100644 --- a/frontend/src/locales/de_DE.json +++ b/frontend/src/locales/de_DE.json @@ -965,7 +965,9 @@ "partialIntervals": "Einige Intervalle dieses Zeitraums sind unvollständig: ein Statistik-Reset, ein Neustart oder ein Major-Upgrade hat die Reihe unterbrochen. Sie werden als Lücke gezeichnet, nicht als Null.", "noData": "Für die aktuelle Auswahl wurde keine I/O-Aktivität erfasst.", "historyUnavailable": "Nur Live-Modus — kein Snapshot-Speicher konfiguriert", - "trackIoTimingOff": "track_io_timing ist auf diesem Host aus — es werden keine I/O-Zeiten erfasst. Zum Nutzen des Zeit-Modus einschalten.", + "trackIoTimingOff": "track_io_timing und track_wal_io_timing sind auf diesem Host aus — es werden keine I/O-Zeiten erfasst.", + "trackIoTimingWalOnly": "track_io_timing ist aus — Zeiten liegen nur für Write-Ahead-Log-Zeilen vor.", + "trackWalIoTimingOff": "track_wal_io_timing ist aus — Write-Ahead-Log-Zeilen haben keine Zeiten.", "mode": { "history": "Verlauf", "live": "Live" diff --git a/frontend/src/locales/en_US.json b/frontend/src/locales/en_US.json index bc0b2e32..3e80d9fd 100644 --- a/frontend/src/locales/en_US.json +++ b/frontend/src/locales/en_US.json @@ -1003,7 +1003,9 @@ "partialIntervals": "Some intervals in this period are incomplete: a statistics reset, a restart or a major upgrade broke the series. They are drawn as gaps, not as zeros.", "noData": "No I/O activity recorded for the current selection.", "historyUnavailable": "Live mode only — snapshot storage is not configured", - "trackIoTimingOff": "track_io_timing is off on this host — no I/O times are collected. Turn it on to use the Time mode.", + "trackIoTimingOff": "track_io_timing and track_wal_io_timing are off on this host — no I/O times are collected.", + "trackIoTimingWalOnly": "track_io_timing is off — times cover write-ahead log rows only.", + "trackWalIoTimingOff": "track_wal_io_timing is off — write-ahead log rows carry no times.", "mode": { "history": "History", "live": "Live" diff --git a/frontend/src/locales/ru_RU.json b/frontend/src/locales/ru_RU.json index b915cc8e..a62dffc1 100644 --- a/frontend/src/locales/ru_RU.json +++ b/frontend/src/locales/ru_RU.json @@ -1003,7 +1003,9 @@ "partialIntervals": "Часть интервалов в этом периоде неполная: сброс статистики, рестарт или мажорный апгрейд разорвали ряд. Они показаны разрывом, а не нулём.", "noData": "Для текущей выборки активности ввода-вывода нет.", "historyUnavailable": "Только живой режим — хранилище снимков не настроено", - "trackIoTimingOff": "На хосте выключен track_io_timing — тайминги не собираются. Включите его для режима «Время».", + "trackIoTimingOff": "На хосте выключены track_io_timing и track_wal_io_timing — тайминги не собираются.", + "trackIoTimingWalOnly": "track_io_timing выключен — тайминги есть только у строк журнала предзаписи.", + "trackWalIoTimingOff": "track_wal_io_timing выключен — у строк журнала предзаписи нет таймингов.", "mode": { "history": "История", "live": "Live" diff --git a/frontend/src/views/IOView.vue b/frontend/src/views/IOView.vue index 2d7aafc0..fb5b3f64 100644 --- a/frontend/src/views/IOView.vue +++ b/frontend/src/views/IOView.vue @@ -51,6 +51,7 @@ const { rows: liveRows, windowSeconds: liveWindowSeconds, trackIoTiming: liveTrackIoTiming, + trackWalIoTiming: liveTrackWalIoTiming, waiting: liveWaiting, lastAt: liveLastAt, loading: liveLoading, @@ -71,6 +72,7 @@ const { backendTypes, windowSeconds, trackIoTiming, + trackWalIoTiming, partial, noData, } = useIoRows({ @@ -78,6 +80,7 @@ const { liveRows, liveWindowSeconds, liveTrackIoTiming, + liveTrackWalIoTiming, history, matrix, backendType: backendTypeFilter, @@ -201,6 +204,7 @@ watch(availableObjects, (objects) => { :backend-types="backendTypes" :ranges="Object.keys(IO_RANGES)" :track-io-timing="trackIoTiming" + :track-wal-io-timing="trackWalIoTiming" :live-active="liveActive" :live-remaining="liveRemaining" :live-loading="liveLoading" From e01ef8f005794241660f607fbe829b1787633579 Mon Sep 17 00:00:00 2001 From: "Dmitry V. Bulashev" Date: Sun, 30 Aug 2026 10:22:48 +0500 Subject: [PATCH 3/3] review fixes --- backend/internal/http/v1_io.go | 28 +++++++++++++++---- backend/internal/http/v1_io_test.go | 28 +++++++++++++++++-- .../internal/mcpserver/kb/en/pg-stat-io.md | 19 +++++++------ .../internal/mcpserver/kb/ru/pg-stat-io.md | 17 +++++------ backend/internal/mcpserver/tools.go | 19 ++++++++----- backend/internal/statio/statio.go | 11 ++++---- backend/internal/storage/migrate.go | 7 +++-- backend/internal/storage/statio.go | 9 ++++-- .../storage/statio_integration_test.go | 3 +- 9 files changed, 99 insertions(+), 42 deletions(-) diff --git a/backend/internal/http/v1_io.go b/backend/internal/http/v1_io.go index 484aa7d7..2f378ff9 100644 --- a/backend/internal/http/v1_io.go +++ b/backend/internal/http/v1_io.go @@ -170,25 +170,43 @@ func ioHistoryMeta(instance string, earliest, latest *time.Time, metas []statio. } meta.TrackIoTiming = metas[len(metas)-1].TrackIOTiming - meta.TrackWalIoTiming = metas[len(metas)-1].TrackWALIOTiming for _, m := range metas[1:] { if m.TrackIOTiming != metas[0].TrackIOTiming { meta.TrackIoTimingChanged = true } - if m.TrackWALIOTiming != metas[0].TrackWALIOTiming { - meta.TrackWalIoTimingChanged = true - } - if m.VersionNum/10000 != metas[0].VersionNum/10000 { meta.VersionChanged = true } } + setWALTiming(&meta, metas) + return meta } +// Captures older than the track_wal_io_timing column carry no value at all; +// they are skipped, so an unknown state never counts as a toggle. +func setWALTiming(meta *serverhttp.IOHistoryMeta, metas []statio.Meta) { + var first *bool + + for _, m := range metas { + if m.TrackWALIOTiming == nil { + continue + } + + switch { + case first == nil: + first = m.TrackWALIOTiming + case *m.TrackWALIOTiming != *first: + meta.TrackWalIoTimingChanged = true + } + + meta.TrackWalIoTiming = *m.TrackWALIOTiming + } +} + func ioSnapshotToAPI(instance string, snap statio.Snapshot) serverhttp.IOSnapshot { out := serverhttp.IOSnapshot{ Instance: instance, diff --git a/backend/internal/http/v1_io_test.go b/backend/internal/http/v1_io_test.go index 8be3f7ec..e4838d1b 100644 --- a/backend/internal/http/v1_io_test.go +++ b/backend/internal/http/v1_io_test.go @@ -154,9 +154,10 @@ func TestIOHistoryMetaSeparatesWALTiming(t *testing.T) { at := time.Date(2026, 8, 22, 3, 0, 0, 0, time.UTC) earliest, latest := at.Add(-24*time.Hour), at + on := true metas := []statio.Meta{ - {CapturedAt: at.Add(-2 * time.Minute), VersionNum: 180001, TrackWALIOTiming: true}, - {CapturedAt: at.Add(-time.Minute), VersionNum: 180001, TrackWALIOTiming: true}, + {CapturedAt: at.Add(-2 * time.Minute), VersionNum: 180001, TrackWALIOTiming: &on}, + {CapturedAt: at.Add(-time.Minute), VersionNum: 180001, TrackWALIOTiming: &on}, } meta := ioHistoryMeta("h1", &earliest, &latest, metas) @@ -174,6 +175,29 @@ func TestIOHistoryMetaSeparatesWALTiming(t *testing.T) { } } +func TestIOHistoryMetaIgnoresUnrecordedWALTiming(t *testing.T) { + t.Parallel() + + at := time.Date(2026, 8, 22, 3, 0, 0, 0, time.UTC) + earliest, latest := at.Add(-24*time.Hour), at + + on := true + metas := []statio.Meta{ + {CapturedAt: at.Add(-2 * time.Minute), VersionNum: 180001}, + {CapturedAt: at.Add(-time.Minute), VersionNum: 180001, TrackWALIOTiming: &on}, + } + + meta := ioHistoryMeta("h1", &earliest, &latest, metas) + + if meta.TrackWalIoTimingChanged { + t.Error("a capture that recorded no track_wal_io_timing is not a toggle") + } + + if !meta.TrackWalIoTiming { + t.Error("the newest recorded value must be reported") + } +} + func TestIOHistoryMetaWithoutSnapshots(t *testing.T) { t.Parallel() diff --git a/backend/internal/mcpserver/kb/en/pg-stat-io.md b/backend/internal/mcpserver/kb/en/pg-stat-io.md index 37d8a7a1..e31e7f49 100644 --- a/backend/internal/mcpserver/kb/en/pg-stat-io.md +++ b/backend/internal/mcpserver/kb/en/pg-stat-io.md @@ -108,9 +108,9 @@ days" cannot be drawn from a capped window. ## Empty results An empty result is not an answer until `empty_reason` says which one it is. -Only `no_io` means the instance was idle across the whole window, -`no_io_in_measured_part` answers for part of it, and the rest mean the question -went unanswered: +Only `no_io` means there was no classifiable physical I/O across the whole +window, `no_io_in_measured_part` answers for part of it, and the rest mean the +question went unanswered: - **unsupported_version** — the host runs PostgreSQL 15 or older and has no `pg_stat_io`. Nothing about its I/O can be read this way; fall back to @@ -143,12 +143,13 @@ went unanswered: `coverage_pct` per bucket. Not an all-clear for the window as asked. - **no_io** — snapshots cover the window, they are comparable, and none of the counters the tool classifies moved. For `io_summary` that is every counter - except `hits`. `io_trend` classifies `reads`, `writes`, `extends` and their - byte and time counters only, so `fsyncs`, `evictions`, `reuses` and - `writebacks` may be non-zero behind an `io_trend` `no_io` — read `io_summary` - over the same window before calling the instance quiet. This is the only - value that is a full answer, and even here `totals` may show heavy cache - activity. + except `hits`. `io_trend` classifies `reads`, `read_bytes`, `writes`, + `write_bytes`, `extends` and, where timing was measured, `read_time` and + `write_time`, so `fsyncs`, `evictions`, `reuses` and `writebacks` may be + non-zero behind an `io_trend` `no_io` — read `io_summary` over the same + window before calling the instance quiet. This is the only value that + answers for the whole window, and even it is not idleness: `totals` may show + heavy cache activity. ## Where to go next diff --git a/backend/internal/mcpserver/kb/ru/pg-stat-io.md b/backend/internal/mcpserver/kb/ru/pg-stat-io.md index 8284870b..0e754c0f 100644 --- a/backend/internal/mcpserver/kb/ru/pg-stat-io.md +++ b/backend/internal/mcpserver/kb/ru/pg-stat-io.md @@ -112,9 +112,9 @@ ## Пустые ответы Пустой результат не является ответом, пока `empty_reason` не скажет, какой это -из случаев. Только `no_io` означает, что инстанс простаивал всё окно, -`no_io_in_measured_part` отвечает за его часть, остальные — что на вопрос не -ответили: +из случаев. Только `no_io` означает, что классифицируемого физического I/O за +всё окно не было, `no_io_in_measured_part` отвечает за его часть, остальные — +что на вопрос не ответили: - **unsupported_version** — на хосте PostgreSQL 15 или старше, `pg_stat_io` там нет. Про его I/O этим путём ничего не узнать; остаются `wait_events` и @@ -147,11 +147,12 @@ `coverage_pct` у бакета. Это не «всё чисто» по запрошенному окну. - **no_io** — снимки окно покрывают, они сравнимы, и ни один из счётчиков, которые инструмент классифицирует, не сдвинулся. У `io_summary` это все - счётчики, кроме `hits`. `io_trend` классифицирует только `reads`, `writes`, - `extends` и их байтовые и временные счётчики, поэтому за `no_io` от - `io_trend` могут стоять ненулевые `fsyncs`, `evictions`, `reuses` и - `writebacks` — прежде чем называть инстанс спокойным, прочитать `io_summary` - за то же окно. Только это значение — полный ответ, и даже здесь `totals` + счётчики, кроме `hits`. `io_trend` классифицирует только `reads`, + `read_bytes`, `writes`, `write_bytes`, `extends` и, при измерении, + `read_time` и `write_time`, поэтому за `no_io` от `io_trend` могут стоять + ненулевые `fsyncs`, `evictions`, `reuses` и `writebacks` — прежде чем + называть инстанс спокойным, прочитать `io_summary` за то же окно. Только + это значение отвечает за всё окно, но и оно не означает простоя: `totals` может показывать активную работу с кэшем. ## Куда идти дальше diff --git a/backend/internal/mcpserver/tools.go b/backend/internal/mcpserver/tools.go index eaf5762d..161dd240 100644 --- a/backend/internal/mcpserver/tools.go +++ b/backend/internal/mcpserver/tools.go @@ -698,8 +698,9 @@ func registerTools(s *mcp.Server, c *DashaClient) { "spills). group_by=context (default) is the cheapest answer to 'whose I/O'; 'full' breaks it down " + "by backend_type x object x context and needs top. Requires PostgreSQL 16 or newer: older hosts have " + "no pg_stat_io at all and come back empty with empty_reason='unsupported_version', which does NOT " + - "mean 'no I/O'. Every empty answer carries an empty_reason, and only 'no_io' means the instance was " + - "idle across the whole window: 'no_io_in_measured_part' means a counter epoch broke inside it and " + + "mean 'no I/O'. Every empty answer carries an empty_reason, and only 'no_io' means no classifiable " + + "physical I/O across the whole window (cache hits can still be heavy): 'no_io_in_measured_part' " + + "means a counter epoch broke inside it and " + "the quiet covers the measured part alone, while 'no_snapshots_in_window', " + "'no_comparable_snapshots', 'window_after_history' and 'no_io_matching_filter' all mean the " + "question went unanswered — check totals and meta before reporting an all-clear. Time counters " + @@ -707,7 +708,10 @@ func registerTools(s *mcp.Server, c *DashaClient) { "track_wal_io_timing for the 'wal' object (PostgreSQL 18+). Under whichever is off every time " + "metric is zero by construction — a missing measurement, not a missing load; meta.track_io_timing " + "and meta.track_wal_io_timing say which, and avg_read_ms/avg_write_ms are absent rather than 0. " + - "A window longer than 31 days is cut back to it and flagged " + + "Both flags report the newest capture: meta.track_io_timing_changed and " + + "meta.track_wal_io_timing_changed mark a setting toggled inside the window, so earlier captures " + + "can carry real times under a setting that now reads off and the timing covers part of the " + + "window only. A window longer than 31 days is cut back to it and flagged " + "window_capped. pg_stat_io is instance-wide: there is no database parameter. A counter absent " + "from values is zero. Needs snapshot storage (501 otherwise). " + "Read dasha://kb/pg-stat-io before interpreting the numbers.", @@ -724,9 +728,10 @@ func registerTools(s *mcp.Server, c *DashaClient) { "coverage_pct: its counters are real but measure only that share of the bucket's span, so it is not " + "comparable with a complete point and a lower number there is not a drop in load (incomplete_points " + "counts such buckets). An incomplete point with no values at all measured nothing. In a complete " + - "point an absent metric is zero. This series carries reads, writes, extends and their byte and " + - "time counters only, so its empty_reason='no_io' does not rule out fsyncs, evictions, reuses or " + - "writebacks — confirm with io_summary over the same window. Same preconditions as io_summary, " + + "point an absent metric is zero. This series carries reads, read_bytes, writes, write_bytes and " + + "extends, plus read_time and write_time where timing was measured, so its empty_reason='no_io' " + + "does not rule out fsyncs, evictions, reuses or writebacks — confirm with io_summary over the same " + + "window. Same preconditions as io_summary, " + "including empty_reason on an empty answer: PostgreSQL 16+, instance-wide (no database), snapshot " + "storage required (501 otherwise). Time metrics are carried when track_io_timing or " + "track_wal_io_timing is on; grouping is by context, which merges WAL and relation rows, so with " + @@ -835,7 +840,7 @@ func parseSince(since string) (time.Duration, error) { return 0, err } - if n > maxSinceDays { + if n > maxSinceDays || n < -maxSinceDays { return 0, errors.New("day count out of range") } diff --git a/backend/internal/statio/statio.go b/backend/internal/statio/statio.go index 8969e193..d962470f 100644 --- a/backend/internal/statio/statio.go +++ b/backend/internal/statio/statio.go @@ -50,10 +50,11 @@ type Snapshot struct { // Meta is one capture's header: what a read plan needs before any matrix body // is fetched. type Meta struct { - CapturedAt time.Time - VersionNum int - TrackIOTiming bool - TrackWALIOTiming bool + CapturedAt time.Time + VersionNum int + TrackIOTiming bool + // nil for a capture taken before the setting was recorded: unknown, not off. + TrackWALIOTiming *bool StatsReset *time.Time } @@ -62,7 +63,7 @@ func (s Snapshot) Meta() Meta { CapturedAt: s.CapturedAt, VersionNum: s.VersionNum, TrackIOTiming: s.TrackIOTiming, - TrackWALIOTiming: s.TrackWALIOTiming, + TrackWALIOTiming: &s.TrackWALIOTiming, StatsReset: s.StatsReset, } } diff --git a/backend/internal/storage/migrate.go b/backend/internal/storage/migrate.go index a97cb9e9..3a3130a8 100644 --- a/backend/internal/storage/migrate.go +++ b/backend/internal/storage/migrate.go @@ -279,15 +279,18 @@ CREATE TABLE IF NOT EXISTS io_snapshot ( version_num int NOT NULL, op_bytes int, track_io_timing boolean NOT NULL, - track_wal_io_timing boolean NOT NULL DEFAULT false, + track_wal_io_timing boolean, stats_reset timestamptz, rows jsonb NOT NULL, CONSTRAINT io_snapshot_pkey PRIMARY KEY (cluster_name, instance, captured_at) ) PARTITION BY RANGE (captured_at)` + // Nullable: captures taken before the column existed recorded no + // track_wal_io_timing, and a backfilled false would read as the setting + // having been off on a PostgreSQL 18 host. addIOSnapshotWALTimingSQL = ` ALTER TABLE io_snapshot - ADD COLUMN IF NOT EXISTS track_wal_io_timing boolean NOT NULL DEFAULT false` + ADD COLUMN IF NOT EXISTS track_wal_io_timing boolean` addAutosnapshotIOConfigSQL = ` ALTER TABLE autosnapshot_config_global diff --git a/backend/internal/storage/statio.go b/backend/internal/storage/statio.go index b6774d00..d68723e4 100644 --- a/backend/internal/storage/statio.go +++ b/backend/internal/storage/statio.go @@ -159,15 +159,18 @@ func (s *Storage) GetIOSnapshotsAt( for rows.Next() { var ( - snap statio.Snapshot - body []byte + snap statio.Snapshot + walTime *bool + body []byte ) if err := rows.Scan(&snap.CapturedAt, &snap.VersionNum, &snap.OpBytes, - &snap.TrackIOTiming, &snap.TrackWALIOTiming, &snap.StatsReset, &body); err != nil { + &snap.TrackIOTiming, &walTime, &snap.StatsReset, &body); err != nil { return nil, fmt.Errorf("storage: scan io snapshot: %w", err) } + snap.TrackWALIOTiming = walTime != nil && *walTime + var stored []ioRowJSON if err := json.Unmarshal(body, &stored); err != nil { return nil, fmt.Errorf("storage: unmarshal io rows: %w", err) diff --git a/backend/internal/storage/statio_integration_test.go b/backend/internal/storage/statio_integration_test.go index 721dca91..6ae584ba 100644 --- a/backend/internal/storage/statio_integration_test.go +++ b/backend/internal/storage/statio_integration_test.go @@ -71,7 +71,8 @@ func TestIOSnapshotRoundTrip(t *testing.T) { assert.Equal(t, 170004, metas[0].VersionNum) assert.True(t, metas[0].TrackIOTiming) - assert.True(t, metas[0].TrackWALIOTiming) + require.NotNil(t, metas[0].TrackWALIOTiming) + assert.True(t, *metas[0].TrackWALIOTiming) require.NotNil(t, metas[0].StatsReset) assert.WithinDuration(t, reset, *metas[0].StatsReset, time.Second)