diff --git a/pkg/operator/triage.go b/pkg/operator/triage.go new file mode 100644 index 0000000..236e71d --- /dev/null +++ b/pkg/operator/triage.go @@ -0,0 +1,487 @@ +package operator + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + + "github.com/wandb/wsm/pkg/kubectl" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/dynamic" +) + +const ( + // DefaultTriageAction is selected when a caller does not specify an action. + DefaultTriageAction = "default" + + triageRunManagedByLabel = "app.kubernetes.io/managed-by" + triageRunManagedByValue = "wsm" + triageRunNameSuffix = "-triage-" + + // Leave room below the Kubernetes 253-character object-name limit for the + // API server's generated suffix. + maxTriageRunGenerateNameLength = 240 +) + +var ErrTriageRunNotTerminal = errors.New("TriageRun is not terminal") + +var triageRunsV2GVR = schema.GroupVersionResource{ + Group: "apps.wandb.com", + Version: "v2", + Resource: "triageruns", +} + +// TriageRunRequest describes one immutable request to diagnose an Application. +// Namespace and ApplicationName are required; Action defaults to "default". +type TriageRunRequest struct { + Namespace string `json:"namespace"` + ApplicationName string `json:"applicationName"` + Action string `json:"action,omitempty"` +} + +// TriageRunRef identifies the newly created TriageRun. It is intentionally +// small so Watchtower and other SDK consumers do not need to import operator +// API types just to trigger a run. +type TriageRunRef struct { + Namespace string `json:"namespace"` + Name string `json:"name"` +} + +// ListTriageRunsRequest scopes run history to one namespace and, optionally, +// one Application. +type ListTriageRunsRequest struct { + Namespace string `json:"namespace"` + ApplicationName string `json:"applicationName,omitempty"` +} + +// TriageRunSummary contains the aggregate diagnostic verdict counts reported by +// the operator. +type TriageRunSummary struct { + Total int64 `json:"total,omitempty"` + Pass int64 `json:"pass,omitempty"` + Warn int64 `json:"warn,omitempty"` + Fail int64 `json:"fail,omitempty"` + Error int64 `json:"error,omitempty"` + OverallSeverity string `json:"overallSeverity,omitempty"` +} + +// TriageCheckResult is one structured diagnostic result reported by the +// application-owned check command. +type TriageCheckResult struct { + Name string `json:"name"` + Umbrella string `json:"umbrella,omitempty"` + Severity string `json:"severity"` + Message string `json:"message,omitempty"` + Evidence any `json:"evidence,omitempty"` + Remediation string `json:"remediation,omitempty"` + StartedAt string `json:"startedAt,omitempty"` + EndedAt string `json:"endedAt,omitempty"` + DurationMilliseconds int64 `json:"durationMs,omitempty"` +} + +// TriageRun is the caller-facing view of an immutable TriageRun and its latest +// operator-reported status. +type TriageRun struct { + Namespace string `json:"namespace"` + Name string `json:"name"` + ApplicationName string `json:"applicationName"` + Action string `json:"action"` + Phase string `json:"phase,omitempty"` + CreatedAt string `json:"createdAt,omitempty"` + StartedAt string `json:"startedAt,omitempty"` + CompletedAt string `json:"completedAt,omitempty"` + Summary *TriageRunSummary `json:"summary,omitempty"` + Results []TriageCheckResult `json:"results,omitempty"` +} + +// CreateTriageRun creates a fresh TriageRun through wsm's configured dynamic +// Kubernetes client. It always uses generateName: repeated calls represent +// distinct execution requests rather than updates or retries of an earlier run. +func CreateTriageRun(ctx context.Context, request TriageRunRequest) (TriageRunRef, error) { + _, dynamicClient, err := kubectl.GetDynamicClientset() + if err != nil { + return TriageRunRef{}, err + } + return createTriageRun(ctx, dynamicClient, request) +} + +// ListTriageRuns returns newest-first run history in one namespace. Supplying +// ApplicationName filters the history without excluding runs created outside +// wsm. +func ListTriageRuns(ctx context.Context, request ListTriageRunsRequest) ([]TriageRun, error) { + _, dynamicClient, err := kubectl.GetDynamicClientset() + if err != nil { + return nil, err + } + return listTriageRuns(ctx, dynamicClient, request) +} + +// GetTriageRun returns one run and its latest status. +func GetTriageRun(ctx context.Context, namespace, name string) (TriageRun, error) { + _, dynamicClient, err := kubectl.GetDynamicClientset() + if err != nil { + return TriageRun{}, err + } + return getTriageRun(ctx, dynamicClient, namespace, name) +} + +// DeleteTriageRun removes a completed run. Pending and Running runs are not +// deleted because deletion would implicitly cancel their owned Job; cancellation +// needs a separate explicit contract. +func DeleteTriageRun(ctx context.Context, namespace, name string) error { + _, dynamicClient, err := kubectl.GetDynamicClientset() + if err != nil { + return err + } + return deleteTriageRun(ctx, dynamicClient, namespace, name) +} + +func createTriageRun( + ctx context.Context, + dynamicClient dynamic.Interface, + request TriageRunRequest, +) (TriageRunRef, error) { + if err := validateTriageRunRequest(request); err != nil { + return TriageRunRef{}, err + } + if request.Action == "" { + request.Action = DefaultTriageAction + } + + created, err := dynamicClient.Resource(triageRunsV2GVR).Namespace(request.Namespace).Create( + ctx, + newTriageRun(request), + metav1.CreateOptions{FieldManager: "wsm"}, + ) + if err != nil { + return TriageRunRef{}, fmt.Errorf( + "failed to create TriageRun for Application %s/%s: %w", + request.Namespace, + request.ApplicationName, + err, + ) + } + + return TriageRunRef{ + Namespace: created.GetNamespace(), + Name: created.GetName(), + }, nil +} + +func listTriageRuns( + ctx context.Context, + dynamicClient dynamic.Interface, + request ListTriageRunsRequest, +) ([]TriageRun, error) { + if err := validateTriageRunNamespace(request.Namespace); err != nil { + return nil, err + } + if request.ApplicationName != "" { + if err := validateTriageApplicationName(request.ApplicationName); err != nil { + return nil, err + } + } + + list, err := dynamicClient.Resource(triageRunsV2GVR).Namespace(request.Namespace).List( + ctx, + metav1.ListOptions{}, + ) + if err != nil { + return nil, fmt.Errorf("failed to list TriageRuns in namespace %s: %w", request.Namespace, err) + } + + runs := make([]TriageRun, 0, len(list.Items)) + for i := range list.Items { + run, err := parseTriageRun(&list.Items[i]) + if err != nil { + return nil, err + } + if request.ApplicationName != "" && run.ApplicationName != request.ApplicationName { + continue + } + runs = append(runs, run) + } + sort.SliceStable(runs, func(i, j int) bool { + if runs[i].CreatedAt == runs[j].CreatedAt { + return runs[i].Name > runs[j].Name + } + return runs[i].CreatedAt > runs[j].CreatedAt + }) + return runs, nil +} + +func getTriageRun( + ctx context.Context, + dynamicClient dynamic.Interface, + namespace string, + name string, +) (TriageRun, error) { + if err := validateTriageRunIdentity(namespace, name); err != nil { + return TriageRun{}, err + } + obj, err := dynamicClient.Resource(triageRunsV2GVR).Namespace(namespace).Get( + ctx, + name, + metav1.GetOptions{}, + ) + if err != nil { + return TriageRun{}, fmt.Errorf("failed to get TriageRun %s/%s: %w", namespace, name, err) + } + return parseTriageRun(obj) +} + +func deleteTriageRun( + ctx context.Context, + dynamicClient dynamic.Interface, + namespace string, + name string, +) error { + if err := validateTriageRunIdentity(namespace, name); err != nil { + return err + } + obj, err := dynamicClient.Resource(triageRunsV2GVR).Namespace(namespace).Get( + ctx, + name, + metav1.GetOptions{}, + ) + if err != nil { + return fmt.Errorf("failed to get TriageRun %s/%s before deletion: %w", namespace, name, err) + } + phase, _, err := unstructured.NestedString(obj.Object, "status", "phase") + if err != nil { + return fmt.Errorf("failed to read TriageRun %s/%s phase: %w", namespace, name, err) + } + if phase == "" { + phase = "Pending" + } + if phase != "Succeeded" && phase != "Failed" { + return fmt.Errorf("%w: %s/%s has phase %q", ErrTriageRunNotTerminal, namespace, name, phase) + } + + uid := obj.GetUID() + if err := dynamicClient.Resource(triageRunsV2GVR).Namespace(namespace).Delete( + ctx, + name, + metav1.DeleteOptions{ + Preconditions: &metav1.Preconditions{UID: &uid}, + }, + ); err != nil { + return fmt.Errorf("failed to delete TriageRun %s/%s: %w", namespace, name, err) + } + return nil +} + +func parseTriageRun(obj *unstructured.Unstructured) (TriageRun, error) { + applicationName, _, err := unstructured.NestedString( + obj.Object, + "spec", + "applicationRef", + "name", + ) + if err != nil { + return TriageRun{}, fmt.Errorf("failed to read TriageRun %s/%s application: %w", obj.GetNamespace(), obj.GetName(), err) + } + action, _, err := unstructured.NestedString(obj.Object, "spec", "action") + if err != nil { + return TriageRun{}, fmt.Errorf("failed to read TriageRun %s/%s action: %w", obj.GetNamespace(), obj.GetName(), err) + } + if action == "" { + action = DefaultTriageAction + } + phase, _, err := unstructured.NestedString(obj.Object, "status", "phase") + if err != nil { + return TriageRun{}, fmt.Errorf("failed to read TriageRun %s/%s phase: %w", obj.GetNamespace(), obj.GetName(), err) + } + if phase == "" { + phase = "Pending" + } + startedAt, _, err := unstructured.NestedString(obj.Object, "status", "startedAt") + if err != nil { + return TriageRun{}, fmt.Errorf("failed to read TriageRun %s/%s startedAt: %w", obj.GetNamespace(), obj.GetName(), err) + } + completedAt, _, err := unstructured.NestedString(obj.Object, "status", "completedAt") + if err != nil { + return TriageRun{}, fmt.Errorf("failed to read TriageRun %s/%s completedAt: %w", obj.GetNamespace(), obj.GetName(), err) + } + + creationTimestamp := obj.GetCreationTimestamp() + run := TriageRun{ + Namespace: obj.GetNamespace(), + Name: obj.GetName(), + ApplicationName: applicationName, + Action: action, + Phase: phase, + CreatedAt: creationTimestamp.UTC().Format("2006-01-02T15:04:05Z07:00"), + StartedAt: startedAt, + CompletedAt: completedAt, + } + if creationTimestamp.IsZero() { + run.CreatedAt = "" + } + + summaryMap, found, err := unstructured.NestedMap(obj.Object, "status", "summary") + if err != nil { + return TriageRun{}, fmt.Errorf("failed to read TriageRun %s/%s summary: %w", obj.GetNamespace(), obj.GetName(), err) + } + if found { + run.Summary, err = parseTriageRunSummary(summaryMap) + if err != nil { + return TriageRun{}, fmt.Errorf("failed to parse TriageRun %s/%s summary: %w", obj.GetNamespace(), obj.GetName(), err) + } + } + + resultItems, found, err := unstructured.NestedSlice(obj.Object, "status", "results") + if err != nil { + return TriageRun{}, fmt.Errorf("failed to read TriageRun %s/%s results: %w", obj.GetNamespace(), obj.GetName(), err) + } + if found { + run.Results = make([]TriageCheckResult, 0, len(resultItems)) + for i, item := range resultItems { + resultMap, ok := item.(map[string]any) + if !ok { + return TriageRun{}, fmt.Errorf("TriageRun %s/%s result %d is %T, want object", obj.GetNamespace(), obj.GetName(), i, item) + } + result, err := parseTriageCheckResult(resultMap) + if err != nil { + return TriageRun{}, fmt.Errorf("failed to parse TriageRun %s/%s result %d: %w", obj.GetNamespace(), obj.GetName(), i, err) + } + run.Results = append(run.Results, result) + } + } + return run, nil +} + +func parseTriageRunSummary(summary map[string]any) (*TriageRunSummary, error) { + result := &TriageRunSummary{} + var err error + if result.Total, _, err = unstructured.NestedInt64(summary, "total"); err != nil { + return nil, err + } + if result.Pass, _, err = unstructured.NestedInt64(summary, "pass"); err != nil { + return nil, err + } + if result.Warn, _, err = unstructured.NestedInt64(summary, "warn"); err != nil { + return nil, err + } + if result.Fail, _, err = unstructured.NestedInt64(summary, "fail"); err != nil { + return nil, err + } + if result.Error, _, err = unstructured.NestedInt64(summary, "error"); err != nil { + return nil, err + } + if result.OverallSeverity, _, err = unstructured.NestedString(summary, "overallSeverity"); err != nil { + return nil, err + } + return result, nil +} + +func parseTriageCheckResult(item map[string]any) (TriageCheckResult, error) { + result := TriageCheckResult{} + var err error + if result.Name, _, err = unstructured.NestedString(item, "name"); err != nil { + return TriageCheckResult{}, err + } + if result.Umbrella, _, err = unstructured.NestedString(item, "umbrella"); err != nil { + return TriageCheckResult{}, err + } + if result.Severity, _, err = unstructured.NestedString(item, "severity"); err != nil { + return TriageCheckResult{}, err + } + if result.Message, _, err = unstructured.NestedString(item, "message"); err != nil { + return TriageCheckResult{}, err + } + if result.Remediation, _, err = unstructured.NestedString(item, "remediation"); err != nil { + return TriageCheckResult{}, err + } + if result.StartedAt, _, err = unstructured.NestedString(item, "startedAt"); err != nil { + return TriageCheckResult{}, err + } + if result.EndedAt, _, err = unstructured.NestedString(item, "endedAt"); err != nil { + return TriageCheckResult{}, err + } + if result.DurationMilliseconds, _, err = unstructured.NestedInt64(item, "durationMs"); err != nil { + return TriageCheckResult{}, err + } + if evidence, ok := item["evidence"]; ok { + result.Evidence = evidence + } + return result, nil +} + +func validateTriageRunRequest(request TriageRunRequest) error { + if err := validateTriageRunNamespace(request.Namespace); err != nil { + return err + } + return validateTriageApplicationName(request.ApplicationName) +} + +func validateTriageRunIdentity(namespace, name string) error { + if err := validateTriageRunNamespace(namespace); err != nil { + return err + } + if name == "" { + return errors.New("TriageRun name is required") + } + if problems := validation.IsDNS1123Subdomain(name); len(problems) > 0 { + return fmt.Errorf("invalid TriageRun name %q: %s", name, strings.Join(problems, "; ")) + } + return nil +} + +func validateTriageRunNamespace(namespace string) error { + if namespace == "" { + return errors.New("namespace is required to access TriageRuns") + } + if problems := validation.IsDNS1123Label(namespace); len(problems) > 0 { + return fmt.Errorf("invalid TriageRun namespace %q: %s", namespace, strings.Join(problems, "; ")) + } + return nil +} + +func validateTriageApplicationName(applicationName string) error { + if applicationName == "" { + return errors.New("application name is required to access TriageRuns") + } + if problems := validation.IsDNS1123Subdomain(applicationName); len(problems) > 0 { + return fmt.Errorf("invalid Application name %q: %s", applicationName, strings.Join(problems, "; ")) + } + return nil +} + +func newTriageRun(request TriageRunRequest) *unstructured.Unstructured { + action := request.Action + if action == "" { + action = DefaultTriageAction + } + + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "apps.wandb.com/v2", + "kind": "TriageRun", + "metadata": map[string]any{ + "generateName": triageRunGenerateName(request.ApplicationName), + "namespace": request.Namespace, + "labels": map[string]any{ + triageRunManagedByLabel: triageRunManagedByValue, + }, + }, + "spec": map[string]any{ + "applicationRef": map[string]any{ + "name": request.ApplicationName, + }, + "action": action, + }, + }} +} + +func triageRunGenerateName(applicationName string) string { + maxApplicationLength := maxTriageRunGenerateNameLength - len(triageRunNameSuffix) + if len(applicationName) > maxApplicationLength { + applicationName = strings.TrimRight(applicationName[:maxApplicationLength], "-.") + } + return applicationName + triageRunNameSuffix +} diff --git a/pkg/operator/triage_test.go b/pkg/operator/triage_test.go new file mode 100644 index 0000000..d840ceb --- /dev/null +++ b/pkg/operator/triage_test.go @@ -0,0 +1,391 @@ +package operator + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic/fake" + k8stesting "k8s.io/client-go/testing" +) + +func TestCreateTriageRun(t *testing.T) { + t.Parallel() + + client := fake.NewSimpleDynamicClient(runtime.NewScheme()) + client.PrependReactor("create", "triageruns", func(action k8stesting.Action) (bool, runtime.Object, error) { + createAction := action.(k8stesting.CreateAction) + obj := createAction.GetObject().(*unstructured.Unstructured).DeepCopy() + + if got := action.GetNamespace(); got != "wandb" { + t.Fatalf("namespace = %q, want wandb", got) + } + if got := obj.GetGenerateName(); got != "weave-trace-triage-" { + t.Fatalf("generateName = %q, want weave-trace-triage-", got) + } + if got := obj.GetLabels()[triageRunManagedByLabel]; got != triageRunManagedByValue { + t.Fatalf("managed-by label = %q, want %q", got, triageRunManagedByValue) + } + if got, _, _ := unstructured.NestedString( + obj.Object, "spec", "applicationRef", "name", + ); got != "weave-trace" { + t.Fatalf("applicationRef.name = %q, want weave-trace", got) + } + if got, _, _ := unstructured.NestedString(obj.Object, "spec", "action"); got != "default" { + t.Fatalf("action = %q, want default", got) + } + + obj.SetName(obj.GetGenerateName() + "abcde") + return true, obj, nil + }) + + ref, err := createTriageRun(context.Background(), client, TriageRunRequest{ + Namespace: "wandb", + ApplicationName: "weave-trace", + }) + if err != nil { + t.Fatalf("create TriageRun: %v", err) + } + if ref.Namespace != "wandb" || ref.Name != "weave-trace-triage-abcde" { + t.Fatalf("ref = %#v", ref) + } +} + +func TestCreateTriageRunPreservesExplicitAction(t *testing.T) { + t.Parallel() + + client := fake.NewSimpleDynamicClient(runtime.NewScheme()) + client.PrependReactor("create", "triageruns", func(action k8stesting.Action) (bool, runtime.Object, error) { + obj := action.(k8stesting.CreateAction).GetObject().(*unstructured.Unstructured).DeepCopy() + got, _, _ := unstructured.NestedString(obj.Object, "spec", "action") + if got != "dependencies" { + t.Fatalf("action = %q, want dependencies", got) + } + obj.SetName("weave-trace-triage-explicit") + return true, obj, nil + }) + + _, err := createTriageRun(context.Background(), client, TriageRunRequest{ + Namespace: "wandb", + ApplicationName: "weave-trace", + Action: "dependencies", + }) + if err != nil { + t.Fatalf("create TriageRun: %v", err) + } +} + +func TestCreateTriageRunValidatesRequestBeforeCallingCluster(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + request TriageRunRequest + want string + }{ + { + name: "namespace required", + request: TriageRunRequest{ApplicationName: "weave-trace"}, + want: "namespace is required", + }, + { + name: "namespace valid", + request: TriageRunRequest{Namespace: "Not Valid", ApplicationName: "weave-trace"}, + want: "invalid TriageRun namespace", + }, + { + name: "application required", + request: TriageRunRequest{Namespace: "wandb"}, + want: "application name is required", + }, + { + name: "application valid", + request: TriageRunRequest{Namespace: "wandb", ApplicationName: "Not Valid"}, + want: "invalid Application name", + }, + } + + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + _, err := createTriageRun(context.Background(), nil, test.request) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("error = %v, want containing %q", err, test.want) + } + }) + } +} + +func TestCreateTriageRunWrapsCreateError(t *testing.T) { + t.Parallel() + + client := fake.NewSimpleDynamicClient(runtime.NewScheme()) + wantErr := errors.New("triageruns.apps.wandb.com not found") + client.PrependReactor("create", "triageruns", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, wantErr + }) + + _, err := createTriageRun(context.Background(), client, TriageRunRequest{ + Namespace: "wandb", + ApplicationName: "weave-trace", + }) + if !errors.Is(err, wantErr) { + t.Fatalf("error = %v, want wrapped %v", err, wantErr) + } +} + +func TestTriageRunGenerateNameIsBounded(t *testing.T) { + t.Parallel() + + name := strings.Repeat("a", 252) + got := triageRunGenerateName(name) + if len(got) != maxTriageRunGenerateNameLength { + t.Fatalf("generateName length = %d, want %d", len(got), maxTriageRunGenerateNameLength) + } + if !strings.HasSuffix(got, triageRunNameSuffix) { + t.Fatalf("generateName = %q, want suffix %q", got, triageRunNameSuffix) + } +} + +func TestTriageRunGVR(t *testing.T) { + t.Parallel() + + want := schema.GroupVersionResource{ + Group: "apps.wandb.com", + Version: "v2", + Resource: "triageruns", + } + if triageRunsV2GVR != want { + t.Fatalf("GVR = %#v, want %#v", triageRunsV2GVR, want) + } +} + +func TestNewTriageRunHasNamespacedMetadata(t *testing.T) { + t.Parallel() + + obj := newTriageRun(TriageRunRequest{ + Namespace: "wandb", + ApplicationName: "weave-trace", + Action: "default", + }) + if obj.GetNamespace() != "wandb" || obj.GetCreationTimestamp() != (metav1.Time{}) { + t.Fatalf("unexpected metadata: %#v", obj.Object["metadata"]) + } +} + +func TestListTriageRunsFiltersSortsAndParsesStatus(t *testing.T) { + older := testTriageRun( + t, + "weave-trace-triage-older", + "weave-trace", + "Succeeded", + "2026-07-28T18:00:00Z", + ) + newer := testTriageRun( + t, + "weave-trace-triage-newer", + "weave-trace", + "Failed", + "2026-07-28T19:00:00Z", + ) + other := testTriageRun( + t, + "gorilla-triage-other", + "gorilla", + "Succeeded", + "2026-07-28T20:00:00Z", + ) + if err := unstructured.SetNestedMap(newer.Object, map[string]any{ + "phase": "Failed", + "startedAt": "2026-07-28T19:00:01Z", + "completedAt": "2026-07-28T19:00:05Z", + "summary": map[string]any{ + "total": int64(2), + "pass": int64(1), + "warn": int64(0), + "fail": int64(1), + "error": int64(0), + "overallSeverity": "fail", + }, + "results": []any{ + map[string]any{ + "name": "clickhouse-reachable", + "umbrella": "clickhouse", + "severity": "pass", + "message": "connected", + "durationMs": int64(12), + "evidence": map[string]any{"host": "clickhouse"}, + "remediation": "", + }, + }, + }, "status"); err != nil { + t.Fatalf("set status: %v", err) + } + + client := newTriageRunFakeClient(older, newer, other) + runs, err := listTriageRuns(context.Background(), client, ListTriageRunsRequest{ + Namespace: "wandb", + ApplicationName: "weave-trace", + }) + if err != nil { + t.Fatalf("list TriageRuns: %v", err) + } + if len(runs) != 2 { + t.Fatalf("len(runs) = %d, want 2", len(runs)) + } + if runs[0].Name != newer.GetName() || runs[1].Name != older.GetName() { + t.Fatalf("run order = [%s, %s]", runs[0].Name, runs[1].Name) + } + if runs[0].Summary == nil || runs[0].Summary.OverallSeverity != "fail" || runs[0].Summary.Total != 2 { + t.Fatalf("summary = %#v", runs[0].Summary) + } + if len(runs[0].Results) != 1 || runs[0].Results[0].Name != "clickhouse-reachable" { + t.Fatalf("results = %#v", runs[0].Results) + } + evidence, ok := runs[0].Results[0].Evidence.(map[string]any) + if !ok || evidence["host"] != "clickhouse" { + t.Fatalf("evidence = %#v", runs[0].Results[0].Evidence) + } +} + +func TestGetTriageRun(t *testing.T) { + obj := testTriageRun( + t, + "weave-trace-triage-get", + "weave-trace", + "Running", + "2026-07-28T19:00:00Z", + ) + client := newTriageRunFakeClient(obj) + + run, err := getTriageRun(context.Background(), client, "wandb", obj.GetName()) + if err != nil { + t.Fatalf("get TriageRun: %v", err) + } + if run.Name != obj.GetName() || run.ApplicationName != "weave-trace" || run.Phase != "Running" { + t.Fatalf("run = %#v", run) + } +} + +func TestDeleteTriageRunRequiresTerminalPhase(t *testing.T) { + running := testTriageRun( + t, + "weave-trace-triage-running", + "weave-trace", + "Running", + "2026-07-28T19:00:00Z", + ) + client := newTriageRunFakeClient(running) + + err := deleteTriageRun(context.Background(), client, "wandb", running.GetName()) + if !errors.Is(err, ErrTriageRunNotTerminal) { + t.Fatalf("error = %v, want ErrTriageRunNotTerminal", err) + } + if _, err := client.Resource(triageRunsV2GVR).Namespace("wandb").Get( + context.Background(), + running.GetName(), + metav1.GetOptions{}, + ); err != nil { + t.Fatalf("running TriageRun was deleted: %v", err) + } +} + +func TestDeleteTriageRunDefaultsEmptyPhaseToPending(t *testing.T) { + pending := testTriageRun( + t, + "weave-trace-triage-pending", + "weave-trace", + "", + "2026-07-28T19:00:00Z", + ) + client := newTriageRunFakeClient(pending) + + err := deleteTriageRun(context.Background(), client, "wandb", pending.GetName()) + if !errors.Is(err, ErrTriageRunNotTerminal) { + t.Fatalf("error = %v, want ErrTriageRunNotTerminal", err) + } + if !strings.Contains(err.Error(), `phase "Pending"`) { + t.Fatalf("error = %q, want normalized Pending phase", err) + } + if _, err := client.Resource(triageRunsV2GVR).Namespace("wandb").Get( + context.Background(), + pending.GetName(), + metav1.GetOptions{}, + ); err != nil { + t.Fatalf("pending TriageRun was deleted: %v", err) + } +} + +func TestDeleteTerminalTriageRun(t *testing.T) { + completed := testTriageRun( + t, + "weave-trace-triage-completed", + "weave-trace", + "Succeeded", + "2026-07-28T19:00:00Z", + ) + client := newTriageRunFakeClient(completed) + + if err := deleteTriageRun(context.Background(), client, "wandb", completed.GetName()); err != nil { + t.Fatalf("delete TriageRun: %v", err) + } + _, err := client.Resource(triageRunsV2GVR).Namespace("wandb").Get( + context.Background(), + completed.GetName(), + metav1.GetOptions{}, + ) + if !apierrors.IsNotFound(err) { + t.Fatalf("get after delete error = %v, want NotFound", err) + } +} + +func testTriageRun( + t *testing.T, + name string, + applicationName string, + phase string, + createdAt string, +) *unstructured.Unstructured { + t.Helper() + obj := newTriageRun(TriageRunRequest{ + Namespace: "wandb", + ApplicationName: applicationName, + Action: "default", + }) + obj.SetName(name) + obj.SetGenerateName("") + obj.SetUID(types.UID(name + "-uid")) + obj.SetGroupVersionKind(schema.GroupVersionKind{ + Group: triageRunsV2GVR.Group, + Version: triageRunsV2GVR.Version, + Kind: "TriageRun", + }) + timestamp, err := time.Parse(time.RFC3339, createdAt) + if err != nil { + t.Fatalf("parse creation timestamp: %v", err) + } + obj.SetCreationTimestamp(metav1.NewTime(timestamp)) + if err := unstructured.SetNestedField(obj.Object, phase, "status", "phase"); err != nil { + t.Fatalf("set phase: %v", err) + } + return obj +} + +func newTriageRunFakeClient(objects ...runtime.Object) *fake.FakeDynamicClient { + return fake.NewSimpleDynamicClientWithCustomListKinds( + runtime.NewScheme(), + map[schema.GroupVersionResource]string{ + triageRunsV2GVR: "TriageRunList", + }, + objects..., + ) +}