Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
298 changes: 262 additions & 36 deletions pkg/operator/triage.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,30 @@ var triageRunsV2GVR = schema.GroupVersionResource{
Resource: "triageruns",
}

var applicationsV2GVR = schema.GroupVersionResource{
Group: "apps.wandb.com",
Version: "v2",
Resource: "applications",
}

// TriageAction is one action advertised by an Application.
type TriageAction struct {
Name string `json:"name"`
Description string `json:"description,omitempty"`
}

// TriageActionReference selects an advertised Application action by name.
type TriageActionReference struct {
Name string `json:"name"`
}

// TriageRunRequest describes one immutable request to diagnose an Application.
// Namespace and ApplicationName are required; Action defaults to "default".
// Namespace and ApplicationName are required; Actions defaults to [{name:
// "default"}].
type TriageRunRequest struct {
Namespace string `json:"namespace"`
ApplicationName string `json:"applicationName"`
Action string `json:"action,omitempty"`
Namespace string `json:"namespace"`
ApplicationName string `json:"applicationName"`
Actions []TriageActionReference `json:"actions,omitempty"`
}

// TriageRunRef identifies the newly created TriageRun. It is intentionally
Expand Down Expand Up @@ -84,19 +102,33 @@ type TriageCheckResult struct {
DurationMilliseconds int64 `json:"durationMs,omitempty"`
}

// TriageActionStatus contains the execution and results for one selected
// Application action.
type TriageActionStatus struct {
Action string `json:"action"`
Phase string `json:"phase,omitempty"`
JobName string `json:"jobName,omitempty"`
StartedAt string `json:"startedAt,omitempty"`
CompletedAt string `json:"completedAt,omitempty"`
Summary *TriageRunSummary `json:"summary,omitempty"`
Results []TriageCheckResult `json:"results,omitempty"`
}

// TriageRun is the caller-facing view of an immutable TriageRun and its latest
// operator-reported status.
type TriageRun struct {
Namespace string `json:"namespace"`
Name string `json:"name"`
ApplicationName string `json:"applicationName"`
Action string `json:"action"`
Phase string `json:"phase,omitempty"`
CreatedAt string `json:"createdAt,omitempty"`
StartedAt string `json:"startedAt,omitempty"`
CompletedAt string `json:"completedAt,omitempty"`
Summary *TriageRunSummary `json:"summary,omitempty"`
Results []TriageCheckResult `json:"results,omitempty"`
Namespace string `json:"namespace"`
Name string `json:"name"`
ApplicationName string `json:"applicationName"`
Actions []TriageActionReference `json:"actions"`
Phase string `json:"phase,omitempty"`
CreatedAt string `json:"createdAt,omitempty"`
StartedAt string `json:"startedAt,omitempty"`
CompletedAt string `json:"completedAt,omitempty"`
Summary *TriageRunSummary `json:"summary,omitempty"`
ActionStatuses []TriageActionStatus `json:"actionStatuses,omitempty"`
// Results is a flattened compatibility view of all action results.
Results []TriageCheckResult `json:"results,omitempty"`
}

// CreateTriageRun creates a fresh TriageRun through wsm's configured dynamic
Expand All @@ -110,6 +142,16 @@ func CreateTriageRun(ctx context.Context, request TriageRunRequest) (TriageRunRe
return createTriageRun(ctx, dynamicClient, request)
}

// ListTriageActions returns the sorted action metadata declared by one
// Application for Watchtower and other SDK consumers.
func ListTriageActions(ctx context.Context, namespace, applicationName string) ([]TriageAction, error) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This function isn't used anywhere in wsm? Is this for watchtower? You also have a function called listTriageActions (lowercase l) at line 222, that's super confusing. Which function is supposed to be here?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

renamed it to listTriageActionsWithClient

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ListTriageActions is invoked by watchtower when the UI check loads.

_, dynamicClient, err := kubectl.GetDynamicClientset()
if err != nil {
return nil, err
}
return listTriageActionsWithClient(ctx, dynamicClient, namespace, applicationName)
}

// ListTriageRuns returns newest-first run history in one namespace. Supplying
// ApplicationName filters the history without excluding runs created outside
// wsm.
Expand Down Expand Up @@ -149,9 +191,7 @@ func createTriageRun(
if err := validateTriageRunRequest(request); err != nil {
return TriageRunRef{}, err
}
if request.Action == "" {
request.Action = DefaultTriageAction
}
request.Actions = normalizedTriageActions(request)

created, err := dynamicClient.Resource(triageRunsV2GVR).Namespace(request.Namespace).Create(
ctx,
Expand All @@ -173,6 +213,71 @@ func createTriageRun(
}, nil
}

func listTriageActionsWithClient(
ctx context.Context,
dynamicClient dynamic.Interface,
namespace string,
applicationName string,
) ([]TriageAction, error) {
if err := validateTriageRunNamespace(namespace); err != nil {
return nil, err
}
if err := validateTriageApplicationName(applicationName); err != nil {
return nil, err
}
application, err := dynamicClient.Resource(applicationsV2GVR).Namespace(namespace).Get(
ctx,
applicationName,
metav1.GetOptions{},
)
if err != nil {
return nil, fmt.Errorf("failed to get Application %s/%s: %w", namespace, applicationName, err)
}
rawActions, found, err := unstructured.NestedFieldNoCopy(
application.Object, "spec", "triage", "actions")
if err != nil {
return nil, fmt.Errorf(
"failed to read Application %s/%s triage actions: %w", namespace, applicationName, err)
}
if !found {
return []TriageAction{}, nil
}
actionItems, ok := rawActions.([]any)
if !ok {
return nil, fmt.Errorf(
"application %s/%s triage actions are %T, want array", namespace, applicationName, rawActions)
}
actions := make([]TriageAction, 0, len(actionItems))
for i, item := range actionItems {
actionMap, ok := item.(map[string]any)
if !ok {
return nil, fmt.Errorf(
"application %s/%s triage action %d is %T, want object",
namespace, applicationName, i, item)
}
name, _, nameErr := unstructured.NestedString(actionMap, "name")
if nameErr != nil {
return nil, fmt.Errorf(
"application %s/%s triage action %d has invalid name: %w",
namespace, applicationName, i, nameErr)
}
if strings.TrimSpace(name) == "" {
return nil, fmt.Errorf(
"application %s/%s triage action %d has an empty name",
namespace, applicationName, i)
}
description, _, descriptionErr := unstructured.NestedString(actionMap, "description")
if descriptionErr != nil {
return nil, fmt.Errorf(
"application %s/%s triage action %q has invalid description: %w",
namespace, applicationName, name, descriptionErr)
}
actions = append(actions, TriageAction{Name: name, Description: description})
}
sort.Slice(actions, func(i, j int) bool { return actions[i].Name < actions[j].Name })
return actions, nil
}

func listTriageRuns(
ctx context.Context,
dynamicClient dynamic.Interface,
Expand Down Expand Up @@ -286,12 +391,14 @@ func parseTriageRun(obj *unstructured.Unstructured) (TriageRun, error) {
if err != nil {
return TriageRun{}, fmt.Errorf("failed to read TriageRun %s/%s application: %w", obj.GetNamespace(), obj.GetName(), err)
}
action, _, err := unstructured.NestedString(obj.Object, "spec", "action")
actions, foundActions, err := parseTriageActionReferences(obj.Object, "spec", "actions")
if err != nil {
return TriageRun{}, fmt.Errorf("failed to read TriageRun %s/%s action: %w", obj.GetNamespace(), obj.GetName(), err)
return TriageRun{}, fmt.Errorf("failed to read TriageRun %s/%s actions: %w", obj.GetNamespace(), obj.GetName(), err)
}
if action == "" {
action = DefaultTriageAction
if !foundActions {
return TriageRun{}, fmt.Errorf(
"failed to read TriageRun %s/%s actions: field is required",
obj.GetNamespace(), obj.GetName())
}
phase, _, err := unstructured.NestedString(obj.Object, "status", "phase")
if err != nil {
Expand All @@ -314,7 +421,7 @@ func parseTriageRun(obj *unstructured.Unstructured) (TriageRun, error) {
Namespace: obj.GetNamespace(),
Name: obj.GetName(),
ApplicationName: applicationName,
Action: action,
Actions: actions,
Phase: phase,
CreatedAt: creationTimestamp.UTC().Format("2006-01-02T15:04:05Z07:00"),
StartedAt: startedAt,
Expand All @@ -335,27 +442,122 @@ func parseTriageRun(obj *unstructured.Unstructured) (TriageRun, error) {
}
}

resultItems, found, err := unstructured.NestedSlice(obj.Object, "status", "results")
actionStatusItems, found, err := unstructured.NestedSlice(obj.Object, "status", "actionStatuses")
if err != nil {
return TriageRun{}, fmt.Errorf("failed to read TriageRun %s/%s results: %w", obj.GetNamespace(), obj.GetName(), err)
return TriageRun{}, fmt.Errorf("failed to read TriageRun %s/%s action statuses: %w", obj.GetNamespace(), obj.GetName(), err)
}
if found {
run.Results = make([]TriageCheckResult, 0, len(resultItems))
for i, item := range resultItems {
resultMap, ok := item.(map[string]any)
run.ActionStatuses = make([]TriageActionStatus, 0, len(actionStatusItems))
for i, item := range actionStatusItems {
statusMap, ok := item.(map[string]any)
if !ok {
return TriageRun{}, fmt.Errorf("TriageRun %s/%s result %d is %T, want object", obj.GetNamespace(), obj.GetName(), i, item)
return TriageRun{}, fmt.Errorf(
"TriageRun %s/%s action status %d is %T, want object",
obj.GetNamespace(), obj.GetName(), i, item)
}
result, err := parseTriageCheckResult(resultMap)
status, err := parseTriageActionStatus(statusMap)
if err != nil {
return TriageRun{}, fmt.Errorf("failed to parse TriageRun %s/%s result %d: %w", obj.GetNamespace(), obj.GetName(), i, err)
return TriageRun{}, fmt.Errorf(
"failed to parse TriageRun %s/%s action status %d: %w",
obj.GetNamespace(), obj.GetName(), i, err)
}
run.Results = append(run.Results, result)
run.ActionStatuses = append(run.ActionStatuses, status)
run.Results = append(run.Results, status.Results...)
}
} else {
legacyResults, resultErr := parseTriageCheckResults(obj.Object, "status", "results")
if resultErr != nil {
return TriageRun{}, fmt.Errorf(
"failed to parse TriageRun %s/%s legacy results: %w",
obj.GetNamespace(), obj.GetName(), resultErr)
}
run.Results = legacyResults
}
return run, nil
}

func parseTriageActionReferences(
object map[string]any,
fields ...string,
) ([]TriageActionReference, bool, error) {
items, found, err := unstructured.NestedSlice(object, fields...)
if err != nil || !found {
return nil, found, err
}
if len(items) == 0 {
return nil, true, errors.New("actions must contain at least one action")
}
actions := make([]TriageActionReference, 0, len(items))
for i, item := range items {
actionMap, ok := item.(map[string]any)
if !ok {
return nil, true, fmt.Errorf("action %d is %T, want object", i, item)
}
name, _, nameErr := unstructured.NestedString(actionMap, "name")
if nameErr != nil {
return nil, true, fmt.Errorf("action %d name: %w", i, nameErr)
}
if strings.TrimSpace(name) == "" {
return nil, true, fmt.Errorf("action %d name must not be empty", i)
}
actions = append(actions, TriageActionReference{Name: name})
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
return actions, true, nil
}

func parseTriageActionStatus(item map[string]any) (TriageActionStatus, error) {
status := TriageActionStatus{}
var err error
if status.Action, _, err = unstructured.NestedString(item, "action"); err != nil {
return TriageActionStatus{}, err
}
if status.Phase, _, err = unstructured.NestedString(item, "phase"); err != nil {
return TriageActionStatus{}, err
}
if status.JobName, _, err = unstructured.NestedString(item, "jobRef", "name"); err != nil {
return TriageActionStatus{}, err
}
if status.StartedAt, _, err = unstructured.NestedString(item, "startedAt"); err != nil {
return TriageActionStatus{}, err
}
if status.CompletedAt, _, err = unstructured.NestedString(item, "completedAt"); err != nil {
return TriageActionStatus{}, err
}
if summary, found, summaryErr := unstructured.NestedMap(item, "summary"); summaryErr != nil {
return TriageActionStatus{}, summaryErr
} else if found {
status.Summary, err = parseTriageRunSummary(summary)
if err != nil {
return TriageActionStatus{}, err
}
}
status.Results, err = parseTriageCheckResults(item, "results")
if err != nil {
return TriageActionStatus{}, err
}
return status, nil
}

func parseTriageCheckResults(object map[string]any, fields ...string) ([]TriageCheckResult, error) {
items, found, err := unstructured.NestedSlice(object, fields...)
if err != nil || !found {
return nil, err
}
results := make([]TriageCheckResult, 0, len(items))
for i, item := range items {
resultMap, ok := item.(map[string]any)
if !ok {
return nil, fmt.Errorf("result %d is %T, want object", i, item)
}
result, err := parseTriageCheckResult(resultMap)
if err != nil {
return nil, fmt.Errorf("result %d: %w", i, err)
}
results = append(results, result)
}
return results, nil
}

func parseTriageRunSummary(summary map[string]any) (*TriageRunSummary, error) {
result := &TriageRunSummary{}
var err error
Expand Down Expand Up @@ -417,7 +619,30 @@ func validateTriageRunRequest(request TriageRunRequest) error {
if err := validateTriageRunNamespace(request.Namespace); err != nil {
return err
}
return validateTriageApplicationName(request.ApplicationName)
if err := validateTriageApplicationName(request.ApplicationName); err != nil {
return err
}
if request.Actions != nil && len(request.Actions) == 0 {
return errors.New("actions must contain at least one action")
}
seen := make(map[string]struct{})
for i, action := range normalizedTriageActions(request) {
if strings.TrimSpace(action.Name) == "" {
return fmt.Errorf("triage action %d must not be empty", i)
}
if _, exists := seen[action.Name]; exists {
return fmt.Errorf("triage action %q is selected more than once", action.Name)
}
seen[action.Name] = struct{}{}
}
return nil
}

func normalizedTriageActions(request TriageRunRequest) []TriageActionReference {
if request.Actions == nil {
return []TriageActionReference{{Name: DefaultTriageAction}}
}
return append([]TriageActionReference(nil), request.Actions...)
}

func validateTriageRunIdentity(namespace, name string) error {
Expand Down Expand Up @@ -454,9 +679,10 @@ func validateTriageApplicationName(applicationName string) error {
}

func newTriageRun(request TriageRunRequest) *unstructured.Unstructured {
action := request.Action
if action == "" {
action = DefaultTriageAction
actions := normalizedTriageActions(request)
unstructuredActions := make([]any, len(actions))
for i := range actions {
unstructuredActions[i] = map[string]any{"name": actions[i].Name}
}

return &unstructured.Unstructured{Object: map[string]any{
Expand All @@ -473,7 +699,7 @@ func newTriageRun(request TriageRunRequest) *unstructured.Unstructured {
"applicationRef": map[string]any{
"name": request.ApplicationName,
},
"action": action,
"actions": unstructuredActions,
},
}}
}
Expand Down
Loading
Loading