lox-cli/eval-report.json at main · eisber/lox-cli · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
{
  "summary": {
    "total": 1,
    "passed": 1,
    "failed": 0,
    "pass_rate": 1.0,
    "xml_valid": 1,
    "validation_pass": 1,
    "avg_overall_score": 1.0,
    "avg_weighted_score": 1.5,
    "avg_block_f1": 1.0,
    "avg_wiring_accuracy": 1.0,
    "avg_wiring_precision": 1.0,
    "avg_param_accuracy": 1.0,
    "avg_ux_score": 1.0,
    "total_cli_invocations": 14,
    "total_retries": 0,
    "total_tokens_input": 250,
    "total_tokens_output": 1385,
    "avg_cli_per_case": 14.0,
    "avg_tokens_per_case": 1635.0
  },
  "by_difficulty": {
    "medium": {
      "count": 1,
      "passed": 1,
      "avg_score": 1.0,
      "avg_block_f1": 1.0,
      "avg_wiring": 1.0,
      "avg_params": 1.0,
      "avg_cli_invocations": 14.0,
      "avg_tokens": 1385.0
    }
  },
  "by_pattern": {
    "threshold": {
      "count": 1,
      "passed": 1,
      "avg_score": 1.0
    },
    "combined-condition": {
      "count": 1,
      "passed": 1,
      "avg_score": 1.0
    }
  },
  "cases": [
    {
      "pass": true,
      "overall_score": 1.0,
      "weighted_score": 1.5,
      "xml_valid": true,
      "validation_pass": true,
      "validation_errors": 0,
      "metrics": {
        "blocks": {
          "precision": 1.0,
          "recall": 1.0,
          "f1": 1.0,
          "true_positives": 2,
          "false_positives": 0,
          "false_negatives": 0
        },
        "wiring": {
          "accuracy": 1.0,
          "precision": 1.0,
          "correct": 4,
          "total": 4,
          "extra": 0
        },
        "params": {
          "accuracy": 1.0,
          "correct": 1,
          "total": 1
        },
        "ux": {
          "score": 1.0,
          "issues": []
        }
      },
      "case_id": "s01-piano-protection",
      "difficulty": "medium",
      "patterns": [
        "threshold",
        "combined-condition"
      ],
      "utterance": "Lower blinds in the living room when it's sunny and above 20 degrees to protect the piano",
      "cli_invocations": 14,
      "retries": 0,
      "validation_runs": 1,
      "tokens": {
        "input_chars": 1003,
        "output_chars": 5543,
        "input_tokens_est": 250,
        "output_tokens_est": 1385
      }
    }
  ]
}