Repository navigation
293 lines (267 loc) · 16.8 KB
/
Copy pathtest.yml
File metadata and controls
293 lines (267 loc) · 16.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
name: Integration tests
on:
push:
branches-ignore:
- "dependabot/**"
pull_request:
# Cancel superseded runs of the same workflow on the same ref (e.g. after a
# force-push), so an in-progress run does not linger and surface as a spurious
# failure. push and pull_request runs for the same commit remain independent.
concurrency:
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}
cancel-in-progress: true
jobs:
test:
runs-on: ubuntu-latest
strategy:
matrix:
# https://endoflife.date/python
# https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2204-Readme.md#python
python-version:
- "3.11"
- "3.12"
- "3.13"
- "3.14"
- "3.15"
steps:
- name: start mongodb
run: docker run --name mongo -p 127.0.0.1:27017:27017 -d mongo:latest
- uses: actions/checkout@v7
- name: set up python ${{ matrix.python-version }}
uses: actions/setup-python@v7
with:
python-version: ${{ matrix.python-version }}
- run: pip install -r requirements.txt
- name: add test data to mongodb
run: |
echo 'db.messages.insertOne({
user: "John Doe",
createdAt: ISODate("3025-01-15T21:31:09.989Z"),
updatedAt: ISODate("3025-01-15T21:31:09.989Z")
});
db.users.insertOne({
username: "jdoe",
createdAt: ISODate("3025-01-15T21:31:09.989Z")
});
db.transactions.insertMany([
{ user: ObjectId("000000000000000000000001"), model: "gemini-flash", context: "message",
tokenType: "prompt", rawAmount: -100, inputTokens: -50, writeTokens: -10, readTokens: -40,
tokenValue: -300, createdAt: ISODate("3025-01-15T21:31:09.989Z") },
{ user: ObjectId("000000000000000000000002"), model: "local-llm", context: "message",
tokenType: "prompt", rawAmount: -100, createdAt: ISODate("3025-01-15T21:31:09.989Z") },
{ user: ObjectId("000000000000000000000002"), model: "gemini-flash", context: "message",
tokenType: "prompt", rawAmount: -100, tokenValue: -300, createdAt: ISODate("3025-01-15T21:31:09.989Z") },
{ user: ObjectId("000000000000000000000003"), model: "gemini-flash", context: "title",
tokenType: "prompt", rawAmount: -100, tokenValue: -300, createdAt: ISODate("3025-01-15T21:31:09.989Z") },
{ user: ObjectId("000000000000000000000004"), model: "gemini-flash", context: "abort",
tokenType: "prompt", rawAmount: -100, tokenValue: -300, createdAt: ISODate("3025-01-15T21:31:09.989Z") },
{ user: ObjectId("000000000000000000000004"), model: "gemini-flash", context: "subagent",
tokenType: "completion", rawAmount: -50, tokenValue: -500, createdAt: ISODate("3025-01-15T21:31:09.989Z") },
{ user: ObjectId("000000000000000000000005"), context: "autoRefill",
tokenType: "credits", rawAmount: 5000000, tokenValue: 5000000, createdAt: ISODate("3025-01-15T21:31:09.989Z") }
]);
db.messages.insertOne({
isCreatedByUser: false,
endpoint: "custom",
model: "local-llm",
createdAt: ISODate("3025-01-15T21:31:09.989Z")
});
db.messages.insertMany([
{ isCreatedByUser: false, endpoint: "custom", model: "local-llm", error: false,
content: [{ type: "text", text: "partial" }, { type: "error", error: "upstream failed" }],
createdAt: ISODate("3025-01-15T21:31:09.989Z") },
{ isCreatedByUser: false, endpoint: "google", model: "gemini-flash", error: true,
text: "{\"type\":\"token_balance\"}", createdAt: ISODate("3025-01-15T21:31:09.989Z") },
{ isCreatedByUser: false, endpoint: "custom", model: "legacy-model", error: true,
text: "Something went wrong", createdAt: ISODate("3025-01-15T21:31:09.989Z") },
{ isCreatedByUser: false, endpoint: "google", model: "gemini-flash", unfinished: true,
createdAt: ISODate("3025-01-15T21:31:09.989Z") },
{ isCreatedByUser: false, endpoint: "google", model: "gemini-flash",
content: [
{ type: "tool_call", tool_call: { name: "search_mcp_mcp-uos", output: "found" } },
{ type: "tool_call", tool_call: { name: "search_mcp_mcp-uos", output: "Error processing tool search" } },
{ type: "tool_call", tool_call: { name: "fetch_mcp_mcp-uos", output: "Error calling tool 'fetch': 404 Not Found" } },
{ type: "tool_call", tool_call: { name: "web_search", output: "found" } },
{ type: "tool_call", tool_call: { name: "web_search", output: "Found 3 results about error handling" } },
{ type: "tool_call", tool_call: { name: "web_search",
output: "Error: Received tool input did not match expected schema\n Please fix your mistakes." } },
{ type: "tool_call", tool_call: { name: "web_search", output: "Error: [timeout] tool call failed: no response" } },
{ type: "tool_call", tool_call: { name: "bash_tool",
output: "Error: Interrupted by a user message. Cancellation was requested; external effects may already have occurred. Do not repeat this operation automatically.\n Please fix your mistakes." } }
],
createdAt: ISODate("3025-01-15T21:31:09.989Z") }
]);
db.messages.insertOne({
isCreatedByUser: false, endpoint: "agents", model: "local-llm",
content: [
{ type: "tool_call", tool_call: { name: "web_search", output: "" } },
{ type: "tool_call", tool_call: { name: "web_search", output: "=== Web Results, Turn 0 ===" } },
{ type: "tool_call", tool_call: { id: "srvtoolu_01", name: "web_search" } },
{ type: "tool_call", tool_call: { name: "bash_tool", output: "ok" } }
],
updatedAt: ISODate("3025-01-15T21:31:09.989Z")
});
db.sessions.insertMany([
{ user: ObjectId("000000000000000000000001"), expiration: ISODate("3025-01-15T21:31:09.989Z") },
{ user: ObjectId("000000000000000000000001"), expiration: ISODate("3025-01-15T21:31:09.989Z") },
{ user: ObjectId("000000000000000000000002"), expiration: ISODate("2020-01-15T21:31:09.989Z") }
]);
db.balances.insertMany([
{ user: ObjectId("000000000000000000000001"), tokenCredits: 0, refillAmount: 1000 },
{ user: ObjectId("000000000000000000000002"), tokenCredits: 50, refillAmount: 1000 },
{ user: ObjectId("000000000000000000000003"), tokenCredits: 500, refillAmount: 1000 },
{ user: ObjectId("000000000000000000000004"), tokenCredits: 50 }
]);
db.memoryentries.insertMany([
{ userId: ObjectId("000000000000000000000001"), key: "a", value: "x" },
{ userId: ObjectId("000000000000000000000001"), key: "b", value: "y" },
{ userId: ObjectId("000000000000000000000002"), key: "a", value: "z" }
]);
db.agents.insertMany([
{ id: "agent_a", model: "gemini-flash", category: "general" },
{ id: "agent_b", model: "gemini-flash", category: "general" },
{ id: "agent_c", model: "local-llm" }
]);
db.skills.insertOne({ name: "summarize" });
db.sharedlinks.insertMany([{ shareId: "s1" }, { shareId: "s2" }]);
db.mcpservers.insertOne({ serverName: "mcp-uos" });
db.agentapikeys.insertMany([
{ name: "used", lastUsedAt: ISODate("3025-01-15T21:31:09.989Z") },
{ name: "unused" }
]);
db.conversations.insertMany([
{ conversationId: "temporary", isTemporary: true, expiredAt: ISODate("3025-01-15T21:31:09.989Z") },
{ conversationId: "temporary-before-v0.8.6", expiredAt: ISODate("3025-01-15T21:31:09.989Z") },
{ conversationId: "retained", isTemporary: false, expiredAt: ISODate("3025-01-15T21:31:09.989Z") }
]);
db.scheduleruns.insertMany([
{ status: "success", createdAt: ISODate("3025-01-15T21:31:09.989Z") },
{ status: "success", createdAt: ISODate("3025-01-15T21:31:09.989Z") },
{ status: "error", createdAt: ISODate("3025-01-15T21:31:09.989Z") }
]);
quit;' | docker exec -i mongo mongosh LibreChat
- name: run exporter
env:
MONGODB_URI: "mongodb://127.0.0.1:27017/"
LOGGING_LEVEL: debug
METRICS_CACHE_ENABLED: "false"
LIBRECHAT_URL: "http://localhost:8000"
METRICS_TIMEZONE: "Asia/Tokyo"
EXTERNAL_MODEL_REGEX: "gemini"
run: python metrics.py &> exporter.log &
- name: get metrics
run: >
curl -fisS --retry 60 --retry-delay 1 --retry-all-errors
http://localhost:8000 -o metrics
# Debug helpers — tolerate a missing file so a cancelled run is not
# flipped to "failure" by these always() steps when the metrics file
# was never written.
- name: show metrics
if: ${{ always() }}
run: cat metrics || true
- name: show log
if: ${{ always() }}
run: cat exporter.log || true
- name: probe for valid output
run: grep 'librechat_registered_users_total 1.0' metrics
- name: probe for librechat status in output
run: grep 'librechat_status_code' metrics
# Errors are either flagged on the message (rejected requests) or stored as
# an error content part (failures during generation in the agents flow).
- name: probe for error metrics
run: |
grep '^librechat_error_messages_total 3.0' metrics
grep '^librechat_errors_per_model_total{model="local-llm"} 1.0' metrics
grep '^librechat_errors_per_model_total{model="gemini-flash"} 1.0' metrics
grep '^librechat_error_messages_5m 3.0' metrics
grep '^librechat_errors_per_model_5m{model="local-llm"} 1.0' metrics
# Requests rejected before generation, by the type LibreChat records.
- name: probe for rejected request metrics
run: |
grep '^librechat_window_rejected_requests{' metrics | grep 'reason="token_balance"' | grep 'window="1d"' | grep ' 1.0$'
grep '^librechat_window_rejected_requests{' metrics | grep 'reason="other"' | grep 'window="1d"' | grep ' 1.0$'
# Label order in the output depends on the prometheus-client version, so
# match each label separately. The title transaction must not count, the
# abort and subagent ones must. The failed answer must not count.
- name: probe for usage window metrics
run: |
grep '^librechat_window_unique_users{window="1d"} 3.0' metrics
grep '^librechat_window_users_by_model_class{' metrics | grep 'window="1d"' | grep 'usage="both"' | grep ' 1.0$'
grep '^librechat_window_users_by_model_class{' metrics | grep 'window="1d"' | grep 'usage="external_only"' | grep ' 2.0$'
grep '^librechat_window_answers_per_model{' metrics | grep 'model="local-llm"' | grep 'class="local"' | grep 'window="1d"' | grep ' 1.0$'
grep '^librechat_window_tokens_per_model{' metrics | grep 'model="gemini-flash"' | grep 'type="input"' | grep 'window="1d"' | grep ' 300.0$'
grep '^librechat_window_tokens_per_model{' metrics | grep 'model="gemini-flash"' | grep 'type="output"' | grep 'window="1d"' | grep ' 50.0$'
- name: probe for cache token metrics
run: |
grep '^librechat_window_cache_tokens_per_model{' metrics | grep 'model="gemini-flash"' | grep 'type="read"' | grep 'window="1d"' | grep ' 40.0$'
grep '^librechat_window_cache_tokens_per_model{' metrics | grep 'model="gemini-flash"' | grep 'type="write"' | grep 'window="1d"' | grep ' 10.0$'
grep '^librechat_window_cache_tokens_per_model{' metrics | grep 'model="local-llm"' | grep 'type="read"' | grep 'window="1d"' | grep ' 0.0$'
# Every context is charged, credit refills are not.
- name: probe for credit metrics
run: |
credits() { grep '^librechat_window_credits_per_model{' metrics | grep 'model="gemini-flash"' | grep 'window="1d"' | grep "context=\"$1\"" | grep "type=\"$2\"" | grep " $3$"; }
credits message input 600.0
credits title input 300.0
credits abort input 300.0
credits subagent output 500.0
if grep -q 'context="autoRefill"' metrics; then echo "credit refills must not count"; exit 1; fi
# Background requests only; chat requests are in librechat_window_tokens_per_model.
- name: probe for background token metrics
run: |
grep '^librechat_window_background_tokens_per_model{' metrics | grep 'model="gemini-flash"' | grep 'context="title"' | grep 'type="input"' | grep 'window="1d"' | grep ' 100.0$'
if grep '^librechat_window_background_tokens_per_model{' metrics | grep -q 'context="message"'; then
echo "chat requests must not count as background"; exit 1
fi
- name: probe for unfinished answer metrics
run: |
grep '^librechat_window_unfinished_answers_per_model{' metrics | grep 'model="gemini-flash"' | grep 'window="1d"' | grep ' 1.0$'
grep '^librechat_window_unfinished_answers_per_model{' metrics | grep 'model="local-llm"' | grep 'window="1d"' | grep ' 0.0$'
# Exhausted: no credits left. Low: below 10% of the refill amount.
- name: probe for balance metrics
run: |
grep '^librechat_balance_users{state="exhausted"} 1.0' metrics
grep '^librechat_balance_users{state="low"} 1.0' metrics
grep '^librechat_balance_users{state="ok"} 2.0' metrics
# Failed tool calls, in the forms LibreChat and MCP servers report them. An
# output that merely mentions an error is not a failure, and neither is a
# call the user interrupted by sending a message.
- name: probe for tool error metrics
run: |
grep '^librechat_tool_call_errors_total 4.0' metrics
grep '^librechat_tool_call_errors_per_tool_total{tool_name="web_search"} 2.0' metrics
grep '^librechat_tool_call_errors_per_tool_total{tool_name="fetch_mcp_mcp-uos"} 1.0' metrics
grep '^librechat_tool_success_rate_per_tool{tool_name="bash_tool"} 100.0' metrics
if grep -q 'librechat_tool_call_errors_per_tool_total{tool_name="bash_tool"}' metrics; then
echo "a call interrupted by a user message is not a failure"; exit 1
fi
# LibreChat's web search stores an empty output when the search returned no
# results. A call with no output at all, such as a provider's own web search,
# does not count as empty. Every tool used in the window gets a value.
- name: probe for empty tool output metrics
run: |
grep '^librechat_tool_call_empty_outputs_per_tool_5m{tool_name="web_search"} 1.0' metrics
grep '^librechat_tool_call_empty_outputs_per_tool_5m{tool_name="bash_tool"} 0.0' metrics
grep '^librechat_tool_calls_per_tool_5m{tool_name="web_search"} 3.0' metrics
# MCP tools are named <tool>_mcp_<server>.
- name: probe for MCP metrics
run: |
grep '^librechat_mcp_tool_calls_per_server{server="mcp-uos"} 3.0' metrics
grep '^librechat_mcp_tool_call_errors_per_server{server="mcp-uos"} 2.0' metrics
# A collection the LibreChat version does not have yields no metric.
- name: probe for feature adoption metrics
run: |
grep '^librechat_logged_in_users 1.0' metrics
grep '^librechat_window_new_users{window="1d"} 1.0' metrics
grep '^librechat_memory_entries_total 3.0' metrics
grep '^librechat_memory_users 2.0' metrics
grep '^librechat_agents_total{category="general"} 2.0' metrics
grep '^librechat_agents_total{category="uncategorized"} 1.0' metrics
grep '^librechat_skills_total 1.0' metrics
grep '^librechat_shared_links_total 2.0' metrics
grep '^librechat_mcp_servers_total 1.0' metrics
grep '^librechat_agent_api_keys_total 2.0' metrics
grep '^librechat_window_agent_api_keys_used{window="1d"} 1.0' metrics
grep '^librechat_temporary_conversations 2.0' metrics
grep '^librechat_window_schedule_runs{' metrics | grep 'status="success"' | grep 'window="1d"' | grep ' 2.0$'
grep '^librechat_window_schedule_runs{' metrics | grep 'status="error"' | grep 'window="1d"' | grep ' 1.0$'
if grep -q '^librechat_chat_projects_total' metrics; then echo "a missing collection must not be reported"; exit 1; fi