From 9021a43d7f5511406dcd828cf82db6fd75f59604 Mon Sep 17 00:00:00 2001 From: JR Boos Date: Tue, 7 Jul 2026 13:30:44 -0400 Subject: [PATCH 1/4] e2e(hitl): added e2e tests for HITL --- .../lightspeed-stack-mcp-approvals.yaml | 46 ++++ .../lightspeed-stack-mcp-approvals.yaml | 47 ++++ tests/e2e/features/approvals.feature | 227 ++++++++++++++++++ 3 files changed, 320 insertions(+) create mode 100644 tests/e2e/configuration/library-mode/lightspeed-stack-mcp-approvals.yaml create mode 100644 tests/e2e/configuration/server-mode/lightspeed-stack-mcp-approvals.yaml create mode 100644 tests/e2e/features/approvals.feature diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-approvals.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-approvals.yaml new file mode 100644 index 000000000..a78b4c484 --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-approvals.yaml @@ -0,0 +1,46 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds llama-stack as library + use_as_library_client: true + library_client_config_path: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop-with-token" +inference: + default_provider: openai + default_model: gpt-4o-mini +approvals: + approval_timeout_seconds: 60 + approval_retention_days: 7 +mcp_servers: + - name: "mcp-approval-always" + provider_id: "model-context-protocol" + url: "http://mock-mcp:3000" + require_approval: "always" + - name: "mcp-approval-never" + provider_id: "model-context-protocol" + url: "http://mock-mcp:3000" + require_approval: "never" + - name: "mcp-approval-granular" + provider_id: "model-context-protocol" + url: "http://mock-mcp:3000" + require_approval: + always: + - "" + never: + - "" \ No newline at end of file diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-approvals.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-approvals.yaml new file mode 100644 index 000000000..1e0acad01 --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-approvals.yaml @@ -0,0 +1,47 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to separate llama-stack service + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop-with-token" +inference: + default_provider: openai + default_model: gpt-4o-mini +approvals: + approval_timeout_seconds: 60 + approval_retention_days: 7 +mcp_servers: + - name: "mcp-approval-always" + provider_id: "model-context-protocol" + url: "http://mock-mcp:3000" + require_approval: "always" + - name: "mcp-approval-never" + provider_id: "model-context-protocol" + url: "http://mock-mcp:3000" + require_approval: "never" + - name: "mcp-approval-granular" + provider_id: "model-context-protocol" + url: "http://mock-mcp:3000" + require_approval: + always: + - "" + never: + - "" \ No newline at end of file diff --git a/tests/e2e/features/approvals.feature b/tests/e2e/features/approvals.feature new file mode 100644 index 000000000..56bf63375 --- /dev/null +++ b/tests/e2e/features/approvals.feature @@ -0,0 +1,227 @@ +Feature: Human-in-the-Loop MCP approval tests + + Background: + Given The service is started locally + And The system is in default state + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + And REST API service prefix is /v1 + And the Lightspeed stack configuration directory is "tests/e2e/configuration" + + + # --- require_approval: "never" returns successful query --- + + Scenario: Query with require_approval "never" returns successful response + Given MCP toolgroups are reset for a new MCP configuration + And The service uses the lightspeed-stack-mcp-approvals.yaml configuration + And The service is restarted + When I use "query" to ask question with authorization header + """ + {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The body of the response does not contain requires_action + + Scenario: Streaming query with require_approval "never" returns successful response + Given MCP toolgroups are reset for a new MCP configuration + And The service uses the lightspeed-stack-mcp-approvals.yaml configuration + And The service is restarted + When I use "streaming_query" to ask question with authorization header + """ + {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + When I wait for the response to be completed + Then The status code of the response is 200 + And The body of the response does not contain approval_required + + + # --- require_approval: "always" returns requires_action --- + + Scenario: Query with require_approval "always" returns requires_action status + Given MCP toolgroups are reset for a new MCP configuration + And The service uses the lightspeed-stack-mcp-approvals.yaml configuration + And The service is restarted + When I use "query" to ask question with authorization header + """ + {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The body of the response contains requires_action + And The body of the response contains mcp_approval + + Scenario: Streaming query with require_approval "always" returns approval_required event + Given MCP toolgroups are reset for a new MCP configuration + And The service uses the lightspeed-stack-mcp-approvals.yaml configuration + And The service is restarted + When I use "streaming_query" to ask question with authorization header + """ + {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + When I wait for the response to be completed + Then The status code of the response is 200 + And The body of the response contains approval_required + + + # --- require_approval: granular (always/never filter) --- + + Scenario: Query with granular approval filter returns requires_action for "always" tool + Given MCP toolgroups are reset for a new MCP configuration + And The service uses the lightspeed-stack-mcp-approvals.yaml configuration + And The service is restarted + When I use "query" to ask question with authorization header + """ + {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The body of the response contains requires_action + And The body of the response contains mcp_approval + + Scenario: Query with granular approval filter returns successful response for "never" tool + Given MCP toolgroups are reset for a new MCP configuration + And The service uses the lightspeed-stack-mcp-approvals.yaml configuration + And The service is restarted + When I use "query" to ask question with authorization header + """ + {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The body of the response does not contain requires_action + + Scenario: Streaming query with granular approval filter returns approval_required for "always" tool + Given MCP toolgroups are reset for a new MCP configuration + And The service uses the lightspeed-stack-mcp-approvals.yaml configuration + And The service is restarted + When I use "streaming_query" to ask question with authorization header + """ + {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The body of the response contains approval_required + + Scenario: Streaming query with granular approval filter returns successful response for "never" tool + Given MCP toolgroups are reset for a new MCP configuration + And The service uses the lightspeed-stack-mcp-approvals.yaml configuration + And The service is restarted + When I use "streaming_query" to ask question with authorization header + """ + {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + When I wait for the response to be completed + Then The status code of the response is 200 + And The body of the response does not contain approval_required + + + # --- Approve a pending approval --- + + Scenario: Approve a pending approval via POST /approvals/{id} on query + Given MCP toolgroups are reset for a new MCP configuration + And The service uses the lightspeed-stack-mcp-approvals.yaml configuration + And The service is restarted + When I use "query" to ask question with authorization header + """ + {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The body of the response contains requires_action + When I extract the approval id from the response + And I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP POST method + """ + {"approve": true} + """ + Then The status code of the response is 200 + And The body of the response contains approved + + Scenario: Approve a pending approval via POST /approvals/{id} on streaming_query + Given MCP toolgroups are reset for a new MCP configuration + And The service uses the lightspeed-stack-mcp-approvals.yaml configuration + And The service is restarted + When I use "streaming_query" to ask question with authorization header + """ + {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The body of the response contains approval_required + When I extract the approval id from the response + And I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP POST method + """ + {"approve": true} + """ + Then The status code of the response is 200 + And The body of the response contains approved + + + # --- Deny a pending approval --- + + Scenario: Deny a pending approval via POST /approvals/{id} on query + Given MCP toolgroups are reset for a new MCP configuration + And The service uses the lightspeed-stack-mcp-approvals.yaml configuration + And The service is restarted + When I use "query" to ask question with authorization header + """ + {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The body of the response contains requires_action + When I extract the approval id from the response + And I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP POST method + """ + {"approve": false} + """ + Then The status code of the response is 200 + And The body of the response contains denied + + Scenario: Deny a pending approval via POST /approvals/{id} on streaming_query + Given MCP toolgroups are reset for a new MCP configuration + And The service uses the lightspeed-stack-mcp-approvals.yaml configuration + And The service is restarted + When I use "streaming_query" to ask question with authorization header + """ + {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The body of the response contains approval_required + When I extract the approval id from the response + And I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP POST method + """ + {"approve": false} + """ + Then The status code of the response is 200 + And The body of the response contains denied + + + # --- GET /approvals returns all approvals --- + + @MCPApprovalsConfig @flaky + Scenario: GET /approvals returns list of pending approvals + Given MCP toolgroups are reset for a new MCP configuration + And The service uses the lightspeed-stack-mcp-approvals.yaml configuration + And The service is restarted + When I use "query" to ask question with authorization header + """ + {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The body of the response contains requires_action + When I access REST API endpoint "approvals" using HTTP GET method + Then The status code of the response is 200 + And The body of the response contains approvals + And The body of the response contains pending + + + # --- GET /approvals/{id} returns a single approval --- + + @MCPApprovalsConfig @flaky + Scenario: GET /approvals/{id} returns a specific approval + Given MCP toolgroups are reset for a new MCP configuration + And The service uses the lightspeed-stack-mcp-approvals.yaml configuration + And The service is restarted + When I use "query" to ask question with authorization header + """ + {"query": "Use the mcp-approval-always server to list repos", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The body of the response contains requires_action + When I extract the approval id from the response + And I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP GET method + Then The status code of the response is 200 + And The body of the response contains pending + And The body of the response contains mcp-approval-always From cc8eacadfb6149d9557de16f852918f36b961853 Mon Sep 17 00:00:00 2001 From: JR Boos Date: Fri, 10 Jul 2026 15:55:31 -0400 Subject: [PATCH 2/4] e2e(hitl): added tests for retention/expiry --- ...d-stack-mcp-approvals-short-retention.yaml | 34 ++++ ...eed-stack-mcp-approvals-short-timeout.yaml | 34 ++++ ...d-stack-mcp-approvals-short-retention.yaml | 35 +++++ ...eed-stack-mcp-approvals-short-timeout.yaml | 35 +++++ tests/e2e/features/approvals.feature | 145 ++++++++++++++++++ 5 files changed, 283 insertions(+) create mode 100644 tests/e2e/configuration/library-mode/lightspeed-stack-mcp-approvals-short-retention.yaml create mode 100644 tests/e2e/configuration/library-mode/lightspeed-stack-mcp-approvals-short-timeout.yaml create mode 100644 tests/e2e/configuration/server-mode/lightspeed-stack-mcp-approvals-short-retention.yaml create mode 100644 tests/e2e/configuration/server-mode/lightspeed-stack-mcp-approvals-short-timeout.yaml diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-approvals-short-retention.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-approvals-short-retention.yaml new file mode 100644 index 000000000..23adf124b --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-approvals-short-retention.yaml @@ -0,0 +1,34 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds llama-stack as library + use_as_library_client: true + library_client_config_path: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop-with-token" +inference: + default_provider: openai + default_model: gpt-4o-mini +approvals: + approval_timeout_seconds: 60 + approval_retention_seconds: 5 +mcp_servers: + - name: "mcp-approval-always" + provider_id: "model-context-protocol" + url: "http://mock-mcp:3000" + require_approval: "always" diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-approvals-short-timeout.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-approvals-short-timeout.yaml new file mode 100644 index 000000000..402e37012 --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-approvals-short-timeout.yaml @@ -0,0 +1,34 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds llama-stack as library + use_as_library_client: true + library_client_config_path: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop-with-token" +inference: + default_provider: openai + default_model: gpt-4o-mini +approvals: + approval_timeout_seconds: 5 + approval_retention_seconds: 5 +mcp_servers: + - name: "mcp-approval-always" + provider_id: "model-context-protocol" + url: "http://mock-mcp:3000" + require_approval: "always" diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-approvals-short-retention.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-approvals-short-retention.yaml new file mode 100644 index 000000000..f824bd59d --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-approvals-short-retention.yaml @@ -0,0 +1,35 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to separate llama-stack service + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop-with-token" +inference: + default_provider: openai + default_model: gpt-4o-mini +approvals: + approval_timeout_seconds: 60 + approval_retention_seconds: 5 +mcp_servers: + - name: "mcp-approval-always" + provider_id: "model-context-protocol" + url: "http://mock-mcp:3000" + require_approval: "always" diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-approvals-short-timeout.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-approvals-short-timeout.yaml new file mode 100644 index 000000000..8959daabe --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-approvals-short-timeout.yaml @@ -0,0 +1,35 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to separate llama-stack service + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" +authentication: + module: "noop-with-token" +inference: + default_provider: openai + default_model: gpt-4o-mini +approvals: + approval_timeout_seconds: 5 + approval_retention_seconds: 5 +mcp_servers: + - name: "mcp-approval-always" + provider_id: "model-context-protocol" + url: "http://mock-mcp:3000" + require_approval: "always" diff --git a/tests/e2e/features/approvals.feature b/tests/e2e/features/approvals.feature index 56bf63375..3529ec411 100644 --- a/tests/e2e/features/approvals.feature +++ b/tests/e2e/features/approvals.feature @@ -225,3 +225,148 @@ Feature: Human-in-the-Loop MCP approval tests Then The status code of the response is 200 And The body of the response contains pending And The body of the response contains mcp-approval-always + + # --- Approval timeout / expiry --- + + Scenario: Expired approval returns 410 when attempting to approve + Given MCP toolgroups are reset for a new MCP configuration + And The service uses the lightspeed-stack-mcp-approvals-short-timeout.yaml configuration + And The service is restarted + When I use "query" to ask question with authorization header + """ + {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The body of the response contains requires_action + When I extract the approval id from the response + And I wait for 6 seconds + And I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP POST method + """ + {"approve": true} + """ + Then The status code of the response is 410 + And The body of the response contains approval_expired + + Scenario: Expired approval returns 410 when attempting to deny + Given MCP toolgroups are reset for a new MCP configuration + And The service uses the lightspeed-stack-mcp-approvals-short-timeout.yaml configuration + And The service is restarted + When I use "query" to ask question with authorization header + """ + {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The body of the response contains requires_action + When I extract the approval id from the response + And I wait for 6 seconds + And I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP POST method + """ + {"approve": false} + """ + Then The status code of the response is 410 + And The body of the response contains approval_expired + + # --- Retention cleanup: decided approvals purged after approval_retention_seconds --- + + Scenario: Approved approval is purged after retention period expires + Given MCP toolgroups are reset for a new MCP configuration + And The service uses the lightspeed-stack-mcp-approvals-short-retention.yaml configuration + And The service is restarted + When I use "query" to ask question with authorization header + """ + {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The body of the response contains requires_action + When I extract the approval id from the response + And I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP POST method + """ + {"approve": true} + """ + Then The status code of the response is 200 + And The body of the response contains approved + When I wait for 6 seconds + And I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP GET method + Then The status code of the response is 404 + And The body of the response contains approval_not_found + + Scenario: Denied approval is purged after retention period expires + Given MCP toolgroups are reset for a new MCP configuration + And The service uses the lightspeed-stack-mcp-approvals-short-retention.yaml configuration + And The service is restarted + When I use "query" to ask question with authorization header + """ + {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The body of the response contains requires_action + When I extract the approval id from the response + And I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP POST method + """ + {"approve": false} + """ + Then The status code of the response is 200 + And The body of the response contains denied + When I wait for 6 seconds + And I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP GET method + Then The status code of the response is 404 + And The body of the response contains approval_not_found + + Scenario: Expired approval is purged after retention period + Given MCP toolgroups are reset for a new MCP configuration + And The service uses the lightspeed-stack-mcp-approvals-short-timeout.yaml configuration + And The service is restarted + When I use "query" to ask question with authorization header + """ + {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The body of the response contains requires_action + When I extract the approval id from the response + And I wait for 11 seconds + And I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP GET method + Then The status code of the response is 404 + And The body of the response contains approval_not_found + + Scenario: Decided approval remains queryable within retention period + Given MCP toolgroups are reset for a new MCP configuration + And The service uses the lightspeed-stack-mcp-approvals-short-retention.yaml configuration + And The service is restarted + When I use "query" to ask question with authorization header + """ + {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The body of the response contains requires_action + When I extract the approval id from the response + And I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP POST method + """ + {"approve": true} + """ + Then The status code of the response is 200 + And The body of the response contains approved + When I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP GET method + Then The status code of the response is 200 + And The body of the response contains approved + And The body of the response contains decided_at + + # --- Approval not found returns 404 --- + + Scenario: GET on non-existent approval returns 404 + Given MCP toolgroups are reset for a new MCP configuration + And The service uses the lightspeed-stack-mcp-approvals.yaml configuration + And The service is restarted + When I access REST API endpoint "approvals/non-existent-id-12345" using HTTP GET method + Then The status code of the response is 404 + And The body of the response contains approval_not_found + + Scenario: POST to non-existent approval returns 404 + Given MCP toolgroups are reset for a new MCP configuration + And The service uses the lightspeed-stack-mcp-approvals.yaml configuration + And The service is restarted + When I access REST API endpoint "approvals/non-existent-id-12345" using HTTP POST method + """ + {"approve": true} + """ + Then The status code of the response is 404 + And The body of the response contains approval_not_found \ No newline at end of file From c578620c29d74a75bc3bb0ff916f1fcc7ca39eff Mon Sep 17 00:00:00 2001 From: JR Boos Date: Fri, 10 Jul 2026 16:02:11 -0400 Subject: [PATCH 3/4] e2e(hitl): skipped tests --- tests/e2e/features/approvals.feature | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/e2e/features/approvals.feature b/tests/e2e/features/approvals.feature index 3529ec411..2b5b5fd68 100644 --- a/tests/e2e/features/approvals.feature +++ b/tests/e2e/features/approvals.feature @@ -1,3 +1,4 @@ +@skip Feature: Human-in-the-Loop MCP approval tests Background: From 880ac393dae7f9335a803acdf149966cf58338f1 Mon Sep 17 00:00:00 2001 From: JR Boos Date: Mon, 24 Aug 2026 11:27:35 -0400 Subject: [PATCH 4/4] addressed comments --- ...d-stack-mcp-approvals-short-retention.yaml | 4 +- ...eed-stack-mcp-approvals-short-timeout.yaml | 4 +- .../lightspeed-stack-mcp-approvals.yaml | 8 +- ...d-stack-mcp-approvals-short-retention.yaml | 4 +- ...eed-stack-mcp-approvals-short-timeout.yaml | 4 +- .../lightspeed-stack-mcp-approvals.yaml | 8 +- tests/e2e/features/approvals.feature | 186 ++++++------------ 7 files changed, 81 insertions(+), 137 deletions(-) diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-approvals-short-retention.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-approvals-short-retention.yaml index 23adf124b..12803e6a6 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-approvals-short-retention.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-approvals-short-retention.yaml @@ -25,8 +25,8 @@ inference: default_provider: openai default_model: gpt-4o-mini approvals: - approval_timeout_seconds: 60 - approval_retention_seconds: 5 + approval_timeout: 5 second + approval_retention: 10 seconds mcp_servers: - name: "mcp-approval-always" provider_id: "model-context-protocol" diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-approvals-short-timeout.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-approvals-short-timeout.yaml index 402e37012..81c7835a5 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-approvals-short-timeout.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-approvals-short-timeout.yaml @@ -25,8 +25,8 @@ inference: default_provider: openai default_model: gpt-4o-mini approvals: - approval_timeout_seconds: 5 - approval_retention_seconds: 5 + approval_timeout: 5 seconds + approval_retention: 1 days mcp_servers: - name: "mcp-approval-always" provider_id: "model-context-protocol" diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-approvals.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-approvals.yaml index a78b4c484..d86f094a5 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-approvals.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-approvals.yaml @@ -25,8 +25,8 @@ inference: default_provider: openai default_model: gpt-4o-mini approvals: - approval_timeout_seconds: 60 - approval_retention_days: 7 + approval_timeout: 60 seconds + approval_retention: 7 days mcp_servers: - name: "mcp-approval-always" provider_id: "model-context-protocol" @@ -41,6 +41,6 @@ mcp_servers: url: "http://mock-mcp:3000" require_approval: always: - - "" + - "add" never: - - "" \ No newline at end of file + - "subtract" \ No newline at end of file diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-approvals-short-retention.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-approvals-short-retention.yaml index f824bd59d..31b8a992f 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-approvals-short-retention.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-approvals-short-retention.yaml @@ -26,8 +26,8 @@ inference: default_provider: openai default_model: gpt-4o-mini approvals: - approval_timeout_seconds: 60 - approval_retention_seconds: 5 + approval_timeout: 5 second + approval_retention: 10 seconds mcp_servers: - name: "mcp-approval-always" provider_id: "model-context-protocol" diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-approvals-short-timeout.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-approvals-short-timeout.yaml index 8959daabe..64d9b0649 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-approvals-short-timeout.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-approvals-short-timeout.yaml @@ -26,8 +26,8 @@ inference: default_provider: openai default_model: gpt-4o-mini approvals: - approval_timeout_seconds: 5 - approval_retention_seconds: 5 + approval_timeout: 5 seconds + approval_retention: 1 days mcp_servers: - name: "mcp-approval-always" provider_id: "model-context-protocol" diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-approvals.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-approvals.yaml index 1e0acad01..e86589ea7 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-approvals.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-approvals.yaml @@ -26,8 +26,8 @@ inference: default_provider: openai default_model: gpt-4o-mini approvals: - approval_timeout_seconds: 60 - approval_retention_days: 7 + approval_timeout: 60 seconds + approval_retention: 7 days mcp_servers: - name: "mcp-approval-always" provider_id: "model-context-protocol" @@ -42,6 +42,6 @@ mcp_servers: url: "http://mock-mcp:3000" require_approval: always: - - "" + - "add" never: - - "" \ No newline at end of file + - "subtract" \ No newline at end of file diff --git a/tests/e2e/features/approvals.feature b/tests/e2e/features/approvals.feature index 2b5b5fd68..795029a15 100644 --- a/tests/e2e/features/approvals.feature +++ b/tests/e2e/features/approvals.feature @@ -9,7 +9,7 @@ Feature: Human-in-the-Loop MCP approval tests And the Lightspeed stack configuration directory is "tests/e2e/configuration" - # --- require_approval: "never" returns successful query --- + # --- require_approval: "never" returns successful query, and no pending approvals are recorded --- Scenario: Query with require_approval "never" returns successful response Given MCP toolgroups are reset for a new MCP configuration @@ -17,10 +17,16 @@ Feature: Human-in-the-Loop MCP approval tests And The service is restarted When I use "query" to ask question with authorization header """ - {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + {"query": "Use the divide tool from the mcp-approval-never server to divide 10 by 2.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 And The body of the response does not contain requires_action + When I access REST API endpoint "approvals" using HTTP GET method + Then The status code of the response is 200 + And The body of the response is the following + """ + {"approvals": []} + """ Scenario: Streaming query with require_approval "never" returns successful response Given MCP toolgroups are reset for a new MCP configuration @@ -28,38 +34,17 @@ Feature: Human-in-the-Loop MCP approval tests And The service is restarted When I use "streaming_query" to ask question with authorization header """ - {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + {"query": "Use the divide tool from the mcp-approval-never server to divide 10 by 2.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ When I wait for the response to be completed Then The status code of the response is 200 And The body of the response does not contain approval_required - - - # --- require_approval: "always" returns requires_action --- - - Scenario: Query with require_approval "always" returns requires_action status - Given MCP toolgroups are reset for a new MCP configuration - And The service uses the lightspeed-stack-mcp-approvals.yaml configuration - And The service is restarted - When I use "query" to ask question with authorization header - """ - {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} - """ - Then The status code of the response is 200 - And The body of the response contains requires_action - And The body of the response contains mcp_approval - - Scenario: Streaming query with require_approval "always" returns approval_required event - Given MCP toolgroups are reset for a new MCP configuration - And The service uses the lightspeed-stack-mcp-approvals.yaml configuration - And The service is restarted - When I use "streaming_query" to ask question with authorization header - """ - {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} - """ - When I wait for the response to be completed + When I access REST API endpoint "approvals" using HTTP GET method Then The status code of the response is 200 - And The body of the response contains approval_required + And The body of the response is the following + """ + {"approvals": []} + """ # --- require_approval: granular (always/never filter) --- @@ -70,7 +55,7 @@ Feature: Human-in-the-Loop MCP approval tests And The service is restarted When I use "query" to ask question with authorization header """ - {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + {"query": "Use the add tool from the mcp-approval-granular server to add 2 and 3.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 And The body of the response contains requires_action @@ -82,10 +67,16 @@ Feature: Human-in-the-Loop MCP approval tests And The service is restarted When I use "query" to ask question with authorization header """ - {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + {"query": "Use the subtract tool from the mcp-approval-granular server to subtract 1 from 5.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 And The body of the response does not contain requires_action + When I access REST API endpoint "approvals" using HTTP GET method + Then The status code of the response is 200 + And The body of the response is the following + """ + {"approvals": []} + """ Scenario: Streaming query with granular approval filter returns approval_required for "always" tool Given MCP toolgroups are reset for a new MCP configuration @@ -93,8 +84,9 @@ Feature: Human-in-the-Loop MCP approval tests And The service is restarted When I use "streaming_query" to ask question with authorization header """ - {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + {"query": "Use the add tool from the mcp-approval-granular server to add 2 and 3.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ + When I wait for the response to be completed Then The status code of the response is 200 And The body of the response contains approval_required @@ -104,11 +96,17 @@ Feature: Human-in-the-Loop MCP approval tests And The service is restarted When I use "streaming_query" to ask question with authorization header """ - {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + {"query": "Use the subtract tool from the mcp-approval-granular server to subtract 1 from 5.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ When I wait for the response to be completed Then The status code of the response is 200 And The body of the response does not contain approval_required + When I access REST API endpoint "approvals" using HTTP GET method + Then The status code of the response is 200 + And The body of the response is the following + """ + {"approvals": []} + """ # --- Approve a pending approval --- @@ -119,17 +117,19 @@ Feature: Human-in-the-Loop MCP approval tests And The service is restarted When I use "query" to ask question with authorization header """ - {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + {"query": "Use the multiply tool from the mcp-approval-always server to multiply 2 and 3.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 And The body of the response contains requires_action - When I extract the approval id from the response And I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP POST method """ {"approve": true} """ Then The status code of the response is 200 And The body of the response contains approved + When I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP GET method + Then The status code of the response is 200 + And The status message of the response is "approved" Scenario: Approve a pending approval via POST /approvals/{id} on streaming_query Given MCP toolgroups are reset for a new MCP configuration @@ -137,17 +137,20 @@ Feature: Human-in-the-Loop MCP approval tests And The service is restarted When I use "streaming_query" to ask question with authorization header """ - {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + {"query": "Use the multiply tool from the mcp-approval-always server to multiply 2 and 3.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ + When I wait for the response to be completed Then The status code of the response is 200 And The body of the response contains approval_required - When I extract the approval id from the response And I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP POST method """ {"approve": true} """ Then The status code of the response is 200 And The body of the response contains approved + When I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP GET method + Then The status code of the response is 200 + And The status message of the response is "approved" # --- Deny a pending approval --- @@ -158,17 +161,19 @@ Feature: Human-in-the-Loop MCP approval tests And The service is restarted When I use "query" to ask question with authorization header """ - {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + {"query": "Use the multiply tool from the mcp-approval-always server to multiply 2 and 3.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 And The body of the response contains requires_action - When I extract the approval id from the response And I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP POST method """ {"approve": false} """ Then The status code of the response is 200 And The body of the response contains denied + When I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP GET method + Then The status code of the response is 200 + And The status message of the response is "denied" Scenario: Deny a pending approval via POST /approvals/{id} on streaming_query Given MCP toolgroups are reset for a new MCP configuration @@ -176,17 +181,20 @@ Feature: Human-in-the-Loop MCP approval tests And The service is restarted When I use "streaming_query" to ask question with authorization header """ - {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + {"query": "Use the multiply tool from the mcp-approval-always server to multiply 2 and 3.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ + When I wait for the response to be completed Then The status code of the response is 200 And The body of the response contains approval_required - When I extract the approval id from the response And I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP POST method """ {"approve": false} """ Then The status code of the response is 200 And The body of the response contains denied + When I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP GET method + Then The status code of the response is 200 + And The status message of the response is "denied" # --- GET /approvals returns all approvals --- @@ -198,7 +206,7 @@ Feature: Human-in-the-Loop MCP approval tests And The service is restarted When I use "query" to ask question with authorization header """ - {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + {"query": "Use the multiply tool from the mcp-approval-always server to multiply 2 and 3.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 And The body of the response contains requires_action @@ -217,11 +225,10 @@ Feature: Human-in-the-Loop MCP approval tests And The service is restarted When I use "query" to ask question with authorization header """ - {"query": "Use the mcp-approval-always server to list repos", "model": "{MODEL}", "provider": "{PROVIDER}"} + {"query": "Use the multiply tool from the mcp-approval-always server to multiply 2 and 3.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 And The body of the response contains requires_action - When I extract the approval id from the response And I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP GET method Then The status code of the response is 200 And The body of the response contains pending @@ -229,17 +236,16 @@ Feature: Human-in-the-Loop MCP approval tests # --- Approval timeout / expiry --- - Scenario: Expired approval returns 410 when attempting to approve + Scenario: Expired approval returns 410 when attempting to decide Given MCP toolgroups are reset for a new MCP configuration And The service uses the lightspeed-stack-mcp-approvals-short-timeout.yaml configuration And The service is restarted When I use "query" to ask question with authorization header """ - {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + {"query": "Use the multiply tool from the mcp-approval-always server to multiply 2 and 3.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 And The body of the response contains requires_action - When I extract the approval id from the response And I wait for 6 seconds And I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP POST method """ @@ -248,123 +254,61 @@ Feature: Human-in-the-Loop MCP approval tests Then The status code of the response is 410 And The body of the response contains approval_expired - Scenario: Expired approval returns 410 when attempting to deny - Given MCP toolgroups are reset for a new MCP configuration - And The service uses the lightspeed-stack-mcp-approvals-short-timeout.yaml configuration - And The service is restarted - When I use "query" to ask question with authorization header - """ - {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} - """ - Then The status code of the response is 200 - And The body of the response contains requires_action - When I extract the approval id from the response - And I wait for 6 seconds - And I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP POST method - """ - {"approve": false} - """ - Then The status code of the response is 410 - And The body of the response contains approval_expired - # --- Retention cleanup: decided approvals purged after approval_retention_seconds --- - Scenario: Approved approval is purged after retention period expires + Scenario: Decided approval remains queryable within retention period then purged after it expires Given MCP toolgroups are reset for a new MCP configuration And The service uses the lightspeed-stack-mcp-approvals-short-retention.yaml configuration And The service is restarted When I use "query" to ask question with authorization header """ - {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + {"query": "Use the multiply tool from the mcp-approval-always server to multiply 2 and 3.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 And The body of the response contains requires_action - When I extract the approval id from the response And I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP POST method """ {"approve": true} """ Then The status code of the response is 200 And The body of the response contains approved - When I wait for 6 seconds - And I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP GET method - Then The status code of the response is 404 - And The body of the response contains approval_not_found - - Scenario: Denied approval is purged after retention period expires - Given MCP toolgroups are reset for a new MCP configuration - And The service uses the lightspeed-stack-mcp-approvals-short-retention.yaml configuration - And The service is restarted - When I use "query" to ask question with authorization header - """ - {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} - """ + When I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP GET method Then The status code of the response is 200 - And The body of the response contains requires_action - When I extract the approval id from the response - And I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP POST method - """ - {"approve": false} - """ + And The body of the response contains approved + And The body of the response contains decided_at + When I wait for 3 seconds + And I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP GET method Then The status code of the response is 200 - And The body of the response contains denied - When I wait for 6 seconds + And The body of the response contains approved + When I wait for 8 seconds And I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP GET method Then The status code of the response is 404 And The body of the response contains approval_not_found Scenario: Expired approval is purged after retention period Given MCP toolgroups are reset for a new MCP configuration - And The service uses the lightspeed-stack-mcp-approvals-short-timeout.yaml configuration + And The service uses the lightspeed-stack-mcp-approvals-short-retention.yaml configuration And The service is restarted When I use "query" to ask question with authorization header """ - {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} + {"query": "Use the multiply tool from the mcp-approval-always server to multiply 2 and 3.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 And The body of the response contains requires_action - When I extract the approval id from the response And I wait for 11 seconds And I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP GET method Then The status code of the response is 404 And The body of the response contains approval_not_found - Scenario: Decided approval remains queryable within retention period - Given MCP toolgroups are reset for a new MCP configuration - And The service uses the lightspeed-stack-mcp-approvals-short-retention.yaml configuration - And The service is restarted - When I use "query" to ask question with authorization header - """ - {"query": "", "model": "{MODEL}", "provider": "{PROVIDER}"} - """ - Then The status code of the response is 200 - And The body of the response contains requires_action - When I extract the approval id from the response - And I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP POST method - """ - {"approve": true} - """ - Then The status code of the response is 200 - And The body of the response contains approved - When I access REST API endpoint "approvals/{APPROVAL_ID}" using HTTP GET method - Then The status code of the response is 200 - And The body of the response contains approved - And The body of the response contains decided_at - # --- Approval not found returns 404 --- - Scenario: GET on non-existent approval returns 404 + Scenario: Requests for a non-existent approval return 404 Given MCP toolgroups are reset for a new MCP configuration And The service uses the lightspeed-stack-mcp-approvals.yaml configuration And The service is restarted When I access REST API endpoint "approvals/non-existent-id-12345" using HTTP GET method Then The status code of the response is 404 And The body of the response contains approval_not_found - - Scenario: POST to non-existent approval returns 404 - Given MCP toolgroups are reset for a new MCP configuration - And The service uses the lightspeed-stack-mcp-approvals.yaml configuration - And The service is restarted When I access REST API endpoint "approvals/non-existent-id-12345" using HTTP POST method """ {"approve": true}