diff --git a/docker-compose.yaml b/docker-compose.yaml index 03bd6450d..4ee0d30c1 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -82,6 +82,7 @@ services: volumes: - ./lightspeed-stack.yaml:/app-root/lightspeed-stack.yaml:z - ./tests/e2e/secrets/mcp-token:/tmp/mcp-secret-token:ro + - ./tests/e2e/secrets/invalid-mcp-token:/tmp/invalid-mcp-secret-token:ro environment: - OPENAI_API_KEY=${OPENAI_API_KEY} # Azure Entra ID credentials (AZURE_API_KEY is obtained dynamically) diff --git a/docs/e2e_testing.md b/docs/e2e_testing.md index 4035a17d0..64eff79b2 100644 --- a/docs/e2e_testing.md +++ b/docs/e2e_testing.md @@ -58,7 +58,7 @@ tests/e2e/ ├── utils/ │ ├── utils.py # restart_container, switch_config, wait_for_container_health, etc. │ ├── prow_utils.py # Prow/OpenShift helpers (restore_llama_stack_pod, etc.) -│ └── llama_stack_shields.py # Shield unregister/register (server mode, optional) +│ └── llama_stack_utils.py # Toolgroups + shield unregister/register (server mode, optional) ├── mock_mcp_server/ # Mock MCP server for MCP tests └── rag/ # RAG test data (e.g. for FAISS) ``` diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-invalid-mcp-file-auth.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-invalid-mcp-file-auth.yaml new file mode 100644 index 000000000..483e32b73 --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-invalid-mcp-file-auth.yaml @@ -0,0 +1,24 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds llama-stack as library + use_as_library_client: true + library_client_config_path: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +mcp_servers: + - name: "mcp-file" + url: "http://mock-mcp:3001" + authorization_headers: + Authorization: "/tmp/invalid-mcp-secret-token" diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-client-auth.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-client-auth.yaml new file mode 100644 index 000000000..05f304a5d --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-client-auth.yaml @@ -0,0 +1,24 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds llama-stack as library + use_as_library_client: true + library_client_config_path: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +mcp_servers: + - name: "mcp-client" + url: "http://mock-mcp:3001" + authorization_headers: + Authorization: "client" diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-file-auth.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-file-auth.yaml index 1ff0d425e..79a8807ec 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-file-auth.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-file-auth.yaml @@ -18,8 +18,7 @@ user_data_collection: authentication: module: "noop" mcp_servers: - - name: "mcp-file-auth" - provider_id: "model-context-protocol" + - name: "mcp-file" url: "http://mock-mcp:3001" authorization_headers: Authorization: "/tmp/mcp-secret-token" diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-kubernetes-auth.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-kubernetes-auth.yaml new file mode 100644 index 000000000..2d79f1f9d --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-kubernetes-auth.yaml @@ -0,0 +1,24 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds llama-stack as library + use_as_library_client: true + library_client_config_path: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +mcp_servers: + - name: "mcp-kubernetes" + url: "http://mock-mcp:3001" + authorization_headers: + Authorization: "kubernetes" diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-oauth-auth.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-oauth-auth.yaml new file mode 100644 index 000000000..3294ac708 --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-oauth-auth.yaml @@ -0,0 +1,24 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds llama-stack as library + use_as_library_client: true + library_client_config_path: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +mcp_servers: + - name: "mcp-oauth" + url: "http://mock-mcp:3001" + authorization_headers: + Authorization: "oauth" diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-mcp.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp.yaml index 0656aa87c..647a2cae9 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-mcp.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp.yaml @@ -19,7 +19,18 @@ authentication: module: "noop" mcp_servers: - name: "mcp-oauth" - provider_id: "model-context-protocol" url: "http://mock-mcp:3001" authorization_headers: - Authorization: "oauth" \ No newline at end of file + Authorization: "oauth" + - name: "mcp-kubernetes" + url: "http://mock-mcp:3001" + authorization_headers: + Authorization: "kubernetes" + - name: "mcp-file" + url: "http://mock-mcp:3001" + authorization_headers: + Authorization: "/tmp/mcp-secret-token" + - name: "mcp-client" + url: "http://mock-mcp:3001" + authorization_headers: + Authorization: "client" \ No newline at end of file diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-invalid-mcp-file-auth.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-invalid-mcp-file-auth.yaml new file mode 100644 index 000000000..05ec86fdf --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-invalid-mcp-file-auth.yaml @@ -0,0 +1,25 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to separate llama-stack service + use_as_library_client: false + url: http://llama-stack:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +mcp_servers: + - name: "mcp-file" + url: "http://mock-mcp:3001" + authorization_headers: + Authorization: "/tmp/invalid-mcp-secret-token" diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-client-auth.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-client-auth.yaml new file mode 100644 index 000000000..e0f952fc3 --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-client-auth.yaml @@ -0,0 +1,25 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to separate llama-stack service + use_as_library_client: false + url: http://llama-stack:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +mcp_servers: + - name: "mcp-client" + url: "http://mock-mcp:3001" + authorization_headers: + Authorization: "client" diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-file-auth.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-file-auth.yaml index d39f55399..aca5c6ef2 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-file-auth.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-file-auth.yaml @@ -19,8 +19,7 @@ user_data_collection: authentication: module: "noop" mcp_servers: - - name: "mcp-file-auth" - provider_id: "model-context-protocol" + - name: "mcp-file" url: "http://mock-mcp:3001" authorization_headers: Authorization: "/tmp/mcp-secret-token" diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-kubernetes-auth.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-kubernetes-auth.yaml new file mode 100644 index 000000000..66dc7f87b --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-kubernetes-auth.yaml @@ -0,0 +1,25 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to separate llama-stack service + use_as_library_client: false + url: http://llama-stack:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +mcp_servers: + - name: "mcp-kubernetes" + url: "http://mock-mcp:3001" + authorization_headers: + Authorization: "kubernetes" diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-oauth-auth.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-oauth-auth.yaml new file mode 100644 index 000000000..b9125de8e --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-oauth-auth.yaml @@ -0,0 +1,25 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to separate llama-stack service + use_as_library_client: false + url: http://llama-stack:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +mcp_servers: + - name: "mcp-oauth" + url: "http://mock-mcp:3001" + authorization_headers: + Authorization: "oauth" diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp.yaml index a598ce441..e35535f42 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp.yaml @@ -20,7 +20,18 @@ authentication: module: "noop" mcp_servers: - name: "mcp-oauth" - provider_id: "model-context-protocol" url: "http://mock-mcp:3001" authorization_headers: - Authorization: "oauth" \ No newline at end of file + Authorization: "oauth" + - name: "mcp-kubernetes" + url: "http://mock-mcp:3001" + authorization_headers: + Authorization: "kubernetes" + - name: "mcp-file" + url: "http://mock-mcp:3001" + authorization_headers: + Authorization: "/tmp/mcp-secret-token" + - name: "mcp-client" + url: "http://mock-mcp:3001" + authorization_headers: + Authorization: "client" \ No newline at end of file diff --git a/tests/e2e/features/environment.py b/tests/e2e/features/environment.py index 0ca8781d0..58a9afe80 100644 --- a/tests/e2e/features/environment.py +++ b/tests/e2e/features/environment.py @@ -10,17 +10,20 @@ import os import subprocess import time +from typing import Optional import requests from behave.model import Feature, Scenario from tests.e2e.utils.prow_utils import restore_llama_stack_pod from behave.runner import Context -from tests.e2e.utils.llama_stack_shields import ( +from tests.e2e.utils.llama_stack_utils import ( register_shield, + unregister_mcp_toolgroups, unregister_shield, ) from tests.e2e.utils.utils import ( + clear_llama_stack_storage, create_config_backup, is_prow_environment, remove_config_backup, @@ -57,6 +60,22 @@ "tests/e2e/configuration/{mode_dir}/lightspeed-stack-mcp-file-auth.yaml", "tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp-file-auth.yaml", ), + "invalid-mcp-file-auth": ( + "tests/e2e/configuration/{mode_dir}/lightspeed-stack-invalid-mcp-file-auth.yaml", + "tests/e2e-prow/rhoai/configs/lightspeed-stack-invalid-mcp-file-auth.yaml", + ), + "mcp-kubernetes-auth": ( + "tests/e2e/configuration/{mode_dir}/lightspeed-stack-mcp-kubernetes-auth.yaml", + "tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp-kubernetes-auth.yaml", + ), + "mcp-client-auth": ( + "tests/e2e/configuration/{mode_dir}/lightspeed-stack-mcp-client-auth.yaml", + "tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp-client-auth.yaml", + ), + "mcp-oauth-auth": ( + "tests/e2e/configuration/{mode_dir}/lightspeed-stack-mcp-oauth-auth.yaml", + "tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp-oauth-auth.yaml", + ), } @@ -207,6 +226,27 @@ def before_scenario(context: Context, scenario: Scenario) -> None: switch_config(context.scenario_config) restart_container("lightspeed-stack") + config_name: Optional[str] = None + if "MCPFileAuthConfig" in scenario.effective_tags: + config_name = "mcp-file-auth" + elif "InvalidMCPFileAuthConfig" in scenario.effective_tags: + config_name = "invalid-mcp-file-auth" + elif "MCPKubernetesAuthConfig" in scenario.effective_tags: + config_name = "mcp-kubernetes-auth" + elif "MCPClientAuthConfig" in scenario.effective_tags: + config_name = "mcp-client-auth" + elif "MCPOAuthAuthConfig" in scenario.effective_tags: + config_name = "mcp-oauth-auth" + + if config_name is not None: + if not context.is_library_mode: + unregister_mcp_toolgroups() + else: + clear_llama_stack_storage() + context.scenario_config = _get_config_path(config_name, mode_dir) + switch_config(context.scenario_config) + restart_container("lightspeed-stack") + def after_scenario(context: Context, scenario: Scenario) -> None: """Run after each scenario is run. @@ -241,7 +281,15 @@ def after_scenario(context: Context, scenario: Scenario) -> None: context.llama_stack_was_running = False # Tags that require config restoration after scenario - config_restore_tags = {"InvalidFeedbackStorageConfig", "NoCacheConfig"} + config_restore_tags = { + "InvalidFeedbackStorageConfig", + "NoCacheConfig", + "MCPFileAuthConfig", + "InvalidMCPFileAuthConfig", + "MCPKubernetesAuthConfig", + "MCPClientAuthConfig", + "MCPOAuthAuthConfig", + } if config_restore_tags & set(scenario.effective_tags): switch_config(context.feature_config) restart_container("lightspeed-stack") diff --git a/tests/e2e/features/mcp.feature b/tests/e2e/features/mcp.feature index 90d6c5cf8..b7cbc2285 100644 --- a/tests/e2e/features/mcp.feature +++ b/tests/e2e/features/mcp.feature @@ -5,7 +5,358 @@ Feature: MCP tests Given The service is started locally And REST API service prefix is /v1 - Scenario: Check if tools endpoint reports error when MCP requires authentication + +# File-based + @skip #TODO: LCORE-1461 + @MCPFileAuthConfig + Scenario: Check if tools endpoint succeeds when MCP file-based auth token is passed + Given The system is in default state + When I access REST API endpoint "tools" using HTTP GET method + Then The status code of the response is 200 + And The body of the response contains mcp-file + + @skip-in-library-mode #TODO: LCORE-1428 + @MCPFileAuthConfig + Scenario: Check if query endpoint succeeds when MCP file-based auth token is passed + Given The system is in default state + And I capture the current token metrics + When I use "query" to ask question + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The response should contain following fragments + | Fragments in LLM response | + | Hello | + And The token metrics should have increased + + @skip-in-library-mode #TODO: LCORE-1428 + @MCPFileAuthConfig + Scenario: Check if streaming_query endpoint succeeds when MCP file-based auth token is passed + Given The system is in default state + And I capture the current token metrics + When I use "streaming_query" to ask question + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + When I wait for the response to be completed + Then The status code of the response is 200 + And The streamed response should contain following fragments + | Fragments in LLM response | + | Hello | + And The token metrics should have increased + + @skip #TODO: LCORE-1461 + @InvalidMCPFileAuthConfig + Scenario: Check if tools endpoint reports error when MCP file-based invalid auth token is passed + Given The system is in default state + When I access REST API endpoint "tools" using HTTP GET method + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3001 requires OAuth" + } + } + """ + + @skip #TODO: LCORE-1463 + @InvalidMCPFileAuthConfig + Scenario: Check if query endpoint reports error when MCP file-based invalid auth token is passed + Given The system is in default state + When I use "query" to ask question + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3001 requires OAuth" + } + } + """ + + @skip #TODO: LCORE-1463 + @InvalidMCPFileAuthConfig + Scenario: Check if streaming_query endpoint reports error when MCP file-based invalid auth token is passed + Given The system is in default state + When I use "streaming_query" to ask question + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3001 requires OAuth" + } + } + """ + +# Kubernetes + @skip #TODO: LCORE-1461 + @MCPKubernetesAuthConfig + Scenario: Check if tools endpoint succeeds when MCP kubernetes auth token is passed + Given The system is in default state + And I set the Authorization header to Bearer kubernetes-test-token + When I access REST API endpoint "tools" using HTTP GET method + Then The status code of the response is 200 + And The body of the response contains mcp-kubernetes + + @skip-in-library-mode #TODO: LCORE-1428 + @MCPKubernetesAuthConfig + Scenario: Check if query endpoint succeeds when MCP kubernetes auth token is passed + Given The system is in default state + And I set the Authorization header to Bearer kubernetes-test-token + And I capture the current token metrics + When I use "query" to ask question with authorization header + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The response should contain following fragments + | Fragments in LLM response | + | Hello | + And The token metrics should have increased + + @skip-in-library-mode #TODO: LCORE-1428 + @MCPKubernetesAuthConfig + Scenario: Check if streaming_query endpoint succeeds when MCP kubernetes auth token is passed + Given The system is in default state + And I set the Authorization header to Bearer kubernetes-test-token + And I capture the current token metrics + When I use "streaming_query" to ask question with authorization header + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + When I wait for the response to be completed + Then The status code of the response is 200 + And The streamed response should contain following fragments + | Fragments in LLM response | + | Hello | + And The token metrics should have increased + + @skip #TODO: LCORE-1461 + @MCPKubernetesAuthConfig + Scenario: Check if tools endpoint reports error when MCP kubernetes invalid auth token is passed + Given The system is in default state + And I set the Authorization header to Bearer kubernetes-invalid-token + When I access REST API endpoint "tools" using HTTP GET method + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3001 requires OAuth" + } + } + """ + + @skip #TODO: LCORE-1463 + @MCPKubernetesAuthConfig + Scenario: Check if query endpoint reports error when MCP kubernetes invalid auth token is passed + Given The system is in default state + And I set the Authorization header to Bearer kubernetes-invalid-token + When I use "query" to ask question with authorization header + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3001 requires OAuth" + } + } + """ + + @skip #TODO: LCORE-1463 + @MCPKubernetesAuthConfig + Scenario: Check if streaming_query endpoint reports error when MCP kubernetes invalid auth token is passed + Given The system is in default state + And I set the Authorization header to Bearer kubernetes-invalid-token + When I use "streaming_query" to ask question with authorization header + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3001 requires OAuth" + } + } + """ + +# Client-provided + @skip #TODO: LCORE-1462 + @MCPClientAuthConfig + Scenario: Check if tools endpoint succeeds by skipping when MCP client-provided auth token is omitted + Given The system is in default state + When I access REST API endpoint "tools" using HTTP GET method + Then The status code of the response is 200 + And The body of the response does not contain mcp-client + + @MCPClientAuthConfig + Scenario: Check if query endpoint succeeds by skipping when MCP client-provided auth token is omitted + Given The system is in default state + And I capture the current token metrics + When I use "query" to ask question + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The body of the response does not contain mcp-client + And The response should contain following fragments + | Fragments in LLM response | + | Hello | + And The token metrics should have increased + + @MCPClientAuthConfig + Scenario: Check if streaming_query endpoint succeeds by skipping when MCP client-provided auth token is omitted + Given The system is in default state + And I capture the current token metrics + When I use "streaming_query" to ask question + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + When I wait for the response to be completed + Then The status code of the response is 200 + And The body of the response does not contain mcp-client + And The streamed response should contain following fragments + | Fragments in LLM response | + | Hello | + And The token metrics should have increased + + @MCPClientAuthConfig + Scenario: Check if tools endpoint succeeds when MCP client-provided auth token is passed + Given The system is in default state + And I set the "MCP-HEADERS" header to + """ + {"mcp-client": {"Authorization": "Bearer client-test-token"}} + """ + When I access REST API endpoint "tools" using HTTP GET method + Then The status code of the response is 200 + And The body of the response contains mcp-client + + @skip-in-library-mode #TODO: LCORE-1428 + @MCPClientAuthConfig + Scenario: Check if query endpoint succeeds when MCP client-provided auth token is passed + Given The system is in default state + And I set the "MCP-HEADERS" header to + """ + {"mcp-client": {"Authorization": "Bearer client-test-token"}} + """ + And I capture the current token metrics + When I use "query" to ask question with authorization header + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The response should contain following fragments + | Fragments in LLM response | + | Hello | + And The token metrics should have increased + + @skip-in-library-mode #TODO: LCORE-1428 + @MCPClientAuthConfig + Scenario: Check if streaming_query endpoint succeeds when MCP client-provided auth token is passed + Given The system is in default state + And I set the "MCP-HEADERS" header to + """ + {"mcp-client": {"Authorization": "Bearer client-test-token"}} + """ + And I capture the current token metrics + When I use "streaming_query" to ask question with authorization header + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + When I wait for the response to be completed + Then The status code of the response is 200 + And The streamed response should contain following fragments + | Fragments in LLM response | + | Hello | + And The token metrics should have increased + + @MCPClientAuthConfig + Scenario: Check if tools endpoint reports error when MCP client-provided invalid auth token is passed + Given The system is in default state + And I set the "MCP-HEADERS" header to + """ + {"mcp-client": {"Authorization": "Bearer client-invalid-token"}} + """ + When I access REST API endpoint "tools" using HTTP GET method + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3001 requires OAuth" + } + } + """ + + @MCPClientAuthConfig + Scenario: Check if query endpoint reports error when MCP client-provided invalid auth token is passed + Given The system is in default state + And I set the "MCP-HEADERS" header to + """ + {"mcp-client": {"Authorization": "Bearer client-invalid-token"}} + """ + When I use "query" to ask question with authorization header + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3001 requires OAuth" + } + } + """ + + @MCPClientAuthConfig + Scenario: Check if streaming_query endpoint reports error when MCP client-provided invalid auth token is passed + Given The system is in default state + And I set the "MCP-HEADERS" header to + """ + {"mcp-client": {"Authorization": "Bearer client-invalid-token"}} + """ + When I use "streaming_query" to ask question with authorization header + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3001 requires OAuth" + } + } + """ + +# OAuth + + @MCPOAuthAuthConfig + Scenario: Check if tools endpoint reports error when MCP OAuth requires authentication Given The system is in default state When I access REST API endpoint "tools" using HTTP GET method Then The status code of the response is 401 @@ -20,7 +371,8 @@ Feature: MCP tests """ And The headers of the response contains the following header "www-authenticate" - Scenario: Check if query endpoint reports error when MCP requires authentication + @MCPOAuthAuthConfig + Scenario: Check if query endpoint reports error when MCP OAuth requires authentication Given The system is in default state When I use "query" to ask question """ @@ -38,7 +390,8 @@ Feature: MCP tests """ And The headers of the response contains the following header "www-authenticate" - Scenario: Check if streaming_query endpoint reports error when MCP requires authentication + @MCPOAuthAuthConfig + Scenario: Check if streaming_query endpoint reports error when MCP OAuth requires authentication Given The system is in default state When I use "streaming_query" to ask question """ @@ -56,22 +409,24 @@ Feature: MCP tests """ And The headers of the response contains the following header "www-authenticate" - Scenario: Check if tools endpoint succeeds when MCP auth token is passed + @MCPOAuthAuthConfig + Scenario: Check if tools endpoint succeeds when MCP OAuth auth token is passed Given The system is in default state And I set the "MCP-HEADERS" header to """ - {"mcp-oauth": {"Authorization": "Bearer test-token"}} + {"mcp-oauth": {"Authorization": "Bearer oauth-test-token"}} """ When I access REST API endpoint "tools" using HTTP GET method Then The status code of the response is 200 And The body of the response contains mcp-oauth - @skip-in-library-mode # will be fixed in LCORE-1428 - Scenario: Check if query endpoint succeeds when MCP auth token is passed + @skip-in-library-mode #TODO: LCORE-1428 + @MCPOAuthAuthConfig + Scenario: Check if query endpoint succeeds when MCP OAuth auth token is passed Given The system is in default state And I set the "MCP-HEADERS" header to """ - {"mcp-oauth": {"Authorization": "Bearer test-token"}} + {"mcp-oauth": {"Authorization": "Bearer oauth-test-token"}} """ And I capture the current token metrics When I use "query" to ask question with authorization header @@ -84,12 +439,13 @@ Feature: MCP tests | Hello | And The token metrics should have increased - @skip-in-library-mode # will be fixed in LCORE-1428 - Scenario: Check if streaming_query endpoint succeeds when MCP auth token is passed + @skip-in-library-mode #TODO: LCORE-1428 + @MCPOAuthAuthConfig + Scenario: Check if streaming_query endpoint succeeds when MCP OAuth auth token is passed Given The system is in default state And I set the "MCP-HEADERS" header to """ - {"mcp-oauth": {"Authorization": "Bearer test-token"}} + {"mcp-oauth": {"Authorization": "Bearer oauth-test-token"}} """ And I capture the current token metrics When I use "streaming_query" to ask question with authorization header @@ -103,11 +459,12 @@ Feature: MCP tests | Hello | And The token metrics should have increased - Scenario: Check if tools endpoint reports error when MCP invalid auth token is passed + @MCPOAuthAuthConfig + Scenario: Check if tools endpoint reports error when MCP OAuth invalid auth token is passed Given The system is in default state And I set the "MCP-HEADERS" header to """ - {"mcp-oauth": {"Authorization": "Bearer invalid-token"}} + {"mcp-oauth": {"Authorization": "Bearer oauth-invalid-token"}} """ When I access REST API endpoint "tools" using HTTP GET method Then The status code of the response is 401 @@ -122,12 +479,12 @@ Feature: MCP tests """ And The headers of the response contains the following header "www-authenticate" - @skip # will be fixed in LCORE-1366 - Scenario: Check if query endpoint reports error when MCP invalid auth token is passed + @MCPOAuthAuthConfig + Scenario: Check if query endpoint reports error when MCP OAuth invalid auth token is passed Given The system is in default state And I set the "MCP-HEADERS" header to """ - {"mcp-oauth": {"Authorization": "Bearer invalid-token"}} + {"mcp-oauth": {"Authorization": "Bearer oauth-invalid-token"}} """ When I use "query" to ask question with authorization header """ @@ -145,11 +502,12 @@ Feature: MCP tests """ And The headers of the response contains the following header "www-authenticate" - Scenario: Check if streaming_query endpoint reports error when MCP invalid auth token is passed + @MCPOAuthAuthConfig + Scenario: Check if streaming_query endpoint reports error when MCP OAuth invalid auth token is passed Given The system is in default state And I set the "MCP-HEADERS" header to """ - {"mcp-oauth": {"Authorization": "Bearer invalid-token"}} + {"mcp-oauth": {"Authorization": "Bearer oauth-invalid-token"}} """ When I use "streaming_query" to ask question with authorization header """ diff --git a/tests/e2e/features/mcp_file_auth.feature b/tests/e2e/features/mcp_file_auth.feature deleted file mode 100644 index 455f0740c..000000000 --- a/tests/e2e/features/mcp_file_auth.feature +++ /dev/null @@ -1,20 +0,0 @@ -@MCPFileAuth -Feature: MCP file-based authorization tests - - Regression tests for LCORE-1414: MCP authorization tokens configured via - file-based authorization_headers must survive model_dump() serialization - and reach the MCP server as a valid Bearer token. - - Background: - Given The service is started locally - And REST API service prefix is /v1 - - @skip-in-library-mode - Scenario: Query succeeds with file-based MCP authorization - Given The system is in default state - When I use "query" to ask question - """ - {"query": "Use the mock_tool_e2e tool to send the message 'hello'", "model": "{MODEL}", "provider": "{PROVIDER}"} - """ - Then The status code of the response is 200 - And The body of the response contains mock_tool_e2e diff --git a/tests/e2e/features/steps/common_http.py b/tests/e2e/features/steps/common_http.py index 016791fb6..284acfeb0 100644 --- a/tests/e2e/features/steps/common_http.py +++ b/tests/e2e/features/steps/common_http.py @@ -178,6 +178,15 @@ def check_response_body_contains(context: Context, substring: str) -> None: ), f"The response text '{context.response.text}' doesn't contain '{expected}'" +@then("The body of the response does not contain {substring}") +def check_response_body_does_not_contain(context: Context, substring: str) -> None: + """Check that response body does not contain a substring.""" + assert context.response is not None, "Request needs to be performed first" + assert ( + substring not in context.response.text + ), f"The response text '{context.response.text}' contains '{substring}'" + + @then("The body of the response is the following") def check_prediction_result(context: Context) -> None: """Check the content of the response to be exactly the same. diff --git a/tests/e2e/secrets/invalid-mcp-token b/tests/e2e/secrets/invalid-mcp-token new file mode 100644 index 000000000..3707272a2 --- /dev/null +++ b/tests/e2e/secrets/invalid-mcp-token @@ -0,0 +1 @@ +invalid-token \ No newline at end of file diff --git a/tests/e2e/test_list.txt b/tests/e2e/test_list.txt index 583cf387b..0da5cae41 100644 --- a/tests/e2e/test_list.txt +++ b/tests/e2e/test_list.txt @@ -16,5 +16,4 @@ features/rlsapi_v1_errors.feature features/streaming_query.feature features/rest_api.feature features/mcp.feature -features/mcp_file_auth.feature features/models.feature diff --git a/tests/e2e/utils/llama_stack_shields.py b/tests/e2e/utils/llama_stack_utils.py similarity index 62% rename from tests/e2e/utils/llama_stack_shields.py rename to tests/e2e/utils/llama_stack_utils.py index 4f793c0bf..2a8c66670 100644 --- a/tests/e2e/utils/llama_stack_shields.py +++ b/tests/e2e/utils/llama_stack_utils.py @@ -1,9 +1,12 @@ -"""E2E helpers to unregister and re-register Llama Stack shields via the client API. +"""E2E test utilities for Llama Stack (toolgroups and shields). -Used by the @disable-shields tag: before the scenario we call client.shields.delete() -to unregister the shield; after the scenario we call client.shields.register() -to restore it. Only applies in server mode (Llama Stack as a separate service). -Requires E2E_LLAMA_STACK_URL or E2E_LLAMA_HOSTNAME/E2E_LLAMA_PORT. +This module provides functions to manage MCP toolgroups and shields on a running +Llama Stack instance during end-to-end tests: unregister MCP toolgroups when +switching configurations or testing MCP auth, and unregister/re-register shields +(e.g. for the @disable-shields tag). + +Only applies when running Llama Stack as a separate service (server mode). +Requires E2E_LLAMA_STACK_URL or E2E_LLAMA_HOSTNAME and E2E_LLAMA_PORT. """ import asyncio @@ -29,6 +32,54 @@ def _get_llama_stack_client() -> AsyncLlamaStackClient: return AsyncLlamaStackClient(base_url=base_url, api_key=api_key, timeout=timeout) +# ----------------------------------------------------------------------------- +# Toolgroups +# ----------------------------------------------------------------------------- + + +async def _unregister_toolgroup_async(identifier: str) -> None: + """Unregister a toolgroup by identifier; return (provider_id, provider_shield_id) for restore.""" + client = _get_llama_stack_client() + try: + await client.toolgroups.unregister(identifier) + except APIConnectionError: + raise + except APIStatusError as e: + # 400 "not found": toolgroup already absent, scenario can proceed + if e.status_code == 400 and "not found" in str(e).lower(): + return None + raise + finally: + await client.close() + + +async def _unregister_mcp_toolgroups_async() -> None: + """Unregister all MCP toolgroups.""" + client = _get_llama_stack_client() + try: + toolgroups = await client.toolgroups.list() + for toolgroup in toolgroups: + if ( + toolgroup.identifier + and toolgroup.provider_id == "model-context-protocol" + ): + await _unregister_toolgroup_async(toolgroup.identifier) + except APIConnectionError: + raise + finally: + await client.close() + + +def unregister_mcp_toolgroups() -> None: + """Unregister all MCP toolgroups.""" + asyncio.run(_unregister_mcp_toolgroups_async()) + + +# ----------------------------------------------------------------------------- +# Shields +# ----------------------------------------------------------------------------- + + async def _unregister_shield_async(identifier: str) -> Optional[tuple[str, str]]: """Unregister a shield by identifier; return (provider_id, provider_shield_id) for restore.""" client = _get_llama_stack_client() diff --git a/tests/e2e/utils/utils.py b/tests/e2e/utils/utils.py index bc9b8b966..580250bff 100644 --- a/tests/e2e/utils/utils.py +++ b/tests/e2e/utils/utils.py @@ -246,6 +246,36 @@ def remove_config_backup(backup_path: str) -> None: print(f"Warning: Could not remove backup file {backup_path}: {e}") +def clear_llama_stack_storage(container_name: str = "lightspeed-stack") -> None: + """Clear Llama Stack storage in library mode (embedded Llama Stack). + + Removes the ~/.llama directory so that toolgroups and other persisted + state are reset. Used before MCP config scenarios when not running in + server mode (no separate Llama Stack to unregister toolgroups from). + Only runs when using Docker (skipped in Prow). + + Parameters: + container_name (str): Docker container name (default "lightspeed-stack"). + + Returns: + None + """ + if is_prow_environment(): + return + + try: + subprocess.run( + ["docker", "exec", container_name, "sh", "-c", "rm -rf ~/.llama"], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + except subprocess.TimeoutExpired as e: + print(f"Failed to clear Llama Stack storage: {e}") + raise + + def restart_container(container_name: str) -> None: """Restart a Docker container by name and wait until it is healthy.