From a589492972effca7c7df2734cbd44b42e9af7d90 Mon Sep 17 00:00:00 2001 From: JR Boos Date: Mon, 9 Mar 2026 15:25:12 -0400 Subject: [PATCH] Add new MCP authentication configurations and update tests - Introduced new YAML configuration files for MCP authentication methods: file, Kubernetes, and OAuth. - Updated existing MCP configuration to include new authentication methods. - Enhanced end-to-end tests to cover scenarios for file-based, Kubernetes, and OAuth authentication. - Added utility functions for unregistering MCP toolgroups in the testing environment. - Implemented new step definitions for checking response body content in tests. Add invalid MCP token configuration and update tests - Added a new invalid MCP token file for testing purposes. - Updated the Docker Compose configuration to mount the invalid MCP token. - Introduced a new YAML configuration for testing invalid MCP file authentication. - Enhanced the test scenarios to include checks for invalid MCP file authentication. - Updated feature files to reflect the new authentication configurations. Add additional MCP authentication configurations and update scenario handling - Added new configurations for invalid, Kubernetes, client, and OAuth MCP authentication methods. - Updated scenario handling to reference the new configuration paths for Kubernetes, client, and OAuth authentication. - Enhanced the testing environment to support the new authentication configurations. fixed errored tests skipped failing scenarios and added library mode configs fixed library mode tests error fixed black skipped acidentally missed failing test Refactor Llama Stack utilities and update E2E tests - Renamed `llama_stack_shields.py` to `llama_stack_utils.py` and expanded its functionality to manage both toolgroups and shields. - Removed the deprecated `llama_stack_tools.py` file. - Updated E2E test scenarios to utilize the new utility functions for unregistering toolgroups and clearing Llama Stack storage. - Enhanced feature files to include comments indicating pending fixes for skipped scenarios. Enhance MCP feature tests and streamline Llama Stack storage clearing - Added checks to ensure the response body does not contain 'mcp-client' in MCP feature tests. - Simplified the `clear_llama_stack_storage` function to remove the entire `~/.llama` directory instead of specific files, improving clarity and efficiency. - Updated comments to reflect the changes in storage clearing logic. Remove deprecated MCP file-based authorization feature and update test list addressed code rabbit Add new MCP authentication configurations and update tests - Introduced new YAML configuration files for MCP authentication methods: file, Kubernetes, and OAuth. - Updated existing MCP configuration to include new authentication methods. - Enhanced end-to-end tests to cover scenarios for file-based, Kubernetes, and OAuth authentication. - Added utility functions for unregistering MCP toolgroups in the testing environment. - Implemented new step definitions for checking response body content in tests. fixed errored tests fixed library mode tests error fixed black Refactor Llama Stack utilities and update E2E tests - Renamed `llama_stack_shields.py` to `llama_stack_utils.py` and expanded its functionality to manage both toolgroups and shields. - Removed the deprecated `llama_stack_tools.py` file. - Updated E2E test scenarios to utilize the new utility functions for unregistering toolgroups and clearing Llama Stack storage. - Enhanced feature files to include comments indicating pending fixes for skipped scenarios. Enhance MCP feature tests and streamline Llama Stack storage clearing - Added checks to ensure the response body does not contain 'mcp-client' in MCP feature tests. - Simplified the `clear_llama_stack_storage` function to remove the entire `~/.llama` directory instead of specific files, improving clarity and efficiency. - Updated comments to reflect the changes in storage clearing logic. Remove deprecated MCP file-based authorization feature and update test list addressed code rabbit fixed ruff addressed comments --- docker-compose.yaml | 1 + docs/e2e_testing.md | 2 +- ...ightspeed-stack-invalid-mcp-file-auth.yaml | 24 ++ .../lightspeed-stack-mcp-client-auth.yaml | 24 ++ .../lightspeed-stack-mcp-file-auth.yaml | 3 +- .../lightspeed-stack-mcp-kubernetes-auth.yaml | 24 ++ .../lightspeed-stack-mcp-oauth-auth.yaml | 24 ++ .../library-mode/lightspeed-stack-mcp.yaml | 15 +- ...ightspeed-stack-invalid-mcp-file-auth.yaml | 25 ++ .../lightspeed-stack-mcp-client-auth.yaml | 25 ++ .../lightspeed-stack-mcp-file-auth.yaml | 3 +- .../lightspeed-stack-mcp-kubernetes-auth.yaml | 25 ++ .../lightspeed-stack-mcp-oauth-auth.yaml | 25 ++ .../server-mode/lightspeed-stack-mcp.yaml | 15 +- tests/e2e/features/environment.py | 52 ++- tests/e2e/features/mcp.feature | 394 +++++++++++++++++- tests/e2e/features/mcp_file_auth.feature | 20 - tests/e2e/features/steps/common_http.py | 9 + tests/e2e/secrets/invalid-mcp-token | 1 + tests/e2e/test_list.txt | 1 - ..._stack_shields.py => llama_stack_utils.py} | 61 ++- tests/e2e/utils/utils.py | 30 ++ 22 files changed, 748 insertions(+), 55 deletions(-) create mode 100644 tests/e2e/configuration/library-mode/lightspeed-stack-invalid-mcp-file-auth.yaml create mode 100644 tests/e2e/configuration/library-mode/lightspeed-stack-mcp-client-auth.yaml create mode 100644 tests/e2e/configuration/library-mode/lightspeed-stack-mcp-kubernetes-auth.yaml create mode 100644 tests/e2e/configuration/library-mode/lightspeed-stack-mcp-oauth-auth.yaml create mode 100644 tests/e2e/configuration/server-mode/lightspeed-stack-invalid-mcp-file-auth.yaml create mode 100644 tests/e2e/configuration/server-mode/lightspeed-stack-mcp-client-auth.yaml create mode 100644 tests/e2e/configuration/server-mode/lightspeed-stack-mcp-kubernetes-auth.yaml create mode 100644 tests/e2e/configuration/server-mode/lightspeed-stack-mcp-oauth-auth.yaml delete mode 100644 tests/e2e/features/mcp_file_auth.feature create mode 100644 tests/e2e/secrets/invalid-mcp-token rename tests/e2e/utils/{llama_stack_shields.py => llama_stack_utils.py} (62%) diff --git a/docker-compose.yaml b/docker-compose.yaml index 03bd6450d..4ee0d30c1 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -82,6 +82,7 @@ services: volumes: - ./lightspeed-stack.yaml:/app-root/lightspeed-stack.yaml:z - ./tests/e2e/secrets/mcp-token:/tmp/mcp-secret-token:ro + - ./tests/e2e/secrets/invalid-mcp-token:/tmp/invalid-mcp-secret-token:ro environment: - OPENAI_API_KEY=${OPENAI_API_KEY} # Azure Entra ID credentials (AZURE_API_KEY is obtained dynamically) diff --git a/docs/e2e_testing.md b/docs/e2e_testing.md index 4035a17d0..64eff79b2 100644 --- a/docs/e2e_testing.md +++ b/docs/e2e_testing.md @@ -58,7 +58,7 @@ tests/e2e/ ├── utils/ │ ├── utils.py # restart_container, switch_config, wait_for_container_health, etc. │ ├── prow_utils.py # Prow/OpenShift helpers (restore_llama_stack_pod, etc.) -│ └── llama_stack_shields.py # Shield unregister/register (server mode, optional) +│ └── llama_stack_utils.py # Toolgroups + shield unregister/register (server mode, optional) ├── mock_mcp_server/ # Mock MCP server for MCP tests └── rag/ # RAG test data (e.g. for FAISS) ``` diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-invalid-mcp-file-auth.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-invalid-mcp-file-auth.yaml new file mode 100644 index 000000000..483e32b73 --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-invalid-mcp-file-auth.yaml @@ -0,0 +1,24 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds llama-stack as library + use_as_library_client: true + library_client_config_path: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +mcp_servers: + - name: "mcp-file" + url: "http://mock-mcp:3001" + authorization_headers: + Authorization: "/tmp/invalid-mcp-secret-token" diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-client-auth.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-client-auth.yaml new file mode 100644 index 000000000..05f304a5d --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-client-auth.yaml @@ -0,0 +1,24 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds llama-stack as library + use_as_library_client: true + library_client_config_path: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +mcp_servers: + - name: "mcp-client" + url: "http://mock-mcp:3001" + authorization_headers: + Authorization: "client" diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-file-auth.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-file-auth.yaml index 1ff0d425e..79a8807ec 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-file-auth.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-file-auth.yaml @@ -18,8 +18,7 @@ user_data_collection: authentication: module: "noop" mcp_servers: - - name: "mcp-file-auth" - provider_id: "model-context-protocol" + - name: "mcp-file" url: "http://mock-mcp:3001" authorization_headers: Authorization: "/tmp/mcp-secret-token" diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-kubernetes-auth.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-kubernetes-auth.yaml new file mode 100644 index 000000000..2d79f1f9d --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-kubernetes-auth.yaml @@ -0,0 +1,24 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds llama-stack as library + use_as_library_client: true + library_client_config_path: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +mcp_servers: + - name: "mcp-kubernetes" + url: "http://mock-mcp:3001" + authorization_headers: + Authorization: "kubernetes" diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-oauth-auth.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-oauth-auth.yaml new file mode 100644 index 000000000..3294ac708 --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp-oauth-auth.yaml @@ -0,0 +1,24 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds llama-stack as library + use_as_library_client: true + library_client_config_path: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +mcp_servers: + - name: "mcp-oauth" + url: "http://mock-mcp:3001" + authorization_headers: + Authorization: "oauth" diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-mcp.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp.yaml index 0656aa87c..647a2cae9 100644 --- a/tests/e2e/configuration/library-mode/lightspeed-stack-mcp.yaml +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-mcp.yaml @@ -19,7 +19,18 @@ authentication: module: "noop" mcp_servers: - name: "mcp-oauth" - provider_id: "model-context-protocol" url: "http://mock-mcp:3001" authorization_headers: - Authorization: "oauth" \ No newline at end of file + Authorization: "oauth" + - name: "mcp-kubernetes" + url: "http://mock-mcp:3001" + authorization_headers: + Authorization: "kubernetes" + - name: "mcp-file" + url: "http://mock-mcp:3001" + authorization_headers: + Authorization: "/tmp/mcp-secret-token" + - name: "mcp-client" + url: "http://mock-mcp:3001" + authorization_headers: + Authorization: "client" \ No newline at end of file diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-invalid-mcp-file-auth.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-invalid-mcp-file-auth.yaml new file mode 100644 index 000000000..05ec86fdf --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-invalid-mcp-file-auth.yaml @@ -0,0 +1,25 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to separate llama-stack service + use_as_library_client: false + url: http://llama-stack:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +mcp_servers: + - name: "mcp-file" + url: "http://mock-mcp:3001" + authorization_headers: + Authorization: "/tmp/invalid-mcp-secret-token" diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-client-auth.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-client-auth.yaml new file mode 100644 index 000000000..e0f952fc3 --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-client-auth.yaml @@ -0,0 +1,25 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to separate llama-stack service + use_as_library_client: false + url: http://llama-stack:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +mcp_servers: + - name: "mcp-client" + url: "http://mock-mcp:3001" + authorization_headers: + Authorization: "client" diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-file-auth.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-file-auth.yaml index d39f55399..aca5c6ef2 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-file-auth.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-file-auth.yaml @@ -19,8 +19,7 @@ user_data_collection: authentication: module: "noop" mcp_servers: - - name: "mcp-file-auth" - provider_id: "model-context-protocol" + - name: "mcp-file" url: "http://mock-mcp:3001" authorization_headers: Authorization: "/tmp/mcp-secret-token" diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-kubernetes-auth.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-kubernetes-auth.yaml new file mode 100644 index 000000000..66dc7f87b --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-kubernetes-auth.yaml @@ -0,0 +1,25 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to separate llama-stack service + use_as_library_client: false + url: http://llama-stack:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +mcp_servers: + - name: "mcp-kubernetes" + url: "http://mock-mcp:3001" + authorization_headers: + Authorization: "kubernetes" diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-oauth-auth.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-oauth-auth.yaml new file mode 100644 index 000000000..b9125de8e --- /dev/null +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp-oauth-auth.yaml @@ -0,0 +1,25 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to separate llama-stack service + use_as_library_client: false + url: http://llama-stack:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +mcp_servers: + - name: "mcp-oauth" + url: "http://mock-mcp:3001" + authorization_headers: + Authorization: "oauth" diff --git a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp.yaml b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp.yaml index a598ce441..e35535f42 100644 --- a/tests/e2e/configuration/server-mode/lightspeed-stack-mcp.yaml +++ b/tests/e2e/configuration/server-mode/lightspeed-stack-mcp.yaml @@ -20,7 +20,18 @@ authentication: module: "noop" mcp_servers: - name: "mcp-oauth" - provider_id: "model-context-protocol" url: "http://mock-mcp:3001" authorization_headers: - Authorization: "oauth" \ No newline at end of file + Authorization: "oauth" + - name: "mcp-kubernetes" + url: "http://mock-mcp:3001" + authorization_headers: + Authorization: "kubernetes" + - name: "mcp-file" + url: "http://mock-mcp:3001" + authorization_headers: + Authorization: "/tmp/mcp-secret-token" + - name: "mcp-client" + url: "http://mock-mcp:3001" + authorization_headers: + Authorization: "client" \ No newline at end of file diff --git a/tests/e2e/features/environment.py b/tests/e2e/features/environment.py index 0ca8781d0..58a9afe80 100644 --- a/tests/e2e/features/environment.py +++ b/tests/e2e/features/environment.py @@ -10,17 +10,20 @@ import os import subprocess import time +from typing import Optional import requests from behave.model import Feature, Scenario from tests.e2e.utils.prow_utils import restore_llama_stack_pod from behave.runner import Context -from tests.e2e.utils.llama_stack_shields import ( +from tests.e2e.utils.llama_stack_utils import ( register_shield, + unregister_mcp_toolgroups, unregister_shield, ) from tests.e2e.utils.utils import ( + clear_llama_stack_storage, create_config_backup, is_prow_environment, remove_config_backup, @@ -57,6 +60,22 @@ "tests/e2e/configuration/{mode_dir}/lightspeed-stack-mcp-file-auth.yaml", "tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp-file-auth.yaml", ), + "invalid-mcp-file-auth": ( + "tests/e2e/configuration/{mode_dir}/lightspeed-stack-invalid-mcp-file-auth.yaml", + "tests/e2e-prow/rhoai/configs/lightspeed-stack-invalid-mcp-file-auth.yaml", + ), + "mcp-kubernetes-auth": ( + "tests/e2e/configuration/{mode_dir}/lightspeed-stack-mcp-kubernetes-auth.yaml", + "tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp-kubernetes-auth.yaml", + ), + "mcp-client-auth": ( + "tests/e2e/configuration/{mode_dir}/lightspeed-stack-mcp-client-auth.yaml", + "tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp-client-auth.yaml", + ), + "mcp-oauth-auth": ( + "tests/e2e/configuration/{mode_dir}/lightspeed-stack-mcp-oauth-auth.yaml", + "tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp-oauth-auth.yaml", + ), } @@ -207,6 +226,27 @@ def before_scenario(context: Context, scenario: Scenario) -> None: switch_config(context.scenario_config) restart_container("lightspeed-stack") + config_name: Optional[str] = None + if "MCPFileAuthConfig" in scenario.effective_tags: + config_name = "mcp-file-auth" + elif "InvalidMCPFileAuthConfig" in scenario.effective_tags: + config_name = "invalid-mcp-file-auth" + elif "MCPKubernetesAuthConfig" in scenario.effective_tags: + config_name = "mcp-kubernetes-auth" + elif "MCPClientAuthConfig" in scenario.effective_tags: + config_name = "mcp-client-auth" + elif "MCPOAuthAuthConfig" in scenario.effective_tags: + config_name = "mcp-oauth-auth" + + if config_name is not None: + if not context.is_library_mode: + unregister_mcp_toolgroups() + else: + clear_llama_stack_storage() + context.scenario_config = _get_config_path(config_name, mode_dir) + switch_config(context.scenario_config) + restart_container("lightspeed-stack") + def after_scenario(context: Context, scenario: Scenario) -> None: """Run after each scenario is run. @@ -241,7 +281,15 @@ def after_scenario(context: Context, scenario: Scenario) -> None: context.llama_stack_was_running = False # Tags that require config restoration after scenario - config_restore_tags = {"InvalidFeedbackStorageConfig", "NoCacheConfig"} + config_restore_tags = { + "InvalidFeedbackStorageConfig", + "NoCacheConfig", + "MCPFileAuthConfig", + "InvalidMCPFileAuthConfig", + "MCPKubernetesAuthConfig", + "MCPClientAuthConfig", + "MCPOAuthAuthConfig", + } if config_restore_tags & set(scenario.effective_tags): switch_config(context.feature_config) restart_container("lightspeed-stack") diff --git a/tests/e2e/features/mcp.feature b/tests/e2e/features/mcp.feature index 90d6c5cf8..b7cbc2285 100644 --- a/tests/e2e/features/mcp.feature +++ b/tests/e2e/features/mcp.feature @@ -5,7 +5,358 @@ Feature: MCP tests Given The service is started locally And REST API service prefix is /v1 - Scenario: Check if tools endpoint reports error when MCP requires authentication + +# File-based + @skip #TODO: LCORE-1461 + @MCPFileAuthConfig + Scenario: Check if tools endpoint succeeds when MCP file-based auth token is passed + Given The system is in default state + When I access REST API endpoint "tools" using HTTP GET method + Then The status code of the response is 200 + And The body of the response contains mcp-file + + @skip-in-library-mode #TODO: LCORE-1428 + @MCPFileAuthConfig + Scenario: Check if query endpoint succeeds when MCP file-based auth token is passed + Given The system is in default state + And I capture the current token metrics + When I use "query" to ask question + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The response should contain following fragments + | Fragments in LLM response | + | Hello | + And The token metrics should have increased + + @skip-in-library-mode #TODO: LCORE-1428 + @MCPFileAuthConfig + Scenario: Check if streaming_query endpoint succeeds when MCP file-based auth token is passed + Given The system is in default state + And I capture the current token metrics + When I use "streaming_query" to ask question + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + When I wait for the response to be completed + Then The status code of the response is 200 + And The streamed response should contain following fragments + | Fragments in LLM response | + | Hello | + And The token metrics should have increased + + @skip #TODO: LCORE-1461 + @InvalidMCPFileAuthConfig + Scenario: Check if tools endpoint reports error when MCP file-based invalid auth token is passed + Given The system is in default state + When I access REST API endpoint "tools" using HTTP GET method + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3001 requires OAuth" + } + } + """ + + @skip #TODO: LCORE-1463 + @InvalidMCPFileAuthConfig + Scenario: Check if query endpoint reports error when MCP file-based invalid auth token is passed + Given The system is in default state + When I use "query" to ask question + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3001 requires OAuth" + } + } + """ + + @skip #TODO: LCORE-1463 + @InvalidMCPFileAuthConfig + Scenario: Check if streaming_query endpoint reports error when MCP file-based invalid auth token is passed + Given The system is in default state + When I use "streaming_query" to ask question + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3001 requires OAuth" + } + } + """ + +# Kubernetes + @skip #TODO: LCORE-1461 + @MCPKubernetesAuthConfig + Scenario: Check if tools endpoint succeeds when MCP kubernetes auth token is passed + Given The system is in default state + And I set the Authorization header to Bearer kubernetes-test-token + When I access REST API endpoint "tools" using HTTP GET method + Then The status code of the response is 200 + And The body of the response contains mcp-kubernetes + + @skip-in-library-mode #TODO: LCORE-1428 + @MCPKubernetesAuthConfig + Scenario: Check if query endpoint succeeds when MCP kubernetes auth token is passed + Given The system is in default state + And I set the Authorization header to Bearer kubernetes-test-token + And I capture the current token metrics + When I use "query" to ask question with authorization header + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The response should contain following fragments + | Fragments in LLM response | + | Hello | + And The token metrics should have increased + + @skip-in-library-mode #TODO: LCORE-1428 + @MCPKubernetesAuthConfig + Scenario: Check if streaming_query endpoint succeeds when MCP kubernetes auth token is passed + Given The system is in default state + And I set the Authorization header to Bearer kubernetes-test-token + And I capture the current token metrics + When I use "streaming_query" to ask question with authorization header + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + When I wait for the response to be completed + Then The status code of the response is 200 + And The streamed response should contain following fragments + | Fragments in LLM response | + | Hello | + And The token metrics should have increased + + @skip #TODO: LCORE-1461 + @MCPKubernetesAuthConfig + Scenario: Check if tools endpoint reports error when MCP kubernetes invalid auth token is passed + Given The system is in default state + And I set the Authorization header to Bearer kubernetes-invalid-token + When I access REST API endpoint "tools" using HTTP GET method + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3001 requires OAuth" + } + } + """ + + @skip #TODO: LCORE-1463 + @MCPKubernetesAuthConfig + Scenario: Check if query endpoint reports error when MCP kubernetes invalid auth token is passed + Given The system is in default state + And I set the Authorization header to Bearer kubernetes-invalid-token + When I use "query" to ask question with authorization header + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3001 requires OAuth" + } + } + """ + + @skip #TODO: LCORE-1463 + @MCPKubernetesAuthConfig + Scenario: Check if streaming_query endpoint reports error when MCP kubernetes invalid auth token is passed + Given The system is in default state + And I set the Authorization header to Bearer kubernetes-invalid-token + When I use "streaming_query" to ask question with authorization header + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3001 requires OAuth" + } + } + """ + +# Client-provided + @skip #TODO: LCORE-1462 + @MCPClientAuthConfig + Scenario: Check if tools endpoint succeeds by skipping when MCP client-provided auth token is omitted + Given The system is in default state + When I access REST API endpoint "tools" using HTTP GET method + Then The status code of the response is 200 + And The body of the response does not contain mcp-client + + @MCPClientAuthConfig + Scenario: Check if query endpoint succeeds by skipping when MCP client-provided auth token is omitted + Given The system is in default state + And I capture the current token metrics + When I use "query" to ask question + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The body of the response does not contain mcp-client + And The response should contain following fragments + | Fragments in LLM response | + | Hello | + And The token metrics should have increased + + @MCPClientAuthConfig + Scenario: Check if streaming_query endpoint succeeds by skipping when MCP client-provided auth token is omitted + Given The system is in default state + And I capture the current token metrics + When I use "streaming_query" to ask question + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + When I wait for the response to be completed + Then The status code of the response is 200 + And The body of the response does not contain mcp-client + And The streamed response should contain following fragments + | Fragments in LLM response | + | Hello | + And The token metrics should have increased + + @MCPClientAuthConfig + Scenario: Check if tools endpoint succeeds when MCP client-provided auth token is passed + Given The system is in default state + And I set the "MCP-HEADERS" header to + """ + {"mcp-client": {"Authorization": "Bearer client-test-token"}} + """ + When I access REST API endpoint "tools" using HTTP GET method + Then The status code of the response is 200 + And The body of the response contains mcp-client + + @skip-in-library-mode #TODO: LCORE-1428 + @MCPClientAuthConfig + Scenario: Check if query endpoint succeeds when MCP client-provided auth token is passed + Given The system is in default state + And I set the "MCP-HEADERS" header to + """ + {"mcp-client": {"Authorization": "Bearer client-test-token"}} + """ + And I capture the current token metrics + When I use "query" to ask question with authorization header + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And The response should contain following fragments + | Fragments in LLM response | + | Hello | + And The token metrics should have increased + + @skip-in-library-mode #TODO: LCORE-1428 + @MCPClientAuthConfig + Scenario: Check if streaming_query endpoint succeeds when MCP client-provided auth token is passed + Given The system is in default state + And I set the "MCP-HEADERS" header to + """ + {"mcp-client": {"Authorization": "Bearer client-test-token"}} + """ + And I capture the current token metrics + When I use "streaming_query" to ask question with authorization header + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + When I wait for the response to be completed + Then The status code of the response is 200 + And The streamed response should contain following fragments + | Fragments in LLM response | + | Hello | + And The token metrics should have increased + + @MCPClientAuthConfig + Scenario: Check if tools endpoint reports error when MCP client-provided invalid auth token is passed + Given The system is in default state + And I set the "MCP-HEADERS" header to + """ + {"mcp-client": {"Authorization": "Bearer client-invalid-token"}} + """ + When I access REST API endpoint "tools" using HTTP GET method + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3001 requires OAuth" + } + } + """ + + @MCPClientAuthConfig + Scenario: Check if query endpoint reports error when MCP client-provided invalid auth token is passed + Given The system is in default state + And I set the "MCP-HEADERS" header to + """ + {"mcp-client": {"Authorization": "Bearer client-invalid-token"}} + """ + When I use "query" to ask question with authorization header + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3001 requires OAuth" + } + } + """ + + @MCPClientAuthConfig + Scenario: Check if streaming_query endpoint reports error when MCP client-provided invalid auth token is passed + Given The system is in default state + And I set the "MCP-HEADERS" header to + """ + {"mcp-client": {"Authorization": "Bearer client-invalid-token"}} + """ + When I use "streaming_query" to ask question with authorization header + """ + {"query": "Say hello", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "MCP server at http://mock-mcp:3001 requires OAuth" + } + } + """ + +# OAuth + + @MCPOAuthAuthConfig + Scenario: Check if tools endpoint reports error when MCP OAuth requires authentication Given The system is in default state When I access REST API endpoint "tools" using HTTP GET method Then The status code of the response is 401 @@ -20,7 +371,8 @@ Feature: MCP tests """ And The headers of the response contains the following header "www-authenticate" - Scenario: Check if query endpoint reports error when MCP requires authentication + @MCPOAuthAuthConfig + Scenario: Check if query endpoint reports error when MCP OAuth requires authentication Given The system is in default state When I use "query" to ask question """ @@ -38,7 +390,8 @@ Feature: MCP tests """ And The headers of the response contains the following header "www-authenticate" - Scenario: Check if streaming_query endpoint reports error when MCP requires authentication + @MCPOAuthAuthConfig + Scenario: Check if streaming_query endpoint reports error when MCP OAuth requires authentication Given The system is in default state When I use "streaming_query" to ask question """ @@ -56,22 +409,24 @@ Feature: MCP tests """ And The headers of the response contains the following header "www-authenticate" - Scenario: Check if tools endpoint succeeds when MCP auth token is passed + @MCPOAuthAuthConfig + Scenario: Check if tools endpoint succeeds when MCP OAuth auth token is passed Given The system is in default state And I set the "MCP-HEADERS" header to """ - {"mcp-oauth": {"Authorization": "Bearer test-token"}} + {"mcp-oauth": {"Authorization": "Bearer oauth-test-token"}} """ When I access REST API endpoint "tools" using HTTP GET method Then The status code of the response is 200 And The body of the response contains mcp-oauth - @skip-in-library-mode # will be fixed in LCORE-1428 - Scenario: Check if query endpoint succeeds when MCP auth token is passed + @skip-in-library-mode #TODO: LCORE-1428 + @MCPOAuthAuthConfig + Scenario: Check if query endpoint succeeds when MCP OAuth auth token is passed Given The system is in default state And I set the "MCP-HEADERS" header to """ - {"mcp-oauth": {"Authorization": "Bearer test-token"}} + {"mcp-oauth": {"Authorization": "Bearer oauth-test-token"}} """ And I capture the current token metrics When I use "query" to ask question with authorization header @@ -84,12 +439,13 @@ Feature: MCP tests | Hello | And The token metrics should have increased - @skip-in-library-mode # will be fixed in LCORE-1428 - Scenario: Check if streaming_query endpoint succeeds when MCP auth token is passed + @skip-in-library-mode #TODO: LCORE-1428 + @MCPOAuthAuthConfig + Scenario: Check if streaming_query endpoint succeeds when MCP OAuth auth token is passed Given The system is in default state And I set the "MCP-HEADERS" header to """ - {"mcp-oauth": {"Authorization": "Bearer test-token"}} + {"mcp-oauth": {"Authorization": "Bearer oauth-test-token"}} """ And I capture the current token metrics When I use "streaming_query" to ask question with authorization header @@ -103,11 +459,12 @@ Feature: MCP tests | Hello | And The token metrics should have increased - Scenario: Check if tools endpoint reports error when MCP invalid auth token is passed + @MCPOAuthAuthConfig + Scenario: Check if tools endpoint reports error when MCP OAuth invalid auth token is passed Given The system is in default state And I set the "MCP-HEADERS" header to """ - {"mcp-oauth": {"Authorization": "Bearer invalid-token"}} + {"mcp-oauth": {"Authorization": "Bearer oauth-invalid-token"}} """ When I access REST API endpoint "tools" using HTTP GET method Then The status code of the response is 401 @@ -122,12 +479,12 @@ Feature: MCP tests """ And The headers of the response contains the following header "www-authenticate" - @skip # will be fixed in LCORE-1366 - Scenario: Check if query endpoint reports error when MCP invalid auth token is passed + @MCPOAuthAuthConfig + Scenario: Check if query endpoint reports error when MCP OAuth invalid auth token is passed Given The system is in default state And I set the "MCP-HEADERS" header to """ - {"mcp-oauth": {"Authorization": "Bearer invalid-token"}} + {"mcp-oauth": {"Authorization": "Bearer oauth-invalid-token"}} """ When I use "query" to ask question with authorization header """ @@ -145,11 +502,12 @@ Feature: MCP tests """ And The headers of the response contains the following header "www-authenticate" - Scenario: Check if streaming_query endpoint reports error when MCP invalid auth token is passed + @MCPOAuthAuthConfig + Scenario: Check if streaming_query endpoint reports error when MCP OAuth invalid auth token is passed Given The system is in default state And I set the "MCP-HEADERS" header to """ - {"mcp-oauth": {"Authorization": "Bearer invalid-token"}} + {"mcp-oauth": {"Authorization": "Bearer oauth-invalid-token"}} """ When I use "streaming_query" to ask question with authorization header """ diff --git a/tests/e2e/features/mcp_file_auth.feature b/tests/e2e/features/mcp_file_auth.feature deleted file mode 100644 index 455f0740c..000000000 --- a/tests/e2e/features/mcp_file_auth.feature +++ /dev/null @@ -1,20 +0,0 @@ -@MCPFileAuth -Feature: MCP file-based authorization tests - - Regression tests for LCORE-1414: MCP authorization tokens configured via - file-based authorization_headers must survive model_dump() serialization - and reach the MCP server as a valid Bearer token. - - Background: - Given The service is started locally - And REST API service prefix is /v1 - - @skip-in-library-mode - Scenario: Query succeeds with file-based MCP authorization - Given The system is in default state - When I use "query" to ask question - """ - {"query": "Use the mock_tool_e2e tool to send the message 'hello'", "model": "{MODEL}", "provider": "{PROVIDER}"} - """ - Then The status code of the response is 200 - And The body of the response contains mock_tool_e2e diff --git a/tests/e2e/features/steps/common_http.py b/tests/e2e/features/steps/common_http.py index 016791fb6..284acfeb0 100644 --- a/tests/e2e/features/steps/common_http.py +++ b/tests/e2e/features/steps/common_http.py @@ -178,6 +178,15 @@ def check_response_body_contains(context: Context, substring: str) -> None: ), f"The response text '{context.response.text}' doesn't contain '{expected}'" +@then("The body of the response does not contain {substring}") +def check_response_body_does_not_contain(context: Context, substring: str) -> None: + """Check that response body does not contain a substring.""" + assert context.response is not None, "Request needs to be performed first" + assert ( + substring not in context.response.text + ), f"The response text '{context.response.text}' contains '{substring}'" + + @then("The body of the response is the following") def check_prediction_result(context: Context) -> None: """Check the content of the response to be exactly the same. diff --git a/tests/e2e/secrets/invalid-mcp-token b/tests/e2e/secrets/invalid-mcp-token new file mode 100644 index 000000000..3707272a2 --- /dev/null +++ b/tests/e2e/secrets/invalid-mcp-token @@ -0,0 +1 @@ +invalid-token \ No newline at end of file diff --git a/tests/e2e/test_list.txt b/tests/e2e/test_list.txt index 583cf387b..0da5cae41 100644 --- a/tests/e2e/test_list.txt +++ b/tests/e2e/test_list.txt @@ -16,5 +16,4 @@ features/rlsapi_v1_errors.feature features/streaming_query.feature features/rest_api.feature features/mcp.feature -features/mcp_file_auth.feature features/models.feature diff --git a/tests/e2e/utils/llama_stack_shields.py b/tests/e2e/utils/llama_stack_utils.py similarity index 62% rename from tests/e2e/utils/llama_stack_shields.py rename to tests/e2e/utils/llama_stack_utils.py index 4f793c0bf..2a8c66670 100644 --- a/tests/e2e/utils/llama_stack_shields.py +++ b/tests/e2e/utils/llama_stack_utils.py @@ -1,9 +1,12 @@ -"""E2E helpers to unregister and re-register Llama Stack shields via the client API. +"""E2E test utilities for Llama Stack (toolgroups and shields). -Used by the @disable-shields tag: before the scenario we call client.shields.delete() -to unregister the shield; after the scenario we call client.shields.register() -to restore it. Only applies in server mode (Llama Stack as a separate service). -Requires E2E_LLAMA_STACK_URL or E2E_LLAMA_HOSTNAME/E2E_LLAMA_PORT. +This module provides functions to manage MCP toolgroups and shields on a running +Llama Stack instance during end-to-end tests: unregister MCP toolgroups when +switching configurations or testing MCP auth, and unregister/re-register shields +(e.g. for the @disable-shields tag). + +Only applies when running Llama Stack as a separate service (server mode). +Requires E2E_LLAMA_STACK_URL or E2E_LLAMA_HOSTNAME and E2E_LLAMA_PORT. """ import asyncio @@ -29,6 +32,54 @@ def _get_llama_stack_client() -> AsyncLlamaStackClient: return AsyncLlamaStackClient(base_url=base_url, api_key=api_key, timeout=timeout) +# ----------------------------------------------------------------------------- +# Toolgroups +# ----------------------------------------------------------------------------- + + +async def _unregister_toolgroup_async(identifier: str) -> None: + """Unregister a toolgroup by identifier; return (provider_id, provider_shield_id) for restore.""" + client = _get_llama_stack_client() + try: + await client.toolgroups.unregister(identifier) + except APIConnectionError: + raise + except APIStatusError as e: + # 400 "not found": toolgroup already absent, scenario can proceed + if e.status_code == 400 and "not found" in str(e).lower(): + return None + raise + finally: + await client.close() + + +async def _unregister_mcp_toolgroups_async() -> None: + """Unregister all MCP toolgroups.""" + client = _get_llama_stack_client() + try: + toolgroups = await client.toolgroups.list() + for toolgroup in toolgroups: + if ( + toolgroup.identifier + and toolgroup.provider_id == "model-context-protocol" + ): + await _unregister_toolgroup_async(toolgroup.identifier) + except APIConnectionError: + raise + finally: + await client.close() + + +def unregister_mcp_toolgroups() -> None: + """Unregister all MCP toolgroups.""" + asyncio.run(_unregister_mcp_toolgroups_async()) + + +# ----------------------------------------------------------------------------- +# Shields +# ----------------------------------------------------------------------------- + + async def _unregister_shield_async(identifier: str) -> Optional[tuple[str, str]]: """Unregister a shield by identifier; return (provider_id, provider_shield_id) for restore.""" client = _get_llama_stack_client() diff --git a/tests/e2e/utils/utils.py b/tests/e2e/utils/utils.py index bc9b8b966..580250bff 100644 --- a/tests/e2e/utils/utils.py +++ b/tests/e2e/utils/utils.py @@ -246,6 +246,36 @@ def remove_config_backup(backup_path: str) -> None: print(f"Warning: Could not remove backup file {backup_path}: {e}") +def clear_llama_stack_storage(container_name: str = "lightspeed-stack") -> None: + """Clear Llama Stack storage in library mode (embedded Llama Stack). + + Removes the ~/.llama directory so that toolgroups and other persisted + state are reset. Used before MCP config scenarios when not running in + server mode (no separate Llama Stack to unregister toolgroups from). + Only runs when using Docker (skipped in Prow). + + Parameters: + container_name (str): Docker container name (default "lightspeed-stack"). + + Returns: + None + """ + if is_prow_environment(): + return + + try: + subprocess.run( + ["docker", "exec", container_name, "sh", "-c", "rm -rf ~/.llama"], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + except subprocess.TimeoutExpired as e: + print(f"Failed to clear Llama Stack storage: {e}") + raise + + def restart_container(container_name: str) -> None: """Restart a Docker container by name and wait until it is healthy.