LCORE-1329: Adding new MCP E2E Tests

docker-compose.yaml

@@ -82,6 +82,7 @@ services:
     volumes:
       - ./lightspeed-stack.yaml:/app-root/lightspeed-stack.yaml:z
       - ./tests/e2e/secrets/mcp-token:/tmp/mcp-secret-token:ro
+      - ./tests/e2e/secrets/invalid-mcp-token:/tmp/invalid-mcp-secret-token:ro
     environment:
       - OPENAI_API_KEY=${OPENAI_API_KEY}
       # Azure Entra ID credentials (AZURE_API_KEY is obtained dynamically)

docs/e2e_testing.md

@@ -58,7 +58,7 @@ tests/e2e/
 ├── utils/
 │   ├── utils.py                 # restart_container, switch_config, wait_for_container_health, etc.
 │   ├── prow_utils.py            # Prow/OpenShift helpers (restore_llama_stack_pod, etc.)
-│   └── llama_stack_shields.py   # Shield unregister/register (server mode, optional)
+│   └── llama_stack_utils.py    # Toolgroups + shield unregister/register (server mode, optional)
 ├── mock_mcp_server/             # Mock MCP server for MCP tests
 └── rag/                         # RAG test data (e.g. for FAISS)
 ```

tests/e2e/configuration/library-mode/lightspeed-stack-invalid-mcp-file-auth.yaml

@@ -0,0 +1,24 @@
+name: Lightspeed Core Service (LCS)
+service:
+  host: 0.0.0.0
+  port: 8080
+  auth_enabled: false
+  workers: 1
+  color_log: true
+  access_log: true
+llama_stack:
+  # Library mode - embeds llama-stack as library
+  use_as_library_client: true
+  library_client_config_path: run.yaml
+user_data_collection:
+  feedback_enabled: true
+  feedback_storage: "/tmp/data/feedback"
+  transcripts_enabled: true
+  transcripts_storage: "/tmp/data/transcripts"
+authentication:
+  module: "noop"
+mcp_servers:
+  - name: "mcp-file"
+    url: "http://mock-mcp:3001"
+    authorization_headers:
+      Authorization: "/tmp/invalid-mcp-secret-token"

tests/e2e/configuration/library-mode/lightspeed-stack-mcp-client-auth.yaml

@@ -0,0 +1,24 @@
+name: Lightspeed Core Service (LCS)
+service:
+  host: 0.0.0.0
+  port: 8080
+  auth_enabled: false
+  workers: 1
+  color_log: true
+  access_log: true
+llama_stack:
+  # Library mode - embeds llama-stack as library
+  use_as_library_client: true
+  library_client_config_path: run.yaml
+user_data_collection:
+  feedback_enabled: true
+  feedback_storage: "/tmp/data/feedback"
+  transcripts_enabled: true
+  transcripts_storage: "/tmp/data/transcripts"
+authentication:
+  module: "noop"
+mcp_servers:
+  - name: "mcp-client"
+    url: "http://mock-mcp:3001"
+    authorization_headers:
+      Authorization: "client"

tests/e2e/configuration/library-mode/lightspeed-stack-mcp-file-auth.yaml

@@ -18,8 +18,7 @@ user_data_collection:
 authentication:
   module: "noop"
 mcp_servers:
-  - name: "mcp-file-auth"
-    provider_id: "model-context-protocol"
+  - name: "mcp-file"
     url: "http://mock-mcp:3001"
     authorization_headers:
       Authorization: "/tmp/mcp-secret-token"

tests/e2e/configuration/library-mode/lightspeed-stack-mcp-kubernetes-auth.yaml

@@ -0,0 +1,24 @@
+name: Lightspeed Core Service (LCS)
+service:
+  host: 0.0.0.0
+  port: 8080
+  auth_enabled: false
+  workers: 1
+  color_log: true
+  access_log: true
+llama_stack:
+  # Library mode - embeds llama-stack as library
+  use_as_library_client: true
+  library_client_config_path: run.yaml
+user_data_collection:
+  feedback_enabled: true
+  feedback_storage: "/tmp/data/feedback"
+  transcripts_enabled: true
+  transcripts_storage: "/tmp/data/transcripts"
+authentication:
+  module: "noop"
+mcp_servers:
+  - name: "mcp-kubernetes"
+    url: "http://mock-mcp:3001"
+    authorization_headers:
+      Authorization: "kubernetes"

tests/e2e/configuration/library-mode/lightspeed-stack-mcp-oauth-auth.yaml

@@ -0,0 +1,24 @@
+name: Lightspeed Core Service (LCS)
+service:
+  host: 0.0.0.0
+  port: 8080
+  auth_enabled: false
+  workers: 1
+  color_log: true
+  access_log: true
+llama_stack:
+  # Library mode - embeds llama-stack as library
+  use_as_library_client: true
+  library_client_config_path: run.yaml
+user_data_collection:
+  feedback_enabled: true
+  feedback_storage: "/tmp/data/feedback"
+  transcripts_enabled: true
+  transcripts_storage: "/tmp/data/transcripts"
+authentication:
+  module: "noop"
+mcp_servers:
+  - name: "mcp-oauth"
+    url: "http://mock-mcp:3001"
+    authorization_headers:
+      Authorization: "oauth"

tests/e2e/configuration/library-mode/lightspeed-stack-mcp.yaml

@@ -19,7 +19,18 @@ authentication:
   module: "noop"
 mcp_servers:
   - name: "mcp-oauth"
-    provider_id: "model-context-protocol"
     url: "http://mock-mcp:3001"
     authorization_headers:
-      Authorization: "oauth"
+      Authorization: "oauth"
+  - name: "mcp-kubernetes"
+    url: "http://mock-mcp:3001"
+    authorization_headers:
+      Authorization: "kubernetes"
+  - name: "mcp-file"
+    url: "http://mock-mcp:3001"
+    authorization_headers:
+      Authorization: "/tmp/mcp-secret-token"
+  - name: "mcp-client"
+    url: "http://mock-mcp:3001"
+    authorization_headers:
+      Authorization: "client"

tests/e2e/configuration/server-mode/lightspeed-stack-invalid-mcp-file-auth.yaml

@@ -0,0 +1,25 @@
+name: Lightspeed Core Service (LCS)
+service:
+  host: 0.0.0.0
+  port: 8080
+  auth_enabled: false
+  workers: 1
+  color_log: true
+  access_log: true
+llama_stack:
+  # Server mode - connects to separate llama-stack service
+  use_as_library_client: false
+  url: http://llama-stack:8321
+  api_key: xyzzy
+user_data_collection:
+  feedback_enabled: true
+  feedback_storage: "/tmp/data/feedback"
+  transcripts_enabled: true
+  transcripts_storage: "/tmp/data/transcripts"
+authentication:
+  module: "noop"
+mcp_servers:
+  - name: "mcp-file"
+    url: "http://mock-mcp:3001"
+    authorization_headers:
+      Authorization: "/tmp/invalid-mcp-secret-token"

tests/e2e/configuration/server-mode/lightspeed-stack-mcp-client-auth.yaml

@@ -0,0 +1,25 @@
+name: Lightspeed Core Service (LCS)
+service:
+  host: 0.0.0.0
+  port: 8080
+  auth_enabled: false
+  workers: 1
+  color_log: true
+  access_log: true
+llama_stack:
+  # Server mode - connects to separate llama-stack service
+  use_as_library_client: false
+  url: http://llama-stack:8321
+  api_key: xyzzy
+user_data_collection:
+  feedback_enabled: true
+  feedback_storage: "/tmp/data/feedback"
+  transcripts_enabled: true
+  transcripts_storage: "/tmp/data/transcripts"
+authentication:
+  module: "noop"
+mcp_servers:
+  - name: "mcp-client"
+    url: "http://mock-mcp:3001"
+    authorization_headers:
+      Authorization: "client"

tests/e2e/configuration/server-mode/lightspeed-stack-mcp-file-auth.yaml

@@ -19,8 +19,7 @@ user_data_collection:
 authentication:
   module: "noop"
 mcp_servers:
-  - name: "mcp-file-auth"
-    provider_id: "model-context-protocol"
+  - name: "mcp-file"
     url: "http://mock-mcp:3001"
     authorization_headers:
       Authorization: "/tmp/mcp-secret-token"

tests/e2e/configuration/server-mode/lightspeed-stack-mcp-kubernetes-auth.yaml

@@ -0,0 +1,25 @@
+name: Lightspeed Core Service (LCS)
+service:
+  host: 0.0.0.0
+  port: 8080
+  auth_enabled: false
+  workers: 1
+  color_log: true
+  access_log: true
+llama_stack:
+  # Server mode - connects to separate llama-stack service
+  use_as_library_client: false
+  url: http://llama-stack:8321
+  api_key: xyzzy
+user_data_collection:
+  feedback_enabled: true
+  feedback_storage: "/tmp/data/feedback"
+  transcripts_enabled: true
+  transcripts_storage: "/tmp/data/transcripts"
+authentication:
+  module: "noop"
+mcp_servers:
+  - name: "mcp-kubernetes"
+    url: "http://mock-mcp:3001"
+    authorization_headers:
+      Authorization: "kubernetes"

tests/e2e/configuration/server-mode/lightspeed-stack-mcp-oauth-auth.yaml

@@ -0,0 +1,25 @@
+name: Lightspeed Core Service (LCS)
+service:
+  host: 0.0.0.0
+  port: 8080
+  auth_enabled: false
+  workers: 1
+  color_log: true
+  access_log: true
+llama_stack:
+  # Server mode - connects to separate llama-stack service
+  use_as_library_client: false
+  url: http://llama-stack:8321
+  api_key: xyzzy
+user_data_collection:
+  feedback_enabled: true
+  feedback_storage: "/tmp/data/feedback"
+  transcripts_enabled: true
+  transcripts_storage: "/tmp/data/transcripts"
+authentication:
+  module: "noop"
+mcp_servers:
+  - name: "mcp-oauth"
+    url: "http://mock-mcp:3001"
+    authorization_headers:
+      Authorization: "oauth"

tests/e2e/configuration/server-mode/lightspeed-stack-mcp.yaml

@@ -20,7 +20,18 @@ authentication:
   module: "noop"
 mcp_servers:
   - name: "mcp-oauth"
-    provider_id: "model-context-protocol"
     url: "http://mock-mcp:3001"
     authorization_headers:
-      Authorization: "oauth"
+      Authorization: "oauth"
+  - name: "mcp-kubernetes"
+    url: "http://mock-mcp:3001"
+    authorization_headers:
+      Authorization: "kubernetes"
+  - name: "mcp-file"
+    url: "http://mock-mcp:3001"
+    authorization_headers:
+      Authorization: "/tmp/mcp-secret-token"
+  - name: "mcp-client"
+    url: "http://mock-mcp:3001"
+    authorization_headers:
+      Authorization: "client"

tests/e2e/features/environment.py

@@ -10,17 +10,20 @@
 import os
 import subprocess
 import time
+from typing import Optional
 
 import requests
 from behave.model import Feature, Scenario
 from tests.e2e.utils.prow_utils import restore_llama_stack_pod
 from behave.runner import Context
 
-from tests.e2e.utils.llama_stack_shields import (
+from tests.e2e.utils.llama_stack_utils import (
     register_shield,
+    unregister_mcp_toolgroups,
     unregister_shield,
 )
 from tests.e2e.utils.utils import (
+    clear_llama_stack_storage,
     create_config_backup,
     is_prow_environment,
     remove_config_backup,
@@ -57,6 +60,22 @@
         "tests/e2e/configuration/{mode_dir}/lightspeed-stack-mcp-file-auth.yaml",
         "tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp-file-auth.yaml",
     ),
+    "invalid-mcp-file-auth": (
+        "tests/e2e/configuration/{mode_dir}/lightspeed-stack-invalid-mcp-file-auth.yaml",
+        "tests/e2e-prow/rhoai/configs/lightspeed-stack-invalid-mcp-file-auth.yaml",
+    ),
+    "mcp-kubernetes-auth": (
+        "tests/e2e/configuration/{mode_dir}/lightspeed-stack-mcp-kubernetes-auth.yaml",
+        "tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp-kubernetes-auth.yaml",
+    ),
+    "mcp-client-auth": (
+        "tests/e2e/configuration/{mode_dir}/lightspeed-stack-mcp-client-auth.yaml",
+        "tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp-client-auth.yaml",
+    ),
+    "mcp-oauth-auth": (
+        "tests/e2e/configuration/{mode_dir}/lightspeed-stack-mcp-oauth-auth.yaml",
+        "tests/e2e-prow/rhoai/configs/lightspeed-stack-mcp-oauth-auth.yaml",
+    ),
 }
 
 
@@ -207,6 +226,27 @@ def before_scenario(context: Context, scenario: Scenario) -> None:
         switch_config(context.scenario_config)
         restart_container("lightspeed-stack")
 
+    config_name: Optional[str] = None
+    if "MCPFileAuthConfig" in scenario.effective_tags:
+        config_name = "mcp-file-auth"
+    elif "InvalidMCPFileAuthConfig" in scenario.effective_tags:
+        config_name = "invalid-mcp-file-auth"
+    elif "MCPKubernetesAuthConfig" in scenario.effective_tags:
+        config_name = "mcp-kubernetes-auth"
+    elif "MCPClientAuthConfig" in scenario.effective_tags:
+        config_name = "mcp-client-auth"
+    elif "MCPOAuthAuthConfig" in scenario.effective_tags:
+        config_name = "mcp-oauth-auth"
+
+    if config_name is not None:
+        if not context.is_library_mode:
+            unregister_mcp_toolgroups()
+        else:
+            clear_llama_stack_storage()
+        context.scenario_config = _get_config_path(config_name, mode_dir)
+        switch_config(context.scenario_config)
+        restart_container("lightspeed-stack")
+
 
 def after_scenario(context: Context, scenario: Scenario) -> None:
     """Run after each scenario is run.
@@ -241,7 +281,15 @@ def after_scenario(context: Context, scenario: Scenario) -> None:
         context.llama_stack_was_running = False
 
     # Tags that require config restoration after scenario
-    config_restore_tags = {"InvalidFeedbackStorageConfig", "NoCacheConfig"}
+    config_restore_tags = {
+        "InvalidFeedbackStorageConfig",
+        "NoCacheConfig",
+        "MCPFileAuthConfig",
+        "InvalidMCPFileAuthConfig",
+        "MCPKubernetesAuthConfig",
+        "MCPClientAuthConfig",
+        "MCPOAuthAuthConfig",
+    }
     if config_restore_tags & set(scenario.effective_tags):
         switch_config(context.feature_config)
         restart_container("lightspeed-stack")