diff --git a/tests/perf_tests/Makefile b/tests/perf_tests/Makefile index bf8fd833f..f35335033 100644 --- a/tests/perf_tests/Makefile +++ b/tests/perf_tests/Makefile @@ -1,7 +1,29 @@ +USERS ?= 250 +SPAWN_RATE ?= 50 +RUN_TIME ?= 5m + +# Interactive UI — single process, useful for smoke tests test: poetry run locust -f src/locustfile.py +# Interactive UI — search only test-read-only: poetry run locust -f src/locustfile.py SearchUser -.PHONY: test test-read-only +# Headless distributed — master + N workers, runs for 5 minutes then exits +test-headless: + poetry run locust -f src/locustfile.py \ + --headless \ + --users $(USERS) \ + --spawn-rate $(SPAWN_RATE) \ + --run-time $(RUN_TIME) + +# Search-only headless distributed +test-read-only-headless: + poetry run locust -f src/locustfile.py SearchUser \ + --headless \ + --users $(USERS) \ + --spawn-rate $(SPAWN_RATE) \ + --run-time $(RUN_TIME) + +.PHONY: test test-read-only test-headless test-read-only-headless \ No newline at end of file diff --git a/tests/perf_tests/README.md b/tests/perf_tests/README.md index 5ddede2dd..8a696acc9 100644 --- a/tests/perf_tests/README.md +++ b/tests/perf_tests/README.md @@ -5,5 +5,6 @@ This project contains Locust performance tests for the Immunisation FHIR API. To run them, ensure you have the `APIGEE_ENVIRONMENT` : Currently, only the ref environment is supported. `PERF_CREATE_RPS_PER_USER` : numeric +`PERF_SEARCH_RPS_PER_USER` : numeric -env vars set, and call `make test`. +env vars set, and call `PERF_SEARCH_RPS_PER_USER=1 make test-read-only-headless`. diff --git a/tests/perf_tests/src/locustfile.py b/tests/perf_tests/src/locustfile.py index 5a4fd0986..6a6dc45e1 100644 --- a/tests/perf_tests/src/locustfile.py +++ b/tests/perf_tests/src/locustfile.py @@ -1,15 +1,23 @@ import json import os import random +import sys +import time import uuid from pathlib import Path from urllib.parse import urlencode +import boto3 +import gevent.lock import pandas as pd -from locust import HttpUser, constant_throughput, task +from botocore.config import Config +from locust import constant_throughput, events, task +from locust.contrib.fasthttp import FastHttpUser +from locust.runners import WorkerRunner from common.api_clients.authentication import AppRestrictedAuth -from common.clients import get_secrets_manager_client + +# from common.clients import get_secrets_manager_client from common.models.constants import Urls from objectModels import patient_loader from objectModels.api_immunization_builder import create_immunization_object @@ -21,7 +29,18 @@ if not APIGEE_ENVIRONMENT: raise ValueError("APIGEE_ENVIRONMENT must be set") +_BOTO_CONFIG = Config( + max_pool_connections=50, # default is 10; needs to exceed max concurrent Locust users + retries={"mode": "standard", "max_attempts": 1}, +) +_secrets_client = boto3.client( + "secretsmanager", + region_name=os.getenv("AWS_REGION", "eu-west-2"), + config=_BOTO_CONFIG, +) + PERF_CREATE_TASK_RPS_PER_USER = float(os.getenv("PERF_CREATE_RPS_PER_USER", "1")) +PERF_SEARCH_RPS_PER_USER = float(os.getenv("PERF_SEARCH_RPS_PER_USER", "1")) IMMUNIZATION_TARGETS = [ "3IN1", @@ -53,29 +72,106 @@ def _load_valid_patients(): VALID_PATIENT_IDS = _load_valid_patients() +_TOKEN_LOCK = gevent.lock.Semaphore(1) -class BaseImmunizationUser(HttpUser): - abstract = True - authenticator = AppRestrictedAuth( - get_secrets_manager_client(), +class LocustTokenManager: + """Serialises token refreshes across all Locust greenlets (double-checked locking pattern).""" + + def __init__(self, auth: AppRestrictedAuth): + self._auth = auth + + def get_access_token(self) -> str: + now = int(time.time()) + # Fast path — no lock needed, reads are safe if the token is already cached + if ( + self._auth.cached_access_token + and self._auth.cached_access_token_expiry_time is not None + and self._auth.cached_access_token_expiry_time > now + 30 # ACCESS_TOKEN_MIN_ACCEPTABLE_LIFETIME_SECONDS + ): + return self._auth.cached_access_token + + # Slow path — exactly one greenlet refreshes; all others wait then hit the fast path + with _TOKEN_LOCK: + now = int(time.time()) # re-read after acquiring the lock + if ( + self._auth.cached_access_token + and self._auth.cached_access_token_expiry_time is not None + and self._auth.cached_access_token_expiry_time > now + 30 + ): + return self._auth.cached_access_token + return self._auth.get_access_token() + + +# Module-level singleton — pre-warmed before any user spawns +_shared_token_manager = LocustTokenManager( + AppRestrictedAuth( + _secrets_client, APIGEE_ENVIRONMENT, f"imms/perf-tests/{APIGEE_ENVIRONMENT}/jwt-secrets", ) +) + + +@events.init.add_listener +def _pre_warm_auth(environment, **kwargs): + """Fetch token once before users spawn so all users start with a cached token. + Only runs on master/standalone — workers fetch lazily on first request, + staggered by the on_start jitter, avoiding simultaneous Secrets Manager calls. + """ + if isinstance(environment.runner, WorkerRunner): + return + + try: + token = _shared_token_manager.get_access_token() + print(f"[perf] Auth pre-warm complete. Token length: {len(token)}") + except Exception as exc: + error_text = str(exc) + is_credential_error = any( + kw in error_text for kw in ("ForbiddenException", "ExpiredToken", "No access", "TokenExpired") + ) + if is_credential_error: + print( + "\n[perf] FATAL: AWS credentials expired or inaccessible.\n" + f" Error: {exc}\n\n" + " Fix: run one of the following, then retry 'make test':\n" + " aws sso login --profile \n", + file=sys.stderr, + ) + sys.exit(1) + raise + + +class BaseImmunizationUser(FastHttpUser): + abstract = True + + # token_manager = LocustTokenManager( + # AppRestrictedAuth( + # _secrets_client, + # APIGEE_ENVIRONMENT, + # f"imms/perf-tests/{APIGEE_ENVIRONMENT}/jwt-secrets", + # ) + # ) + + token_manager = _shared_token_manager host = f"https://{APIGEE_ENVIRONMENT}.api.service.nhs.uk/immunisation-fhir-api/FHIR/R4" def get_headers(self): return { "Accept": CONTENT_TYPE_FHIR_JSON, - "Authorization": f"Bearer {self.authenticator.get_access_token()}", + "Authorization": f"Bearer {self.token_manager.get_access_token()}", "Content-Type": CONTENT_TYPE_FHIR_JSON, "X-Correlation-ID": str(uuid.uuid4()), "X-Request-ID": str(uuid.uuid4()), } + def on_start(self): + # Jitter each user's start by up to 2 s to avoid simultaneous first-request burst + gevent.sleep(random.uniform(0, 2.0)) + def _build_create_payload(self): immunization_target = random.choice(IMMUNIZATION_TARGETS) - patient = load_patient_by_id(random.choice(VALID_PATIENT_IDS)) + patient = load_patient_by_id("Valid_NHS") immunization = create_immunization_object(patient, immunization_target) return json.loads(immunization.json(exclude_none=True)) @@ -94,7 +190,7 @@ def _delete_created_immunization(self, immunization_id: str): class SearchUser(BaseImmunizationUser): - wait_time = constant_throughput(1) + wait_time = constant_throughput(PERF_SEARCH_RPS_PER_USER) @task def search_single_vacc_type(self):