From 13f1d8c5312876745c5469bb283debb8d518c9a2 Mon Sep 17 00:00:00 2001 From: Andrew Lahiff Date: Thu, 26 Feb 2026 17:15:28 +0000 Subject: [PATCH 1/4] Ensure we don't get an error if a POST is being retried --- simvue/run.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/simvue/run.py b/simvue/run.py index b7e5f385..52d76f65 100644 --- a/simvue/run.py +++ b/simvue/run.py @@ -379,7 +379,9 @@ def _get_internal_metrics( # Set join on fail to false as if an error is thrown # join would be called on this thread and a thread cannot # join itself! - if self.status == "running": + if self.status == "running" and ( + self._shutdown_event and not self._shutdown_event.is_set() + ): self._add_metrics_to_dispatch( _current_system_measure.to_dict(), join_on_fail=False, From 2113b433a2c1d68758f2258cfdebbdedddbdfd18 Mon Sep 17 00:00:00 2001 From: Andrew Lahiff Date: Thu, 26 Feb 2026 17:24:52 +0000 Subject: [PATCH 2/4] Ensure retries work for 5xx errors --- simvue/api/request.py | 50 +++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/simvue/api/request.py b/simvue/api/request.py index 4133fa95..385dc7c9 100644 --- a/simvue/api/request.py +++ b/simvue/api/request.py @@ -17,6 +17,7 @@ from tenacity import ( retry, retry_if_exception, + retry_if_exception_type, stop_after_attempt, wait_exponential, ) @@ -29,14 +30,7 @@ RETRY_MAX = 10 RETRY_STOP = 5 MAX_ENTRIES_PER_PAGE: int = 100 -RETRY_STATUS_CODES = ( - http.HTTPStatus.BAD_REQUEST, - http.HTTPStatus.SERVICE_UNAVAILABLE, - http.HTTPStatus.GATEWAY_TIMEOUT, - http.HTTPStatus.REQUEST_TIMEOUT, - http.HTTPStatus.TOO_EARLY, -) -RETRY_EXCEPTION_TYPES = (RuntimeError, requests.exceptions.ConnectionError) +RETRY_STATUSES = {502, 503, 504} def set_json_header(headers: dict[str, str]) -> dict[str, str]: @@ -49,18 +43,14 @@ def set_json_header(headers: dict[str, str]) -> dict[str, str]: return headers -def is_retryable_exception(exception: Exception) -> bool: - """Returns if the given exception should lead to a retry being called""" - if isinstance(exception, requests.HTTPError): - return exception.response.status_code in RETRY_STATUS_CODES - - return isinstance(exception, RETRY_EXCEPTION_TYPES) +class RetryableHTTPError(Exception): + pass @retry( wait=wait_exponential(multiplier=RETRY_MULTIPLIER, min=RETRY_MIN, max=RETRY_MAX), stop=stop_after_attempt(RETRY_STOP), - retry=retry_if_exception(is_retryable_exception), + retry=retry_if_exception_type(RetryableHTTPError), reraise=True, ) def post( @@ -114,12 +104,15 @@ def post( f"Validation error for '{url}' [{response.status_code}]:\n{_parsed_response}" ) + if response.status_code in RETRY_STATUSES: + raise RetryableHTTPError(f"Received status code {response.status_code} from server") + return response @retry( wait=wait_exponential(multiplier=RETRY_MULTIPLIER, min=RETRY_MIN, max=RETRY_MAX), - retry=retry_if_exception(is_retryable_exception), + retry=retry_if_exception_type(RetryableHTTPError), stop=stop_after_attempt(RETRY_STOP), reraise=True, ) @@ -161,14 +154,19 @@ def put( logging.debug(f"PUT: {url}\n\tdata={data_sent}\n\tjson={json}") - return requests.put( + response = requests.put( url, headers=headers, data=data_sent, timeout=timeout, json=json ) + if response.status_code in RETRY_STATUSES: + raise RetryableHTTPError(f"Received status code {response.status_code} from server") + + return response + @retry( wait=wait_exponential(multiplier=RETRY_MULTIPLIER, min=RETRY_MIN, max=RETRY_MAX), - retry=retry_if_exception(is_retryable_exception), + retry=retry_if_exception_type(RetryableHTTPError), stop=stop_after_attempt(RETRY_STOP), reraise=True, ) @@ -198,12 +196,17 @@ def get( response from executing GET """ logging.debug(f"GET: {url}\n\tparams={params}") - return requests.get(url, headers=headers, timeout=timeout, params=params, json=json) + response = requests.get(url, headers=headers, timeout=timeout, params=params, json=json) + + if response.status_code in RETRY_STATUSES: + raise RetryableHTTPError(f"Received status code {response.status_code} from server") + + return response @retry( wait=wait_exponential(multiplier=RETRY_MULTIPLIER, min=RETRY_MIN, max=RETRY_MAX), - retry=retry_if_exception(is_retryable_exception), + retry=retry_if_exception_type(RetryableHTTPError), stop=stop_after_attempt(RETRY_STOP), reraise=True, ) @@ -232,7 +235,12 @@ def delete( response from executing DELETE """ logging.debug(f"DELETE: {url}\n\tparams={params}") - return requests.delete(url, headers=headers, timeout=timeout, params=params) + response = requests.delete(url, headers=headers, timeout=timeout, params=params) + + if response.status_code in RETRY_STATUSES: + raise RetryableHTTPError(f"Received status code {response.status_code} from server") + + return response def get_json_from_response( From e4e33d31dac7e731aca6bc8afe343cf94cb5f2da Mon Sep 17 00:00:00 2001 From: Andrew Lahiff Date: Thu, 26 Feb 2026 17:50:22 +0000 Subject: [PATCH 3/4] Handle timeouts and connection errors too --- simvue/api/request.py | 52 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 43 insertions(+), 9 deletions(-) diff --git a/simvue/api/request.py b/simvue/api/request.py index 385dc7c9..d0379daf 100644 --- a/simvue/api/request.py +++ b/simvue/api/request.py @@ -50,7 +50,13 @@ class RetryableHTTPError(Exception): @retry( wait=wait_exponential(multiplier=RETRY_MULTIPLIER, min=RETRY_MIN, max=RETRY_MAX), stop=stop_after_attempt(RETRY_STOP), - retry=retry_if_exception_type(RetryableHTTPError), + retry=retry_if_exception_type( + ( + RetryableHTTPError, + requests.exceptions.Timeout, + requests.exceptions.ConnectionError, + ) + ), reraise=True, ) def post( @@ -105,14 +111,22 @@ def post( ) if response.status_code in RETRY_STATUSES: - raise RetryableHTTPError(f"Received status code {response.status_code} from server") + raise RetryableHTTPError( + f"Received status code {response.status_code} from server" + ) return response @retry( wait=wait_exponential(multiplier=RETRY_MULTIPLIER, min=RETRY_MIN, max=RETRY_MAX), - retry=retry_if_exception_type(RetryableHTTPError), + retry=retry_if_exception_type( + ( + RetryableHTTPError, + requests.exceptions.Timeout, + requests.exceptions.ConnectionError, + ) + ), stop=stop_after_attempt(RETRY_STOP), reraise=True, ) @@ -159,14 +173,22 @@ def put( ) if response.status_code in RETRY_STATUSES: - raise RetryableHTTPError(f"Received status code {response.status_code} from server") + raise RetryableHTTPError( + f"Received status code {response.status_code} from server" + ) return response @retry( wait=wait_exponential(multiplier=RETRY_MULTIPLIER, min=RETRY_MIN, max=RETRY_MAX), - retry=retry_if_exception_type(RetryableHTTPError), + retry=retry_if_exception_type( + ( + RetryableHTTPError, + requests.exceptions.Timeout, + requests.exceptions.ConnectionError, + ) + ), stop=stop_after_attempt(RETRY_STOP), reraise=True, ) @@ -196,17 +218,27 @@ def get( response from executing GET """ logging.debug(f"GET: {url}\n\tparams={params}") - response = requests.get(url, headers=headers, timeout=timeout, params=params, json=json) + response = requests.get( + url, headers=headers, timeout=timeout, params=params, json=json + ) if response.status_code in RETRY_STATUSES: - raise RetryableHTTPError(f"Received status code {response.status_code} from server") + raise RetryableHTTPError( + f"Received status code {response.status_code} from server" + ) return response @retry( wait=wait_exponential(multiplier=RETRY_MULTIPLIER, min=RETRY_MIN, max=RETRY_MAX), - retry=retry_if_exception_type(RetryableHTTPError), + retry=retry_if_exception_type( + ( + RetryableHTTPError, + requests.exceptions.Timeout, + requests.exceptions.ConnectionError, + ) + ), stop=stop_after_attempt(RETRY_STOP), reraise=True, ) @@ -238,7 +270,9 @@ def delete( response = requests.delete(url, headers=headers, timeout=timeout, params=params) if response.status_code in RETRY_STATUSES: - raise RetryableHTTPError(f"Received status code {response.status_code} from server") + raise RetryableHTTPError( + f"Received status code {response.status_code} from server" + ) return response From 682552e50f990bf0bb6ce5cfdb2b4361133510a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Zar=C4=99bski?= Date: Fri, 27 Feb 2026 08:07:18 +0000 Subject: [PATCH 4/4] Fix linting issues --- simvue/api/request.py | 1 - 1 file changed, 1 deletion(-) diff --git a/simvue/api/request.py b/simvue/api/request.py index d0379daf..5a568588 100644 --- a/simvue/api/request.py +++ b/simvue/api/request.py @@ -16,7 +16,6 @@ import requests from tenacity import ( retry, - retry_if_exception, retry_if_exception_type, stop_after_attempt, wait_exponential,