From a98e5f9ef0b8ea94a5ac2614683cc70a0fc310a9 Mon Sep 17 00:00:00 2001 From: Benjamin Pelletier Date: Fri, 20 Mar 2026 20:56:38 +0000 Subject: [PATCH 1/2] Add global timeout and run-to-completion requirement --- .basedpyright/baseline.json | 16 - .../configurations/configuration.py | 5 +- .../dev/f3548_self_contained.yaml | 2 + .../reports/tested_requirements/breakdown.py | 433 +++++++++++------- .../reports/tested_requirements/data_types.py | 15 +- .../interuss/automated_testing/execution.md | 7 + monitoring/uss_qualifier/suites/suite.py | 21 + .../configuration/ExecutionConfiguration.json | 8 + 8 files changed, 333 insertions(+), 174 deletions(-) create mode 100644 monitoring/uss_qualifier/requirements/interuss/automated_testing/execution.md diff --git a/.basedpyright/baseline.json b/.basedpyright/baseline.json index 861fa1b68c..eb24fe91a6 100644 --- a/.basedpyright/baseline.json +++ b/.basedpyright/baseline.json @@ -5571,22 +5571,6 @@ } ], "./monitoring/uss_qualifier/reports/tested_requirements/breakdown.py": [ - { - "code": "reportOptionalMemberAccess", - "range": { - "startColumn": 43, - "endColumn": 50, - "lineCount": 1 - } - }, - { - "code": "reportOptionalMemberAccess", - "range": { - "startColumn": 49, - "endColumn": 56, - "lineCount": 1 - } - }, { "code": "reportOptionalMemberAccess", "range": { diff --git a/monitoring/uss_qualifier/configurations/configuration.py b/monitoring/uss_qualifier/configurations/configuration.py index 7ad43a8f19..5b80109c65 100644 --- a/monitoring/uss_qualifier/configurations/configuration.py +++ b/monitoring/uss_qualifier/configurations/configuration.py @@ -2,7 +2,7 @@ from collections.abc import Iterable -from implicitdict import ImplicitDict, Optional +from implicitdict import ImplicitDict, Optional, StringBasedTimeDelta from monitoring.monitorlib.dicts import JSONAddress from monitoring.uss_qualifier.action_generators.definitions import GeneratorTypeName @@ -140,6 +140,9 @@ class ExecutionConfiguration(ImplicitDict): scenarios_filter: str | None """Filter test scenarios by scenario type using a regex. If the filter regex does not match within the scenario type, the scenario is skipped. When empty, all scenarios are executed. Useful for targeted debugging. Overridden by --filter""" + stop_after: Optional[StringBasedTimeDelta] + """If specified, stop the test run at the next earliest convenience (generally just after completion of the current test scenario) if it has been running at least this long.""" + class TestConfiguration(ImplicitDict): action: TestSuiteActionDeclaration diff --git a/monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml b/monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml index ee8b860f34..aa5d2355d5 100644 --- a/monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml +++ b/monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml @@ -370,6 +370,7 @@ v1: - astm.f3548.v21.DSS0210,A2-7-2,7 - astm.f3548.v21.DSS0215 - astm.f3548.v21.DSS0300 + - interuss.automated_testing.execution.RunToCompletion - interuss.automated_testing.flight_planning.ClearArea - interuss.automated_testing.flight_planning.DeleteFlightSuccess - interuss.automated_testing.flight_planning.ExpectedBehavior @@ -404,6 +405,7 @@ v1: - astm.f3548.v21.USS0105,1 - astm.f3548.v21.USS0105,3 - astm.f3548.v21.USS0105,4 + - interuss.automated_testing.execution.RunToCompletion - interuss.automated_testing.flight_planning.ClearArea - interuss.automated_testing.flight_planning.DeleteFlightSuccess - interuss.automated_testing.flight_planning.ExpectedBehavior diff --git a/monitoring/uss_qualifier/reports/tested_requirements/breakdown.py b/monitoring/uss_qualifier/reports/tested_requirements/breakdown.py index e3c3c05573..4469f3ddf6 100644 --- a/monitoring/uss_qualifier/reports/tested_requirements/breakdown.py +++ b/monitoring/uss_qualifier/reports/tested_requirements/breakdown.py @@ -18,6 +18,7 @@ from monitoring.uss_qualifier.reports.report import ( FailedCheck, PassedCheck, + SkippedActionReport, TestCaseReport, TestRunReport, TestScenarioReport, @@ -47,6 +48,11 @@ TestSuiteActionDeclaration, TestSuiteDefinition, ) +from monitoring.uss_qualifier.suites.suite import TEST_RUN_TIMEOUT_SKIP_REASON + +REQ_RUN_TO_COMPLETION = RequirementID( + "interuss.automated_testing.execution.RunToCompletion" +) def make_breakdown( @@ -83,6 +89,40 @@ def make_breakdown( ) if participant_reqs is not None: _populate_breakdown_with_req_set(participant_breakdown, participant_reqs) + if REQ_RUN_TO_COMPLETION in participant_reqs: + # Add a success to REQ_RUN_TO_COMPLETION if nothing caused it to fail + tested_requirement = _tested_requirement_for( + REQ_RUN_TO_COMPLETION, participant_breakdown + ) + if not tested_requirement.scenarios: + tested_requirement.scenarios.append( + TestedScenario( + type="uss_qualifier.execution", + name="N/A", + url="", + cases=[ + TestedCase( + name="N/A", + url="", + steps=[ + TestedStep( + name="N/A", + url="", + checks=[ + TestedCheck( + name="Test run completed normally", + url="", + has_todo=False, + is_finding_acceptable=False, + successes=1, + ) + ], + ) + ], + ) + ], + ) + ) sort_breakdown(participant_breakdown) return participant_breakdown @@ -91,24 +131,7 @@ def _populate_breakdown_with_req_set( breakdown: TestedBreakdown, req_set: set[RequirementID] ) -> None: for req_id in req_set: - package_id = req_id.package() - matches = [p for p in breakdown.packages if p.id == package_id] - if matches: - tested_package = matches[0] - else: - url = repo_url_of(package_id.md_file_path()) - tested_package = TestedPackage( - id=package_id, url=url, name=package_id, requirements=[] - ) - breakdown.packages.append(tested_package) - - short_req_id = req_id.split(".")[-1] - matches = [r for r in tested_package.requirements if r.id == short_req_id] - if matches: - tested_requirement = matches[0] - else: - tested_requirement = TestedRequirement(id=short_req_id, scenarios=[]) - tested_package.requirements.append(tested_requirement) + _tested_requirement_for(req_id, breakdown) def _populate_breakdown_with_action_report( @@ -118,9 +141,7 @@ def _populate_breakdown_with_action_report( participant_ids: Iterable[ParticipantID], req_set: set[RequirementID] | None, ) -> None: - test_suite, test_scenario, action_generator = action.get_applicable_report() - if test_scenario: - assert action.test_scenario + if "test_scenario" in action and action.test_scenario: return _populate_breakdown_with_scenario_report( breakdown, action.test_scenario, @@ -128,18 +149,84 @@ def _populate_breakdown_with_action_report( participant_ids, req_set, ) - elif test_suite: + elif "test_suite" in action and action.test_suite: for subaction in action.test_suite.actions: _populate_breakdown_with_action_report( breakdown, subaction, acceptable_findings, participant_ids, req_set ) - elif action_generator: + elif "action_generator" in action and action.action_generator: for subaction in action.action_generator.actions: _populate_breakdown_with_action_report( breakdown, subaction, acceptable_findings, participant_ids, req_set ) + elif "skipped_action" in action and action.skipped_action: + if ( + req_set is not None + and REQ_RUN_TO_COMPLETION in req_set + and action.skipped_action.reason == TEST_RUN_TIMEOUT_SKIP_REASON + ): + _populate_breakdown_with_timeout_skip(breakdown, action.skipped_action) + else: + raise ValueError( + "Unrecognized or unspecified oneof option in TestSuiteActionReport" + ) + + +def _populate_breakdown_with_timeout_skip( + breakdown: TestedBreakdown, skipped_action: SkippedActionReport +) -> None: + declaration = skipped_action.declaration + if "test_scenario" in declaration and declaration.test_scenario: + doc = get_documentation( + get_scenario_type_by_name(declaration.test_scenario.scenario_type) + ) + default_scenario = TestedScenario( + type=declaration.test_scenario.scenario_type, + name=doc.name, + documentation_url=doc.url, + cases=[], + ) + elif "test_suite" in declaration and declaration.test_suite: + type_name = declaration.test_suite.type_name + default_scenario = TestedScenario( + name=f"(Test suite) {type_name}", type=type_name, url="", cases=[] + ) + elif "action_generator" in declaration and declaration.action_generator: + type_name = declaration.action_generator.generator_type + default_scenario = TestedScenario( + name=f"(Action generator) {type_name}", type=type_name, url="", cases=[] + ) + else: + raise ValueError( + "Unrecognized or unspecified oneof option in TestSuiteActionDeclaration" + ) + tested_requirement = _tested_requirement_for(REQ_RUN_TO_COMPLETION, breakdown) + tested_scenario = _tested_scenario_for(default_scenario, tested_requirement) + # Assume each TestedScenario for the the TestedRequirement for this requirement should only ever have 1 case with 1 step with 1 check + if not tested_scenario.cases: + tested_scenario.cases.append( + TestedCase( + name="N/A", + url="", + steps=[ + TestedStep( + name="N/A", + url="", + checks=[ + TestedCheck( + name="Test run completed normally", + url="", + has_todo=False, + is_finding_acceptable=False, + failures=1, + ) + ], + ) + ], + ) + ) else: - pass # Skipped action + tested_scenario.cases[0].steps[0].checks[0].failures += 1 def _populate_breakdown_with_scenario_report( @@ -149,7 +236,6 @@ def _populate_breakdown_with_scenario_report( participant_ids: Iterable[ParticipantID], req_set: set[RequirementID] | None, ) -> None: - scenario_type_name = scenario_report.scenario_type steps: list[tuple[TestCaseReport | None, TestStepReport]] = [] for case in scenario_report.cases: for step in case.steps: @@ -162,101 +248,168 @@ def _populate_breakdown_with_scenario_report( if not any(pid in check.participants for pid in participant_ids): continue for req_id in check.requirements: - if req_set is not None and req_id not in req_set: - continue - package_id = req_id.package() - package_name = "
.".join(package_id.split(".")) - matches = [p for p in breakdown.packages if p.id == package_id] - if matches: - tested_package = matches[0] - else: - # TODO: Improve name of package by using title of page - url = repo_url_of(package_id.md_file_path()) - tested_package = TestedPackage( - id=package_id, url=url, name=package_name, requirements=[] - ) - breakdown.packages.append(tested_package) - - short_req_id = req_id.split(".")[-1] - matches = [ - r for r in tested_package.requirements if r.id == short_req_id - ] - if matches: - tested_requirement = matches[0] - else: - tested_requirement = TestedRequirement( - id=short_req_id, scenarios=[] + if req_set is None or req_id in req_set: + _add_check_to_breakdown_for_req( + req_id, + scenario_report, + case, + step, + check, + breakdown, + acceptable_findings, ) - tested_package.requirements.append(tested_requirement) - - matches = [ - s - for s in tested_requirement.scenarios - if are_scenario_types_equal(s.type, scenario_type_name) - ] - if matches: - tested_scenario = matches[0] - else: - tested_scenario = TestedScenario( - type=scenario_type_name, - name=scenario_report.name, - url=scenario_report.documentation_url, - cases=[], - ) - tested_requirement.scenarios.append(tested_scenario) - - if case: - case_name = case.name - case_url = case.documentation_url - else: - case_name = "Cleanup" - case_url = step.documentation_url - matches = [c for c in tested_scenario.cases if c.name == case_name] - if matches: - tested_case = matches[0] - else: - tested_case = TestedCase(name=case_name, url=case_url, steps=[]) - tested_scenario.cases.append(tested_case) - - matches = [s for s in tested_case.steps if s.name == step.name] - if matches: - tested_step = matches[0] - else: - tested_step = TestedStep( - name=step.name, url=step.documentation_url, checks=[] - ) - tested_case.steps.append(tested_step) - - matches = [c for c in tested_step.checks if c.name == check.name] - if matches: - tested_check = matches[0] - else: - current_check = FullyQualifiedCheck( - scenario_type=scenario_type_name, - test_case_name=case_name, - test_step_name=step.name, - check_name=check.name, - ) - tested_check = TestedCheck( - name=check.name, - url="", - has_todo=False, - is_finding_acceptable=fully_qualified_check_in_collection( - current_check, acceptable_findings - ), - ) # TODO: Consider populating has_todo with documentation instead - if isinstance(check, FailedCheck): - tested_check.url = check.documentation_url - tested_step.checks.append(tested_check) - if isinstance(check, PassedCheck): - tested_check.successes += 1 - elif isinstance(check, FailedCheck): - if check.severity == Severity.Low: - tested_check.findings += 1 - else: - tested_check.failures += 1 - else: - raise ValueError("Check is neither PassedCheck nor FailedCheck") + if ( + req_set is not None + and REQ_RUN_TO_COMPLETION in req_set + and "severity" in check + and check.severity == Severity.Critical + ): + _add_check_to_breakdown_for_req( + REQ_RUN_TO_COMPLETION, + scenario_report, + case, + step, + check, + breakdown, + acceptable_findings, + ) + + +def _tested_requirement_for( + req_id: RequirementID, breakdown: TestedBreakdown +) -> TestedRequirement: + """Retrieves the TestedRequirement for the specified ID in the breakdown, creating an empty one if necessary.""" + package_id = req_id.package() + package_name = "
.".join(package_id.split(".")) + matches = [p for p in breakdown.packages if p.id == package_id] + if matches: + tested_package = matches[0] + else: + # TODO: Improve name of package by using title of page + url = repo_url_of(package_id.md_file_path()) + tested_package = TestedPackage( + id=package_id, url=url, name=package_name, requirements=[] + ) + breakdown.packages.append(tested_package) + + short_req_id = req_id.split(".")[-1] + matches = [r for r in tested_package.requirements if r.id == short_req_id] + if matches: + tested_requirement = matches[0] + else: + tested_requirement = TestedRequirement(id=short_req_id, scenarios=[]) + tested_package.requirements.append(tested_requirement) + + return tested_requirement + + +class ScenarioInfo(ImplicitDict): + """Limited subset of a full TestScenarioReport that still contains enough information to produce a TestedScenario.""" + + name: str + scenario_type: TestScenarioTypeName + documentation_url: str + + +def _same_tested_scenario_types(s1: TestedScenario, s2: TestedScenario) -> bool: + if s1.type.startswith("scenarios.") and s2.type.startswith("scenarios."): + return are_scenario_types_equal(s1.type, s2.type) + else: + return s1.type == s2.type + + +def _tested_scenario_for( + default_scenario: TestedScenario, tested_requirement: TestedRequirement +) -> TestedScenario: + """Retrieves the TestedScenario for the specified scenario within the specified requirement, creating an empty one if necessary. + + Args: + * default_scenario: The TestedScenario information to use if no pre-existing TestedScenario is found. + * tested_requirement: The requirement breakdown level for which the scenario is being found. + """ + matches = [ + s + for s in tested_requirement.scenarios + if _same_tested_scenario_types(s, default_scenario) + ] + if matches: + tested_scenario = matches[0] + else: + tested_scenario = TestedScenario( + type=default_scenario.type, + name=default_scenario.name, + url=default_scenario.url, + cases=[], + ) + tested_requirement.scenarios.append(tested_scenario) + + return tested_scenario + + +def _add_check_to_breakdown_for_req( + req_id: RequirementID, + scenario_report: TestScenarioReport, + case: TestCaseReport | None, + step: TestStepReport, + check: PassedCheck | FailedCheck, + breakdown: TestedBreakdown, + acceptable_findings: list[FullyQualifiedCheck], +): + tested_requirement = _tested_requirement_for(req_id, breakdown) + tested_scenario = _tested_scenario_for( + TestedScenario.from_scenario_report(scenario_report), tested_requirement + ) + + if case: + case_name = case.name + case_url = case.documentation_url + else: + case_name = "Cleanup" + case_url = step.documentation_url + matches = [c for c in tested_scenario.cases if c.name == case_name] + if matches: + tested_case = matches[0] + else: + tested_case = TestedCase(name=case_name, url=case_url, steps=[]) + tested_scenario.cases.append(tested_case) + + matches = [s for s in tested_case.steps if s.name == step.name] + if matches: + tested_step = matches[0] + else: + tested_step = TestedStep(name=step.name, url=step.documentation_url, checks=[]) + tested_case.steps.append(tested_step) + + matches = [c for c in tested_step.checks if c.name == check.name] + if matches: + tested_check = matches[0] + else: + current_check = FullyQualifiedCheck( + scenario_type=scenario_report.scenario_type, + test_case_name=case_name, + test_step_name=step.name, + check_name=check.name, + ) + tested_check = TestedCheck( + name=check.name, + url="", + has_todo=False, + is_finding_acceptable=fully_qualified_check_in_collection( + current_check, acceptable_findings + ), + ) # TODO: Consider populating has_todo with documentation instead + if isinstance(check, FailedCheck): + tested_check.url = check.documentation_url + tested_step.checks.append(tested_check) + if isinstance(check, PassedCheck): + tested_check.successes += 1 + elif isinstance(check, FailedCheck): + if check.severity == Severity.Low: + tested_check.findings += 1 + else: + tested_check.failures += 1 + else: + raise ValueError("Check is neither PassedCheck nor FailedCheck") def _populate_breakdown_with_action_declaration( @@ -316,46 +469,16 @@ def _populate_breakdown_with_scenario( for req_id in check.applicable_requirements: if req_set is not None and req_id not in req_set: continue - package_id = req_id.package() - package_name = "
.".join(package_id.split(".")) - matches = [p for p in breakdown.packages if p.id == package_id] - if matches: - tested_package = matches[0] - else: - # TODO: Improve name of package by using title of page - url = repo_url_of(package_id.md_file_path()) - tested_package = TestedPackage( - id=package_id, url=url, name=package_name, requirements=[] - ) - breakdown.packages.append(tested_package) - - short_req_id = req_id.split(".")[-1] - matches = [ - r for r in tested_package.requirements if r.id == short_req_id - ] - if matches: - tested_requirement = matches[0] - else: - tested_requirement = TestedRequirement( - id=short_req_id, scenarios=[] - ) - tested_package.requirements.append(tested_requirement) - - matches = [ - s - for s in tested_requirement.scenarios - if are_scenario_types_equal(s.type, scenario_type_name) - ] - if matches: - tested_scenario = matches[0] - else: - tested_scenario = TestedScenario( + tested_requirement = _tested_requirement_for(req_id, breakdown) + tested_scenario = _tested_scenario_for( + TestedScenario( type=scenario_type_name, name=scenario_doc.name, url=scenario_doc.url, cases=[], - ) - tested_requirement.scenarios.append(tested_scenario) + ), + tested_requirement, + ) matches = [c for c in tested_scenario.cases if c.name == case.name] if matches: diff --git a/monitoring/uss_qualifier/reports/tested_requirements/data_types.py b/monitoring/uss_qualifier/reports/tested_requirements/data_types.py index ef13c29d0b..75e2b96212 100644 --- a/monitoring/uss_qualifier/reports/tested_requirements/data_types.py +++ b/monitoring/uss_qualifier/reports/tested_requirements/data_types.py @@ -1,11 +1,13 @@ +from __future__ import annotations + from collections.abc import Iterable from enum import Enum from implicitdict import ImplicitDict, Optional from monitoring.uss_qualifier.configurations.configuration import ParticipantID +from monitoring.uss_qualifier.reports.report import TestScenarioReport from monitoring.uss_qualifier.requirements.definitions import PackageID -from monitoring.uss_qualifier.scenarios.definitions import TestScenarioTypeName PASS_CLASS = "pass_result" FINDINGS_CLASS = "findings_result" @@ -88,7 +90,7 @@ def rows(self) -> int: class TestedScenario(ImplicitDict): - type: TestScenarioTypeName + type: str name: str url: str cases: list[TestedCase] @@ -97,6 +99,15 @@ class TestedScenario(ImplicitDict): def rows(self) -> int: return sum(c.rows for c in self.cases) + @staticmethod + def from_scenario_report(report: TestScenarioReport) -> TestedScenario: + return TestedScenario( + type=report.scenario_type, + name=report.name, + url=report.documentation_url, + cases=[], + ) + class TestedRequirementStatus(str, Enum): Pass = "Pass" diff --git a/monitoring/uss_qualifier/requirements/interuss/automated_testing/execution.md b/monitoring/uss_qualifier/requirements/interuss/automated_testing/execution.md new file mode 100644 index 0000000000..47ea77c0e5 --- /dev/null +++ b/monitoring/uss_qualifier/requirements/interuss/automated_testing/execution.md @@ -0,0 +1,7 @@ +# InterUSS automated test execution requirements + +## Requirements + +### RunToCompletion + +When a test configuration designer specifies that an automated test must run to completion (by including this requirement), all applicable participants in the automated test will fail this requirement if the automated test does not run to completion. Not running to completion may be due to exceeding the maximum allowed test run time in ExecutionConfiguration.stop_after or because of a failed check with Critical severity. diff --git a/monitoring/uss_qualifier/suites/suite.py b/monitoring/uss_qualifier/suites/suite.py index a374481af5..91e6ab8596 100644 --- a/monitoring/uss_qualifier/suites/suite.py +++ b/monitoring/uss_qualifier/suites/suite.py @@ -63,6 +63,8 @@ TestSuiteDefinition, ) +TEST_RUN_TIMEOUT_SKIP_REASON = "Maximum test run time has been exceeded" + def _print_failed_check(failed_check: FailedCheck) -> None: yaml_lines = yaml.dump(json.loads(json.dumps(failed_check))).split("\n") @@ -357,6 +359,15 @@ def _run_actions( for a, action in enumerate(actions): if isinstance(action, SkippedActionReport): action_report = TestSuiteActionReport(skipped_action=action) + elif context.should_stop_early_now(): + assert context.current_frame + action_report = TestSuiteActionReport( + skipped_action=SkippedActionReport( + timestamp=StringBasedDateTime(arrow.utcnow().datetime), + reason=TEST_RUN_TIMEOUT_SKIP_REASON, + declaration=context.current_frame.action.declaration, + ) + ) else: action_report = action.run(context) report.actions.append(action_report) @@ -491,6 +502,16 @@ def stop_fast( return True return False + def should_stop_early_now(self) -> bool: + if ( + not self.config + or "stop_after" not in self.config + or not self.config.stop_after + ): + return False + dt = arrow.utcnow() - self.start_time + return dt >= self.config.stop_after.timedelta + def _compute_n_of( self, target: TestSuiteAction, condition: TestSuiteActionSelectionCondition ) -> int: diff --git a/schemas/monitoring/uss_qualifier/configurations/configuration/ExecutionConfiguration.json b/schemas/monitoring/uss_qualifier/configurations/configuration/ExecutionConfiguration.json index 5796d3040d..f06fce3833 100644 --- a/schemas/monitoring/uss_qualifier/configurations/configuration/ExecutionConfiguration.json +++ b/schemas/monitoring/uss_qualifier/configurations/configuration/ExecutionConfiguration.json @@ -41,6 +41,14 @@ "null" ] }, + "stop_after": { + "description": "If specified, stop the test run at the next earliest convenience (generally just after completion of the current test scenario) if it has been running at least this long.", + "format": "duration", + "type": [ + "string", + "null" + ] + }, "stop_fast": { "description": "If true, escalate the Severity of any failed check to Critical in order to end the test run early.", "type": [ From 2dd79b5dbe8221d4ac0322a12c4686af83e56d77 Mon Sep 17 00:00:00 2001 From: Benjamin Pelletier Date: Tue, 24 Mar 2026 17:19:30 +0000 Subject: [PATCH 2/2] Address comments --- .../uss_qualifier/reports/tested_requirements/breakdown.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/monitoring/uss_qualifier/reports/tested_requirements/breakdown.py b/monitoring/uss_qualifier/reports/tested_requirements/breakdown.py index 4469f3ddf6..6767d3d406 100644 --- a/monitoring/uss_qualifier/reports/tested_requirements/breakdown.py +++ b/monitoring/uss_qualifier/reports/tested_requirements/breakdown.py @@ -202,7 +202,7 @@ def _populate_breakdown_with_timeout_skip( ) tested_requirement = _tested_requirement_for(REQ_RUN_TO_COMPLETION, breakdown) tested_scenario = _tested_scenario_for(default_scenario, tested_requirement) - # Assume each TestedScenario for the the TestedRequirement for this requirement should only ever have 1 case with 1 step with 1 check + # Assume each TestedScenario for the TestedRequirement for this requirement should only ever have 1 case with 1 step with 1 check if not tested_scenario.cases: tested_scenario.cases.append( TestedCase( @@ -226,6 +226,10 @@ def _populate_breakdown_with_timeout_skip( ) ) else: + if len(tested_scenario.cases) > 1: + raise ValueError( + f"TestedScenario {tested_scenario.name} ({tested_scenario.type}) for requirement {tested_requirement.id} was expected to only have one N/A case, but instead had {len(tested_scenario.cases)} cases: {', '.join(c.name for c in tested_scenario.cases)}" + ) tested_scenario.cases[0].steps[0].checks[0].failures += 1