diff --git a/src/integrationtest/data_classes.py b/src/integrationtest/data_classes.py index d66b5c8..bf2bef1 100755 --- a/src/integrationtest/data_classes.py +++ b/src/integrationtest/data_classes.py @@ -1,5 +1,5 @@ from dataclasses import dataclass, field - +from enum import Enum @dataclass class DROMap_config: @@ -47,32 +47,67 @@ class list_element_addition(config_substitution): additional_object_id: str = "" +class ConnSvcControl(Enum): + INTEGRATIONTEST = "integrationtest" + RUNCONTROL = "runcontrol" + NONE = "none" + +@dataclass +class integtest_param_base_class: + # daq_session_name can be specified; it is automatically populated if not specified + daq_session_name: str = None + + # config substitutions can be made to both generated and predefined dunedaq configs + config_substitutions: list[config_substitution] = field(default_factory=list) + + # the cleanup of leftover RunControl or ConnSvc processes is available to all types of integtests + attempt_cleanup: bool = False + + # parameter(s) related to the startup of the Connectivity Service + connsvc_port: int = 0 + connsvc_debug_level: int = None + @dataclass -class drunc_config: +class integtest_params_for_generated_dunedaq_config(integtest_param_base_class): + # *** Parameters that are needed for both generated and predefined configs, + # *** and benefit from different default values + # - for generated configs, these two params do not need to have specific values op_env: str = "integtest" config_session_name: str = "integtest" - daq_session_name: str = None + + # *** Parameters that are only needed for generated dunedaq configurations + # - databases that are needed to support config generation + object_databases: list[str] = field(default_factory=list) + # - parameters that control what the generators produce dro_map_config: DROMap_config = field(default_factory=lambda: DROMap_config(1)) - frame_file: str = "asset://?checksum=370df564205290d27cab47e44ae4ca47" + frame_file: str = "asset://?checksum=370df564205290d27cab47e44ae4ca47" # wib_link_67.bin tpg_enabled: bool = False trmon_app_enabled: bool = False fake_hsi_enabled: bool = False use_fakedataprod: bool = False fake_data_fragment_type: str = "" - config_db: str = "" n_df_apps: int = 1 n_data_writers: int = 1 - object_databases: list[str] = field(default_factory=list) - config_substitutions: list[config_substitution] = field(default_factory=list) - attempt_cleanup: bool = False - drunc_connsvc: bool = False - connsvc_port: int = 0 - connsvc_debug_level: int = 0 + # - control over how the Connectivity Service is started + connsvc_control: ConnSvcControl = ConnSvcControl.INTEGRATIONTEST + +@dataclass +class integtest_params_for_predefined_dunedaq_config(integtest_param_base_class): + # *** Parameters that are needed for both generated and predefined configs, + # *** and benefit from different default values + # - for a predefined config, the following two parameters must contain values that + # match what is in that config; they are likely reassigned in integtest files + op_env: str = "test" + config_session_name: str = "local-1x1-config" + + # *** Parameters that are unique to predefined dunedaq configurations + # - the predefined configuration that should be used + predefined_config_db: str = "" @dataclass class CreateConfigResult: - config: drunc_config + config: integtest_param_base_class config_dir: str config_file: str log_file: str diff --git a/src/integrationtest/integrationtest_commandline.py b/src/integrationtest/integrationtest_commandline.py index 44625d9..ac9de62 100755 --- a/src/integrationtest/integrationtest_commandline.py +++ b/src/integrationtest/integrationtest_commandline.py @@ -22,7 +22,7 @@ def pytest_addoption(parser): required=False ) parser.addoption( - "--disable-connectivity-service", + "--no-integtest-connsvc", action="store_true", default=False, help="Whether to disable the Connectivity Service for this test", diff --git a/src/integrationtest/integrationtest_drunc.py b/src/integrationtest/integrationtest_drunc.py index de27c8f..ec7a09a 100755 --- a/src/integrationtest/integrationtest_drunc.py +++ b/src/integrationtest/integrationtest_drunc.py @@ -20,6 +20,9 @@ relationship_substitution, list_element_substitution, list_element_addition, + ConnSvcControl, + integtest_params_for_generated_dunedaq_config, + integtest_params_for_predefined_dunedaq_config, ) from daqconf.generate_hwmap import generate_hwmap from daqconf.generate import ( @@ -201,19 +204,28 @@ def create_config_files(request, tmp_path_factory, check_system_resources): """ dummy_resource_check = check_system_resources - drunc_config = request.param + integtest_params = request.param + + #if isinstance(integtest_params, integtest_params_for_generated_dunedaq_config): + # print("*** integtest_params is of type integtest_params_for_generated_dunedaq_config") + #if isinstance(integtest_params, integtest_params_for_predefined_dunedaq_config): + # print("*** integtest_params is of type integtest_params_for_predefined_dunedaq_config") + if not isinstance(integtest_params, integtest_params_for_generated_dunedaq_config) \ + and not isinstance(integtest_params, integtest_params_for_predefined_dunedaq_config): + fail_msg = f"The integtest configuration object has an invalid type: {type(integtest_params)}" + pytest.fail(fail_msg, pytrace=False) + + no_integtest_connsvc = request.config.getoption("--no-integtest-connsvc") + skip_resource_checks = request.config.getoption("--skip-resource-checks") - disable_connectivity_service = request.config.getoption( - "--disable-connectivity-service" - ) - skip_resource_checks = request.config.getoption( - "--skip-resource-checks" - ) + if no_integtest_connsvc and \ + isinstance(integtest_params, integtest_params_for_generated_dunedaq_config): + integtest_params.connsvc_control = ConnSvcControl.NONE # 06-Mar-2026, KAB: if the DAQ session name has not explicitly been set by the # user, set it here so that we can make use of it from this point onward. - if not drunc_config.daq_session_name: - drunc_config.daq_session_name = drunc_config.config_session_name + if not integtest_params.daq_session_name: + integtest_params.daq_session_name = integtest_params.config_session_name # 26-Mar-2026, KAB: suppress output messages, if requested integtest_verbosity_level = int(request.config.getoption("--integtest-verbosity")) @@ -229,30 +241,30 @@ def create_config_files(request, tmp_path_factory, check_system_resources): sys.stdout = catcher = StringIO() config_dir = tmp_path_factory.mktemp("config") - boot_file = config_dir / "boot.json" - configfile = config_dir / "config.json" - dro_map_file = config_dir / "ReadoutMap.data.xml" - readout_db = config_dir / "readout-segment.data.xml" - dataflow_db = config_dir / "df-segment.data.xml" - trigger_db = config_dir / "trg-segment.data.xml" - hsi_db = config_dir / "hsi-segment.data.xml" config_db = config_dir / "integtest-session-resolved.data.xml" temp_config_db = config_dir / "integtest-session.data.xml" logfile = tmp_path_factory.getbasetemp() / f"stdouterr{request.param_index}.txt" - integtest_conf = drunc_config.config_db + if isinstance(integtest_params, integtest_params_for_predefined_dunedaq_config): + integtest_conf = integtest_params.predefined_config_db + if file_exists(integtest_conf): + print(f"Integtest preconfigured config file: {integtest_conf}") + consolidate_files(str(temp_config_db), integtest_conf) + else: + fail_msg = f"The file containing the predefined dunedaq configuration \"{integtest_conf}\" could not be found." + pytest.fail(fail_msg, pytrace=False) + else: + dro_map_file = config_dir / "ReadoutMap.data.xml" + readout_db = config_dir / "readout-segment.data.xml" + dataflow_db = config_dir / "df-segment.data.xml" + trigger_db = config_dir / "trg-segment.data.xml" + hsi_db = config_dir / "hsi-segment.data.xml" - object_databases = getattr(request.module, "object_databases", []) - local_object_databases = copy_configuration(config_dir, object_databases) + local_object_databases = copy_configuration(config_dir, integtest_params.object_databases) - #print() # Blank line - if file_exists(integtest_conf): - print(f"Integtest preconfigured config file: {integtest_conf}") - consolidate_files(str(temp_config_db), integtest_conf, *local_object_databases) - else: - if not drunc_config.use_fakedataprod: + if not integtest_params.use_fakedataprod: if not file_exists(dro_map_file): - dro_map_config = drunc_config.dro_map_config + dro_map_config = integtest_params.dro_map_config if dro_map_config != None: generate_hwmap( str(dro_map_file), @@ -272,28 +284,28 @@ def create_config_files(request, tmp_path_factory, check_system_resources): oksfile=str(readout_db), include=local_object_databases, generate_segment=True, - emulated_file_name=drunc_config.frame_file, - tpg_enabled=drunc_config.tpg_enabled, + emulated_file_name=integtest_params.frame_file, + tpg_enabled=integtest_params.tpg_enabled, ) elif not file_exists(readout_db): generate_fakedata( oksfile=str(readout_db), include=local_object_databases, generate_segment=True, - n_streams=drunc_config.dro_map_config.n_streams, - n_apps=drunc_config.dro_map_config.n_apps, - det_id=drunc_config.dro_map_config.det_id, - fragment_type=drunc_config.fake_data_fragment_type, + n_streams=integtest_params.dro_map_config.n_streams, + n_apps=integtest_params.dro_map_config.n_apps, + det_id=integtest_params.dro_map_config.det_id, + fragment_type=integtest_params.fake_data_fragment_type, ) generate_trigger( oksfile=str(trigger_db), include=local_object_databases, generate_segment=True, - tpg_enabled=drunc_config.tpg_enabled, - hsi_enabled=drunc_config.fake_hsi_enabled, + tpg_enabled=integtest_params.tpg_enabled, + hsi_enabled=integtest_params.fake_hsi_enabled, ) - if drunc_config.fake_hsi_enabled: + if integtest_params.fake_hsi_enabled: generate_hsi( oksfile=str(hsi_db), include=local_object_databases, @@ -302,39 +314,40 @@ def create_config_files(request, tmp_path_factory, check_system_resources): generate_dataflow( oksfile=str(dataflow_db), include=local_object_databases, - n_dfapps=drunc_config.n_df_apps, - tpwriting_enabled=drunc_config.tpg_enabled, + n_dfapps=integtest_params.n_df_apps, + tpwriting_enabled=integtest_params.tpg_enabled, generate_segment=True, - n_data_writers=drunc_config.n_data_writers, - trmon_app=drunc_config.trmon_app_enabled, + n_data_writers=integtest_params.n_data_writers, + trmon_app=integtest_params.trmon_app_enabled, ) + runcontrol_starts_connsvc = integtest_params.connsvc_control == ConnSvcControl.RUNCONTROL generate_session( oksfile=str(temp_config_db), include=local_object_databases + [str(readout_db), str(trigger_db), str(dataflow_db)] - + ([str(hsi_db)] if drunc_config.fake_hsi_enabled else []), - session_name=drunc_config.config_session_name, - op_env=drunc_config.op_env, - connectivity_service_is_infrastructure_app=drunc_config.drunc_connsvc, - disable_connectivity_service=disable_connectivity_service, + + ([str(hsi_db)] if integtest_params.fake_hsi_enabled else []), + session_name=integtest_params.config_session_name, + op_env=integtest_params.op_env, + connectivity_service_is_infrastructure_app=runcontrol_starts_connsvc, + disable_connectivity_service=no_integtest_connsvc, ) consolidate_db(str(temp_config_db), str(config_db)) - if drunc_config.connsvc_port is not None: - drunc_config.connsvc_port = set_connectivity_service_port( + if integtest_params.connsvc_port is not None: + integtest_params.connsvc_port = set_connectivity_service_port( oksfile=str(config_db), - session_name=drunc_config.config_session_name, - connsvc_port=drunc_config.connsvc_port, # Default is 0, which causes random port to be selected + session_name=integtest_params.config_session_name, + connsvc_port=integtest_params.connsvc_port, # Default is 0, which causes random port to be selected ) # 05-Nov-2025, KAB, MiR: added the setting of a random RC port - set_rc_controller_port(oksfile=str(config_db), session_name=drunc_config.config_session_name, rc_port=0) + set_rc_controller_port(oksfile=str(config_db), session_name=integtest_params.config_session_name, rc_port=0) # 03-Jul-2025, KAB: added the setting of the TRACE_FILE env var in the OKS Session, # if it is set in the user's environment, and if it is not already set in the configuration. try: trace_file_env_var = os.environ["TRACE_FILE"] - set_session_env_var(str(config_db), drunc_config.config_session_name, "TRACE_FILE", trace_file_env_var, overwrite=False) + set_session_env_var(str(config_db), integtest_params.config_session_name, "TRACE_FILE", trace_file_env_var, overwrite=False) except KeyError: pass @@ -364,7 +377,7 @@ def apply_update(obj, substitution): db.update_dal(obj) - for substitution in drunc_config.config_substitutions: + for substitution in integtest_params.config_substitutions: if substitution.obj_id != "*": obj = db.get_dal(class_name=substitution.obj_class, uid=substitution.obj_id) apply_update(obj, substitution) @@ -375,15 +388,8 @@ def apply_update(obj, substitution): db.commit() - # For preconfigured tests, disable starting the ConnSvc if the ConnectionService is an ifapp or unused - sessionobj = db.get_dal(class_name="Session", uid=drunc_config.config_session_name) - if sessionobj.connectivity_service is None: - drunc_config.drunc_connsvc = True - for if_app in sessionobj.infrastructure_applications: - if if_app.className() == "ConnectionService": - drunc_config.drunc_connsvc = True - # 30-Dec-2024, KAB: build up the list of directories used for writing raw and TPStream data + sessionobj = db.get_dal(class_name="Session", uid=integtest_params.config_session_name) rawdata_dirs = [] tpstream_dirs = [] trmon_dirs = [] @@ -417,7 +423,7 @@ def apply_update(obj, substitution): pass result = CreateConfigResult( - config=drunc_config, + config=integtest_params, config_dir=config_dir, config_file=config_db, log_file=logfile, @@ -451,11 +457,13 @@ def run_dunerc(request, create_config_files, process_manager_type, tmp_path_fact """ run_control_commands = request.param - disable_connectivity_service = request.config.getoption( - "--disable-connectivity-service" - ) + no_integtest_connsvc = request.config.getoption("--no-integtest-connsvc") integtest_verbosity_level = int(request.config.getoption("--integtest-verbosity")) + if no_integtest_connsvc and \ + isinstance(create_config_files.config, integtest_params_for_generated_dunedaq_config): + create_config_files.config.connsvc_control = ConnSvcControl.NONE + run_dir = tmp_path_factory.mktemp("run") global total_paramtrization_combinations @@ -505,11 +513,11 @@ def run_dunerc(request, create_config_files, process_manager_type, tmp_path_fact # if the expected env var is not set, we simply don't create the bundle info file pass + # start the Connectivity Service, if requested (only supported for generated dune-daq configs, for now) connsvc_obj = None if ( - not disable_connectivity_service - and not create_config_files.config.drunc_connsvc - and create_config_files.config.connsvc_port is not None + isinstance(create_config_files.config, integtest_params_for_generated_dunedaq_config) + and create_config_files.config.connsvc_control == ConnSvcControl.INTEGRATIONTEST ): # start connsvc if integtest_verbosity_level >= IntegtestVerbosityLevels.full_output: @@ -518,9 +526,10 @@ def run_dunerc(request, create_config_files, process_manager_type, tmp_path_fact ) connsvc_env = os.environ.copy() - connsvc_env["CONNECTION_FLASK_DEBUG"] = str( - create_config_files.config.connsvc_debug_level - ) + if create_config_files.config.connsvc_debug_level is not None: + connsvc_env["CONNECTION_FLASK_DEBUG"] = str( + create_config_files.config.connsvc_debug_level + ) connsvc_log = open( run_dir @@ -534,6 +543,10 @@ def run_dunerc(request, create_config_files, process_manager_type, tmp_path_fact env=connsvc_env, ) + elif create_config_files.config.connsvc_debug_level is not None: + set_session_env_var(str(create_config_files.config_file), create_config_files.config.config_session_name, + "CONNECTION_FLASK_DEBUG", create_config_files.config.connsvc_debug_level, overwrite=True) + dunerc = request.config.getoption("--dunerc-path") if dunerc is None: dunerc = "drunc-unified-shell"