Reorganized the integration test parameters that developers use to specify a DUNE DAQ configuration

src/integrationtest/data_classes.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass, field
-
+from enum import Enum
 
 @dataclass
 class DROMap_config:
@@ -47,32 +47,67 @@ class list_element_addition(config_substitution):
     additional_object_id: str = ""
 
 
+class ConnSvcControl(Enum):
+    INTEGRATIONTEST = "integrationtest"
+    RUNCONTROL = "runcontrol"
+    NONE = "none"
+
+@dataclass
+class integtest_param_base_class:
+    # daq_session_name can be specified; it is automatically populated if not specified
+    daq_session_name: str = None
+
+    # config substitutions can be made to both generated and predefined dunedaq configs
+    config_substitutions: list[config_substitution] = field(default_factory=list)
+
+    # the cleanup of leftover RunControl or ConnSvc processes is available to all types of integtests
+    attempt_cleanup: bool = False
+
+    # parameter(s) related to the startup of the Connectivity Service
+    connsvc_port: int = 0
+    connsvc_debug_level: int = None
+
 @dataclass
-class drunc_config:
+class integtest_params_for_generated_dunedaq_config(integtest_param_base_class):
+    # *** Parameters that are needed for both generated and predefined configs,
+    # *** and benefit from different default values
+    # - for generated configs, these two params do not need to have specific values
     op_env: str = "integtest"
     config_session_name: str = "integtest"
-    daq_session_name: str = None
+
+    # *** Parameters that are only needed for generated dunedaq configurations
+    # - databases that are needed to support config generation
+    object_databases: list[str] = field(default_factory=list)
+    # - parameters that control what the generators produce
     dro_map_config: DROMap_config = field(default_factory=lambda: DROMap_config(1))
-    frame_file: str = "asset://?checksum=370df564205290d27cab47e44ae4ca47"
+    frame_file: str = "asset://?checksum=370df564205290d27cab47e44ae4ca47"  # wib_link_67.bin
     tpg_enabled: bool = False
     trmon_app_enabled: bool = False
     fake_hsi_enabled: bool = False
     use_fakedataprod: bool = False
     fake_data_fragment_type: str = ""
-    config_db: str = ""
     n_df_apps: int = 1
     n_data_writers: int = 1
-    object_databases: list[str] = field(default_factory=list)
-    config_substitutions: list[config_substitution] = field(default_factory=list)
-    attempt_cleanup: bool = False
-    drunc_connsvc: bool = False
-    connsvc_port: int = 0
-    connsvc_debug_level: int = 0
+    # - control over how the Connectivity Service is started
+    connsvc_control: ConnSvcControl = ConnSvcControl.INTEGRATIONTEST
+
+@dataclass
+class integtest_params_for_predefined_dunedaq_config(integtest_param_base_class):
+    # *** Parameters that are needed for both generated and predefined configs,
+    # *** and benefit from different default values
+    # - for a predefined config, the following two parameters must contain values that
+    #   match what is in that config; they are likely reassigned in integtest files
+    op_env: str = "test"
+    config_session_name: str = "local-1x1-config"
+
+    # *** Parameters that are unique to predefined dunedaq configurations
+    # - the predefined configuration that should be used
+    predefined_config_db: str = ""
 
 
 @dataclass
 class CreateConfigResult:
-    config: drunc_config
+    config: integtest_param_base_class
     config_dir: str
     config_file: str
     log_file: str

src/integrationtest/integrationtest_commandline.py

@@ -22,7 +22,7 @@ def pytest_addoption(parser):
         required=False
     )
     parser.addoption(
-        "--disable-connectivity-service",
+        "--no-integtest-connsvc",
         action="store_true",
         default=False,
         help="Whether to disable the Connectivity Service for this test",

src/integrationtest/integrationtest_drunc.py

@@ -20,6 +20,9 @@
     relationship_substitution,
     list_element_substitution,
     list_element_addition,
+    ConnSvcControl,
+    integtest_params_for_generated_dunedaq_config,
+    integtest_params_for_predefined_dunedaq_config,
 )
 from daqconf.generate_hwmap import generate_hwmap
 from daqconf.generate import (
@@ -201,19 +204,28 @@ def create_config_files(request, tmp_path_factory, check_system_resources):
 
     """
     dummy_resource_check = check_system_resources
-    drunc_config = request.param
+    integtest_params = request.param
+
+    #if isinstance(integtest_params, integtest_params_for_generated_dunedaq_config):
+    #    print("*** integtest_params is of type integtest_params_for_generated_dunedaq_config")
+    #if isinstance(integtest_params, integtest_params_for_predefined_dunedaq_config):
+    #    print("*** integtest_params is of type integtest_params_for_predefined_dunedaq_config")
+    if not isinstance(integtest_params, integtest_params_for_generated_dunedaq_config) \
+       and not isinstance(integtest_params, integtest_params_for_predefined_dunedaq_config):
+        fail_msg = f"The integtest configuration object has an invalid type: {type(integtest_params)}"
+        pytest.fail(fail_msg, pytrace=False)
+
+    no_integtest_connsvc = request.config.getoption("--no-integtest-connsvc")
+    skip_resource_checks = request.config.getoption("--skip-resource-checks")
 
-    disable_connectivity_service = request.config.getoption(
-        "--disable-connectivity-service"
-    )
-    skip_resource_checks = request.config.getoption(
-        "--skip-resource-checks"
-    )
+    if no_integtest_connsvc and \
+       isinstance(integtest_params, integtest_params_for_generated_dunedaq_config):
+        integtest_params.connsvc_control = ConnSvcControl.NONE
 
     # 06-Mar-2026, KAB: if the DAQ session name has not explicitly been set by the
     # user, set it here so that we can make use of it from this point onward.
-    if not drunc_config.daq_session_name:
-        drunc_config.daq_session_name = drunc_config.config_session_name
+    if not integtest_params.daq_session_name:
+        integtest_params.daq_session_name = integtest_params.config_session_name
 
     # 26-Mar-2026, KAB: suppress output messages, if requested
     integtest_verbosity_level = int(request.config.getoption("--integtest-verbosity"))
@@ -229,30 +241,30 @@ def create_config_files(request, tmp_path_factory, check_system_resources):
         sys.stdout = catcher = StringIO()
 
     config_dir = tmp_path_factory.mktemp("config")
-    boot_file = config_dir / "boot.json"
-    configfile = config_dir / "config.json"
-    dro_map_file = config_dir / "ReadoutMap.data.xml"
-    readout_db = config_dir / "readout-segment.data.xml"
-    dataflow_db = config_dir / "df-segment.data.xml"
-    trigger_db = config_dir / "trg-segment.data.xml"
-    hsi_db = config_dir / "hsi-segment.data.xml"
     config_db = config_dir / "integtest-session-resolved.data.xml"
     temp_config_db = config_dir / "integtest-session.data.xml"
     logfile = tmp_path_factory.getbasetemp() / f"stdouterr{request.param_index}.txt"
 
-    integtest_conf = drunc_config.config_db
+    if isinstance(integtest_params, integtest_params_for_predefined_dunedaq_config):
+        integtest_conf = integtest_params.predefined_config_db
+        if file_exists(integtest_conf):
+            print(f"Integtest preconfigured config file: {integtest_conf}")
+            consolidate_files(str(temp_config_db), integtest_conf)
+        else:
+            fail_msg = f"The file containing the predefined dunedaq configuration \"{integtest_conf}\" could not be found."
+            pytest.fail(fail_msg, pytrace=False)
+    else:
+        dro_map_file = config_dir / "ReadoutMap.data.xml"
+        readout_db = config_dir / "readout-segment.data.xml"
+        dataflow_db = config_dir / "df-segment.data.xml"
+        trigger_db = config_dir / "trg-segment.data.xml"
+        hsi_db = config_dir / "hsi-segment.data.xml"
 
-    object_databases = getattr(request.module, "object_databases", [])
-    local_object_databases = copy_configuration(config_dir, object_databases)
+        local_object_databases = copy_configuration(config_dir, integtest_params.object_databases)
 
-    #print()  # Blank line
-    if file_exists(integtest_conf):
-        print(f"Integtest preconfigured config file: {integtest_conf}")
-        consolidate_files(str(temp_config_db), integtest_conf, *local_object_databases)
-    else:
-        if not drunc_config.use_fakedataprod:
+        if not integtest_params.use_fakedataprod:
             if not file_exists(dro_map_file):
-                dro_map_config = drunc_config.dro_map_config
+                dro_map_config = integtest_params.dro_map_config
                 if dro_map_config != None:
                     generate_hwmap(
                         str(dro_map_file),
@@ -272,28 +284,28 @@ def create_config_files(request, tmp_path_factory, check_system_resources):
                     oksfile=str(readout_db),
                     include=local_object_databases,
                     generate_segment=True,
-                    emulated_file_name=drunc_config.frame_file,
-                    tpg_enabled=drunc_config.tpg_enabled,
+                    emulated_file_name=integtest_params.frame_file,
+                    tpg_enabled=integtest_params.tpg_enabled,
                 )
         elif not file_exists(readout_db):
             generate_fakedata(
                 oksfile=str(readout_db),
                 include=local_object_databases,
                 generate_segment=True,
-                n_streams=drunc_config.dro_map_config.n_streams,
-                n_apps=drunc_config.dro_map_config.n_apps,
-                det_id=drunc_config.dro_map_config.det_id,
-                fragment_type=drunc_config.fake_data_fragment_type,
+                n_streams=integtest_params.dro_map_config.n_streams,
+                n_apps=integtest_params.dro_map_config.n_apps,
+                det_id=integtest_params.dro_map_config.det_id,
+                fragment_type=integtest_params.fake_data_fragment_type,
             )
 
         generate_trigger(
             oksfile=str(trigger_db),
             include=local_object_databases,
             generate_segment=True,
-            tpg_enabled=drunc_config.tpg_enabled,
-            hsi_enabled=drunc_config.fake_hsi_enabled,
+            tpg_enabled=integtest_params.tpg_enabled,
+            hsi_enabled=integtest_params.fake_hsi_enabled,
         )
-        if drunc_config.fake_hsi_enabled:
+        if integtest_params.fake_hsi_enabled:
             generate_hsi(
                 oksfile=str(hsi_db),
                 include=local_object_databases,
@@ -302,39 +314,40 @@ def create_config_files(request, tmp_path_factory, check_system_resources):
         generate_dataflow(
             oksfile=str(dataflow_db),
             include=local_object_databases,
-            n_dfapps=drunc_config.n_df_apps,
-            tpwriting_enabled=drunc_config.tpg_enabled,
+            n_dfapps=integtest_params.n_df_apps,
+            tpwriting_enabled=integtest_params.tpg_enabled,
             generate_segment=True,
-            n_data_writers=drunc_config.n_data_writers,
-            trmon_app=drunc_config.trmon_app_enabled,
+            n_data_writers=integtest_params.n_data_writers,
+            trmon_app=integtest_params.trmon_app_enabled,
         )
 
+        runcontrol_starts_connsvc = integtest_params.connsvc_control == ConnSvcControl.RUNCONTROL
         generate_session(
             oksfile=str(temp_config_db),
             include=local_object_databases
             + [str(readout_db), str(trigger_db), str(dataflow_db)]
-            + ([str(hsi_db)] if drunc_config.fake_hsi_enabled else []),
-            session_name=drunc_config.config_session_name,
-            op_env=drunc_config.op_env,
-            connectivity_service_is_infrastructure_app=drunc_config.drunc_connsvc,
-            disable_connectivity_service=disable_connectivity_service,
+            + ([str(hsi_db)] if integtest_params.fake_hsi_enabled else []),
+            session_name=integtest_params.config_session_name,
+            op_env=integtest_params.op_env,
+            connectivity_service_is_infrastructure_app=runcontrol_starts_connsvc,
+            disable_connectivity_service=no_integtest_connsvc,
         )
 
     consolidate_db(str(temp_config_db), str(config_db))
-    if drunc_config.connsvc_port is not None:
-        drunc_config.connsvc_port = set_connectivity_service_port(
+    if integtest_params.connsvc_port is not None:
+        integtest_params.connsvc_port = set_connectivity_service_port(
             oksfile=str(config_db),
-            session_name=drunc_config.config_session_name,
-            connsvc_port=drunc_config.connsvc_port, # Default is 0, which causes random port to be selected
+            session_name=integtest_params.config_session_name,
+            connsvc_port=integtest_params.connsvc_port, # Default is 0, which causes random port to be selected
         )
     # 05-Nov-2025, KAB, MiR: added the setting of a random RC port
-    set_rc_controller_port(oksfile=str(config_db), session_name=drunc_config.config_session_name, rc_port=0)
+    set_rc_controller_port(oksfile=str(config_db), session_name=integtest_params.config_session_name, rc_port=0)
 
     # 03-Jul-2025, KAB: added the setting of the TRACE_FILE env var in the OKS Session,
     # if it is set in the user's environment, and if it is not already set in the configuration.
     try:
         trace_file_env_var = os.environ["TRACE_FILE"]
-        set_session_env_var(str(config_db), drunc_config.config_session_name, "TRACE_FILE", trace_file_env_var, overwrite=False)
+        set_session_env_var(str(config_db), integtest_params.config_session_name, "TRACE_FILE", trace_file_env_var, overwrite=False)
     except KeyError:
         pass
 
@@ -364,7 +377,7 @@ def apply_update(obj, substitution):
 
         db.update_dal(obj)
 
-    for substitution in drunc_config.config_substitutions:
+    for substitution in integtest_params.config_substitutions:
         if substitution.obj_id != "*":
             obj = db.get_dal(class_name=substitution.obj_class, uid=substitution.obj_id)
             apply_update(obj, substitution)
@@ -375,15 +388,8 @@ def apply_update(obj, substitution):
 
     db.commit()
 
-    # For preconfigured tests, disable starting the ConnSvc if the ConnectionService is an ifapp or unused
-    sessionobj = db.get_dal(class_name="Session", uid=drunc_config.config_session_name)
-    if sessionobj.connectivity_service is None:
-        drunc_config.drunc_connsvc = True
-    for if_app in sessionobj.infrastructure_applications:
-        if if_app.className() == "ConnectionService":
-            drunc_config.drunc_connsvc = True
-
     # 30-Dec-2024, KAB: build up the list of directories used for writing raw and TPStream data
+    sessionobj = db.get_dal(class_name="Session", uid=integtest_params.config_session_name)
     rawdata_dirs = []
     tpstream_dirs = []
     trmon_dirs = []
@@ -417,7 +423,7 @@ def apply_update(obj, substitution):
             pass
 
     result = CreateConfigResult(
-        config=drunc_config,
+        config=integtest_params,
         config_dir=config_dir,
         config_file=config_db,
         log_file=logfile,
@@ -451,11 +457,13 @@ def run_dunerc(request, create_config_files, process_manager_type, tmp_path_fact
     """
     run_control_commands = request.param
 
-    disable_connectivity_service = request.config.getoption(
-        "--disable-connectivity-service"
-    )
+    no_integtest_connsvc = request.config.getoption("--no-integtest-connsvc")
     integtest_verbosity_level = int(request.config.getoption("--integtest-verbosity"))
 
+    if no_integtest_connsvc and \
+       isinstance(create_config_files.config, integtest_params_for_generated_dunedaq_config):
+        create_config_files.config.connsvc_control = ConnSvcControl.NONE
+
     run_dir = tmp_path_factory.mktemp("run")
 
     global total_paramtrization_combinations
@@ -505,11 +513,11 @@ def run_dunerc(request, create_config_files, process_manager_type, tmp_path_fact
         # if the expected env var is not set, we simply don't create the bundle info file
         pass
 
+    # start the Connectivity Service, if requested (only supported for generated dune-daq configs, for now)
     connsvc_obj = None
     if (
-        not disable_connectivity_service
-        and not create_config_files.config.drunc_connsvc
-        and create_config_files.config.connsvc_port is not None
+        isinstance(create_config_files.config, integtest_params_for_generated_dunedaq_config)
+        and create_config_files.config.connsvc_control == ConnSvcControl.INTEGRATIONTEST
     ):
         # start connsvc
         if integtest_verbosity_level >= IntegtestVerbosityLevels.full_output:
@@ -518,9 +526,10 @@ def run_dunerc(request, create_config_files, process_manager_type, tmp_path_fact
             )
 
         connsvc_env = os.environ.copy()
-        connsvc_env["CONNECTION_FLASK_DEBUG"] = str(
-            create_config_files.config.connsvc_debug_level
-        )
+        if create_config_files.config.connsvc_debug_level is not None:
+            connsvc_env["CONNECTION_FLASK_DEBUG"] = str(
+                create_config_files.config.connsvc_debug_level
+            )
 
         connsvc_log = open(
             run_dir
@@ -534,6 +543,10 @@ def run_dunerc(request, create_config_files, process_manager_type, tmp_path_fact
             env=connsvc_env,
         )
 
+    elif create_config_files.config.connsvc_debug_level is not None:
+        set_session_env_var(str(create_config_files.config_file), create_config_files.config.config_session_name,
+                            "CONNECTION_FLASK_DEBUG", create_config_files.config.connsvc_debug_level, overwrite=True)
+
     dunerc = request.config.getoption("--dunerc-path")
     if dunerc is None:
         dunerc = "drunc-unified-shell"