From 2b0676fa9d4f4b95edfe2bae8b41c89ece17504b Mon Sep 17 00:00:00 2001 From: Guillaume Fieni Date: Fri, 20 Mar 2026 15:24:09 +0100 Subject: [PATCH 1/6] feat(cli): Use lazy imports for report processors --- src/powerapi/cli/generator.py | 23 +++++++++++-------- src/powerapi/processor/pre/k8s/__init__.py | 2 -- .../processor/pre/openstack/__init__.py | 1 - 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/powerapi/cli/generator.py b/src/powerapi/cli/generator.py index 0090ffe4..dbff78e1 100644 --- a/src/powerapi/cli/generator.py +++ b/src/powerapi/cli/generator.py @@ -35,8 +35,6 @@ from powerapi.exception import PowerAPIException, ModelNameAlreadyUsed, DatabaseNameDoesNotExist, ModelNameDoesNotExist, \ DatabaseNameAlreadyUsed, ProcessorTypeDoesNotExist, ProcessorTypeAlreadyUsed from powerapi.filter import ReportFilter -from powerapi.processor.pre.k8s import K8sPreProcessorActor, K8sProcessorConfig -from powerapi.processor.pre.openstack import OpenStackPreProcessorActor from powerapi.processor.processor_actor import ProcessorActor from powerapi.puller import PullerActor from powerapi.pusher import PusherActor @@ -405,6 +403,14 @@ def add_processor_factory(self, processor_type: str, processor_factory_function: self.processor_factory[processor_type] = processor_factory_function + def _generate_processor(self, processor_name: str, component_config: dict) -> ProcessorActor: + try: + return self.processor_factory[processor_name](component_config) + except KeyError as exn: + raise PowerAPIException('Configuration error: Invalid processor type: %s', processor_name) from exn + except ImportError as exn: + raise PowerAPIException('Dependencies for %s processor are not installed', processor_name) from exn + def _gen_actor(self, component_config: dict, main_config: dict, component_name: str) -> ProcessorActor: """ Helper method to generate a processor actor from the given configuration. @@ -414,12 +420,9 @@ def _gen_actor(self, component_config: dict, main_config: dict, component_name: :return: Processor actor """ processor_actor_type = component_config[COMPONENT_TYPE_KEY] - if processor_actor_type not in self.processor_factory: - raise PowerAPIException(f'Configuration error: Unknown processor actor type: {processor_actor_type}') - component_config[ACTOR_NAME_KEY] = component_name component_config[GENERAL_CONF_VERBOSE_KEY] = main_config[GENERAL_CONF_VERBOSE_KEY] - return self.processor_factory[processor_actor_type](component_config) + return self._generate_processor(processor_actor_type, component_config) class PreProcessorGenerator(ProcessorGenerator): @@ -431,14 +434,15 @@ def __init__(self): super().__init__('pre-processor', self._get_default_processor_factories()) @staticmethod - def _k8s_pre_processor_factory(processor_config: dict) -> K8sPreProcessorActor: + def _k8s_pre_processor_factory(processor_config: dict) -> ProcessorActor: """ Kubernetes pre-processor actor factory. :param processor_config: Pre-Processor configuration :return: Configured Kubernetes pre-processor actor """ + from powerapi.processor.pre.k8s.actor import K8sPreProcessorActor, K8sProcessorConfig name = processor_config[ACTOR_NAME_KEY] - api_mode = processor_config.get(K8S_API_MODE_KEY, 'manual') # use manual mode by default + api_mode = processor_config[K8S_API_MODE_KEY] api_host = processor_config.get(K8S_API_HOST_KEY, None) api_key = processor_config.get(K8S_API_KEY_KEY, None) level_logger = logging.DEBUG if processor_config[GENERAL_CONF_VERBOSE_KEY] else logging.INFO @@ -446,12 +450,13 @@ def _k8s_pre_processor_factory(processor_config: dict) -> K8sPreProcessorActor: return K8sPreProcessorActor(name, config, level_logger) @staticmethod - def _openstack_pre_processor_factory(processor_config: dict) -> OpenStackPreProcessorActor: + def _openstack_pre_processor_factory(processor_config: dict) -> ProcessorActor: """ Openstack pre-processor actor factory. :param processor_config: Pre-Processor configuration :return: Configured OpenStack pre-processor actor """ + from powerapi.processor.pre.openstack.actor import OpenStackPreProcessorActor name = processor_config[ACTOR_NAME_KEY] level_logger = logging.DEBUG if processor_config[GENERAL_CONF_VERBOSE_KEY] else logging.INFO return OpenStackPreProcessorActor(name, level_logger) diff --git a/src/powerapi/processor/pre/k8s/__init__.py b/src/powerapi/processor/pre/k8s/__init__.py index d7909960..8b676d4d 100644 --- a/src/powerapi/processor/pre/k8s/__init__.py +++ b/src/powerapi/processor/pre/k8s/__init__.py @@ -26,5 +26,3 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from .actor import K8sPreProcessorActor, K8sProcessorConfig diff --git a/src/powerapi/processor/pre/openstack/__init__.py b/src/powerapi/processor/pre/openstack/__init__.py index 9904ea6b..42dd5649 100644 --- a/src/powerapi/processor/pre/openstack/__init__.py +++ b/src/powerapi/processor/pre/openstack/__init__.py @@ -27,4 +27,3 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from powerapi.processor.pre.openstack.actor import OpenStackPreProcessorActor From 219bf911557849e1fef52e4211c6d4223926fed0 Mon Sep 17 00:00:00 2001 From: Guillaume Fieni Date: Fri, 20 Mar 2026 15:28:14 +0100 Subject: [PATCH 2/6] test(unit/cli): Cleanup k8s processor tests - Move k8s processor tests into a separate file - Update test configuration files - Remove unused fixtures --- tests/unit/cli/conftest.py | 8 -- tests/unit/cli/test_generator.py | 62 ------------- tests/unit/cli/test_generator_k8s.py | 87 +++++++++++++++++++ ..._pre_processor_complete_configuration.json | 2 +- 4 files changed, 88 insertions(+), 71 deletions(-) create mode 100644 tests/unit/cli/test_generator_k8s.py diff --git a/tests/unit/cli/conftest.py b/tests/unit/cli/conftest.py index d8dc5bbb..324d88e9 100644 --- a/tests/unit/cli/conftest.py +++ b/tests/unit/cli/conftest.py @@ -146,14 +146,6 @@ def config_without_output(csv_io_postmortem_config): return csv_io_postmortem_config -@pytest.fixture -def k8s_pre_processor_config(): - """ - Configuration with k8s as pre-processor - """ - return load_configuration_from_json_file(file_name='k8s_pre_processor_configuration.json') - - @pytest.fixture def subgroup_parser(): """ diff --git a/tests/unit/cli/test_generator.py b/tests/unit/cli/test_generator.py index 43dc8f0f..28ae9b45 100644 --- a/tests/unit/cli/test_generator.py +++ b/tests/unit/cli/test_generator.py @@ -36,7 +36,6 @@ from powerapi.database.socket import Socket from powerapi.exception import PowerAPIException from powerapi.filter import BroadcastReportFilter -from powerapi.processor.pre.k8s import K8sPreProcessorActor from powerapi.puller import PullerActor from powerapi.pusher import PusherActor from powerapi.report import PowerReport, FormulaReport @@ -194,64 +193,3 @@ def test_generate_pre_processor_from_empty_config_dict_raise_an_exception(): with pytest.raises(PowerAPIException): generator.generate(conf) - - -def check_k8s_pre_processor_infos(pre_processor: K8sPreProcessorActor, processor_name: str): - """ - Check that the infos related to a K8sMonitorAgentActor are correct regarding its related K8SProcessorActor - """ - assert isinstance(pre_processor, K8sPreProcessorActor) - assert pre_processor.name == processor_name - assert len(pre_processor.target_actors) == 0 - - -def test_generate_pre_processor_from_k8s_config(k8s_pre_processor_config): - """ - Test that generation for k8s processor from a config works correctly - """ - generator = PreProcessorGenerator() - processor_name = 'my_processor' - - processors = generator.generate(k8s_pre_processor_config) - - assert len(processors) == len(k8s_pre_processor_config['pre-processor']) - assert processor_name in processors - - processor = processors[processor_name] - - check_k8s_pre_processor_infos(processor, processor_name) - - -def test_generate_several_k8s_pre_processors_from_config(several_k8s_pre_processors_config): - """ - Test that several k8s pre-processors are correctly generated - """ - generator = PreProcessorGenerator() - - processors = generator.generate(several_k8s_pre_processors_config) - - assert len(processors) == len(several_k8s_pre_processors_config['pre-processor']) - - for processor_name in several_k8s_pre_processors_config['pre-processor']: - assert processor_name in processors - - processor = processors[processor_name] - - check_k8s_pre_processor_infos(processor, processor_name) - - -def test_generate_k8s_pre_processor_uses_default_values_with_missing_arguments(several_k8s_pre_processors_without_some_arguments_config): - """ - Test that PreProcessorGenerator generates a pre-processor with default values when arguments are not defined - """ - generator = PreProcessorGenerator() - - processors = generator.generate(several_k8s_pre_processors_without_some_arguments_config) - - assert len(processors) == len(several_k8s_pre_processors_without_some_arguments_config['pre-processor']) - - for pre_processor_name in several_k8s_pre_processors_without_some_arguments_config['pre-processor']: - assert pre_processor_name in processors - - processor = processors[pre_processor_name] - check_k8s_pre_processor_infos(processor, pre_processor_name) diff --git a/tests/unit/cli/test_generator_k8s.py b/tests/unit/cli/test_generator_k8s.py new file mode 100644 index 00000000..ab86a98c --- /dev/null +++ b/tests/unit/cli/test_generator_k8s.py @@ -0,0 +1,87 @@ +# Copyright (c) 2026, Inria +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import pytest + +pytest.importorskip('powerapi.processor.pre.k8s.actor') # The Kubernetes processor requires external dependencies to work. + +from powerapi.cli.generator import PreProcessorGenerator +from powerapi.exception import PowerAPIException +from powerapi.processor.pre.k8s.actor import K8sPreProcessorActor + + +@pytest.fixture +def k8s_processor_config(): + """ + Fixture that provides a configuration with a Kubernetes pre-processor. + """ + return { + 'stream': True, + 'verbose': True, + 'pre-processor': { + 'pytest-k8s-preprocessor': { + 'type': 'k8s', + 'api-mode': 'manual', + 'api-host': 'https://127.0.0.1:36599', + 'api-key': 'pytest-token-powerapi', + 'puller': 'pytest-json-puller' + } + } + } + + +def test_preprocessor_generator_with_valid_k8s_config(k8s_processor_config): + """ + PreProcessorGenerator should generate a K8sPreProcessorActor. + """ + generator = PreProcessorGenerator() + preprocessors = generator.generate(k8s_processor_config) + + assert len(preprocessors) == 1 + assert 'pytest-k8s-preprocessor' in preprocessors + + preprocessor = preprocessors['pytest-k8s-preprocessor'] + assert isinstance(preprocessor, K8sPreProcessorActor) + + expected_preprocessor_attributes = k8s_processor_config['pre-processor']['pytest-k8s-preprocessor'] + assert preprocessor.config.api_mode == expected_preprocessor_attributes['api-mode'] + assert preprocessor.config.api_key == expected_preprocessor_attributes['api-key'] + assert preprocessor.config.api_host == expected_preprocessor_attributes['api-host'] + + +@pytest.mark.parametrize('missing_arg', ['api-mode']) +def test_preprocessor_generator_with_missing_arguments_in_k8s_config(k8s_processor_config, missing_arg): + """ + PreProcessorGenerator should raise an exception when a required argument is missing from the MongoDB config. + """ + generator = PreProcessorGenerator() + + k8s_processor_config['pre-processor']['pytest-k8s-preprocessor'].pop(missing_arg) + + with pytest.raises(PowerAPIException): + generator.generate(k8s_processor_config) diff --git a/tests/utils/cli/k8s_pre_processor_complete_configuration.json b/tests/utils/cli/k8s_pre_processor_complete_configuration.json index 90e84089..151bc026 100644 --- a/tests/utils/cli/k8s_pre_processor_complete_configuration.json +++ b/tests/utils/cli/k8s_pre_processor_complete_configuration.json @@ -19,7 +19,7 @@ "pre-processor": { "my_processor": { "type": "k8s", - "api_mode": "manual", + "api-mode": "manual", "puller": "one_puller" } } From a5d68df0d9d116319d8a6145395aec06a427ab97 Mon Sep 17 00:00:00 2001 From: Guillaume Fieni Date: Mon, 23 Mar 2026 14:25:38 +0100 Subject: [PATCH 3/6] test(unit/processor/k8s): Gracefully handle missing dependencies Update the test modules to automatically skip the tests requiring external dependencies when they are not installed. --- tests/unit/processor/pre/k8s/__init__.py | 12 ++++++------ tests/unit/processor/pre/k8s/conftest.py | 14 ++------------ tests/unit/processor/pre/k8s/test_handlers.py | 2 ++ .../unit/processor/pre/k8s/test_monitor_agent.py | 15 +++++++++++++++ 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/tests/unit/processor/pre/k8s/__init__.py b/tests/unit/processor/pre/k8s/__init__.py index 36c43967..8b676d4d 100644 --- a/tests/unit/processor/pre/k8s/__init__.py +++ b/tests/unit/processor/pre/k8s/__init__.py @@ -1,21 +1,21 @@ -# Copyright (c) 2023, INRIA +# Copyright (c) 2023, Inria # Copyright (c) 2023, University of Lille # All rights reserved. - +# # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: - +# # * Redistributions of source code must retain the above copyright notice, this # list of conditions and the following disclaimer. - +# # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. - +# # * Neither the name of the copyright holder nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. - +# # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE diff --git a/tests/unit/processor/pre/k8s/conftest.py b/tests/unit/processor/pre/k8s/conftest.py index 40dccc7d..0d27119e 100644 --- a/tests/unit/processor/pre/k8s/conftest.py +++ b/tests/unit/processor/pre/k8s/conftest.py @@ -32,11 +32,10 @@ import pytest from powerapi.processor.pre.k8s.metadata_cache_manager import K8sMetadataCacheManager -from powerapi.processor.pre.k8s.monitor_agent import K8sMonitorAgent -@pytest.fixture(name='initialized_metadata_cache_manager') -def fx_initialized_metadata_cache_manager(): +@pytest.fixture +def initialized_metadata_cache_manager(): """ Returns an initialized metadata cache manager. """ @@ -44,12 +43,3 @@ def fx_initialized_metadata_cache_manager(): yield K8sMetadataCacheManager(manager) manager.shutdown() - - -@pytest.fixture -def initialized_monitor_agent(initialized_metadata_cache_manager): - """ - Returns an initialized monitor agent. - """ - agent = K8sMonitorAgent(initialized_metadata_cache_manager, 'manual', '', '') - return agent diff --git a/tests/unit/processor/pre/k8s/test_handlers.py b/tests/unit/processor/pre/k8s/test_handlers.py index 2a5e3765..2bb9a04a 100644 --- a/tests/unit/processor/pre/k8s/test_handlers.py +++ b/tests/unit/processor/pre/k8s/test_handlers.py @@ -32,6 +32,8 @@ import pytest +pytest.importorskip('powerapi.processor.pre.k8s.actor') # The actor module requires external dependencies. + from powerapi.processor.pre.k8s.actor import K8sProcessorState, K8sProcessorConfig from powerapi.processor.pre.k8s.handlers import K8sPreProcessorActorHWPCReportHandler from powerapi.processor.pre.k8s.metadata_cache_manager import K8sContainerMetadata diff --git a/tests/unit/processor/pre/k8s/test_monitor_agent.py b/tests/unit/processor/pre/k8s/test_monitor_agent.py index 5db42bbb..78ffff6c 100644 --- a/tests/unit/processor/pre/k8s/test_monitor_agent.py +++ b/tests/unit/processor/pre/k8s/test_monitor_agent.py @@ -27,8 +27,23 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import pytest + +pytest.importorskip('powerapi.processor.pre.k8s.monitor_agent') # The monitor agent requires external dependencies. + from kubernetes.client import V1Pod, V1ContainerStatus, Configuration, V1ObjectMeta, V1PodStatus +from powerapi.processor.pre.k8s.monitor_agent import K8sMonitorAgent + + +@pytest.fixture +def initialized_monitor_agent(initialized_metadata_cache_manager): + """ + Returns an initialized monitor agent. + """ + agent = K8sMonitorAgent(initialized_metadata_cache_manager, 'manual', '', '') + return agent + def generate_k8s_config_for_tests() -> Configuration: """ From 2dbdad85f65a1ffefc3f9c9a24bbd2dd6ae2fd7e Mon Sep 17 00:00:00 2001 From: Guillaume Fieni Date: Mon, 23 Mar 2026 17:51:13 +0100 Subject: [PATCH 4/6] test(unit/cli): Add tests for OpenStack processor generator --- tests/unit/cli/test_generator_openstack.py | 65 ++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 tests/unit/cli/test_generator_openstack.py diff --git a/tests/unit/cli/test_generator_openstack.py b/tests/unit/cli/test_generator_openstack.py new file mode 100644 index 00000000..a77199f4 --- /dev/null +++ b/tests/unit/cli/test_generator_openstack.py @@ -0,0 +1,65 @@ +# Copyright (c) 2026, Inria +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import pytest + +pytest.importorskip('powerapi.processor.pre.openstack.actor') # The OpenStack processor requires external dependencies to work. + +from powerapi.cli.generator import PreProcessorGenerator +from powerapi.processor.pre.openstack.actor import OpenStackPreProcessorActor + + +@pytest.fixture +def openstack_config(): + """ + Fixture that provides a configuration with an OpenStack pre-processor. + """ + return { + 'stream': True, + 'verbose': True, + 'pre-processor': { + 'pytest-openstack-preprocessor': { + 'type': 'openstack', + 'puller': 'pytest-json-puller' + } + } + } + + +def test_preprocessor_generator_with_valid_openstack_config(openstack_config): + """ + PreProcessorGenerator should generate an OpenStackPreProcessorActor when given a valid config. + """ + generator = PreProcessorGenerator() + preprocessors = generator.generate(openstack_config) + + assert len(preprocessors) == 1 + assert 'pytest-openstack-preprocessor' in preprocessors + + preprocessor = preprocessors['pytest-openstack-preprocessor'] + assert isinstance(preprocessor, OpenStackPreProcessorActor) From 9eaaf8167c69e0cb4559189274bdc68abdafa8dc Mon Sep 17 00:00:00 2001 From: Guillaume Fieni Date: Tue, 24 Mar 2026 17:26:35 +0100 Subject: [PATCH 5/6] refactor(cli): Initialize processor factories through explicit registration --- src/powerapi/cli/generator.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/src/powerapi/cli/generator.py b/src/powerapi/cli/generator.py index dbff78e1..4622d6c0 100644 --- a/src/powerapi/cli/generator.py +++ b/src/powerapi/cli/generator.py @@ -373,14 +373,13 @@ class ProcessorGenerator(Generator): Generator that initializes the processor actor(s) from the configuration. """ - def __init__(self, component_group_name: str, processor_factory: dict[str, Callable[[dict], ProcessorActor]]): + def __init__(self, component_group_name: str): """ :param component_group_name: Name of the component group - :param processor_factory: Dictionary mapping processor type to actor factory """ super().__init__(component_group_name) - self.processor_factory = processor_factory + self.processor_factory: dict[str, Callable[[dict], ProcessorActor]] = {} def remove_processor_factory(self, processor_type: str) -> None: """ @@ -431,7 +430,10 @@ class PreProcessorGenerator(ProcessorGenerator): """ def __init__(self): - super().__init__('pre-processor', self._get_default_processor_factories()) + super().__init__('pre-processor') + + self.add_processor_factory('k8s', self._k8s_pre_processor_factory) + self.add_processor_factory('openstack', self._openstack_pre_processor_factory) @staticmethod def _k8s_pre_processor_factory(processor_config: dict) -> ProcessorActor: @@ -460,12 +462,3 @@ def _openstack_pre_processor_factory(processor_config: dict) -> ProcessorActor: name = processor_config[ACTOR_NAME_KEY] level_logger = logging.DEBUG if processor_config[GENERAL_CONF_VERBOSE_KEY] else logging.INFO return OpenStackPreProcessorActor(name, level_logger) - - def _get_default_processor_factories(self) -> dict[str, Callable[[dict], ProcessorActor]]: - """ - Return the default pre-processors factory. - """ - return { - 'k8s': self._k8s_pre_processor_factory, - 'openstack': self._openstack_pre_processor_factory - } From 2df2f7cdbf7c378a40fa2da28a787ffa472147e4 Mon Sep 17 00:00:00 2001 From: Guillaume Fieni Date: Tue, 24 Mar 2026 17:30:52 +0100 Subject: [PATCH 6/6] test(unit/cli): Rework pre-processor binding manager tests --- tests/unit/cli/conftest.py | 131 +----------- tests/unit/cli/test_binding_manager.py | 190 ++++++++++-------- .../cli/k8s_pre_processor_configuration.json | 10 - ...used_puller_in_bindings_configuration.json | 31 --- ...processor_wrong_binding_configuration.json | 26 --- ...eral_k8s_pre_processors_configuration.json | 35 ---- ..._without_some_arguments_configuration.json | 29 --- 7 files changed, 105 insertions(+), 347 deletions(-) delete mode 100644 tests/utils/cli/k8s_pre_processor_configuration.json delete mode 100644 tests/utils/cli/k8s_pre_processor_with_reused_puller_in_bindings_configuration.json delete mode 100644 tests/utils/cli/k8s_pre_processor_wrong_binding_configuration.json delete mode 100644 tests/utils/cli/several_k8s_pre_processors_configuration.json delete mode 100644 tests/utils/cli/several_k8s_pre_processors_without_some_arguments_configuration.json diff --git a/tests/unit/cli/conftest.py b/tests/unit/cli/conftest.py index 324d88e9..7c041bec 100644 --- a/tests/unit/cli/conftest.py +++ b/tests/unit/cli/conftest.py @@ -80,23 +80,6 @@ def several_inputs_outputs_stream_socket_without_some_arguments_config(several_i return several_inputs_outputs_stream_config -@pytest.fixture -def several_k8s_pre_processors_config(): - """ - Configuration with several k8s processors - """ - return load_configuration_from_json_file(file_name='several_k8s_pre_processors_configuration.json') - - -@pytest.fixture -def several_k8s_pre_processors_without_some_arguments_config(): - """ - Configuration with several k8s processors - """ - return load_configuration_from_json_file( - file_name='several_k8s_pre_processors_without_some_arguments_configuration.json') - - @pytest.fixture def csv_io_postmortem_config(invalid_csv_io_stream_config): """ @@ -419,124 +402,12 @@ def empty_pre_processor_config(pre_processor_complete_configuration): return pre_processor_complete_configuration -@pytest.fixture(params=['k8s_pre_processor_wrong_binding_configuration.json']) -def pre_processor_wrong_binding_configuration(request): - """ - Return a dictionary containing wrong bindings with a pre-processor - """ - return load_configuration_from_json_file(file_name=request.param) - - @pytest.fixture(params=['k8s_pre_processor_with_non_existing_puller_configuration.json']) def pre_processor_with_unexisting_puller_configuration(request): """ Return a dictionary containing a pre-processor with a puller that doesn't exist """ - return load_configuration_from_json_file( - file_name=request.param) - - -@pytest.fixture(params=['k8s_pre_processor_with_reused_puller_in_bindings_configuration.json']) -def pre_processor_with_reused_puller_in_bindings_configuration(request): - """ - Return a dictionary containing a pre-processor with a puller that doesn't exist - """ - return load_configuration_from_json_file( - file_name=request.param) - - -@pytest.fixture -def pre_processor_pullers_and_processors_dictionaries(pre_processor_complete_configuration): - """ - Return a dictionary which contains puller actors, a dictionary of processors as well as the pushers - """ - return get_pre_processor_pullers_and_processors_dictionaries_from_configuration( - configuration=pre_processor_complete_configuration) - - -def get_pre_processor_pullers_and_processors_dictionaries_from_configuration(configuration: dict) -> (dict, dict, dict): - """ - Return a tuple of dictionaries (pullers, processors) created from the given configuration. - :param dict configuration : Dictionary containing the configuration - """ - report_filter = BroadcastReportFilter() - puller_generator = PullerGenerator(report_filter=report_filter) - pullers = puller_generator.generate(main_config=configuration) - - pusher_generator = PusherGenerator() - pushers = pusher_generator.generate(main_config=configuration) - - route_table = RouteTable() - - dispatcher = DispatcherActor('dispatcher', None, pushers, route_table) - - report_filter.register(lambda msg: True, dispatcher.get_proxy()) - - processor_generator = PreProcessorGenerator() - processors = processor_generator.generate(main_config=configuration) - - return pullers, processors, pushers - - -@pytest.fixture -def pre_processor_binding_manager(pre_processor_complete_configuration, pre_processor_pullers_and_processors_dictionaries): - """ - Return a ProcessorBindingManager with a Processor - """ - pullers = pre_processor_pullers_and_processors_dictionaries[0] - processors = pre_processor_pullers_and_processors_dictionaries[1] - - return PreProcessorBindingManager( - config=pre_processor_complete_configuration, - pullers=pullers, - processors=processors - ) - - -@pytest.fixture -def pre_processor_binding_manager_with_wrong_binding_types(pre_processor_wrong_binding_configuration): - """ - Return a PreProcessorBindingManager with wrong target for the pre-processor (a pusher instead of a puller) - """ - _, processors, pushers = get_pre_processor_pullers_and_processors_dictionaries_from_configuration( - configuration=pre_processor_wrong_binding_configuration) - - return PreProcessorBindingManager( - config=pre_processor_wrong_binding_configuration, - pullers=pushers, - processors=processors - ) - - -@pytest.fixture -def pre_processor_binding_manager_with_unexisting_puller(pre_processor_with_unexisting_puller_configuration): - """ - Return a PreProcessorBindingManager with an unexisting target for the pre-processor (a puller that doesn't exist) - """ - pullers, processors, _ = get_pre_processor_pullers_and_processors_dictionaries_from_configuration( - configuration=pre_processor_with_unexisting_puller_configuration) - - return PreProcessorBindingManager( - config=pre_processor_with_unexisting_puller_configuration, - pullers=pullers, - processors=processors - ) - - -@pytest.fixture -def pre_processor_binding_manager_with_reused_puller_in_bindings( - pre_processor_with_reused_puller_in_bindings_configuration): - """ - Return a PreProcessorBindingManager with a puller used by two different pre-processors - """ - pullers, processors, _ = get_pre_processor_pullers_and_processors_dictionaries_from_configuration( - configuration=pre_processor_with_reused_puller_in_bindings_configuration) - - return PreProcessorBindingManager( - config=pre_processor_with_reused_puller_in_bindings_configuration, - pullers=pullers, - processors=processors - ) + return load_configuration_from_json_file(request.param) def get_config_with_longest_argument_names(config: dict, arguments: dict): diff --git a/tests/unit/cli/test_binding_manager.py b/tests/unit/cli/test_binding_manager.py index e7ceb80d..97b8f7e2 100644 --- a/tests/unit/cli/test_binding_manager.py +++ b/tests/unit/cli/test_binding_manager.py @@ -27,143 +27,161 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import copy +from unittest.mock import Mock import pytest +from powerapi.actor import ActorProxy from powerapi.cli.binding_manager import PreProcessorBindingManager -from powerapi.exception import UnsupportedActorTypeException, UnexistingActorException, TargetActorAlreadyUsed +from powerapi.cli.generator import PreProcessorGenerator, PullerGenerator +from powerapi.dispatcher import DispatcherActor +from powerapi.exception import UnexistingActorException, UnsupportedActorTypeException, TargetActorAlreadyUsed +from powerapi.filter import BroadcastReportFilter +from powerapi.processor.processor_actor import PreProcessorActor, ProcessorActor +from powerapi.puller import PullerActor -def test_create_pre_processor_binding_manager_with_actors(pre_processor_complete_configuration, pre_processor_pullers_and_processors_dictionaries): +class NoopPreprocessor(PreProcessorActor): """ - Test that a PreProcessorBindingManager is correctly created when an actor and a processor dictionary are provided + Pre-processor actor that does nothing, for testing purposes. """ - expected_actors_dictionary = copy.copy(pre_processor_pullers_and_processors_dictionaries[0]) - expected_processors_dictionary = copy.copy(pre_processor_pullers_and_processors_dictionaries[1]) - binding_manager = PreProcessorBindingManager( - config=pre_processor_complete_configuration, - pullers=pre_processor_pullers_and_processors_dictionaries[0], - processors=pre_processor_pullers_and_processors_dictionaries[1] - ) - assert binding_manager.actors == expected_actors_dictionary - assert binding_manager.processors == expected_processors_dictionary +@pytest.fixture +def preprocessor_config(): + """ + Fixture that provides a configuration with a socket puller and a no-op pre-processor. + """ + return { + 'verbose': True, + 'stream': True, + 'input': { + 'pytest-socket-puller': { + 'type': 'socket', + 'model': 'HWPCReport', + 'host': 'localhost', + 'port': 8889 + } + }, + 'pre-processor': { + 'pytest-noop-preprocessor': { + 'type': 'noop', + 'puller': 'pytest-socket-puller' + } + } + } -def test_create_processor_binding_manager_without_actors(): +@pytest.fixture +def puller_generator() -> PullerGenerator: """ - Test that a ProcessorBindingManager is correctly created without a dictionary + Fixture that provides a generator for the puller actors. """ - binding_manager = PreProcessorBindingManager(config={}, pullers=None, processors=None) + report_filter = BroadcastReportFilter() + report_filter.register(lambda _: True, Mock(actor_name='pytest-dispatcher', actor_type=DispatcherActor, spec=ActorProxy)) - assert len(binding_manager.actors) == 0 - assert len(binding_manager.processors) == 0 + generator = PullerGenerator(report_filter) + return generator -def test_process_bindings_for_pre_processor(pre_processor_complete_configuration, pre_processor_pullers_and_processors_dictionaries): +@pytest.fixture +def preprocessor_generator() -> PreProcessorGenerator: """ - Test that the bindings between a puller and a processor are correctly created + Fixture to return a preprocessor generator with the noop preprocessor available. """ - pullers = pre_processor_pullers_and_processors_dictionaries[0] - processors = pre_processor_pullers_and_processors_dictionaries[1] - binding_manager = PreProcessorBindingManager(pre_processor_complete_configuration, pullers, processors) - - dispatchers = list(pullers['one_puller'].report_filter.dispatchers()) - assert len(dispatchers) == 1 - assert dispatchers[0].actor_name == 'dispatcher' + generator = PreProcessorGenerator() - binding_manager.process_bindings() - - dispatchers = list(pullers['one_puller'].report_filter.dispatchers()) - assert len(dispatchers) == 1 - assert dispatchers[0].actor_name == 'my_processor' + def _noop_processor_factory(_) -> ProcessorActor: + return NoopPreprocessor('pytest-noop-preprocessor') - assert len(processors['my_processor'].target_actors) == 1 - assert processors['my_processor'].target_actors[0].actor_name == 'dispatcher' + generator.add_processor_factory('noop', _noop_processor_factory) + return generator -def test_process_bindings_for_pre_processor_raise_exception_with_wrong_binding_types(pre_processor_binding_manager_with_wrong_binding_types): +def test_preprocessor_binding_manager_replace_puller_target(puller_generator, preprocessor_generator, preprocessor_config): """ - Test that an exception is raised with a wrong type for the from actor in a binding + The binding manager should replace the target of the puller actor from a dispatcher actor to the pre-processor actor. """ + pullers = puller_generator.generate(preprocessor_config) + processors = preprocessor_generator.generate(preprocessor_config) + binding_manager = PreProcessorBindingManager(preprocessor_config, pullers, processors) - with pytest.raises(UnsupportedActorTypeException): - pre_processor_binding_manager_with_wrong_binding_types.process_bindings() + puller = pullers['pytest-socket-puller'] + assert isinstance(puller, PullerActor) + puller_target_proxy, = puller.report_filter.dispatchers() + assert issubclass(puller_target_proxy.actor_type, DispatcherActor) + assert puller_target_proxy.actor_name == 'pytest-dispatcher' -def test_process_bindings_for_pre_processor_raise_exception_with_no_existing_puller( - pre_processor_binding_manager_with_unexisting_puller): - """ - Test that an exception is raised with a puller that doesn't exist - """ + binding_manager.process_bindings() - with pytest.raises(UnexistingActorException): - pre_processor_binding_manager_with_unexisting_puller.process_bindings() + puller_target_proxy, = puller.report_filter.dispatchers() + assert issubclass(puller_target_proxy.actor_type, PreProcessorActor) + assert puller_target_proxy.actor_name == 'pytest-noop-preprocessor' -def test_process_bindings_for_pre_processor_raise_exception_with_reused_puller_in_bindings( - pre_processor_binding_manager_with_reused_puller_in_bindings): +def test_preprocessor_binding_manager_with_empty_preprocessor_config(puller_generator, preprocessor_config): """ - Test that an exception is raised when the same puller is used by several processors + The binding manager should not replace the target of the puller actor when there is no pre-processor configuration. """ + preprocessor_config['pre-processor'] = {} - with pytest.raises(TargetActorAlreadyUsed): - pre_processor_binding_manager_with_reused_puller_in_bindings.process_bindings() + pullers = puller_generator.generate(preprocessor_config) + processors = {} + binding_manager = PreProcessorBindingManager(preprocessor_config, pullers, processors) + puller = pullers['pytest-socket-puller'] + assert isinstance(puller, PullerActor) -def test_check_processors_targets_are_unique_raise_exception_with_reused_puller_in_bindings( - pre_processor_binding_manager_with_reused_puller_in_bindings): - """ - Test that an exception is raised when the same puller is used by several processors - """ - with pytest.raises(TargetActorAlreadyUsed): - pre_processor_binding_manager_with_reused_puller_in_bindings.check_processors_targets_are_unique() + puller_target_proxy, = puller.report_filter.dispatchers() + assert issubclass(puller_target_proxy.actor_type, DispatcherActor) + assert puller_target_proxy.actor_name == 'pytest-dispatcher' + binding_manager.process_bindings() -def test_check_processors_targets_are_unique_pass_without_reused_puller_in_bindings( - pre_processor_binding_manager): - """ - Test that a correct without repeated target passes the validation - """ - try: - pre_processor_binding_manager.check_processors_targets_are_unique() - except TargetActorAlreadyUsed: - pytest.fail("Processors targets are not unique") + puller_target_proxy, = puller.report_filter.dispatchers() + assert issubclass(puller_target_proxy.actor_type, DispatcherActor) + assert puller_target_proxy.actor_name == 'pytest-dispatcher' -def check_all_processors_targets(preprocessor_binding_manager: PreProcessorBindingManager): +def test_preprocessor_binding_manager_with_invalid_puller_name(puller_generator, preprocessor_generator, preprocessor_config): """ - Helper function that checks the processors targets of the given binding manager. + The binding manager should raise an exception when the puller name of the pre-processor is invalid. """ - for processor_name, processor in preprocessor_binding_manager.processors.items(): - preprocessor_binding_manager.check_processor_targets(processor_name, processor) + preprocessor_config['pre-processor']['pytest-noop-preprocessor']['puller'] = 'invalid-puller-target' + pullers = puller_generator.generate(preprocessor_config) + processors = preprocessor_generator.generate(preprocessor_config) + binding_manager = PreProcessorBindingManager(preprocessor_config, pullers, processors) -def test_check_processor_targets_raise_exception_with_no_existing_puller(pre_processor_binding_manager_with_unexisting_puller): - """ - Test that an exception is raised with a puller that doesn't exist - """ with pytest.raises(UnexistingActorException): - check_all_processors_targets(pre_processor_binding_manager_with_unexisting_puller) + binding_manager.process_bindings() -def test_check_processor_targets_raise_exception_with_raise_exception_with_wrong_binding_types(pre_processor_binding_manager_with_wrong_binding_types): +def test_preprocessor_binding_manager_with_invalid_puller_type(puller_generator, preprocessor_generator, preprocessor_config): """ - Test that an exception is raised with a puller that doesn't exist + The binding manager should raise an exception when the puller type of the pre-processor is invalid. """ + pullers = puller_generator.generate(preprocessor_config) + pullers['pytest-socket-puller'] = Mock(name='pytest-socket-puller') + + processors = preprocessor_generator.generate(preprocessor_config) + binding_manager = PreProcessorBindingManager(preprocessor_config, pullers, processors) + with pytest.raises(UnsupportedActorTypeException): - check_all_processors_targets(pre_processor_binding_manager_with_wrong_binding_types) + binding_manager.process_bindings() -def test_check_processor_targets_pass_with_correct_targets(pre_processor_binding_manager): +def test_preprocessor_binding_manager_with_duplicate_target(puller_generator, preprocessor_generator, preprocessor_config): """ - Test that validation of a configuration with existing targets of the correct type + The binding manager should raise an exception when multiple pre-processor have the same target. """ - try: - check_all_processors_targets(pre_processor_binding_manager) - except UnsupportedActorTypeException as e: - pytest.fail(f'Unsupported actor type: {e}') - except UnexistingActorException as e: - pytest.fail(f'Actor does not exist: {e}') + pre_processors = preprocessor_config['pre-processor'] + pre_processors['pytest-noop-preprocessor-duplicate'] = pre_processors['pytest-noop-preprocessor'] + + pullers = puller_generator.generate(preprocessor_config) + processors = preprocessor_generator.generate(preprocessor_config) + binding_manager = PreProcessorBindingManager(preprocessor_config, pullers, processors) + + with pytest.raises(TargetActorAlreadyUsed): + binding_manager.process_bindings() diff --git a/tests/utils/cli/k8s_pre_processor_configuration.json b/tests/utils/cli/k8s_pre_processor_configuration.json deleted file mode 100644 index 3abb4fa7..00000000 --- a/tests/utils/cli/k8s_pre_processor_configuration.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "verbose": true, - "pre-processor": { - "my_processor": { - "type": "k8s", - "api-mode": "manual", - "puller": "my_puller" - } - } -} diff --git a/tests/utils/cli/k8s_pre_processor_with_reused_puller_in_bindings_configuration.json b/tests/utils/cli/k8s_pre_processor_with_reused_puller_in_bindings_configuration.json deleted file mode 100644 index 8ba84acf..00000000 --- a/tests/utils/cli/k8s_pre_processor_with_reused_puller_in_bindings_configuration.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "verbose": true, - "stream": true, - "input": { - "one_puller": { - "model": "HWPCReport", - "type": "socket", - "host": "localhost", - "port": 8889 - } - }, - "output": { - "one_pusher": { - "type": "json", - "model": "PowerReport", - "filepath": "/tmp/pytest-powerapi-powerrep.jsonl" - } - }, - "pre-processor": { - "my_processor": { - "type": "k8s", - "api-mode": "manual", - "puller": "one_puller" - }, - "my_processor_2": { - "type": "k8s", - "api-mode": "manual", - "puller": "one_puller" - } - } -} diff --git a/tests/utils/cli/k8s_pre_processor_wrong_binding_configuration.json b/tests/utils/cli/k8s_pre_processor_wrong_binding_configuration.json deleted file mode 100644 index fae7ce5a..00000000 --- a/tests/utils/cli/k8s_pre_processor_wrong_binding_configuration.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "verbose": true, - "stream": true, - "input": { - "one_puller": { - "model": "HWPCReport", - "type": "socket", - "host": "localhost", - "port": 8889 - } - }, - "output": { - "one_pusher": { - "type": "json", - "model": "PowerReport", - "filepath": "/tmp/pytest-powerapi-powerrep.jsonl" - } - }, - "pre-processor": { - "my_processor": { - "type": "k8s", - "api-mode": "manual", - "puller": "one_pusher" - } - } -} diff --git a/tests/utils/cli/several_k8s_pre_processors_configuration.json b/tests/utils/cli/several_k8s_pre_processors_configuration.json deleted file mode 100644 index 0ba7a937..00000000 --- a/tests/utils/cli/several_k8s_pre_processors_configuration.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "verbose": true, - "pre-processor": { - "my_processor_1": { - "type": "k8s", - "api-mode": "manual", - "puller": "my_puller_1" - }, - "my_processor_2": { - "type": "k8s", - "api-mode": "manual", - "puller": "my_puller_2" - }, - "my_processor_3": { - "type": "k8s", - "api-mode": "manual", - "puller": "my_puller_3" - }, - "my_processor_4": { - "type": "k8s", - "api-mode": "manual", - "puller": "my_puller_4" - }, - "my_processor_5": { - "type": "k8s", - "api-mode": "manual", - "puller": "my_puller_5" - }, - "my_processor_6": { - "type": "k8s", - "api-mode": "manual", - "puller": "my_puller_6" - } - } -} diff --git a/tests/utils/cli/several_k8s_pre_processors_without_some_arguments_configuration.json b/tests/utils/cli/several_k8s_pre_processors_without_some_arguments_configuration.json deleted file mode 100644 index 1f068c53..00000000 --- a/tests/utils/cli/several_k8s_pre_processors_without_some_arguments_configuration.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "verbose": true, - "pre-processor": { - "my_processor_1": { - "type": "k8s", - "puller": "my_puller_1" - }, - "my_processor_2": { - "type": "k8s", - "puller": "my_puller_2" - }, - "my_processor_3": { - "type": "k8s", - "puller": "my_puller_3" - }, - "my_processor_4": { - "type": "k8s", - "puller": "my_puller_4" - }, - "my_processor_5": { - "type": "k8s", - "puller": "my_puller_5" - }, - "my_processor_6": { - "type": "k8s", - "puller": "my_puller_6" - } - } -} \ No newline at end of file