From 9ae6e900d420836a61eda6d71087a29ac99d2c16 Mon Sep 17 00:00:00 2001 From: Pierre Lasorak Date: Thu, 9 Jan 2025 19:15:57 +0100 Subject: [PATCH 1/6] fix a bug --- elisa-logbook/credmgr.py | 1 + 1 file changed, 1 insertion(+) diff --git a/elisa-logbook/credmgr.py b/elisa-logbook/credmgr.py index 7098df42..b0f2ffea 100644 --- a/elisa-logbook/credmgr.py +++ b/elisa-logbook/credmgr.py @@ -133,6 +133,7 @@ def __init__(self, service:str, username:str, password:str, realm:str): self.username = username self.password = password self.realm = realm + self.log = logging.getLogger(self.__class__.__name__) def generate_cern_sso_cookie(self, website, kerberos_directory, output_directory): env = {'KRB5CCNAME': f'DIR:{kerberos_directory}'} From 8baa986fee2243988118fe66fea183b36f3c134f Mon Sep 17 00:00:00 2001 From: Pierre Lasorak Date: Thu, 9 Jan 2025 19:16:14 +0100 Subject: [PATCH 2/6] we actually dont need the title here --- elisa-logbook/logbook.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elisa-logbook/logbook.py b/elisa-logbook/logbook.py index c8cdb85d..adc0efe5 100644 --- a/elisa-logbook/logbook.py +++ b/elisa-logbook/logbook.py @@ -158,10 +158,10 @@ def new_message(): @app.route('/v1/elisaLogbook/reply_to_message/', methods=["PUT"]) @auth.login_required def reply_to_message(): - if request.json.get('body', "") == "" or request.json.get('title', "") == "" or request.json.get('command', "") == "" or request.json.get('author', "") == "" or request.json.get('id', "") == "": + if request.json.get('body', "") == "" or request.json.get('command', "") == "" or request.json.get('author', "") == "" or request.json.get('id', "") == "": resp = make_response( jsonify( - response = "Body, title, command, author or id cannot be empty!", + response = "Body, command, author or id cannot be empty!", sent_data = request.json ) ) From 733da69e8ee6016ebd3424e53cbbff3107c377c1 Mon Sep 17 00:00:00 2001 From: Pierre Lasorak Date: Thu, 9 Jan 2025 19:16:36 +0100 Subject: [PATCH 3/6] add the tests --- elisa-logbook/README.md | 21 +++ elisa-logbook/conftest.py | 10 ++ elisa-logbook/test_elisa_microservice.py | 218 +++++++++++++++++++++++ 3 files changed, 249 insertions(+) create mode 100644 elisa-logbook/README.md create mode 100644 elisa-logbook/conftest.py create mode 100644 elisa-logbook/test_elisa_microservice.py diff --git a/elisa-logbook/README.md b/elisa-logbook/README.md new file mode 100644 index 00000000..b9ed03f4 --- /dev/null +++ b/elisa-logbook/README.md @@ -0,0 +1,21 @@ +# Elisa Logbook Microservice +## To test the microservice +On a machine at CERN: + - Clone the repository `git clone https://github.com/DUNE-DAQ/microservices.git` + - Clone the repository `git clone https://github.com/DUNE-DAQ/elisa_client_api.git` + - Create a virtual environment `python3 -m venv .venv` + - Activate the virtual environment `source .venv/bin/activate` + - Install the dependencies `pip install -r microservices/dockerfiles/requirements.txt` + - Install the elisa_client_api `pip install ./elisa_client_api` + - `cd` into this directory: `cd microservices/elisa-logbook` + - Run `pytest` + - Once done, deactivate the virtual environment: `deactivate` + +If you don't want to be spamming ELisA logbook, you can run only one test for example: +```bash +python -m pytest -k test_elisa_microservice_conf +``` +or choose a specific system: +```bash +python -m pytest --system pddp -k test_elisa_microservice_locally +``` diff --git a/elisa-logbook/conftest.py b/elisa-logbook/conftest.py new file mode 100644 index 00000000..34b206e9 --- /dev/null +++ b/elisa-logbook/conftest.py @@ -0,0 +1,10 @@ + +def pytest_addoption(parser): + parser.addoption( + '--system', + action="store", + type=str, + nargs='+', + default=['npvd', 'pdsp', 'pddp'], + help="Name of the system(s) to test" + ) \ No newline at end of file diff --git a/elisa-logbook/test_elisa_microservice.py b/elisa-logbook/test_elisa_microservice.py new file mode 100644 index 00000000..afe6fd87 --- /dev/null +++ b/elisa-logbook/test_elisa_microservice.py @@ -0,0 +1,218 @@ +import getpass +import json +import multiprocessing +import os +import pytest +import requests +import socket +import time + +from elisa import ElisaLogbook +from credmgr import CERNSessionHandler, credentials + + +@pytest.fixture +def elisa_logbook_config(): + with open('elisaconf.json', 'r') as f: + return json.load(f) + + +@pytest.fixture +def elisa_microservice_config(): + with open(os.path.expanduser('~/.drunc.json'), 'r') as f: + return json.load(f) + + +@pytest.fixture +def elisa_microservice_local_config(): # a configuration to start the microservice locally + with open(os.path.expanduser('~/.drunc.json'), 'r') as f: + data = json.load(f) + updated_data = {} + port = 61263 + for key, values in data['elisa_configuration'].items(): + values['socket'] = f'http://0.0.0.0:{port}' + port += 1 + updated_data[key] = values + data['elisa_configuration'] = updated_data + return data + + +@pytest.fixture +def systems(request): + return request.config.getoption("--system") + + +@pytest.fixture +def cern_session(): + user_var = os.getenv("USERNAME").strip("\n") + pass_var = os.getenv("PASSWORD").strip("\n") + credentials.add_login("elisa", user_var, pass_var, "CERN.CH") + return CERNSessionHandler(user_var) + + +@pytest.fixture() +def local_microservice(elisa_microservice_local_config): + + def run_app(which_system): + # Configure app for testing + + socket = elisa_microservice_local_config['elisa_configuration'][which_system]['socket'] + port = int(socket.split(':')[-1]) + host = ":".join(socket.split(':')[:-1]) + + def run_with_env(*args, **kwargs): + os.environ["HARDWARE"] = which_system + from logbook import app + app.config['TESTING'] = True + app.run(*args, **kwargs) + + # Start Flask in a separate process + process = multiprocessing.Process( + target = run_with_env, + kwargs = { + 'port': port, + 'host': host.replace('http://', '').replace('https://', ''), + 'debug': True, + } + ) + process.daemon = True + process.start() + + # Give the server a moment to start + time.sleep(1) + return process + + return run_app + + +def test_elisa_configuration( + systems, + elisa_logbook_config): + + for system in systems: + assert system in elisa_logbook_config + + system_config = elisa_logbook_config[system] + + assert 'connection' in system_config + assert 'website' in system_config + assert 'attributes' in system_config + messages_kind = ['start', 'stop', 'message'] + + for message_kind in messages_kind: + assert message_kind in system_config['attributes'] + + assert 'type' in system_config['attributes'][message_kind] + assert 'set_on_reply' in system_config['attributes'][message_kind]['type'] + assert 'set_on_new_thread' in system_config['attributes'][message_kind]['type'] + assert 'value' in system_config['attributes'][message_kind]['type'] + + assert ('RunControl_MessageType' in system_config['attributes'][message_kind] or + 'Automatic_Message_Type' in system_config['attributes'][message_kind]) + + if 'RunControl_MessageType' in system_config['attributes'][message_kind]: + assert 'set_on_reply' in system_config['attributes'][message_kind]['RunControl_MessageType'] + assert 'set_on_new_thread' in system_config['attributes'][message_kind]['RunControl_MessageType'] + assert 'value' in system_config['attributes'][message_kind]['RunControl_MessageType'] + + elif 'Automatic_Message_Type' in system_config['attributes'][message_kind]: + assert 'set_on_reply' in system_config['attributes'][message_kind]['Automatic_Message_Type'] + assert 'set_on_new_thread' in system_config['attributes'][message_kind]['Automatic_Message_Type'] + assert 'value' in system_config['attributes'][message_kind]['Automatic_Message_Type'] + + +def test_elisa_logbook( + systems, + elisa_logbook_config, + cern_session): + + for system in systems: + + elisa_logbook = ElisaLogbook( + elisa_logbook_config[system], + cern_session + ) + assert elisa_logbook is not None + thread_id = elisa_logbook.start_new_thread( + "unit-test", + "test_elisa_logbook: Ignore this message", + "start", + getpass.getuser(), + ['DAQ'] + ) + assert thread_id is not None + thread_id_response = elisa_logbook.reply( + "test_elisa_logbook: Ignore this message", + "stop", + getpass.getuser(), + ['DAQ'], + thread_id + ) + assert thread_id_response is not None + + +def test_elisa_microservice_conf( + systems, + elisa_microservice_config): + + assert "elisa_configuration" in elisa_microservice_config + + elisa_configuration = elisa_microservice_config["elisa_configuration"] + + for system in systems: + assert system in elisa_configuration + + system_config = elisa_configuration[system] + + assert "socket" in system_config + assert "user" in system_config + assert "password" in system_config + + + +def test_elisa_microservice_locally( + systems, + elisa_microservice_local_config, + local_microservice): + + elisa_configuration = elisa_microservice_local_config["elisa_configuration"] + + for system in systems: + lm = local_microservice(system) + system_config = elisa_configuration[system] + usvc_socket = system_config['socket'].replace('0.0.0.0', socket.gethostname()) + + response = requests.get(usvc_socket + '/') + response.raise_for_status() + assert response.status_code == 200 + + response = requests.post( + usvc_socket + '/v1/elisaLogbook/new_message/', + json={ + 'title': 'unit-test', + 'body': 'test_elisa_microservice_locally: Ignore this message', + 'command': 'start', + 'author': getpass.getuser(), + 'systems': ['DAQ'] + }, + auth=(system_config['user'], system_config['password']) + ) + response.raise_for_status() + response_json = response.json() + assert response_json['response'] == "Message thread started successfully" + assert response_json['thread_id'] is not None + response = requests.put( + usvc_socket + '/v1/elisaLogbook/reply_to_message/', + json={ + 'id': response_json['thread_id'], + 'body': 'test_elisa_microservice_locally: Ignore this message', + 'command': 'stop', + 'author': getpass.getuser(), + 'systems': ['DAQ'] + }, + auth=(system_config['user'], system_config['password']) + ) + response.raise_for_status() + response_json = response.json() + assert response_json['response'] == "Message thread replied successfully" + From 68a2730e57e8b33518564a3520f2d2b1b49c1c00 Mon Sep 17 00:00:00 2001 From: Pierre Lasorak Date: Thu, 9 Jan 2025 19:21:34 +0100 Subject: [PATCH 4/6] update the readme --- elisa-logbook/README.md | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/elisa-logbook/README.md b/elisa-logbook/README.md index b9ed03f4..e94cd766 100644 --- a/elisa-logbook/README.md +++ b/elisa-logbook/README.md @@ -1,17 +1,23 @@ # Elisa Logbook Microservice ## To test the microservice On a machine at CERN: - - Clone the repository `git clone https://github.com/DUNE-DAQ/microservices.git` - - Clone the repository `git clone https://github.com/DUNE-DAQ/elisa_client_api.git` - - Create a virtual environment `python3 -m venv .venv` - - Activate the virtual environment `source .venv/bin/activate` - - Install the dependencies `pip install -r microservices/dockerfiles/requirements.txt` - - Install the elisa_client_api `pip install ./elisa_client_api` - - `cd` into this directory: `cd microservices/elisa-logbook` - - Run `pytest` - - Once done, deactivate the virtual environment: `deactivate` +```bash +git clone https://github.com/DUNE-DAQ/microservices.git # Clone this repository +git clone https://github.com/DUNE-DAQ/elisa_client_api.git # Clone the elisa_client_api repository +python3 -m venv .venv # Create a virtual environment +source .venv/bin/activate # Activate the virtual environment +pip install -r microservices/dockerfiles/requirements.txt # Install the dependencies +pip install ./elisa_client_api # Install the elisa_client_api +pip install pytest # Install pytest +cd microservices/elisa-logbook +USERNAME=something PASSWORD=something python -m pytest -s --system pdsp -k test_elisa_microservice_locally +# After running the tests: +deactivate +``` +You need to provide the service account username and password for the ELisA logbook. Ask Pierre Lasorak for the credentials. + -If you don't want to be spamming ELisA logbook, you can run only one test for example: +If you don't want to be spamming the ELisA logbook too much, you can run a subset of tests for example: ```bash python -m pytest -k test_elisa_microservice_conf ``` From a5da140e19a245e2a585ba0739afd9196331bb29 Mon Sep 17 00:00:00 2001 From: Pierre Lasorak Date: Thu, 9 Jan 2025 19:21:54 +0100 Subject: [PATCH 5/6] add a gitignore --- elisa-logbook/.gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 elisa-logbook/.gitignore diff --git a/elisa-logbook/.gitignore b/elisa-logbook/.gitignore new file mode 100644 index 00000000..e40b57d4 --- /dev/null +++ b/elisa-logbook/.gitignore @@ -0,0 +1,2 @@ +./.venv/ +./elisa_client_api/ \ No newline at end of file From f6b40062fa630f2a6d76f7bbbef8c292299b6b0c Mon Sep 17 00:00:00 2001 From: Pierre Lasorak Date: Thu, 9 Jan 2025 19:22:46 +0100 Subject: [PATCH 6/6] small update --- elisa-logbook/.gitignore | 4 ++-- elisa-logbook/README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/elisa-logbook/.gitignore b/elisa-logbook/.gitignore index e40b57d4..d4097a4c 100644 --- a/elisa-logbook/.gitignore +++ b/elisa-logbook/.gitignore @@ -1,2 +1,2 @@ -./.venv/ -./elisa_client_api/ \ No newline at end of file +.venv/ +elisa_client_api/ \ No newline at end of file diff --git a/elisa-logbook/README.md b/elisa-logbook/README.md index e94cd766..ab338ed2 100644 --- a/elisa-logbook/README.md +++ b/elisa-logbook/README.md @@ -3,13 +3,13 @@ On a machine at CERN: ```bash git clone https://github.com/DUNE-DAQ/microservices.git # Clone this repository +cd microservices/elisa-logbook git clone https://github.com/DUNE-DAQ/elisa_client_api.git # Clone the elisa_client_api repository python3 -m venv .venv # Create a virtual environment source .venv/bin/activate # Activate the virtual environment pip install -r microservices/dockerfiles/requirements.txt # Install the dependencies pip install ./elisa_client_api # Install the elisa_client_api pip install pytest # Install pytest -cd microservices/elisa-logbook USERNAME=something PASSWORD=something python -m pytest -s --system pdsp -k test_elisa_microservice_locally # After running the tests: deactivate