From b2423a3ac48019d1b290f5830c4be3efc9698706 Mon Sep 17 00:00:00 2001 From: John Freeman Date: Sat, 1 Nov 2025 10:03:44 -0500 Subject: [PATCH 1/8] JCF: Issue #66: standardize the Python code by moving *.py files into src/druncschema and removing version specifications from pyproject.toml --- pyproject.toml | 27 +++++++++---------- .../druncschema/__init__.py | 0 .../druncschema/apps/__init__.py | 0 .../apps/__main_generate_protos__.py | 0 .../druncschema/authoriser_pb2.py | 0 .../druncschema/authoriser_pb2.pyi | 0 .../druncschema/authoriser_pb2_grpc.py | 0 .../druncschema/broadcast_pb2.py | 0 .../druncschema/broadcast_pb2.pyi | 0 .../druncschema/broadcast_pb2_grpc.py | 0 .../druncschema/controller_pb2.py | 0 .../druncschema/controller_pb2.pyi | 0 .../druncschema/controller_pb2_grpc.py | 0 .../druncschema/description_pb2.py | 0 .../druncschema/description_pb2.pyi | 0 .../druncschema/description_pb2_grpc.py | 0 .../druncschema/generic_pb2.py | 0 .../druncschema/generic_pb2.pyi | 0 .../druncschema/generic_pb2_grpc.py | 0 .../druncschema/opmon/FSM_pb2.py | 0 .../druncschema/opmon/FSM_pb2.pyi | 0 .../druncschema/opmon/FSM_pb2_grpc.py | 0 .../druncschema/opmon/__init__.py | 0 .../druncschema/opmon/generic_pb2.py | 0 .../druncschema/opmon/generic_pb2.pyi | 0 .../druncschema/opmon/generic_pb2_grpc.py | 0 .../druncschema/opmon/process_manager_pb2.py | 0 .../druncschema/opmon/process_manager_pb2.pyi | 0 .../opmon/process_manager_pb2_grpc.py | 0 .../druncschema/process_manager_pb2.py | 0 .../druncschema/process_manager_pb2.pyi | 0 .../druncschema/process_manager_pb2_grpc.py | 0 .../druncschema/py.typed | 0 .../druncschema/request_response_pb2.py | 0 .../druncschema/request_response_pb2.pyi | 0 .../druncschema/request_response_pb2_grpc.py | 0 .../druncschema/session_manager_pb2.py | 0 .../druncschema/session_manager_pb2.pyi | 0 .../druncschema/session_manager_pb2_grpc.py | 0 .../druncschema/token_pb2.py | 0 .../druncschema/token_pb2.pyi | 0 .../druncschema/token_pb2_grpc.py | 0 42 files changed, 13 insertions(+), 14 deletions(-) rename {python_not_for_dunedaq => src}/druncschema/__init__.py (100%) rename {python_not_for_dunedaq => src}/druncschema/apps/__init__.py (100%) rename {python_not_for_dunedaq => src}/druncschema/apps/__main_generate_protos__.py (100%) rename {python_not_for_dunedaq => src}/druncschema/authoriser_pb2.py (100%) rename {python_not_for_dunedaq => src}/druncschema/authoriser_pb2.pyi (100%) rename {python_not_for_dunedaq => src}/druncschema/authoriser_pb2_grpc.py (100%) rename {python_not_for_dunedaq => src}/druncschema/broadcast_pb2.py (100%) rename {python_not_for_dunedaq => src}/druncschema/broadcast_pb2.pyi (100%) rename {python_not_for_dunedaq => src}/druncschema/broadcast_pb2_grpc.py (100%) rename {python_not_for_dunedaq => src}/druncschema/controller_pb2.py (100%) rename {python_not_for_dunedaq => src}/druncschema/controller_pb2.pyi (100%) rename {python_not_for_dunedaq => src}/druncschema/controller_pb2_grpc.py (100%) rename {python_not_for_dunedaq => src}/druncschema/description_pb2.py (100%) rename {python_not_for_dunedaq => src}/druncschema/description_pb2.pyi (100%) rename {python_not_for_dunedaq => src}/druncschema/description_pb2_grpc.py (100%) rename {python_not_for_dunedaq => src}/druncschema/generic_pb2.py (100%) rename {python_not_for_dunedaq => src}/druncschema/generic_pb2.pyi (100%) rename {python_not_for_dunedaq => src}/druncschema/generic_pb2_grpc.py (100%) rename {python_not_for_dunedaq => src}/druncschema/opmon/FSM_pb2.py (100%) rename {python_not_for_dunedaq => src}/druncschema/opmon/FSM_pb2.pyi (100%) rename {python_not_for_dunedaq => src}/druncschema/opmon/FSM_pb2_grpc.py (100%) rename {python_not_for_dunedaq => src}/druncschema/opmon/__init__.py (100%) rename {python_not_for_dunedaq => src}/druncschema/opmon/generic_pb2.py (100%) rename {python_not_for_dunedaq => src}/druncschema/opmon/generic_pb2.pyi (100%) rename {python_not_for_dunedaq => src}/druncschema/opmon/generic_pb2_grpc.py (100%) rename {python_not_for_dunedaq => src}/druncschema/opmon/process_manager_pb2.py (100%) rename {python_not_for_dunedaq => src}/druncschema/opmon/process_manager_pb2.pyi (100%) rename {python_not_for_dunedaq => src}/druncschema/opmon/process_manager_pb2_grpc.py (100%) rename {python_not_for_dunedaq => src}/druncschema/process_manager_pb2.py (100%) rename {python_not_for_dunedaq => src}/druncschema/process_manager_pb2.pyi (100%) rename {python_not_for_dunedaq => src}/druncschema/process_manager_pb2_grpc.py (100%) rename {python_not_for_dunedaq => src}/druncschema/py.typed (100%) rename {python_not_for_dunedaq => src}/druncschema/request_response_pb2.py (100%) rename {python_not_for_dunedaq => src}/druncschema/request_response_pb2.pyi (100%) rename {python_not_for_dunedaq => src}/druncschema/request_response_pb2_grpc.py (100%) rename {python_not_for_dunedaq => src}/druncschema/session_manager_pb2.py (100%) rename {python_not_for_dunedaq => src}/druncschema/session_manager_pb2.pyi (100%) rename {python_not_for_dunedaq => src}/druncschema/session_manager_pb2_grpc.py (100%) rename {python_not_for_dunedaq => src}/druncschema/token_pb2.py (100%) rename {python_not_for_dunedaq => src}/druncschema/token_pb2.pyi (100%) rename {python_not_for_dunedaq => src}/druncschema/token_pb2_grpc.py (100%) diff --git a/pyproject.toml b/pyproject.toml index 02678d3..7eb1230 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,20 +4,23 @@ build-backend = "setuptools.build_meta" [project] name = "druncschema" +description = "Schema definitions and compilation script for drunc" +version = "0.11.3" +readme = "docs/README.md" requires-python = ">=3.10" -dynamic = ["version", "readme"] dependencies = [ - "grpcio==1.68.0", - "grpcio-status==1.68.0", - "grpcio-tools==1.68.0", - "googleapis-common-protos==1.66.0" + "click", + "grpcio", + "grpcio-status", + "grpcio-tools", + "googleapis-common-protos", + "rich", ] authors = [ {name = "Pierre Lasorak", email = "p.lasorak@imperial.ac.uk"}, {name = "Pawel Plesniak", email = "pawel.plesniak15@imperial.ac.uk"}, {name = "Claudia Su", email = "claudia.su@physics.ox.ac.uk"} ] -description = "Schema definitions and compilation script for drunc" [project.optional-dependencies] dev = ["ruff", "pre-commit", "pytest", "mypy-protobuf", "types-protobuf"] @@ -29,18 +32,14 @@ Repository = "https://github.com/DUNE-DAQ/druncschema" [project.scripts] druncschema-generate-protos = "druncschema.apps.__main_generate_protos__:main" -[tool.setuptools.dynamic] -version = {attr = "druncschema.__version__"} -readme = {file = ["docs/README.md"]} - [tool.setuptools.packages.find] -where = ["python_not_for_dunedaq"] +where = ["src"] [tool.ruff] exclude = [ - "python_not_for_dunedaq/druncschema.egg-info", - "python_not_for_dunedaq/*/*_pb2.py*", - "python_not_for_dunedaq/*/*_pb2_grpc.py", + "src/druncschema.egg-info", + "src/druncschema/*_pb2.py*", + "src/druncschema/*_pb2_grpc.py", ] [tool.ruff.lint] diff --git a/python_not_for_dunedaq/druncschema/__init__.py b/src/druncschema/__init__.py similarity index 100% rename from python_not_for_dunedaq/druncschema/__init__.py rename to src/druncschema/__init__.py diff --git a/python_not_for_dunedaq/druncschema/apps/__init__.py b/src/druncschema/apps/__init__.py similarity index 100% rename from python_not_for_dunedaq/druncschema/apps/__init__.py rename to src/druncschema/apps/__init__.py diff --git a/python_not_for_dunedaq/druncschema/apps/__main_generate_protos__.py b/src/druncschema/apps/__main_generate_protos__.py similarity index 100% rename from python_not_for_dunedaq/druncschema/apps/__main_generate_protos__.py rename to src/druncschema/apps/__main_generate_protos__.py diff --git a/python_not_for_dunedaq/druncschema/authoriser_pb2.py b/src/druncschema/authoriser_pb2.py similarity index 100% rename from python_not_for_dunedaq/druncschema/authoriser_pb2.py rename to src/druncschema/authoriser_pb2.py diff --git a/python_not_for_dunedaq/druncschema/authoriser_pb2.pyi b/src/druncschema/authoriser_pb2.pyi similarity index 100% rename from python_not_for_dunedaq/druncschema/authoriser_pb2.pyi rename to src/druncschema/authoriser_pb2.pyi diff --git a/python_not_for_dunedaq/druncschema/authoriser_pb2_grpc.py b/src/druncschema/authoriser_pb2_grpc.py similarity index 100% rename from python_not_for_dunedaq/druncschema/authoriser_pb2_grpc.py rename to src/druncschema/authoriser_pb2_grpc.py diff --git a/python_not_for_dunedaq/druncschema/broadcast_pb2.py b/src/druncschema/broadcast_pb2.py similarity index 100% rename from python_not_for_dunedaq/druncschema/broadcast_pb2.py rename to src/druncschema/broadcast_pb2.py diff --git a/python_not_for_dunedaq/druncschema/broadcast_pb2.pyi b/src/druncschema/broadcast_pb2.pyi similarity index 100% rename from python_not_for_dunedaq/druncschema/broadcast_pb2.pyi rename to src/druncschema/broadcast_pb2.pyi diff --git a/python_not_for_dunedaq/druncschema/broadcast_pb2_grpc.py b/src/druncschema/broadcast_pb2_grpc.py similarity index 100% rename from python_not_for_dunedaq/druncschema/broadcast_pb2_grpc.py rename to src/druncschema/broadcast_pb2_grpc.py diff --git a/python_not_for_dunedaq/druncschema/controller_pb2.py b/src/druncschema/controller_pb2.py similarity index 100% rename from python_not_for_dunedaq/druncschema/controller_pb2.py rename to src/druncschema/controller_pb2.py diff --git a/python_not_for_dunedaq/druncschema/controller_pb2.pyi b/src/druncschema/controller_pb2.pyi similarity index 100% rename from python_not_for_dunedaq/druncschema/controller_pb2.pyi rename to src/druncschema/controller_pb2.pyi diff --git a/python_not_for_dunedaq/druncschema/controller_pb2_grpc.py b/src/druncschema/controller_pb2_grpc.py similarity index 100% rename from python_not_for_dunedaq/druncschema/controller_pb2_grpc.py rename to src/druncschema/controller_pb2_grpc.py diff --git a/python_not_for_dunedaq/druncschema/description_pb2.py b/src/druncschema/description_pb2.py similarity index 100% rename from python_not_for_dunedaq/druncschema/description_pb2.py rename to src/druncschema/description_pb2.py diff --git a/python_not_for_dunedaq/druncschema/description_pb2.pyi b/src/druncschema/description_pb2.pyi similarity index 100% rename from python_not_for_dunedaq/druncschema/description_pb2.pyi rename to src/druncschema/description_pb2.pyi diff --git a/python_not_for_dunedaq/druncschema/description_pb2_grpc.py b/src/druncschema/description_pb2_grpc.py similarity index 100% rename from python_not_for_dunedaq/druncschema/description_pb2_grpc.py rename to src/druncschema/description_pb2_grpc.py diff --git a/python_not_for_dunedaq/druncschema/generic_pb2.py b/src/druncschema/generic_pb2.py similarity index 100% rename from python_not_for_dunedaq/druncschema/generic_pb2.py rename to src/druncschema/generic_pb2.py diff --git a/python_not_for_dunedaq/druncschema/generic_pb2.pyi b/src/druncschema/generic_pb2.pyi similarity index 100% rename from python_not_for_dunedaq/druncschema/generic_pb2.pyi rename to src/druncschema/generic_pb2.pyi diff --git a/python_not_for_dunedaq/druncschema/generic_pb2_grpc.py b/src/druncschema/generic_pb2_grpc.py similarity index 100% rename from python_not_for_dunedaq/druncschema/generic_pb2_grpc.py rename to src/druncschema/generic_pb2_grpc.py diff --git a/python_not_for_dunedaq/druncschema/opmon/FSM_pb2.py b/src/druncschema/opmon/FSM_pb2.py similarity index 100% rename from python_not_for_dunedaq/druncschema/opmon/FSM_pb2.py rename to src/druncschema/opmon/FSM_pb2.py diff --git a/python_not_for_dunedaq/druncschema/opmon/FSM_pb2.pyi b/src/druncschema/opmon/FSM_pb2.pyi similarity index 100% rename from python_not_for_dunedaq/druncschema/opmon/FSM_pb2.pyi rename to src/druncschema/opmon/FSM_pb2.pyi diff --git a/python_not_for_dunedaq/druncschema/opmon/FSM_pb2_grpc.py b/src/druncschema/opmon/FSM_pb2_grpc.py similarity index 100% rename from python_not_for_dunedaq/druncschema/opmon/FSM_pb2_grpc.py rename to src/druncschema/opmon/FSM_pb2_grpc.py diff --git a/python_not_for_dunedaq/druncschema/opmon/__init__.py b/src/druncschema/opmon/__init__.py similarity index 100% rename from python_not_for_dunedaq/druncschema/opmon/__init__.py rename to src/druncschema/opmon/__init__.py diff --git a/python_not_for_dunedaq/druncschema/opmon/generic_pb2.py b/src/druncschema/opmon/generic_pb2.py similarity index 100% rename from python_not_for_dunedaq/druncschema/opmon/generic_pb2.py rename to src/druncschema/opmon/generic_pb2.py diff --git a/python_not_for_dunedaq/druncschema/opmon/generic_pb2.pyi b/src/druncschema/opmon/generic_pb2.pyi similarity index 100% rename from python_not_for_dunedaq/druncschema/opmon/generic_pb2.pyi rename to src/druncschema/opmon/generic_pb2.pyi diff --git a/python_not_for_dunedaq/druncschema/opmon/generic_pb2_grpc.py b/src/druncschema/opmon/generic_pb2_grpc.py similarity index 100% rename from python_not_for_dunedaq/druncschema/opmon/generic_pb2_grpc.py rename to src/druncschema/opmon/generic_pb2_grpc.py diff --git a/python_not_for_dunedaq/druncschema/opmon/process_manager_pb2.py b/src/druncschema/opmon/process_manager_pb2.py similarity index 100% rename from python_not_for_dunedaq/druncschema/opmon/process_manager_pb2.py rename to src/druncschema/opmon/process_manager_pb2.py diff --git a/python_not_for_dunedaq/druncschema/opmon/process_manager_pb2.pyi b/src/druncschema/opmon/process_manager_pb2.pyi similarity index 100% rename from python_not_for_dunedaq/druncschema/opmon/process_manager_pb2.pyi rename to src/druncschema/opmon/process_manager_pb2.pyi diff --git a/python_not_for_dunedaq/druncschema/opmon/process_manager_pb2_grpc.py b/src/druncschema/opmon/process_manager_pb2_grpc.py similarity index 100% rename from python_not_for_dunedaq/druncschema/opmon/process_manager_pb2_grpc.py rename to src/druncschema/opmon/process_manager_pb2_grpc.py diff --git a/python_not_for_dunedaq/druncschema/process_manager_pb2.py b/src/druncschema/process_manager_pb2.py similarity index 100% rename from python_not_for_dunedaq/druncschema/process_manager_pb2.py rename to src/druncschema/process_manager_pb2.py diff --git a/python_not_for_dunedaq/druncschema/process_manager_pb2.pyi b/src/druncschema/process_manager_pb2.pyi similarity index 100% rename from python_not_for_dunedaq/druncschema/process_manager_pb2.pyi rename to src/druncschema/process_manager_pb2.pyi diff --git a/python_not_for_dunedaq/druncschema/process_manager_pb2_grpc.py b/src/druncschema/process_manager_pb2_grpc.py similarity index 100% rename from python_not_for_dunedaq/druncschema/process_manager_pb2_grpc.py rename to src/druncschema/process_manager_pb2_grpc.py diff --git a/python_not_for_dunedaq/druncschema/py.typed b/src/druncschema/py.typed similarity index 100% rename from python_not_for_dunedaq/druncschema/py.typed rename to src/druncschema/py.typed diff --git a/python_not_for_dunedaq/druncschema/request_response_pb2.py b/src/druncschema/request_response_pb2.py similarity index 100% rename from python_not_for_dunedaq/druncschema/request_response_pb2.py rename to src/druncschema/request_response_pb2.py diff --git a/python_not_for_dunedaq/druncschema/request_response_pb2.pyi b/src/druncschema/request_response_pb2.pyi similarity index 100% rename from python_not_for_dunedaq/druncschema/request_response_pb2.pyi rename to src/druncschema/request_response_pb2.pyi diff --git a/python_not_for_dunedaq/druncschema/request_response_pb2_grpc.py b/src/druncschema/request_response_pb2_grpc.py similarity index 100% rename from python_not_for_dunedaq/druncschema/request_response_pb2_grpc.py rename to src/druncschema/request_response_pb2_grpc.py diff --git a/python_not_for_dunedaq/druncschema/session_manager_pb2.py b/src/druncschema/session_manager_pb2.py similarity index 100% rename from python_not_for_dunedaq/druncschema/session_manager_pb2.py rename to src/druncschema/session_manager_pb2.py diff --git a/python_not_for_dunedaq/druncschema/session_manager_pb2.pyi b/src/druncschema/session_manager_pb2.pyi similarity index 100% rename from python_not_for_dunedaq/druncschema/session_manager_pb2.pyi rename to src/druncschema/session_manager_pb2.pyi diff --git a/python_not_for_dunedaq/druncschema/session_manager_pb2_grpc.py b/src/druncschema/session_manager_pb2_grpc.py similarity index 100% rename from python_not_for_dunedaq/druncschema/session_manager_pb2_grpc.py rename to src/druncschema/session_manager_pb2_grpc.py diff --git a/python_not_for_dunedaq/druncschema/token_pb2.py b/src/druncschema/token_pb2.py similarity index 100% rename from python_not_for_dunedaq/druncschema/token_pb2.py rename to src/druncschema/token_pb2.py diff --git a/python_not_for_dunedaq/druncschema/token_pb2.pyi b/src/druncschema/token_pb2.pyi similarity index 100% rename from python_not_for_dunedaq/druncschema/token_pb2.pyi rename to src/druncschema/token_pb2.pyi diff --git a/python_not_for_dunedaq/druncschema/token_pb2_grpc.py b/src/druncschema/token_pb2_grpc.py similarity index 100% rename from python_not_for_dunedaq/druncschema/token_pb2_grpc.py rename to src/druncschema/token_pb2_grpc.py From f9bd062931ac229b61a9c0ecea0f850576fdc3db Mon Sep 17 00:00:00 2001 From: John Freeman Date: Sat, 1 Nov 2025 10:08:01 -0500 Subject: [PATCH 2/8] JCF: Issue #66: simplify the name of __main_generate_protos__.py to generate_protos.py --- pyproject.toml | 2 +- .../apps/{__main_generate_protos__.py => generate_protos.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/druncschema/apps/{__main_generate_protos__.py => generate_protos.py} (100%) diff --git a/pyproject.toml b/pyproject.toml index 7eb1230..c7ab8e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ Documentation = "https://dune-daq-sw.readthedocs.io/en/latest/packages/daq-relea Repository = "https://github.com/DUNE-DAQ/druncschema" [project.scripts] -druncschema-generate-protos = "druncschema.apps.__main_generate_protos__:main" +druncschema-generate-protos = "druncschema.apps.generate_protos:main" [tool.setuptools.packages.find] where = ["src"] diff --git a/src/druncschema/apps/__main_generate_protos__.py b/src/druncschema/apps/generate_protos.py similarity index 100% rename from src/druncschema/apps/__main_generate_protos__.py rename to src/druncschema/apps/generate_protos.py From 118b33acb5460f5d3a729a46dcca48c62dce6167 Mon Sep 17 00:00:00 2001 From: John Freeman Date: Sat, 1 Nov 2025 10:11:24 -0500 Subject: [PATCH 3/8] JCF: Issue #66: update the generate_protos code to account for moving python_not_for_dunedaq/druncschema to src/druncschema --- generate_protos | 10 +++++----- src/druncschema/apps/generate_protos.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/generate_protos b/generate_protos index e56e185..bc7a3aa 100755 --- a/generate_protos +++ b/generate_protos @@ -6,9 +6,9 @@ do python -m grpc_tools.protoc \ --proto_path=./schema/ \ - --python_out=python_not_for_dunedaq/ \ - --grpc_python_out=python_not_for_dunedaq/ \ - --mypy_out=python_not_for_dunedaq/ \ + --python_out=src/ \ + --grpc_python_out=src/ \ + --mypy_out=src/ \ ${FILE} # FILE_NAME=$(basename -- "$FILE") @@ -25,8 +25,8 @@ do python -m grpc_tools.protoc \ --proto_path=./schema/ \ - --python_out=python_not_for_dunedaq/ \ - --mypy_out=python_not_for_dunedaq/ \ + --python_out=src/ \ + --mypy_out=src/ \ ${FILE} # FILE_NAME=$(basename -- "$FILE") diff --git a/src/druncschema/apps/generate_protos.py b/src/druncschema/apps/generate_protos.py index 19c721a..e1822d8 100644 --- a/src/druncschema/apps/generate_protos.py +++ b/src/druncschema/apps/generate_protos.py @@ -199,7 +199,7 @@ def main( druncschema_root = files("druncschema").parents[1] log.debug(f"Found druncschema directory at {druncschema_root}") - output_dir = druncschema_root / "python_not_for_dunedaq/" + output_dir = druncschema_root / "src/" log.debug(f"Set output directory as {output_dir}") source_path = druncschema_root.parents[1] / "env.sh" From 091675ad4e6de1ee54fd2b7c21d3336d8d2d5295 Mon Sep 17 00:00:00 2001 From: John Freeman Date: Sat, 1 Nov 2025 13:37:12 -0500 Subject: [PATCH 4/8] JCF: Issue #66: update generate_protos.py so as to prepare to move it out of the Python-package part of druncschema, as this is meant to be used as a developer tool in a work area --- scripts/generate_protos.py | 227 ++++++++++++++++++++++++ src/druncschema/apps/generate_protos.py | 6 +- 2 files changed, 232 insertions(+), 1 deletion(-) create mode 100644 scripts/generate_protos.py diff --git a/scripts/generate_protos.py b/scripts/generate_protos.py new file mode 100644 index 0000000..e1822d8 --- /dev/null +++ b/scripts/generate_protos.py @@ -0,0 +1,227 @@ +"""Defines command to compile *.proto files. + +Compiles buffers into + - *pb2.py. + - *pb2.pyi. + - pb2_grpc.py. +""" + +import importlib +import logging +import os +import subprocess +from importlib.resources import files +from pathlib import Path + +import click +from rich.console import Console +from rich.logging import RichHandler + +log_levels = { + "CRITICAL": logging.CRITICAL, + "ERROR": logging.ERROR, + "WARNING": logging.WARNING, + "INFO": logging.INFO, + "DEBUG": logging.DEBUG, + "NOTSET": logging.NOTSET, +} +log = logging.getLogger("druncschema-generate-protos") +try: + width = os.get_terminal_size()[0] +except OSError: + width = 150 + +log.addHandler(RichHandler( + console=Console(width=width), + omit_repeated_times=False, + markup=True, + rich_tracebacks=True, + show_path=False, + tracebacks_width=width, +)) + +compiled_extensions = ["_pb2.py", "_pb2.pyi", "_pb2_grpc.py"] +def in_dev_mode(): + """Check if in dev mode. + + Validate dev mode by attempting to import a library that otherwise would not be + a part of the stack. + """ + if importlib.util.find_spec("mypy_protobuf"): + return True + return False + +def generate_protos( + source_path: Path, + druncschema_root: Path, + proto_files: list[Path], + output_dir: Path, + subdir: Path) -> None: + """Compiles the protobuf messages.""" + for proto_file in proto_files: + log.info(f"Processing file {proto_file!s} to output dir {output_dir!s}") + if not str(proto_file).endswith(".proto"): + raise Exception(f"File names must end in a '.proto', received {proto_file}") + try: + cmd = " ".join([ + f"source {source_path}; cd {druncschema_root}; " + "python -m grpc_tools.protoc", + "-I'./schema'", + f"--python_out={output_dir!s}", + f"--grpc_python_out={output_dir!s}", + f"--mypy_out={output_dir!s}", + str(proto_file), + ]) + log.debug(cmd) + subprocess.run( + cmd, + capture_output=True, + text=True, + check=True, + shell=True, + executable="/bin/bash" + ) + except subprocess.CalledProcessError as e: + log.exception(e) + log.error(e.stderr) + output_files = [ + output_dir / Path("druncschema") / subdir / Path( + Path(proto_file.name).stem + extension + ) + for extension in compiled_extensions + ] + for output_file in output_files: + if not output_file.exists(): + log.error(output_file) + e = ValueError(f"Could not find output file {output_file}.") + log.exception(e) + else: + log.debug(f"Generated {output_file}") + +def clear_previous_compiled_schema(output_dir: Path, output_files: list[Path]) -> None: + """Delete previous results of schema compilation.""" + for output_file in output_files: + for compiled_extension in compiled_extensions: + existing_file = output_dir / output_file.with_name( + output_file.stem + compiled_extension) + if existing_file.exists(): + log.info(f"Deleting file {existing_file}") + existing_file.unlink() + +def call_generate_protos( + source_path: Path, + druncschema_root: Path, + output_dir: Path, + subdir: Path, + clean: bool, + do_not_compile: bool + ) -> None: + """Clear existing compiled buffers, compile new buffers. + + List *.proto files. + Clean existing compiled buffers. + Call the compiling function. + """ + proto_relative_path = Path("schema/druncschema") + proto_files = [ + proto_relative_path / subdir / Path(f.name) + for f in (druncschema_root / proto_relative_path / subdir).glob("*.proto") + ] + if not proto_files: + return + if clean: + clear_previous_compiled_schema( + output_dir / Path("druncschema") / subdir, + proto_files + ) + if get_files(output_dir): + e = Exception("Not all files removed, exiting") + log.error(get_files(output_dir)) + log.exception(e) + if not do_not_compile: + generate_protos(source_path, druncschema_root, proto_files, output_dir, subdir) + return + +def get_subdirs(path: Path) ->list[str]: + """Generate list of relevant directories.""" + return [Path(p.name) for p in path.iterdir() + if p.is_dir() and not str(p.name).endswith("__") + and not str(p.name).endswith("apps")] + +def get_files(path: Path) ->list[Path]: + """Generate list of *.proto files.""" + return [Path(p.name) for p in path.iterdir() + if p.is_file() and not str(p.name).endswith("__.py") + and not str(p.name).endswith("typed")] + +@click.command() +@click.option( + "-l", + "--log-level", + type=click.Choice(log_levels.keys(), case_sensitive=False), + default="INFO", + help="Set the log level", +) +@click.option( + "-c", + "--clean", + is_flag=True, + help="Explicitly deletes the existing compiled schemas " \ + "before starting the compile the new ones", +) +@click.option( + "-d", + "--do-not-compile", + is_flag=True, + help="Does not compile the schema. Only allowed with use of the clean flag", +) +def main( + log_level: str, + clean: bool, + do_not_compile: bool, + ) -> None: + """Compile the protobuf message schema into the relevant python code.""" + log.setLevel(log_level) + + if not in_dev_mode(): + e = Exception( + "This command is only available in developer mode." \ + "See the druncschema wiki for further clarification." + ) + log.exception(e) + + if do_not_compile and not clean: + e = Exception( + "Used option -d/--do-not-compile but not -c/--clean, require -c to use -d" + ) + log.exception(e) + + druncschema_root = files("druncschema").parents[1] + log.debug(f"Found druncschema directory at {druncschema_root}") + + output_dir = druncschema_root / "src/" + log.debug(f"Set output directory as {output_dir}") + + source_path = druncschema_root.parents[1] / "env.sh" + log.debug(f"Set source path to {source_path=}") + + subdir = Path() + call_generate_protos( + source_path, + druncschema_root, + output_dir, + subdir, + clean, + do_not_compile + ) + + subdirs = get_subdirs(druncschema_root / Path("schema/druncschema")) + for subdir in subdirs: + call_generate_protos( + source_path, + druncschema_root, + output_dir, + subdir, + clean, + do_not_compile + ) diff --git a/src/druncschema/apps/generate_protos.py b/src/druncschema/apps/generate_protos.py index e1822d8..f335b4c 100644 --- a/src/druncschema/apps/generate_protos.py +++ b/src/druncschema/apps/generate_protos.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + """Defines command to compile *.proto files. Compiles buffers into @@ -196,7 +198,7 @@ def main( ) log.exception(e) - druncschema_root = files("druncschema").parents[1] + druncschema_root = Path(f'{os.environ["DBT_AREA_ROOT"]}/sourcecode/druncschema') log.debug(f"Found druncschema directory at {druncschema_root}") output_dir = druncschema_root / "src/" @@ -225,3 +227,5 @@ def main( clean, do_not_compile ) + +main() From c6325960b84119a1b23bc64e4bff658f4a20a8f5 Mon Sep 17 00:00:00 2001 From: John Freeman Date: Sat, 1 Nov 2025 13:38:54 -0500 Subject: [PATCH 5/8] JCF: Issue #66: make generate_protos.py not a pip-installed app, but a daq-cmake-installed script --- pyproject.toml | 3 - scripts/generate_protos.py | 6 +- src/druncschema/apps/__init__.py | 1 - src/druncschema/apps/generate_protos.py | 231 ------------------------ 4 files changed, 5 insertions(+), 236 deletions(-) delete mode 100644 src/druncschema/apps/__init__.py delete mode 100644 src/druncschema/apps/generate_protos.py diff --git a/pyproject.toml b/pyproject.toml index c7ab8e4..259ad6b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,9 +29,6 @@ dev = ["ruff", "pre-commit", "pytest", "mypy-protobuf", "types-protobuf"] Documentation = "https://dune-daq-sw.readthedocs.io/en/latest/packages/daq-release/team_repos/?h=druncschema" Repository = "https://github.com/DUNE-DAQ/druncschema" -[project.scripts] -druncschema-generate-protos = "druncschema.apps.generate_protos:main" - [tool.setuptools.packages.find] where = ["src"] diff --git a/scripts/generate_protos.py b/scripts/generate_protos.py index e1822d8..f335b4c 100644 --- a/scripts/generate_protos.py +++ b/scripts/generate_protos.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + """Defines command to compile *.proto files. Compiles buffers into @@ -196,7 +198,7 @@ def main( ) log.exception(e) - druncschema_root = files("druncschema").parents[1] + druncschema_root = Path(f'{os.environ["DBT_AREA_ROOT"]}/sourcecode/druncschema') log.debug(f"Found druncschema directory at {druncschema_root}") output_dir = druncschema_root / "src/" @@ -225,3 +227,5 @@ def main( clean, do_not_compile ) + +main() diff --git a/src/druncschema/apps/__init__.py b/src/druncschema/apps/__init__.py deleted file mode 100644 index ad38309..0000000 --- a/src/druncschema/apps/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""app package for druncschema.""" \ No newline at end of file diff --git a/src/druncschema/apps/generate_protos.py b/src/druncschema/apps/generate_protos.py deleted file mode 100644 index f335b4c..0000000 --- a/src/druncschema/apps/generate_protos.py +++ /dev/null @@ -1,231 +0,0 @@ -#!/usr/bin/env python3 - -"""Defines command to compile *.proto files. - -Compiles buffers into - - *pb2.py. - - *pb2.pyi. - - pb2_grpc.py. -""" - -import importlib -import logging -import os -import subprocess -from importlib.resources import files -from pathlib import Path - -import click -from rich.console import Console -from rich.logging import RichHandler - -log_levels = { - "CRITICAL": logging.CRITICAL, - "ERROR": logging.ERROR, - "WARNING": logging.WARNING, - "INFO": logging.INFO, - "DEBUG": logging.DEBUG, - "NOTSET": logging.NOTSET, -} -log = logging.getLogger("druncschema-generate-protos") -try: - width = os.get_terminal_size()[0] -except OSError: - width = 150 - -log.addHandler(RichHandler( - console=Console(width=width), - omit_repeated_times=False, - markup=True, - rich_tracebacks=True, - show_path=False, - tracebacks_width=width, -)) - -compiled_extensions = ["_pb2.py", "_pb2.pyi", "_pb2_grpc.py"] -def in_dev_mode(): - """Check if in dev mode. - - Validate dev mode by attempting to import a library that otherwise would not be - a part of the stack. - """ - if importlib.util.find_spec("mypy_protobuf"): - return True - return False - -def generate_protos( - source_path: Path, - druncschema_root: Path, - proto_files: list[Path], - output_dir: Path, - subdir: Path) -> None: - """Compiles the protobuf messages.""" - for proto_file in proto_files: - log.info(f"Processing file {proto_file!s} to output dir {output_dir!s}") - if not str(proto_file).endswith(".proto"): - raise Exception(f"File names must end in a '.proto', received {proto_file}") - try: - cmd = " ".join([ - f"source {source_path}; cd {druncschema_root}; " - "python -m grpc_tools.protoc", - "-I'./schema'", - f"--python_out={output_dir!s}", - f"--grpc_python_out={output_dir!s}", - f"--mypy_out={output_dir!s}", - str(proto_file), - ]) - log.debug(cmd) - subprocess.run( - cmd, - capture_output=True, - text=True, - check=True, - shell=True, - executable="/bin/bash" - ) - except subprocess.CalledProcessError as e: - log.exception(e) - log.error(e.stderr) - output_files = [ - output_dir / Path("druncschema") / subdir / Path( - Path(proto_file.name).stem + extension - ) - for extension in compiled_extensions - ] - for output_file in output_files: - if not output_file.exists(): - log.error(output_file) - e = ValueError(f"Could not find output file {output_file}.") - log.exception(e) - else: - log.debug(f"Generated {output_file}") - -def clear_previous_compiled_schema(output_dir: Path, output_files: list[Path]) -> None: - """Delete previous results of schema compilation.""" - for output_file in output_files: - for compiled_extension in compiled_extensions: - existing_file = output_dir / output_file.with_name( - output_file.stem + compiled_extension) - if existing_file.exists(): - log.info(f"Deleting file {existing_file}") - existing_file.unlink() - -def call_generate_protos( - source_path: Path, - druncschema_root: Path, - output_dir: Path, - subdir: Path, - clean: bool, - do_not_compile: bool - ) -> None: - """Clear existing compiled buffers, compile new buffers. - - List *.proto files. - Clean existing compiled buffers. - Call the compiling function. - """ - proto_relative_path = Path("schema/druncschema") - proto_files = [ - proto_relative_path / subdir / Path(f.name) - for f in (druncschema_root / proto_relative_path / subdir).glob("*.proto") - ] - if not proto_files: - return - if clean: - clear_previous_compiled_schema( - output_dir / Path("druncschema") / subdir, - proto_files - ) - if get_files(output_dir): - e = Exception("Not all files removed, exiting") - log.error(get_files(output_dir)) - log.exception(e) - if not do_not_compile: - generate_protos(source_path, druncschema_root, proto_files, output_dir, subdir) - return - -def get_subdirs(path: Path) ->list[str]: - """Generate list of relevant directories.""" - return [Path(p.name) for p in path.iterdir() - if p.is_dir() and not str(p.name).endswith("__") - and not str(p.name).endswith("apps")] - -def get_files(path: Path) ->list[Path]: - """Generate list of *.proto files.""" - return [Path(p.name) for p in path.iterdir() - if p.is_file() and not str(p.name).endswith("__.py") - and not str(p.name).endswith("typed")] - -@click.command() -@click.option( - "-l", - "--log-level", - type=click.Choice(log_levels.keys(), case_sensitive=False), - default="INFO", - help="Set the log level", -) -@click.option( - "-c", - "--clean", - is_flag=True, - help="Explicitly deletes the existing compiled schemas " \ - "before starting the compile the new ones", -) -@click.option( - "-d", - "--do-not-compile", - is_flag=True, - help="Does not compile the schema. Only allowed with use of the clean flag", -) -def main( - log_level: str, - clean: bool, - do_not_compile: bool, - ) -> None: - """Compile the protobuf message schema into the relevant python code.""" - log.setLevel(log_level) - - if not in_dev_mode(): - e = Exception( - "This command is only available in developer mode." \ - "See the druncschema wiki for further clarification." - ) - log.exception(e) - - if do_not_compile and not clean: - e = Exception( - "Used option -d/--do-not-compile but not -c/--clean, require -c to use -d" - ) - log.exception(e) - - druncschema_root = Path(f'{os.environ["DBT_AREA_ROOT"]}/sourcecode/druncschema') - log.debug(f"Found druncschema directory at {druncschema_root}") - - output_dir = druncschema_root / "src/" - log.debug(f"Set output directory as {output_dir}") - - source_path = druncschema_root.parents[1] / "env.sh" - log.debug(f"Set source path to {source_path=}") - - subdir = Path() - call_generate_protos( - source_path, - druncschema_root, - output_dir, - subdir, - clean, - do_not_compile - ) - - subdirs = get_subdirs(druncschema_root / Path("schema/druncschema")) - for subdir in subdirs: - call_generate_protos( - source_path, - druncschema_root, - output_dir, - subdir, - clean, - do_not_compile - ) - -main() From 7f7280439a2fe04c27d8183e3de2bf4c7b5940e6 Mon Sep 17 00:00:00 2001 From: John Freeman Date: Sat, 1 Nov 2025 13:45:04 -0500 Subject: [PATCH 6/8] JCF: DUNE-DAQ/daq-deliverables#196: bring in standard tooling, which in practice means add pytest coverage and add the refactoring suggestions section of Ruff --- pyproject.toml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 259ad6b..3255c18 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ authors = [ ] [project.optional-dependencies] -dev = ["ruff", "pre-commit", "pytest", "mypy-protobuf", "types-protobuf"] +dev = ["ruff", "pre-commit", "pytest", "pytest-cov", "mypy-protobuf", "types-protobuf"] [project.urls] Documentation = "https://dune-daq-sw.readthedocs.io/en/latest/packages/daq-release/team_repos/?h=druncschema" @@ -32,6 +32,14 @@ Repository = "https://github.com/DUNE-DAQ/druncschema" [tool.setuptools.packages.find] where = ["src"] +[tool.pytest.ini_options] +addopts = "-v --tb=short --cov=druncschema --cov=src/druncschema tests/" + +[tool.coverage.run] +source = ["druncschema"] +omit = ["tests/*", "scripts/*"] + + [tool.ruff] exclude = [ "src/druncschema.egg-info", @@ -47,5 +55,6 @@ select = [ "I", # isort "UP", # pyupgrade "RUF", # ruff-specific rules + "R", # refactoring suggestions ] pydocstyle.convention = "google" From 8625653d01d99bd31f6fdeddf247a1a1c84d0110 Mon Sep 17 00:00:00 2001 From: John Freeman Date: Sat, 1 Nov 2025 14:06:40 -0500 Subject: [PATCH 7/8] JCF: Issue #66: make sure generate_protos.py is executable --- scripts/generate_protos.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/generate_protos.py diff --git a/scripts/generate_protos.py b/scripts/generate_protos.py old mode 100644 new mode 100755 From 1a39a5a11f2d135dfb2a3da2b7c8e0e71fadd55f Mon Sep 17 00:00:00 2001 From: John Freeman Date: Sun, 2 Nov 2025 08:50:35 -0600 Subject: [PATCH 8/8] JCF: Issue #66: remove version from __init__.py --- src/druncschema/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/druncschema/__init__.py b/src/druncschema/__init__.py index 4d176e4..571ad55 100644 --- a/src/druncschema/__init__.py +++ b/src/druncschema/__init__.py @@ -1,2 +1 @@ # This directory is _NOT_ to be used for dune daq, this is solely when one wants to run standalone without any dunedaq dependency. -__version__='0.11.3'