From 85f32878a91b7a7db0da07b29b9f82655a271d46 Mon Sep 17 00:00:00 2001 From: Alex Ioannidis Date: Mon, 22 Apr 2024 02:57:23 +0300 Subject: [PATCH 01/13] install: convert to group and add "symlink" sub-command --- invenio_cli/cli/install.py | 27 ++++++++++++++++--- invenio_cli/commands/install.py | 46 ++++++++++++++++----------------- 2 files changed, 46 insertions(+), 27 deletions(-) diff --git a/invenio_cli/cli/install.py b/invenio_cli/cli/install.py index 3da6ab41..11c20776 100644 --- a/invenio_cli/cli/install.py +++ b/invenio_cli/cli/install.py @@ -13,7 +13,16 @@ from .utils import pass_cli_config, run_steps -@click.command() +@click.group(invoke_without_command=True) +@click.pass_context +def install(ctx): + """Commands for installing the project.""" + if ctx.invoked_subcommand is None: + # If no sub-command is passed, default to the install all command. + ctx.invoke(install_all) + + +@install.command("all") @click.option( "--pre", default=False, @@ -35,8 +44,8 @@ " statics/assets.", ) @pass_cli_config -def install(cli_config, pre, dev, production): - """Installs the project locally. +def install_all(cli_config, pre, dev, production): + """Installs the project locally. Installs dependencies, creates instance directory, links invenio.cfg + templates, copies images and other statics and finally @@ -52,3 +61,15 @@ def install(cli_config, pre, dev, production): on_success = "Dependencies installed successfully." run_steps(steps, on_fail, on_success) + + +@install.command() +@pass_cli_config +def symlink(cli_config): + """Symlinks project files in the instance directory.""" + commands = InstallCommands(cli_config) + steps = commands.symlink() + on_fail = "Failed to symlink project files and folders." + on_success = "Project ffles and folders symlinked successfully." + + run_steps(steps, on_fail, on_success) diff --git a/invenio_cli/commands/install.py b/invenio_cli/commands/install.py index 2431f594..1a485715 100644 --- a/invenio_cli/commands/install.py +++ b/invenio_cli/commands/install.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. +# Copyright (C) 2025 Graz University of Technology. # # Invenio-Cli is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. @@ -54,42 +55,39 @@ def update_instance_path(self): result.output = "Instance path updated successfully." return result - def symlink_project_file_or_folder(self, target): + def _symlink_project_file_or_folder(self, target): """Create symlink in instance pointing to project file or folder.""" target_path = self.cli_config.get_project_dir() / target link_path = self.cli_config.get_instance_path() / target return filesystem.force_symlink(target_path, link_path) - def install(self, pre, dev=False, debug=False): - """Development installation steps.""" - steps = self.install_py_dependencies(pre=pre, dev=dev) + def symlink(self): + """Sylink all necessary project files and folders.""" + steps = [] steps.append( FunctionStep( func=self.update_instance_path, message="Updating instance path..." ) ) - steps.append( - FunctionStep( - func=self.symlink_project_file_or_folder, - args={"target": "invenio.cfg"}, - message="Symlinking 'invenio.cfg'...", - ) - ) - steps.append( - FunctionStep( - func=self.symlink_project_file_or_folder, - args={"target": "templates"}, - message="Symlinking 'templates'...", - ) - ) - steps.append( - FunctionStep( - func=self.symlink_project_file_or_folder, - args={"target": "app_data"}, - message="Symlinking 'app_data'...", - ) + steps.extend( + [ + FunctionStep( + func=self._symlink_project_file_or_folder, + args={"target": path}, + message=f"Symlinking '{path}'...", + ) + for path in ("invenio.cfg", "templates", "app_data") + ] ) + return steps + + def install(self, pre, dev=False, debug=False): + """Development installation steps.""" + steps = self.install_py_dependencies(pre=pre, dev=dev) + + steps.extend(self.symlink()) + steps.append( FunctionStep( func=self.update_statics_and_assets, From 04df8cb85d1b20e4a9bee15f7c8e1611c2576fa6 Mon Sep 17 00:00:00 2001 From: Alex Ioannidis Date: Thu, 23 May 2024 12:36:20 +0200 Subject: [PATCH 02/13] install: add "python" and "assets" sub-commands --- invenio_cli/cli/install.py | 45 +++++++++++++++++++++++++++++++++ invenio_cli/commands/install.py | 18 +++++++------ 2 files changed, 55 insertions(+), 8 deletions(-) diff --git a/invenio_cli/cli/install.py b/invenio_cli/cli/install.py index 11c20776..e896c0ca 100644 --- a/invenio_cli/cli/install.py +++ b/invenio_cli/cli/install.py @@ -63,6 +63,51 @@ def install_all(cli_config, pre, dev, production): run_steps(steps, on_fail, on_success) +@install.command("python") +@click.option( + "--pre", + default=False, + is_flag=True, + help="If specified, allows the installation of alpha releases", +) +@click.option( + "--dev/--no-dev", + default=True, + is_flag=True, + help="Includes development dependencies.", +) +@pass_cli_config +def install_python(cli_config, pre, dev): + """Install Python dependencies and packages.""" + commands = InstallCommands(cli_config) + steps = commands.install_py_dependencies(pre=pre, dev=dev) + on_fail = "Failed to install Python dependencies." + on_success = "Python dependencies installed successfully." + + run_steps(steps, on_fail, on_success) + + +@install.command("assets") +@click.option( + "--production/--development", + "-p/-d", + default=False, + is_flag=True, + help="Production mode copies statics/assets. Development mode symlinks" + " statics/assets.", +) +@pass_cli_config +def install_assets(cli_config, production): + """Install assets.""" + commands = InstallCommands(cli_config) + flask_env = "production" if production else "development" + steps = commands.install_assets(flask_env) + on_fail = "Failed to install assets." + on_success = "Assets installed successfully." + + run_steps(steps, on_fail, on_success) + + @install.command() @pass_cli_config def symlink(cli_config): diff --git a/invenio_cli/commands/install.py b/invenio_cli/commands/install.py index 1a485715..01391371 100644 --- a/invenio_cli/commands/install.py +++ b/invenio_cli/commands/install.py @@ -70,6 +70,7 @@ def symlink(self): func=self.update_instance_path, message="Updating instance path..." ) ) + steps.extend( [ FunctionStep( @@ -82,18 +83,19 @@ def symlink(self): ) return steps - def install(self, pre, dev=False, debug=False): - """Development installation steps.""" - steps = self.install_py_dependencies(pre=pre, dev=dev) - - steps.extend(self.symlink()) - - steps.append( + def install_assets(self, debug=False): + """Install assets.""" + return [ FunctionStep( func=self.update_statics_and_assets, args={"force": True, "debug": debug}, message="Updating statics and assets...", ) - ) + ] + def install(self, pre, dev=False, flask_env="production"): + """Development installation steps.""" + steps = self.install_py_dependencies(pre=pre, dev=dev) + steps.extend(self.symlink()) + steps.extend(self.install_assets(flask_env)) return steps From dffb53bf75ff483c7f869c0513d1441045deb0dc Mon Sep 17 00:00:00 2001 From: Alex Ioannidis Date: Thu, 23 May 2024 12:36:07 +0200 Subject: [PATCH 03/13] run: add explicit celery worker queues * Explicitly makes the celery worker listen to the "default" and "low" queues. --- invenio_cli/commands/local.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/invenio_cli/commands/local.py b/invenio_cli/commands/local.py index 5f872923..63daa0b6 100644 --- a/invenio_cli/commands/local.py +++ b/invenio_cli/commands/local.py @@ -141,6 +141,8 @@ def signal_handler(sig, frame): "--events", "--loglevel", "INFO", + "--queues", + "celery,low", ] if celery_log_file: From 68f6a6028109b6c03ba93f0ddb48034bcd14035e Mon Sep 17 00:00:00 2001 From: Alex Ioannidis Date: Thu, 23 May 2024 13:16:29 +0200 Subject: [PATCH 04/13] run: split into "web" and "worker" sub-commands * also, reword some help texts and clean up some strings --- invenio_cli/cli/cli.py | 105 ++++++++++++++++----- invenio_cli/cli/install.py | 8 +- invenio_cli/cli/utils.py | 11 +++ invenio_cli/commands/containers.py | 9 +- invenio_cli/commands/local.py | 90 ++++++++++-------- invenio_cli/commands/requirements.py | 3 +- invenio_cli/helpers/cli_config.py | 3 +- invenio_cli/helpers/filesystem.py | 2 +- tests/commands/test_containers.py | 6 +- tests/helpers/test_cookiecutter_wrapper.py | 4 +- 10 files changed, 158 insertions(+), 83 deletions(-) diff --git a/invenio_cli/cli/cli.py b/invenio_cli/cli/cli.py index 3efbb9de..3341f12c 100644 --- a/invenio_cli/cli/cli.py +++ b/invenio_cli/cli/cli.py @@ -30,7 +30,12 @@ from .packages import packages from .services import services from .translations import translations -from .utils import handle_process_response, pass_cli_config, run_steps +from .utils import ( + combine_decorators, + handle_process_response, + pass_cli_config, + run_steps, +) @click.group() @@ -104,8 +109,7 @@ def pyshell(debug): @click.option( "--user-input/--no-input", default=True, - help="If input is disabled, uses the defaults (if --config is" - " also passed, uses values from an .invenio config file).", + help="If input is disabled, uses the defaults (if --config is also passed, uses values from an .invenio config file).", # noqa ) @click.option( "--config", required=False, help="The .invenio configuration file to use." @@ -142,34 +146,87 @@ def init(flavour, template, checkout, user_input, config): cookiecutter_wrapper.remove_config() -@invenio_cli.command() -@click.option("--host", "-h", default="127.0.0.1", help="The interface to bind to.") -@click.option("--port", "-p", default=5000, help="The port to bind to.") -@click.option( - "--debug/--no-debug", - "-d/", - default=True, - is_flag=True, - help="Enable/disable debug mode including auto-reloading " "(default: enabled).", -) -@click.option( +@invenio_cli.group("run", invoke_without_command=True) +@click.pass_context +def run_group(ctx): + """Run command group.""" + # For backward compatibility + if ctx.invoked_subcommand is None: + ctx.invoke(run_all) + + +services_option = click.option( "--services/--no-services", "-s/-n", default=True, is_flag=True, help="Enable/disable dockerized services (default: enabled).", ) -@click.option( - "--celery-log-file", - default=None, - help="Celery log file (default: None, this means logging to stderr)", +web_options = combine_decorators( + click.option("--host", "-h", default="127.0.0.1", help="The interface to bind to."), + click.option("--port", "-p", default=5000, help="The port to bind to."), + click.option( + "--debug/--no-debug", + "-d/", + default=True, + is_flag=True, + help="Enable/disable debug mode including auto-reloading (default: enabled).", + ), ) + + +@run_group.command("web") +@services_option +@web_options @pass_cli_config -def run(cli_config, host, port, debug, services, celery_log_file): - """Starts the local development server. +def run_web(cli_config, host, port, debug, services): + """Starts the local development web server.""" + if services: + cmds = ServicesCommands(cli_config) + response = cmds.ensure_containers_running() + # fail and exit if containers are not running + handle_process_response(response) + + commands = LocalCommands(cli_config) + processes = commands.run_web(host=host, port=str(port), debug=debug) + for proc in processes: + proc.wait() + + +worker_options = combine_decorators( + click.option( + "--celery-log-file", + default=None, + help="Celery log file (default: None, this means logging to stderr)", + ), +) + + +@run_group.command("worker") +@services_option +@worker_options +@pass_cli_config +def run_worker(cli_config, services, celery_log_file): + """Starts the local development server.""" + if services: + cmds = ServicesCommands(cli_config) + response = cmds.ensure_containers_running() + # fail and exit if containers are not running + handle_process_response(response) - NOTE: this only makes sense locally so no --local option - """ + commands = LocalCommands(cli_config) + processes = commands.run_worker(celery_log_file=celery_log_file) + for proc in processes: + proc.wait() + + +@run_group.command("all") +@services_option +@web_options +@worker_options +@pass_cli_config +def run_all(cli_config, host, port, debug, services, celery_log_file): + """Starts web and worker development servers.""" if services: cmds = ServicesCommands(cli_config) response = cmds.ensure_containers_running() @@ -177,13 +234,15 @@ def run(cli_config, host, port, debug, services, celery_log_file): handle_process_response(response) commands = LocalCommands(cli_config) - commands.run( + processes = commands.run_all( host=host, port=str(port), debug=debug, services=services, celery_log_file=celery_log_file, ) + for proc in processes: + proc.wait() @invenio_cli.command() diff --git a/invenio_cli/cli/install.py b/invenio_cli/cli/install.py index e896c0ca..8620b84d 100644 --- a/invenio_cli/cli/install.py +++ b/invenio_cli/cli/install.py @@ -40,8 +40,7 @@ def install(ctx): "-p/-d", default=False, is_flag=True, - help="Production mode copies statics/assets. Development mode symlinks" - " statics/assets.", + help="Production mode copies statics/assets. Development mode symlinks them.", ) @pass_cli_config def install_all(cli_config, pre, dev, production): @@ -93,8 +92,7 @@ def install_python(cli_config, pre, dev): "-p/-d", default=False, is_flag=True, - help="Production mode copies statics/assets. Development mode symlinks" - " statics/assets.", + help="Production mode copies statics/assets. Development mode symlinks them.", ) @pass_cli_config def install_assets(cli_config, production): @@ -115,6 +113,6 @@ def symlink(cli_config): commands = InstallCommands(cli_config) steps = commands.symlink() on_fail = "Failed to symlink project files and folders." - on_success = "Project ffles and folders symlinked successfully." + on_success = "Project files and folders symlinked successfully." run_steps(steps, on_fail, on_success) diff --git a/invenio_cli/cli/utils.py b/invenio_cli/cli/utils.py index 85f3acee..f79f932a 100644 --- a/invenio_cli/cli/utils.py +++ b/invenio_cli/cli/utils.py @@ -48,3 +48,14 @@ def handle_process_response(response, fail_message=None): click.secho(msg, fg="yellow") elif response.output: click.secho(message=response.output, fg="green") + + +def combine_decorators(*decorators): + """Combine multiple decorators.""" + + def _decorator(f): + for dec in reversed(decorators): + f = dec(f) + return f + + return _decorator diff --git a/invenio_cli/commands/containers.py b/invenio_cli/commands/containers.py index f79fc060..8b64d4ae 100644 --- a/invenio_cli/commands/containers.py +++ b/invenio_cli/commands/containers.py @@ -53,8 +53,7 @@ def _cleanup(self, project_shortname="/opt/var/instance/"): func=self.docker_helper.execute_cli_command, args={ "project_shortname": project_shortname, - "command": "invenio shell --no-term-title -c " - "\"import redis; redis.StrictRedis.from_url(app.config['CACHE_REDIS_URL']).flushall(); print('Cache cleared')\"", # noqa + "command": "invenio shell --no-term-title -c \"import redis; redis.StrictRedis.from_url(app.config['CACHE_REDIS_URL']).flushall(); print('Cache cleared')\"", # noqa }, message="Flushing redis cache...", ), @@ -106,9 +105,7 @@ def _setup(self, project_shortname="/opt/var/instance/"): func=self.docker_helper.execute_cli_command, args={ "project_shortname": project_shortname, - "command": "invenio files location create --default " - "default-location " - "${INVENIO_INSTANCE_PATH}/data", + "command": "invenio files location create --default default-location ${INVENIO_INSTANCE_PATH}/data", # noqa }, message="Creating files location...", ), @@ -124,7 +121,7 @@ def _setup(self, project_shortname="/opt/var/instance/"): func=self.docker_helper.execute_cli_command, args={ "project_shortname": project_shortname, - "command": "invenio access allow " "superuser-access role admin", + "command": "invenio access allow superuser-access role admin", }, message="Assigning superuser access to admin role...", ), diff --git a/invenio_cli/commands/local.py b/invenio_cli/commands/local.py index 63daa0b6..60dc982c 100644 --- a/invenio_cli/commands/local.py +++ b/invenio_cli/commands/local.py @@ -115,50 +115,27 @@ def update_statics_and_assets(self, force, debug=False, log_file=None): break return response - def run(self, host, port, debug=True, services=True, celery_log_file=None): - """Run development server and celery queue.""" + def _handle_sigint(self, name, process): + """Terminate services on SIGINT.""" + prev_handler = signal.getsignal(signal.SIGINT) - def signal_handler(sig, frame): - click.secho("Stopping server and worker...", fg="green") - server.terminate() - if services: - worker.terminate() - click.secho("Server and worker stopped...", fg="green") + def _signal_handler(sig, frame): + click.secho(f"Stopping {name}...", fg="green") + process.terminate() + click.secho(f"{name} stopped...", fg="green") + if prev_handler is not None: + prev_handler(sig, frame) - signal.signal(signal.SIGINT, signal_handler) - - if services: - click.secho("Starting celery worker...", fg="green") - - celery_command = [ - "pipenv", - "run", - "celery", - "--app", - "invenio_app.celery", - "worker", - "--beat", - "--events", - "--loglevel", - "INFO", - "--queues", - "celery,low", - ] - - if celery_log_file: - celery_command += [ - "--logfile", - celery_log_file, - ] - - worker = popen(celery_command) + signal.signal(signal.SIGINT, _signal_handler) + def run_web(self, host, port, debug=True): + """Run development server.""" click.secho("Starting up local (development) server...", fg="green") run_env = environ.copy() run_env["FLASK_DEBUG"] = str(debug) run_env["INVENIO_SITE_UI_URL"] = f"https://{host}:{port}" run_env["INVENIO_SITE_API_URL"] = f"https://{host}:{port}/api" - server = popen( + proc = popen( [ "pipenv", "run", @@ -177,6 +154,43 @@ def signal_handler(sig, frame): ], env=run_env, ) - + self._handle_sigint("Web server", proc) click.secho(f"Instance running!\nVisit https://{host}:{port}", fg="green") - server.wait() + return [proc] + + def run_worker(self, celery_log_file=None): + """Run Celery worker.""" + click.secho("Starting celery worker...", fg="green") + + celery_command = [ + "pipenv", + "run", + "celery", + "--app", + "invenio_app.celery", + "worker", + "--beat", + "--events", + "--loglevel", + "INFO", + "--queues", + "celery,low", + ] + + if celery_log_file: + celery_command += [ + "--logfile", + celery_log_file, + ] + + proc = popen(celery_command) + self._handle_sigint("Celery worker", proc) + click.secho("Worker running!", fg="green") + return [proc] + + def run_all(self, host, port, debug=True, services=True, celery_log_file=None): + """Run all services.""" + return [ + *self.run_web(host, port, debug), + *self.run_worker(celery_log_file), + ] diff --git a/invenio_cli/commands/requirements.py b/invenio_cli/commands/requirements.py index ca9e2f28..c75e1722 100644 --- a/invenio_cli/commands/requirements.py +++ b/invenio_cli/commands/requirements.py @@ -36,8 +36,7 @@ def _check_version(cls, binary, version, major, minor=-1, patch=-1, exact=False) if len(parts) != 3: return ProcessResponse( - error=f"{binary} incorrect version format or not found. " - "Check that it is installed correctly", + error=f"{binary} incorrect version format or not found. Check that it is installed correctly", # noqa status_code=1, ) diff --git a/invenio_cli/helpers/cli_config.py b/invenio_cli/helpers/cli_config.py index e839e28f..195ce477 100644 --- a/invenio_cli/helpers/cli_config.py +++ b/invenio_cli/helpers/cli_config.py @@ -49,8 +49,7 @@ def __init__(self, project_dir="./"): self.config.read_file(cfg_file) except FileNotFoundError as e: raise InvenioCLIConfigError( - "Missing '{0}' file in current directory. " - "Are you in the project folder?".format(e.filename), + f"Missing '{e.filename}' file in current directory. Are you in the project folder?", # noqa ) try: diff --git a/invenio_cli/helpers/filesystem.py b/invenio_cli/helpers/filesystem.py index fdb6a623..11bd3203 100644 --- a/invenio_cli/helpers/filesystem.py +++ b/invenio_cli/helpers/filesystem.py @@ -60,6 +60,6 @@ def force_symlink(target, link_name): if e.errno == errno.EEXIST: remove(link_name) symlink(target, link_name) - output = output + "Deleted already existing link." + output = f"{output} Deleted already existing link." return ProcessResponse(output=output, status_code=0) diff --git a/tests/commands/test_containers.py b/tests/commands/test_containers.py index 78966e89..7af2e31f 100644 --- a/tests/commands/test_containers.py +++ b/tests/commands/test_containers.py @@ -23,8 +23,7 @@ def expected_setup_calls(): call("project-shortname", "invenio db init create"), call( "project-shortname", - "invenio files location create --default default-location " - "${INVENIO_INSTANCE_PATH}/data", + "invenio files location create --default default-location ${INVENIO_INSTANCE_PATH}/data", # noqa ), call("project-shortname", "invenio roles create admin"), call("project-shortname", "invenio access allow superuser-access role admin"), @@ -43,8 +42,7 @@ def expected_force_calls(): return [ call( "project-shortname", - "invenio shell --no-term-title -c " - "\"import redis; redis.StrictRedis.from_url(app.config['CACHE_REDIS_URL']).flushall(); print('Cache cleared')\"", # noqa + "invenio shell --no-term-title -c \"import redis; redis.StrictRedis.from_url(app.config['CACHE_REDIS_URL']).flushall(); print('Cache cleared')\"", # noqa ), call("project-shortname", "invenio db destroy --yes-i-know"), call("project-shortname", "invenio index destroy --force --yes-i-know"), diff --git a/tests/helpers/test_cookiecutter_wrapper.py b/tests/helpers/test_cookiecutter_wrapper.py index b932d9ce..85bab9ce 100644 --- a/tests/helpers/test_cookiecutter_wrapper.py +++ b/tests/helpers/test_cookiecutter_wrapper.py @@ -17,8 +17,8 @@ def test_constructor(): assert cookiecutter.tmp_file is None assert cookiecutter.flavour == "RDM" assert ( - cookiecutter.template == "https://github.com/inveniosoftware/" - "cookiecutter-invenio-rdm.git" + cookiecutter.template + == "https://github.com/inveniosoftware/cookiecutter-invenio-rdm.git" ) assert cookiecutter.template_name == "cookiecutter-invenio-rdm" assert cookiecutter.checkout == "master" From 5d6ba66b68c587f9bc30632e7620007416fee3d2 Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Tue, 5 Nov 2024 16:53:08 +0100 Subject: [PATCH 05/13] web: make host, port cli_config configurable --- invenio_cli/cli/cli.py | 21 +++++++++++++++++++-- invenio_cli/helpers/cli_config.py | 8 ++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/invenio_cli/cli/cli.py b/invenio_cli/cli/cli.py index 3341f12c..dd8a0083 100644 --- a/invenio_cli/cli/cli.py +++ b/invenio_cli/cli/cli.py @@ -3,6 +3,7 @@ # Copyright (C) 2019-2021 CERN. # Copyright (C) 2019 Northwestern University. # Copyright (C) 2022 Forschungszentrum Jülich GmbH. +# Copyright (C) 2024 Graz University of Technology. # # Invenio-Cli is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. @@ -163,8 +164,18 @@ def run_group(ctx): help="Enable/disable dockerized services (default: enabled).", ) web_options = combine_decorators( - click.option("--host", "-h", default="127.0.0.1", help="The interface to bind to."), - click.option("--port", "-p", default=5000, help="The port to bind to."), + click.option( + "--host", + "-h", + default=None, + help="The interface to bind to. The default is defined in the CLIConfig.", + ), + click.option( + "--port", + "-p", + default=None, + help="The port to bind to. The default is defined in the CLIConfig.", + ), click.option( "--debug/--no-debug", "-d/", @@ -187,6 +198,9 @@ def run_web(cli_config, host, port, debug, services): # fail and exit if containers are not running handle_process_response(response) + host = host or cli_config.get_web_host() + port = port or cli_config.get_web_port() + commands = LocalCommands(cli_config) processes = commands.run_web(host=host, port=str(port), debug=debug) for proc in processes: @@ -233,6 +247,9 @@ def run_all(cli_config, host, port, debug, services, celery_log_file): # fail and exit if containers are not running handle_process_response(response) + host = host or cli_config.get_web_host() + port = port or cli_config.get_web_port() + commands = LocalCommands(cli_config) processes = commands.run_all( host=host, diff --git a/invenio_cli/helpers/cli_config.py b/invenio_cli/helpers/cli_config.py index 195ce477..6f49fc4b 100644 --- a/invenio_cli/helpers/cli_config.py +++ b/invenio_cli/helpers/cli_config.py @@ -120,6 +120,14 @@ def get_search_host(self): "localhost", ) + def get_web_port(self): + """Returns web port.""" + return self.config[CLIConfig.COOKIECUTTER_SECTION].get("web_port", "5000") + + def get_web_host(self): + """Returns web host.""" + return self.config[CLIConfig.COOKIECUTTER_SECTION].get("web_host", "127.0.0.1") + def get_db_type(self): """Returns the database type (mysql, postgresql).""" return self.config[CLIConfig.COOKIECUTTER_SECTION]["database"] From 3184a05fc1cbf145beeb10cc213c8941b8d2be33 Mon Sep 17 00:00:00 2001 From: Sarah Wiechers Date: Wed, 15 Jan 2025 13:04:56 +0100 Subject: [PATCH 06/13] run: allow changing of celery log level --- invenio_cli/cli/cli.py | 16 ++++++++++++---- invenio_cli/commands/local.py | 16 ++++++++++++---- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/invenio_cli/cli/cli.py b/invenio_cli/cli/cli.py index dd8a0083..b0aa7183 100644 --- a/invenio_cli/cli/cli.py +++ b/invenio_cli/cli/cli.py @@ -3,7 +3,7 @@ # Copyright (C) 2019-2021 CERN. # Copyright (C) 2019 Northwestern University. # Copyright (C) 2022 Forschungszentrum Jülich GmbH. -# Copyright (C) 2024 Graz University of Technology. +# Copyright (C) 2024-2025 Graz University of Technology. # # Invenio-Cli is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. @@ -213,6 +213,11 @@ def run_web(cli_config, host, port, debug, services): default=None, help="Celery log file (default: None, this means logging to stderr)", ), + click.option( + "--celery-log-level", + default="INFO", + help="Celery log level (default: INFO)", + ), ) @@ -220,7 +225,7 @@ def run_web(cli_config, host, port, debug, services): @services_option @worker_options @pass_cli_config -def run_worker(cli_config, services, celery_log_file): +def run_worker(cli_config, services, celery_log_file, celery_log_level): """Starts the local development server.""" if services: cmds = ServicesCommands(cli_config) @@ -229,7 +234,9 @@ def run_worker(cli_config, services, celery_log_file): handle_process_response(response) commands = LocalCommands(cli_config) - processes = commands.run_worker(celery_log_file=celery_log_file) + processes = commands.run_worker( + celery_log_file=celery_log_file, celery_log_level=celery_log_level + ) for proc in processes: proc.wait() @@ -239,7 +246,7 @@ def run_worker(cli_config, services, celery_log_file): @web_options @worker_options @pass_cli_config -def run_all(cli_config, host, port, debug, services, celery_log_file): +def run_all(cli_config, host, port, debug, services, celery_log_file, celery_log_level): """Starts web and worker development servers.""" if services: cmds = ServicesCommands(cli_config) @@ -257,6 +264,7 @@ def run_all(cli_config, host, port, debug, services, celery_log_file): debug=debug, services=services, celery_log_file=celery_log_file, + celery_log_level=celery_log_level, ) for proc in processes: proc.wait() diff --git a/invenio_cli/commands/local.py b/invenio_cli/commands/local.py index 60dc982c..8b7d5b0a 100644 --- a/invenio_cli/commands/local.py +++ b/invenio_cli/commands/local.py @@ -158,7 +158,7 @@ def run_web(self, host, port, debug=True): click.secho(f"Instance running!\nVisit https://{host}:{port}", fg="green") return [proc] - def run_worker(self, celery_log_file=None): + def run_worker(self, celery_log_file=None, celery_log_level="INFO"): """Run Celery worker.""" click.secho("Starting celery worker...", fg="green") @@ -172,7 +172,7 @@ def run_worker(self, celery_log_file=None): "--beat", "--events", "--loglevel", - "INFO", + celery_log_level, "--queues", "celery,low", ] @@ -188,9 +188,17 @@ def run_worker(self, celery_log_file=None): click.secho("Worker running!", fg="green") return [proc] - def run_all(self, host, port, debug=True, services=True, celery_log_file=None): + def run_all( + self, + host, + port, + debug=True, + services=True, + celery_log_file=None, + celery_log_level="INFO", + ): """Run all services.""" return [ *self.run_web(host, port, debug), - *self.run_worker(celery_log_file), + *self.run_worker(celery_log_file, celery_log_level), ] From ecf6fcccd1a4ff69957c92a252f7a022a231a25a Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Sat, 18 Jan 2025 20:54:36 +0100 Subject: [PATCH 07/13] setup: remove upper pins --- setup.cfg | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup.cfg b/setup.cfg index 31219817..2dcab2e5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,7 +2,7 @@ # # Copyright (C) 2019 CERN. # Copyright (C) 2019 Northwestern University. -# Copyright (C) 2022 Graz University of Technology. +# Copyright (C) 2022-2025 Graz University of Technology. # # Invenio-Cli is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. @@ -28,10 +28,10 @@ python_requires = >=3.7 zip_safe = False install_requires = Babel>=2.8 - cookiecutter>=2.0.0,<2.2.0 - click>=7.1.1,<8.2 - click-default-group>=1.2.2,<2.0.0 - docker>=7.1.0,<8.0.0 + cookiecutter>=2.0.0 + click>=7.1.1 + click-default-group>=1.2.2 + docker>=7.1.0 pipfile>=0.0.2 pipenv>=2020.6.2 PyYAML>=5.1.2 From 0e7713efd12a840cb1e01053d4b395402a3a0efb Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Thu, 2 Jan 2025 09:15:43 +0100 Subject: [PATCH 08/13] packages: implement uv as alternative to pipenv * implement `uv` as alternative for `pipenv` as python package management backend - to be configured in the file `.invenio` * handle the detection of the python package manager to be used and the generation of its commands in a central spot to lower the maintenance burden * make some commands (like translations) aware of the CLI config (and thus the configured python package manager) * implement commands for activating and removing the venv as shell scripts for `uv` as it doesn't implement such utilities --- invenio_cli/cli/cli.py | 15 +- invenio_cli/cli/packages.py | 14 +- invenio_cli/cli/translations.py | 7 +- invenio_cli/commands/assets.py | 9 +- invenio_cli/commands/commands.py | 21 ++- invenio_cli/commands/containers.py | 6 +- invenio_cli/commands/install.py | 22 ++- invenio_cli/commands/local.py | 28 ++-- invenio_cli/commands/packages.py | 86 ++++-------- invenio_cli/commands/services.py | 65 ++++----- invenio_cli/commands/translations.py | 35 ++--- invenio_cli/commands/upgrade.py | 28 ++-- invenio_cli/helpers/cli_config.py | 26 +++- invenio_cli/helpers/package_managers.py | 173 ++++++++++++++++++++++++ 14 files changed, 358 insertions(+), 177 deletions(-) create mode 100644 invenio_cli/helpers/package_managers.py diff --git a/invenio_cli/cli/cli.py b/invenio_cli/cli/cli.py index b0aa7183..248d0ff7 100644 --- a/invenio_cli/cli/cli.py +++ b/invenio_cli/cli/cli.py @@ -73,9 +73,10 @@ def check_requirements(development): @invenio_cli.command() -def shell(): +@pass_cli_config +def shell(cli_config): """Shell command.""" - Commands.shell() + Commands(cli_config).shell() @invenio_cli.command() @@ -86,9 +87,10 @@ def shell(): is_flag=True, help="Enable Flask development mode (default: disabled).", ) -def pyshell(debug): +@pass_cli_config +def pyshell(cli_config, debug): """Python shell command.""" - Commands.pyshell(debug=debug) + Commands(cli_config).pyshell(debug=debug) @invenio_cli.command() @@ -290,9 +292,10 @@ def destroy(cli_config): @invenio_cli.command() @click.option("--script", required=True, help="The path of custom migration script.") -def upgrade(script): +@pass_cli_config +def upgrade(cli_config, script): """Upgrades the current instance to a newer version.""" - steps = UpgradeCommands.upgrade(script) + steps = UpgradeCommands(cli_config).upgrade(script) on_fail = "Upgrade failed." on_success = "Upgrade sucessfull." diff --git a/invenio_cli/cli/packages.py b/invenio_cli/cli/packages.py index 18e0ed6e..29d95de5 100644 --- a/invenio_cli/cli/packages.py +++ b/invenio_cli/cli/packages.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Copyright (C) 2020-2021 CERN. -# Copyright (C) 2022 Graz University of Technology. +# Copyright (C) 2022-2025 Graz University of Technology. # # Invenio-Cli is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. @@ -34,7 +34,7 @@ def lock(cli_config, pre, dev): + f"Include dev-packages: {dev}.", fg="green", ) - steps = PackagesCommands.lock(pre, dev) + steps = PackagesCommands(cli_config).lock(pre, dev) on_fail = "Failed to lock dependencies." on_success = "Dependencies locked successfully." @@ -58,7 +58,7 @@ def install(cli_config, packages, skip_build, pip_log_file, node_log_file): if len(packages) < 1: raise click.UsageError("You must specify at least one package.") - steps = PackagesCommands.install_packages(packages, pip_log_file) + steps = PackagesCommands(cli_config).install_packages(packages, pip_log_file) on_fail = f"Failed to install packages {packages}." on_success = f"Packages {packages} installed successfully." @@ -77,7 +77,7 @@ def install(cli_config, packages, skip_build, pip_log_file, node_log_file): @pass_cli_config def outdated(cli_config): """Show outdated Python dependencies.""" - steps = PackagesCommands.outdated_packages() + steps = PackagesCommands(cli_config).outdated_packages() on_fail = "Some of the packages need to be updated." on_success = "All packages are up to date." @@ -94,11 +94,13 @@ def update(cli_config, version=None): db = cli_config.get_db_type() search = cli_config.get_search_type() package = f"invenio-app-rdm[{db},{search}]~=" - steps = PackagesCommands.update_package_new_version(package, version) + steps = PackagesCommands(cli_config).update_package_new_version( + package, version + ) on_fail = f"Failed to update version {version}" on_success = f"Version {version} installed successfully." else: - steps = PackagesCommands.update_packages() + steps = PackagesCommands(cli_config).update_packages() on_fail = "Failed to update packages." on_success = "Packages installed successfully." diff --git a/invenio_cli/cli/translations.py b/invenio_cli/cli/translations.py index d7f213ef..ae39b23d 100644 --- a/invenio_cli/cli/translations.py +++ b/invenio_cli/cli/translations.py @@ -31,7 +31,7 @@ def translations(): def extract(cli_config, babel_ini): """Extract messages for i18n support (translations).""" click.secho("Extracting messages...", fg="green") - steps = TranslationsCommands.extract( + steps = TranslationsCommands(cli_config).extract( msgid_bugs_address=cli_config.get_author_email(), copyright_holder=cli_config.get_author_name(), babel_file=cli_config.get_project_dir() / Path("translations/babel.ini"), @@ -50,7 +50,7 @@ def extract(cli_config, babel_ini): def init(cli_config, locale): """Initialized message catalog for a given locale.""" click.secho("Initializing messages catalog...", fg="green") - steps = TranslationsCommands.init( + steps = TranslationsCommands(cli_config).init( output_dir=cli_config.get_project_dir() / Path("translations/"), input_file=cli_config.get_project_dir() / Path("translations/messages.pot"), locale=locale, @@ -66,7 +66,7 @@ def init(cli_config, locale): def update(cli_config): """Update messages catalog.""" click.secho("Updating messages catalog...", fg="green") - steps = TranslationsCommands.update( + steps = TranslationsCommands(cli_config).update( output_dir=cli_config.get_project_dir() / Path("translations/"), input_file=cli_config.get_project_dir() / Path("translations/messages.pot"), ) @@ -83,6 +83,7 @@ def compile(cli_config, fuzzy): """Compile message catalog.""" click.secho("Compiling catalog...", fg="green") commands = TranslationsCommands( + cli_config, project_path=cli_config.get_project_dir(), instance_path=cli_config.get_instance_path(), ) diff --git a/invenio_cli/commands/assets.py b/invenio_cli/commands/assets.py index 80a6519b..dcdbaff4 100644 --- a/invenio_cli/commands/assets.py +++ b/invenio_cli/commands/assets.py @@ -113,9 +113,12 @@ def _assets_link(assets_pkg, module_pkg): def watch_assets(self): """High-level command to watch assets for changes.""" - # Commands - prefix = ["pipenv", "run"] - watch_cmd = prefix + ["invenio", "webpack", "run", "start"] + watch_cmd = self.cli_config.python_package_manager.run_command( + "invenio", + "webpack", + "run", + "start", + ) with env(FLASK_DEBUG="true"): # Collect into statics/ and assets/ folder diff --git a/invenio_cli/commands/commands.py b/invenio_cli/commands/commands.py index 4327f22e..fe432261 100644 --- a/invenio_cli/commands/commands.py +++ b/invenio_cli/commands/commands.py @@ -7,6 +7,8 @@ """Invenio module to ease the creation and management of applications.""" + +from ..helpers.cli_config import CLIConfig from ..helpers.env import env from ..helpers.process import run_interactive from .steps import CommandStep @@ -15,27 +17,23 @@ class Commands(object): """Abstraction over CLI commands that are either local or containerized.""" - def __init__(self, cli_config): + def __init__(self, cli_config: CLIConfig): """Constructor. :param cli_config: :class:CLIConfig instance """ self.cli_config = cli_config - @classmethod - def shell(cls): + def shell(self): """Start a shell in the virtual environment.""" - command = [ - "pipenv", - "shell", - ] + command = self.cli_config.python_package_manager.start_activated_subshell() return run_interactive(command, env={"PIPENV_VERBOSITY": "-1"}) - @classmethod - def pyshell(cls, debug=False): + def pyshell(self, debug=False): """Start a Python shell.""" + pkg_man = self.cli_config.python_package_manager with env(FLASK_DEBUG=str(debug)): - command = ["pipenv", "run", "invenio", "shell"] + command = pkg_man.run_command("invenio", "shell") return run_interactive(command, env={"PIPENV_VERBOSITY": "-1"}) def destroy(self): @@ -44,9 +42,10 @@ def destroy(self): NOTE: This function has no knowledge of the existence of services. Refer to services.py to destroy services' containers. """ + command = self.cli_config.python_package_manager.remove_venv() steps = [ CommandStep( - cmd=["pipenv", "--rm"], + cmd=command, env={"PIPENV_VERBOSITY": "-1"}, message="Destroying virtual environment", ) diff --git a/invenio_cli/commands/containers.py b/invenio_cli/commands/containers.py index 8b64d4ae..218011a9 100644 --- a/invenio_cli/commands/containers.py +++ b/invenio_cli/commands/containers.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. +# Copyright (C) 2025 Graz University of Technology. # # Invenio-Cli is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. @@ -34,7 +35,7 @@ def build(self, pull=True, cache=True): """ steps = [ FunctionStep( - func=PackagesCommands.is_locked, + func=lambda: PackagesCommands(self.cli_config).is_locked(), message="Checking if dependencies are locked.", ), FunctionStep( @@ -233,6 +234,7 @@ def rdm_fixtures(self, project_shortname): def translations(self, project_shortname): """Steps to compile translations for the instance.""" commands = TranslationsCommands( + self.cli_config, project_path=self.cli_config.get_project_dir(), # we use INVENIO_INSTANCE_PATH that is set in the Dockerfile as # config.instance_path is set only in development `install` command @@ -312,7 +314,7 @@ def start( if lock: # FIXME: Should this params be accepted? sensible defaults? - steps.extend(PackagesCommands.lock(pre=True, dev=True)) + steps.extend(PackagesCommands(self.cli_config).lock(pre=True, dev=True)) if build: steps.extend(self.build()) diff --git a/invenio_cli/commands/install.py b/invenio_cli/commands/install.py index 01391371..91fdbd25 100644 --- a/invenio_cli/commands/install.py +++ b/invenio_cli/commands/install.py @@ -1,13 +1,15 @@ # -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. -# Copyright (C) 2025 Graz University of Technology. +# Copyright (C) 2024-2025 Graz University of Technology. # # Invenio-Cli is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. """Invenio module to ease the creation and management of applications.""" +import sys + from ..helpers import filesystem from ..helpers.process import run_cmd from .local import LocalCommands @@ -26,29 +28,25 @@ def install_py_dependencies(self, pre, dev=False): """Install Python dependencies.""" # If not locked, lock. Then install. steps = [] + packages_commands = PackagesCommands(self.cli_config) - if PackagesCommands.is_locked().status_code > 0: - steps.extend(PackagesCommands.lock(pre, dev)) + if packages_commands.is_locked().status_code > 0: + steps.extend(packages_commands.lock(pre, dev)) - steps.extend(PackagesCommands.install_locked_dependencies(pre, dev)) + steps.extend(packages_commands.install_locked_dependencies(pre, dev)) return steps def update_instance_path(self): """Update path to instance in config.""" - # FIXME: Transform into steps. - # Currently not possible because the second step (update instance - # path) requires the ouptut of the previous step result = run_cmd( - [ - "pipenv", - "run", + self.cli_config.python_package_manager.run_command( "invenio", "shell", "--no-term-title", "-c", "\"print(app.instance_path, end='')\"", - ] + ) ) if result.status_code == 0: self.cli_config.update_instance_path(result.output.strip()) @@ -93,7 +91,7 @@ def install_assets(self, debug=False): ) ] - def install(self, pre, dev=False, flask_env="production"): + def install(self, pre, dev=False, debug=False, flask_env="production"): """Development installation steps.""" steps = self.install_py_dependencies(pre=pre, dev=dev) steps.extend(self.symlink()) diff --git a/invenio_cli/commands/local.py b/invenio_cli/commands/local.py index 8b7d5b0a..daf0b576 100644 --- a/invenio_cli/commands/local.py +++ b/invenio_cli/commands/local.py @@ -10,6 +10,7 @@ import os import signal +import sys from distutils.dir_util import copy_tree from os import environ from pathlib import Path @@ -84,17 +85,16 @@ def update_statics_and_assets(self, force, debug=False, log_file=None): Needed here (parent) because is used by Assets and Install commands. """ # Commands - prefix = ["pipenv", "run", "invenio"] - - ops = [prefix + ["collect", "--verbose"]] + pkg_man = self.cli_config.python_package_manager + ops = [pkg_man.run_command("invenio", "collect", "--verbose")] if force: - ops.append(prefix + ["webpack", "clean", "create"]) - ops.append(prefix + ["webpack", "install"]) + ops.append(pkg_man.run_command("invenio", "webpack", "clean", "create")) + ops.append(pkg_man.run_command("invenio", "webpack", "install")) else: - ops.append(prefix + ["webpack", "create"]) + ops.append(pkg_man.run_command("invenio", "webpack", "create")) ops.append(self._statics) - ops.append(prefix + ["webpack", "build"]) + ops.append(pkg_man.run_command("invenio", "webpack", "build")) # Keep the same messages for some of the operations for backward compatibility messages = { "build": "Building assets...", @@ -135,10 +135,9 @@ def run_web(self, host, port, debug=True): run_env["FLASK_DEBUG"] = str(debug) run_env["INVENIO_SITE_UI_URL"] = f"https://{host}:{port}" run_env["INVENIO_SITE_API_URL"] = f"https://{host}:{port}/api" + pkg_man = self.cli_config.python_package_manager proc = popen( - [ - "pipenv", - "run", + pkg_man.run_command( "invenio", "run", "--cert", @@ -151,7 +150,7 @@ def run_web(self, host, port, debug=True): port, "--extra-files", "invenio.cfg", - ], + ), env=run_env, ) self._handle_sigint("Web server", proc) @@ -162,9 +161,8 @@ def run_worker(self, celery_log_file=None, celery_log_level="INFO"): """Run Celery worker.""" click.secho("Starting celery worker...", fg="green") - celery_command = [ - "pipenv", - "run", + pkg_man = self.cli_config.python_package_manager + celery_command = pkg_man.run_command( "celery", "--app", "invenio_app.celery", @@ -175,7 +173,7 @@ def run_worker(self, celery_log_file=None, celery_log_level="INFO"): celery_log_level, "--queues", "celery,low", - ] + ) if celery_log_file: celery_command += [ diff --git a/invenio_cli/commands/packages.py b/invenio_cli/commands/packages.py index f47514a1..3cf029f0 100644 --- a/invenio_cli/commands/packages.py +++ b/invenio_cli/commands/packages.py @@ -1,15 +1,17 @@ # -*- coding: utf-8 -*- # # Copyright (C) 2020-2021 CERN. -# Copyright (C) 2022 Graz University of Technology. +# Copyright (C) 2022-2025 Graz University of Technology. # # Invenio-Cli is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. """Invenio module to ease the creation and management of applications.""" +import sys from os import listdir +from ..helpers.cli_config import CLIConfig from ..helpers.process import ProcessResponse from .steps import CommandStep @@ -17,17 +19,13 @@ class PackagesCommands(object): """Local installation commands.""" - @staticmethod - def install_packages(packages, log_file=None): - """Steps to install Python packages. - - It is a class method since it does not require any configuration. - """ - prefix = ["pipenv", "run"] - cmd = prefix + ["pip", "install"] - for package in packages: - cmd.extend(["-e", package]) + def __init__(self, cli_config: CLIConfig): + """Construct PackagesCommands.""" + self.cli_config = cli_config + def install_packages(self, packages, log_file=None): + """Steps to install Python packages.""" + cmd = self.cli_config.python_package_manager.editable_dev_install(*packages) steps = [ CommandStep( cmd=cmd, @@ -39,14 +37,9 @@ def install_packages(packages, log_file=None): return steps - @staticmethod - def outdated_packages(): - """Steps to show outdated packages. - - It is a class method since it does not require any configuration. - """ - cmd = ["pipenv", "update", "--outdated"] - + def outdated_packages(self): + """Steps to show outdated packages.""" + cmd = self.cli_config.python_package_manager.list_outdated_packages() steps = [ CommandStep( cmd=cmd, @@ -57,14 +50,9 @@ def outdated_packages(): return steps - @staticmethod - def update_packages(): - """Steps to update all Python packages. - - It is a class method since it does not require any configuration. - """ - cmd = ["pipenv", "update"] - + def update_packages(self): + """Steps to update all Python packages.""" + cmd = self.cli_config.python_package_manager.update_packages() steps = [ CommandStep( cmd=cmd, @@ -75,18 +63,15 @@ def update_packages(): return steps - @staticmethod - def update_package_new_version(package, version): + def update_package_new_version(self, package, version): """Update invenio-app-rdm version. It is a class method since it does not require any configuration. """ - prefix = ["pipenv"] - app = prefix + ["install", package + version] - + cmd = self.cli_config.python_package_manager.install_package(package, version) steps = [ CommandStep( - cmd=app, + cmd=cmd, env={"PIPENV_VERBOSITY": "-1"}, message=f"Updating {package} to version {version}...", ) @@ -94,36 +79,24 @@ def update_package_new_version(package, version): return steps - @staticmethod - def install_locked_dependencies(pre, dev): - """Install dependencies from Pipfile.lock using sync.""" - # NOTE: sync has no interactive process feedback - cmd = ["pipenv", "sync"] - if pre: - cmd += ["--pre"] - if dev: - cmd += ["--dev"] - + def install_locked_dependencies(self, pre, dev): + """Install dependencies from requirements.txt using install.""" + cmd = self.cli_config.python_package_manager.install_locked_deps(pre, dev) steps = [ CommandStep( cmd=cmd, env={"PIPENV_VERBOSITY": "-1"}, - message="Installing python dependencies... Please be " - + "patient, this operation might take some time...", + message=( + "Installing python dependencies... Please be patient, this operation might take some time..." + ), ) ] return steps - @staticmethod - def lock(pre, dev): + def lock(self, pre, dev): """Steps to lock Python dependencies.""" - cmd = ["pipenv", "lock"] - if pre: - cmd += ["--pre"] - if dev: - cmd += ["--dev"] - + cmd = self.cli_config.python_package_manager.lock_dependencies(pre, dev) steps = [ CommandStep( cmd=cmd, @@ -134,10 +107,11 @@ def lock(pre, dev): return steps - @staticmethod - def is_locked(): + def is_locked(self): """Checks if the dependencies have been locked.""" - locked = "Pipfile.lock" in listdir(".") + lock_file_name = self.cli_config.python_package_manager.lock_file_name + locked = lock_file_name in listdir(".") + if not locked: return ProcessResponse( error="Dependencies were not locked. " diff --git a/invenio_cli/commands/services.py b/invenio_cli/commands/services.py index 1add4eb9..1261027b 100644 --- a/invenio_cli/commands/services.py +++ b/invenio_cli/commands/services.py @@ -86,43 +86,40 @@ def services_expected_status(self, expected): def _cleanup(self): """Services cleanup steps.""" + pkg_man = self.cli_config.python_package_manager steps = [ CommandStep( - cmd=[ - "pipenv", - "run", + cmd=pkg_man.run_command( "invenio", "shell", "--no-term-title", "-c", - "import redis; redis.StrictRedis.from_url(app.config['CACHE_REDIS_URL']).flushall(); print('Cache cleared')", - ], # noqa + "import redis; redis.StrictRedis.from_url(app.config['CACHE_REDIS_URL']).flushall(); print('Cache cleared')", # noqa + ), env={"PIPENV_VERBOSITY": "-1"}, message="Flushing Redis...", skippable=True, ), CommandStep( - cmd=["pipenv", "run", "invenio", "db", "destroy", "--yes-i-know"], + cmd=pkg_man.run_command("invenio", "db", "destroy", "--yes-i-know"), env={"PIPENV_VERBOSITY": "-1"}, message="Destroying database...", skippable=True, ), CommandStep( - cmd=[ - "pipenv", - "run", + cmd=pkg_man.run_command( "invenio", "index", "destroy", "--force", "--yes-i-know", - ], + ), env={"PIPENV_VERBOSITY": "-1"}, message="Destroying indices...", skippable=True, ), CommandStep( - cmd=["pipenv", "run", "invenio", "index", "queue", "init", "purge"], + cmd=pkg_man.run_command("invenio", "index", "queue", "init", "purge"), env={"PIPENV_VERBOSITY": "-1"}, message="Purging queues...", skippable=True, @@ -145,6 +142,7 @@ def _default_location_path(self): def _setup(self, demo_data=False): """Services initialization steps.""" + pkg_man = self.cli_config.python_package_manager steps = [ FunctionStep( func=self.services_expected_status, @@ -152,14 +150,12 @@ def _setup(self, demo_data=False): message="Checking services are not setup...", ), CommandStep( - cmd=["pipenv", "run", "invenio", "db", "init", "create"], + cmd=pkg_man.run_command("invenio", "db", "init", "create"), env={"PIPENV_VERBOSITY": "-1"}, message="Creating database...", ), CommandStep( - cmd=[ - "pipenv", - "run", + cmd=pkg_man.run_command( "invenio", "files", "location", @@ -167,31 +163,29 @@ def _setup(self, demo_data=False): "--default", "default-location", self._default_location_path(), - ], + ), env={"PIPENV_VERBOSITY": "-1"}, message="Creating files location...", ), CommandStep( - cmd=["pipenv", "run", "invenio", "roles", "create", "admin"], + cmd=pkg_man.run_command("invenio", "roles", "create", "admin"), env={"PIPENV_VERBOSITY": "-1"}, message="Creating admin role...", ), CommandStep( - cmd=[ - "pipenv", - "run", + cmd=pkg_man.run_command( "invenio", "access", "allow", "superuser-access", "role", "admin", - ], + ), env={"PIPENV_VERBOSITY": "-1"}, message="Allowing superuser access to admin role...", ), CommandStep( - cmd=["pipenv", "run", "invenio", "index", "init"], + cmd=pkg_man.run_command("invenio", "index", "init"), env={"PIPENV_VERBOSITY": "-1"}, message="Creating indices...", ), @@ -203,26 +197,22 @@ def _setup(self, demo_data=False): steps.extend( [ CommandStep( - cmd=[ - "pipenv", - "run", + cmd=pkg_man.run_command( "invenio", "rdm-records", "custom-fields", "init", - ], + ), env={"PIPENV_VERBOSITY": "-1"}, message="Creating custom fields for records...", ), CommandStep( - cmd=[ - "pipenv", - "run", + cmd=pkg_man.run_command( "invenio", "communities", "custom-fields", "init", - ], + ), env={"PIPENV_VERBOSITY": "-1"}, message="Creating custom fields for communities...", ), @@ -240,7 +230,7 @@ def _setup(self, demo_data=False): if demo_data: steps.extend(self.demo()) elif ils_version(): - cmd = ["pipenv", "run", "invenio", "setup", "--verbose"] + cmd = pkg_man.run_command("invenio", "setup", "--verbose") if not demo_data: cmd.append("--skip-demo-data") steps.extend( @@ -265,9 +255,10 @@ def _setup(self, demo_data=False): def demo(self): """Steps to add demo records into the instance.""" + pkg_man = self.cli_config.python_package_manager steps = [ CommandStep( - cmd=["pipenv", "run", "invenio", "rdm-records", "demo"], + cmd=pkg_man.run_command("invenio", "rdm-records", "demo"), env={"PIPENV_VERBOSITY": "-1"}, message="Creating demo records...", ) @@ -277,13 +268,15 @@ def demo(self): def declare_queues(self): """Steps to declare the MQ queues required for statistics, etc.""" - command = ["pipenv", "run", "invenio", "queues", "declare"] + pkg_man = self.cli_config.python_package_manager + command = pkg_man.run_command("invenio", "queues", "declare") steps = [CommandStep(cmd=command, message="Declaring queues...")] return steps def fixtures(self): """Steps to set up the required fixtures for the instance.""" - command = ["pipenv", "run", "invenio", "rdm-records", "fixtures"] + pkg_man = self.cli_config.python_package_manager + command = pkg_man.run_command("invenio", "rdm-records", "fixtures") steps = [ CommandStep( cmd=command, @@ -296,7 +289,8 @@ def fixtures(self): def rdm_fixtures(self): """Steps to set up the rdm fixtures for the instance.""" - command = ["pipenv", "run", "invenio", "rdm", "fixtures"] + pkg_man = self.cli_config.python_package_manager + command = pkg_man.run_command("invenio", "rdm", "fixtures") steps = [ CommandStep( cmd=command, @@ -310,6 +304,7 @@ def rdm_fixtures(self): def translations(self): """Steps to compile translations.""" commands = TranslationsCommands( + self.cli_config, project_path=self.cli_config.get_project_dir(), instance_path=self.cli_config.get_instance_path(), ) diff --git a/invenio_cli/commands/translations.py b/invenio_cli/commands/translations.py index bb5948f2..28038040 100644 --- a/invenio_cli/commands/translations.py +++ b/invenio_cli/commands/translations.py @@ -10,6 +10,7 @@ from pathlib import Path from ..commands import Commands +from ..helpers.cli_config import CLIConfig from ..helpers.filesystem import force_symlink from .steps import CommandStep, FunctionStep @@ -17,16 +18,14 @@ class TranslationsCommands(Commands): """Translations CLI commands.""" - CMD_PREFIX = ["pipenv", "run"] - - def __init__(self, project_path, instance_path): + def __init__(self, cli_config: CLIConfig, project_path=None, instance_path=None): """Constructor.""" + self.cli_config = cli_config self.project_path = project_path self.instance_path = instance_path - @classmethod def extract( - cls, + self, babel_file, output_file, input_dirs, @@ -35,7 +34,8 @@ def extract( add_comments="NOTE", ): """Extract messages from source code and templates.""" - cmd = cls.CMD_PREFIX + [ + pkg_man = self.cli_config.python_package_manager + cmd = pkg_man.run_command( "pybabel", "extract", f"--mapping-file={babel_file}", @@ -44,7 +44,7 @@ def extract( f"--msgid-bugs-address={msgid_bugs_address}", f"--copyright-holder={copyright_holder}", f"--add-comments={add_comments}", - ] + ) return [ CommandStep( @@ -54,16 +54,16 @@ def extract( ) ] - @classmethod - def init(cls, output_dir, input_file, locale): + def init(self, output_dir, input_file, locale): """Initialize a new language catalog.""" - cmd = cls.CMD_PREFIX + [ + pkg_man = self.cli_config.python_package_manager + cmd = pkg_man.run_command( "pybabel", "init", f"--output-dir={output_dir}", f"--input-file={input_file}", f"--locale={locale}", - ] + ) return [ CommandStep( @@ -73,15 +73,15 @@ def init(cls, output_dir, input_file, locale): ) ] - @classmethod - def update(cls, output_dir, input_file): + def update(self, output_dir, input_file): """Update the message catalog.""" - cmd = cls.CMD_PREFIX + [ + pkg_man = self.cli_config.python_package_manager + cmd = pkg_man.run_command( "pybabel", "update", f"--output-dir={output_dir}", f"--input-file={input_file}", - ] + ) return [ CommandStep( @@ -100,12 +100,13 @@ def compile( ): """Compile the message catalog.""" directory = directory or self.project_path / translation_folder + pkg_man = self.cli_config.python_package_manager - cmd = self.CMD_PREFIX + [ + cmd = pkg_man.run_command( "pybabel", "compile", f"--directory={directory}", - ] + ) if fuzzy: cmd.append("--use-fuzzy") diff --git a/invenio_cli/commands/upgrade.py b/invenio_cli/commands/upgrade.py index 5c998e0c..6b0ce53a 100644 --- a/invenio_cli/commands/upgrade.py +++ b/invenio_cli/commands/upgrade.py @@ -7,14 +7,18 @@ """Invenio module to ease the creation and management of applications.""" +from ..helpers.cli_config import CLIConfig from .steps import CommandStep class UpgradeCommands(object): """Local installation commands.""" - @staticmethod - def upgrade(script_path): + def __init__(self, cli_config: CLIConfig): + """Constructor.""" + self.cli_config = cli_config + + def upgrade(self, script_path): """Steps to perform an upgrade of the invenio instance. First, and alembic upgrade is launched to allow alembic to migrate the @@ -23,13 +27,19 @@ def upgrade(script_path): Last, the search indices are destroyed, initialized and rebuilt. It is a class method since it does not require any configuration. """ - prefix = ["pipenv", "run", "invenio"] - alembic_cmd = prefix + ["alembic", "upgrade"] - destroy_index_cmd = prefix + ["index", "destroy", "--yes-i-know"] - init_index_cmd = prefix + ["index", "init"] - rec_rebuild_index_cmd = prefix + ["rdm-records", "rebuild-index"] - comm_rebuild_index_cmd = prefix + ["communities", "rebuild-index"] - script_cmd = prefix + ["shell", script_path] + pkg_man = self.cli_config.python_package_manager + alembic_cmd = pkg_man.run_command("invenio", "alembic", "upgrade") + destroy_index_cmd = pkg_man.run_command( + "invenio", "index", "destroy", "--yes-i-know" + ) + init_index_cmd = pkg_man.run_command("invenio", "index", "init") + rec_rebuild_index_cmd = pkg_man.run_command( + "invenio", "rdm-records", "rebuild-index" + ) + comm_rebuild_index_cmd = pkg_man.run_command( + "invenio", "communities", "rebuild-index" + ) + script_cmd = pkg_man.run_command("invenio", "shell", script_path) steps = [ CommandStep( diff --git a/invenio_cli/helpers/cli_config.py b/invenio_cli/helpers/cli_config.py index 6f49fc4b..9ae2af71 100644 --- a/invenio_cli/helpers/cli_config.py +++ b/invenio_cli/helpers/cli_config.py @@ -15,6 +15,7 @@ from ..errors import InvenioCLIConfigError from .filesystem import get_created_files +from .package_managers import UV, Pipenv, PythonPackageManager from .process import ProcessResponse @@ -39,9 +40,10 @@ def __init__(self, project_dir="./"): :param config_dir: Path to general cli config file. """ - self.config_path = Path(project_dir) / self.CONFIG_FILENAME + self.project_path = Path(project_dir) + self.config_path = self.project_path / self.CONFIG_FILENAME self.config = ConfigParser() - self.private_config_path = Path(project_dir) / self.PRIVATE_CONFIG_FILENAME + self.private_config_path = self.project_path / self.PRIVATE_CONFIG_FILENAME self.private_config = ConfigParser() try: @@ -60,6 +62,26 @@ def __init__(self, project_dir="./"): with open(self.private_config_path) as cfg_file: self.private_config.read_file(cfg_file) + @property + def python_package_manager(self) -> PythonPackageManager: + """Get python packages manager.""" + manager_name = self.config[CLIConfig.CLI_SECTION].get( + "python_package_manager", None + ) + if manager_name == Pipenv.name: + return Pipenv() + elif manager_name == UV.name: + return UV() + + if (self.project_path / "Pipfile").is_file(): + return Pipenv() + elif (self.project_path / "pyproject.toml").is_file(): + return UV() + else: + raise RuntimeError( + "Could not determine the Python package manager, please configure it." + ) + def get_project_dir(self): """Returns path to project directory.""" return self.config_path.parent.resolve() diff --git a/invenio_cli/helpers/package_managers.py b/invenio_cli/helpers/package_managers.py new file mode 100644 index 00000000..a961e636 --- /dev/null +++ b/invenio_cli/helpers/package_managers.py @@ -0,0 +1,173 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2025 TU Wien. +# +# Invenio-Cli is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. + +"""Wrappers around various package managers to be used under the hood.""" + +import os +from abc import ABC +from typing import List + + +class PythonPackageManager(ABC): + """Interface for creating tool-specific Python package management commands.""" + + name: str = None + lock_file_name: str = None + + def run_command(self, *command: str) -> List[str]: + """Generate command to run the given command in the managed environment.""" + raise NotImplementedError() + + def editable_dev_install(self, package: str) -> List[str]: + """Install the local packages as editable, but ignore it for locking.""" + raise NotImplementedError() + + def install_package(self, package: str, version: str = None) -> List[str]: + """Install the package in the specified version.""" + raise NotImplementedError() + + def update_packages(self) -> List[str]: + """Update all updated packages.""" + raise NotImplementedError() + + def list_outdated_packages(self) -> List[str]: + """List outdated installed packages.""" + raise NotImplementedError() + + def install_locked_deps(self, prereleases: bool, devtools: bool) -> List[str]: + """Install the packages according to the lock file.""" + raise NotImplementedError() + + def lock_dependencies(self, prereleases: bool, devtools: bool) -> List[str]: + """Update the lock file.""" + raise NotImplementedError() + + def remove_venv(self) -> List[str]: + """Remove the created virtualenv.""" + raise NotImplementedError() + + def start_activated_subshell(self) -> List[str]: + """Remove the created virtualenv.""" + raise NotImplementedError() + + +class Pipenv(PythonPackageManager): + """Generate ``pipenv`` commands for managing Python packages.""" + + name = "pipenv" + lock_file_name = "Pipfile.lock" + + def run_command(self, *command): + """Generate command to run the given command in the managed environment.""" + return [self.name, "run", *command] + + def editable_dev_install(self, *packages): + """Install the local packages as editable, but ignore it for locking.""" + cmd = [self.name, "run", "pip", "install"] + for package in packages: + cmd += ["-e", package] + return cmd + + def install_package(self, package, version=None): + """Install the package in the specified version.""" + package_version = package if not version else package + version + return [self.name, "install", package_version] + + def update_packages(self): + """Update all updated packages.""" + return [self.name, "update"] + + def list_outdated_packages(self): + """List outdated installed packages.""" + return [self.name, "update", "--outdated"] + + def install_locked_deps(self, prereleases, devtools): + """Install the packages according to the lock file.""" + cmd = [self.name, "sync"] + if prereleases: + cmd += ["--pre"] + if devtools: + cmd += ["--dev"] + return cmd + + def lock_dependencies(self, prereleases, devtools): + """Update the lock file.""" + cmd = [self.name, "lock"] + if prereleases: + cmd += ["--pre"] + if devtools: + cmd += ["--dev"] + return cmd + + def remove_venv(self): + """Remove the created virtualenv.""" + return ["pipenv", "--rm"] + + def start_activated_subshell(self) -> List[str]: + """Remove the created virtualenv.""" + return ["pipenv", "shell"] + + +class UV(PythonPackageManager): + """Generate ``uv`` commands for managing Python packages.""" + + name = "uv" + lock_file_name = "uv.lock" + + def run_command(self, *command): + """Generate command to run the given command in the managed environment.""" + # "--no-sync" is used to not override locally installed editable packages + return [self.name, "run", "--no-sync", *command] + + def editable_dev_install(self, *packages): + """Install the local packages as editable, but ignore it for locking.""" + cmd = [self.name, "pip", "install"] + for package in packages: + cmd += ["-e", package] + return cmd + + def install_package(self, package, version=None): + """Install the package in the specified version.""" + package_version = package if not version else package + version + return [self.name, "add", package_version] + + def update_packages(self): + """Update all updated packages.""" + return [self.name, "sync", "--upgrade"] + + def list_outdated_packages(self): + """List outdated installed packages.""" + return [self.name, "sync", "--upgrade", "--dry-run"] + + def install_locked_deps(self, prereleases, devtools): + """Install the packages according to the lock file.""" + cmd = [self.name, "sync"] + if prereleases: + cmd += ["--prerelease", "allow"] + if not devtools: + cmd += ["--no-dev"] + return cmd + + def lock_dependencies(self, prereleases, devtools): + """Update the lock file.""" + cmd = [self.name, "lock"] + if prereleases: + cmd += ["--prerelease", "allow"] + return cmd + + def remove_venv(self): + """Remove the created virtualenv.""" + # This assumes the default location for the uv venv + return ["rm", "-r", ".venv"] + + def start_activated_subshell(self) -> List[str]: + """Remove the created virtualenv.""" + # This assumes we're using a Unixoid OS... + # Since Invenio doesn't support Windows that should be given + # Also, it has a good chance of not properly setting a PS1... + shell = os.getenv("SHELL") + return [shell, "-c", f"source .venv/bin/activate; exec {shell} -i"] From 3a0eb37c17bf7c966fcaf909e825dc6296f1e9cf Mon Sep 17 00:00:00 2001 From: Maximilian Moser Date: Wed, 26 Feb 2025 15:18:41 +0100 Subject: [PATCH 09/13] global: fix typo in FLASK_DEBUG and harmonize usage --- invenio_cli/commands/assets.py | 2 +- invenio_cli/commands/commands.py | 2 +- invenio_cli/commands/local.py | 4 ++-- tests/commands/test_local.py | 2 +- tests/helpers/test_env.py | 8 ++++---- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/invenio_cli/commands/assets.py b/invenio_cli/commands/assets.py index dcdbaff4..22c06da6 100644 --- a/invenio_cli/commands/assets.py +++ b/invenio_cli/commands/assets.py @@ -120,7 +120,7 @@ def watch_assets(self): "start", ) - with env(FLASK_DEBUG="true"): + with env(FLASK_DEBUG="1"): # Collect into statics/ and assets/ folder click.secho( "Starting assets watching (press CTRL+C to stop)...", fg="green" diff --git a/invenio_cli/commands/commands.py b/invenio_cli/commands/commands.py index fe432261..bb343a0f 100644 --- a/invenio_cli/commands/commands.py +++ b/invenio_cli/commands/commands.py @@ -32,7 +32,7 @@ def shell(self): def pyshell(self, debug=False): """Start a Python shell.""" pkg_man = self.cli_config.python_package_manager - with env(FLASK_DEBUG=str(debug)): + with env(FLASK_DEBUG="1" if debug else "0"): command = pkg_man.run_command("invenio", "shell") return run_interactive(command, env={"PIPENV_VERBOSITY": "-1"}) diff --git a/invenio_cli/commands/local.py b/invenio_cli/commands/local.py index daf0b576..ff573cc8 100644 --- a/invenio_cli/commands/local.py +++ b/invenio_cli/commands/local.py @@ -101,7 +101,7 @@ def update_statics_and_assets(self, force, debug=False, log_file=None): "install": "Installing JS dependencies...", } - with env(FLASK_DEUBG="true" if debug else "false"): + with env(FLASK_DEBUG="1" if debug else "0"): for op in ops: if callable(op): response = op() @@ -132,7 +132,7 @@ def run_web(self, host, port, debug=True): """Run development server.""" click.secho("Starting up local (development) server...", fg="green") run_env = environ.copy() - run_env["FLASK_DEBUG"] = str(debug) + run_env["FLASK_DEBUG"] = "1" if debug else "0" run_env["INVENIO_SITE_UI_URL"] = f"https://{host}:{port}" run_env["INVENIO_SITE_API_URL"] = f"https://{host}:{port}/api" pkg_man = self.cli_config.python_package_manager diff --git a/tests/commands/test_local.py b/tests/commands/test_local.py index ab0e0852..3bb701d2 100644 --- a/tests/commands/test_local.py +++ b/tests/commands/test_local.py @@ -283,7 +283,7 @@ def test_run( commands.run(host=host, port=port, debug=True) run_env = environ.copy() - run_env["FLASK_DEBUG"] = "True" + run_env["FLASK_DEBUG"] = "1" run_env["INVENIO_SITE_HOSTNAME"] = f"{host}:{port}" expected_calls = [ call(["pipenv", "run", "celery", "--app", "invenio_app.celery", "worker"]), diff --git a/tests/helpers/test_env.py b/tests/helpers/test_env.py index 945ac9ab..7a22524a 100644 --- a/tests/helpers/test_env.py +++ b/tests/helpers/test_env.py @@ -18,10 +18,10 @@ def test_env(): assert "SERVER_NAME" not in os.environ assert "FLASK_DEBUG" not in os.environ - os.environ["FLASK_DEBUG"] = "true" - with env(SERVER_NAME="example.com", FLASK_DEBUG="false"): + os.environ["FLASK_DEBUG"] = "1" + with env(SERVER_NAME="example.com", FLASK_DEBUG="0"): assert os.environ["SERVER_NAME"] == "example.com" - assert os.environ["FLASK_DEBUG"] == "false" + assert os.environ["FLASK_DEBUG"] == "0" - assert os.environ["FLASK_DEBUG"] == "true" + assert os.environ["FLASK_DEBUG"] == "1" assert "SERVER_NAME" not in os.environ From f2d2345c3e884f2eca4c90d710264323a3eb3250 Mon Sep 17 00:00:00 2001 From: Maximilian Moser Date: Wed, 26 Feb 2025 15:36:05 +0100 Subject: [PATCH 10/13] chore: fix ruff linter warnings --- docs/conf.py | 2 -- invenio_cli/cli/translations.py | 4 ++-- invenio_cli/commands/install.py | 1 - invenio_cli/commands/local.py | 1 - invenio_cli/commands/packages.py | 1 - invenio_cli/commands/requirements.py | 5 ++--- invenio_cli/commands/translations.py | 5 ++--- invenio_cli/helpers/__init__.py | 7 +++++++ tests/helpers/test_cli_config.py | 1 - 9 files changed, 13 insertions(+), 14 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 0f7a17c2..00cb72ba 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -11,8 +11,6 @@ from __future__ import print_function -import sphinx.environment - from invenio_cli import __version__ # -- General configuration ------------------------------------------------ diff --git a/invenio_cli/cli/translations.py b/invenio_cli/cli/translations.py index ae39b23d..cf898a94 100644 --- a/invenio_cli/cli/translations.py +++ b/invenio_cli/cli/translations.py @@ -70,8 +70,8 @@ def update(cli_config): output_dir=cli_config.get_project_dir() / Path("translations/"), input_file=cli_config.get_project_dir() / Path("translations/messages.pot"), ) - on_fail = f"Failed to update message catalog." - on_success = f"Message catalog updated successfully." + on_fail = "Failed to update message catalog." + on_success = "Message catalog updated successfully." run_steps(steps, on_fail, on_success) diff --git a/invenio_cli/commands/install.py b/invenio_cli/commands/install.py index 91fdbd25..eab744e4 100644 --- a/invenio_cli/commands/install.py +++ b/invenio_cli/commands/install.py @@ -8,7 +8,6 @@ """Invenio module to ease the creation and management of applications.""" -import sys from ..helpers import filesystem from ..helpers.process import run_cmd diff --git a/invenio_cli/commands/local.py b/invenio_cli/commands/local.py index ff573cc8..a6b96d57 100644 --- a/invenio_cli/commands/local.py +++ b/invenio_cli/commands/local.py @@ -10,7 +10,6 @@ import os import signal -import sys from distutils.dir_util import copy_tree from os import environ from pathlib import Path diff --git a/invenio_cli/commands/packages.py b/invenio_cli/commands/packages.py index 3cf029f0..b4f47499 100644 --- a/invenio_cli/commands/packages.py +++ b/invenio_cli/commands/packages.py @@ -8,7 +8,6 @@ """Invenio module to ease the creation and management of applications.""" -import sys from os import listdir from ..helpers.cli_config import CLIConfig diff --git a/invenio_cli/commands/requirements.py b/invenio_cli/commands/requirements.py index c75e1722..938ed2d0 100644 --- a/invenio_cli/commands/requirements.py +++ b/invenio_cli/commands/requirements.py @@ -12,10 +12,9 @@ import json import re import sys -from os import listdir from ..helpers.docker_helper import DockerHelper -from ..helpers.process import ProcessResponse, run_cmd, run_interactive +from ..helpers.process import ProcessResponse, run_cmd from ..helpers.rdm import rdm_version from .steps import FunctionStep @@ -187,7 +186,7 @@ def check_pipenv_installed(cls): return ProcessResponse( output=f"Pipenv OK. Got version {version}.", status_code=0 ) - except Exception as err: + except Exception: return ProcessResponse( error=f"Pipenv not found. Got {result.error}.", status_code=1 ) diff --git a/invenio_cli/commands/translations.py b/invenio_cli/commands/translations.py index 28038040..5fd85c7e 100644 --- a/invenio_cli/commands/translations.py +++ b/invenio_cli/commands/translations.py @@ -7,7 +7,6 @@ """Invenio module to ease the creation and management of applications.""" -from pathlib import Path from ..commands import Commands from ..helpers.cli_config import CLIConfig @@ -87,7 +86,7 @@ def update(self, output_dir, input_file): CommandStep( cmd=cmd, env={"PIPENV_VERBOSITY": "-1"}, - message=f"Updating message catalog...", + message="Updating message catalog...", ) ] @@ -115,7 +114,7 @@ def compile( CommandStep( cmd=cmd, env={"PIPENV_VERBOSITY": "-1"}, - message=f"Compiling message catalog...", + message="Compiling message catalog...", skippable=True, ), ] diff --git a/invenio_cli/helpers/__init__.py b/invenio_cli/helpers/__init__.py index b0eb9abd..a8d9485f 100644 --- a/invenio_cli/helpers/__init__.py +++ b/invenio_cli/helpers/__init__.py @@ -13,3 +13,10 @@ from .docker_helper import DockerHelper from .env import env from .filesystem import get_created_files + +__all__ = ( + "CookiecutterWrapper", + "DockerHelper", + "env", + "get_created_files", +) diff --git a/tests/helpers/test_cli_config.py b/tests/helpers/test_cli_config.py index b5dfef73..be0551be 100644 --- a/tests/helpers/test_cli_config.py +++ b/tests/helpers/test_cli_config.py @@ -10,7 +10,6 @@ import os import tempfile from pathlib import Path -from unittest.mock import patch import pytest From 6eadda8151bee318937980fd8b96efba8b55828b Mon Sep 17 00:00:00 2001 From: Maximilian Moser Date: Fri, 28 Feb 2025 16:58:58 +0100 Subject: [PATCH 11/13] tests: remove pytest marks on fixtures * we got a warning that marks on fixtures don't have an effect --- tests/commands/test_containers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/commands/test_containers.py b/tests/commands/test_containers.py index 7af2e31f..e65fac17 100644 --- a/tests/commands/test_containers.py +++ b/tests/commands/test_containers.py @@ -16,7 +16,6 @@ from invenio_cli.commands import ContainersCommands -@pytest.mark.skip() @pytest.fixture(scope="function") def expected_setup_calls(): return [ @@ -36,7 +35,6 @@ def expected_setup_calls(): ] -@pytest.mark.skip() @pytest.fixture(scope="function") def expected_force_calls(): return [ From 1d19092cfecc43beccdddc7b7c64ac2fd55f2d7b Mon Sep 17 00:00:00 2001 From: Maximilian Moser Date: Fri, 28 Feb 2025 17:19:52 +0100 Subject: [PATCH 12/13] config: move {web,search} {host,port} config to the private config * the private config (`.invenio.private`) is not supposed to be added to version control and thus suited better for developer preferences than the public config (`.invenio`) --- invenio_cli/helpers/cli_config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/invenio_cli/helpers/cli_config.py b/invenio_cli/helpers/cli_config.py index 9ae2af71..46e1092f 100644 --- a/invenio_cli/helpers/cli_config.py +++ b/invenio_cli/helpers/cli_config.py @@ -133,22 +133,22 @@ def get_project_shortname(self): def get_search_port(self): """Returns the search port.""" - return self.config[CLIConfig.COOKIECUTTER_SECTION].get("search_port", "9200") + return self.private_config[CLIConfig.CLI_SECTION].get("search_port", "9200") def get_search_host(self): """Returns the search host.""" - return self.config[CLIConfig.COOKIECUTTER_SECTION].get( + return self.private_config[CLIConfig.CLI_SECTION].get( "search_host", "localhost", ) def get_web_port(self): """Returns web port.""" - return self.config[CLIConfig.COOKIECUTTER_SECTION].get("web_port", "5000") + return self.private_config[CLIConfig.CLI_SECTION].get("web_port", "5000") def get_web_host(self): """Returns web host.""" - return self.config[CLIConfig.COOKIECUTTER_SECTION].get("web_host", "127.0.0.1") + return self.private_config[CLIConfig.CLI_SECTION].get("web_host", "127.0.0.1") def get_db_type(self): """Returns the database type (mysql, postgresql).""" From 3799492fc22001eed9e0f87e9e1ce70ca2deb3b6 Mon Sep 17 00:00:00 2001 From: Maximilian Moser Date: Fri, 28 Feb 2025 17:22:03 +0100 Subject: [PATCH 13/13] release: v1.6.0 --- CHANGES.rst | 8 ++++++++ invenio_cli/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 0c1fa3bd..8a6179a9 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -8,6 +8,14 @@ Changes ======= +Version 1.6.0 (released 2025-02-28) + +- packages: allow use of either `pipenv` or `uv` as python package manager +- flask: replace `FLASK_ENV` with `FLASK_DEBUG` +- celery: allow setting a log level +- config: make host & port for both web and search configurable via `.invenio.private` +- run: introduce more fine-granular "web" and "worker" sub-commands + Version 1.5.0 (released 2024-08-01) - dependencies: update for invenio-app-rdm v12 diff --git a/invenio_cli/__init__.py b/invenio_cli/__init__.py index a047619e..9ff67488 100644 --- a/invenio_cli/__init__.py +++ b/invenio_cli/__init__.py @@ -9,6 +9,6 @@ """Invenio module to ease the creation and management of applications.""" -__version__ = "1.5.0" +__version__ = "1.6.0" __all__ = ("__version__",)