diff --git a/README.rst b/README.rst index a1b9b358..fd0f181e 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,7 @@ .. Copyright (C) 2019-2020 CERN. Copyright (C) 2019-2020 Northwestern University. + Copyright (C) 2025 Graz University of Technology. Invenio-Cli is free software; you can redistribute it and/or modify it under the terms of the MIT License; see LICENSE file for more details. @@ -63,6 +64,59 @@ Local Development environment # Update assets or statics $ invenio-cli update +Customizations +============== + +It is possible to choose between two python package managers: `pipenv` and `uv`. +`pipenv` is the default one. To customize the python package manager it is +necessary to add the line `python_package_manager = VALUE` to the `.invenio` +file. `VALUE` is `uv` or `pipenv`. + +It is possible to choose between two javascript package managers: `npm` and +`pnpm`. `npm` is the default one. To customize the python package manager it is +necessary to add the line `javascript_package_manager = VALUE` to the `.invenio` +file. `VALUE` is `npm` or `pnpm`. + +It is possible to choose between two assets builders: `webpack` and `rspack`. +`webpack` is the default one. To use `rspack` add `WEBPACKEXT_PROJECT = +"invenio_assets.webpack:rspack_project"` to the `invenio.cfg` file. + +It is possible to use a long running invenio rpc-server by starting it with +`invenio rpc-server start --port 5001`. This should only be done if you know the +drawbacks. The server has to be restarted if code which the server uses has been +updated. Further it is not necessary to use a long running invenio rpc-server +since it will be started on every invenio-cli command a new rpc-server +nevertheless if no rpc-server runs already. + +The javascript package manager uses a lock file like the python package manager. +This file `pnpm-lock.yaml` for `pnpm` and `packages-lock.json` for `npm` will be +symlinked to the `var/instance/assets/` directory. + +Hints +===== + +`uv` + +The development with `uv` is a little bit different than with `pipenv`. If there +is a `uv.lock` file packages have to be updated manually or by removing the +`uv.lock` file. The absence of the `uv.lock` file triggeres a new dependency +resolving call which takes into account of new released packages. It would also +be possible to use the `uv sync --upgrade` feature of `uv` but this installs +also beta versions of packages which is not recommended. It may sound strange to +remove the `uv.lock` file, but `uv` is that fast that deleting the `.venv` +directory and the `uv.lock` file is the easiest and fastest and safest way to +upgrade the packages. + +UV uses a local `.venv` directory. This will be created automatically, if not +existing in the same directory as the `pyproject.toml` file. + +UV uses a `pyproject.toml` file. + +`rcp-server` + +To use `pnpm` with the `rpc-server` it is necessary to add +`WEBPACKEXT_NPM_PKG_CLS = "pynpm:PNPMPackage"` to the `invenio.cfg` file. + Containerized 'Production' environment -------------------------------------- diff --git a/invenio_cli/commands/assets.py b/invenio_cli/commands/assets.py index ded3ff90..85a38817 100644 --- a/invenio_cli/commands/assets.py +++ b/invenio_cli/commands/assets.py @@ -2,6 +2,7 @@ # # Copyright (C) 2026 California Institute of Technology. # Copyright (C) 2020 CERN. +# Copyright (C) 2025 Graz University of Technology. # # Invenio-Cli is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. diff --git a/invenio_cli/commands/install.py b/invenio_cli/commands/install.py index b4dbe503..58ec7ea8 100644 --- a/invenio_cli/commands/install.py +++ b/invenio_cli/commands/install.py @@ -38,14 +38,14 @@ def install_py_dependencies(self, pre, dev=False): def update_instance_path(self): """Update path to instance in config.""" result = run_cmd( - self.cli_config.python_package_manager.run_command( + self.cli_config.python_package_manager.send_command( "invenio", "shell", # make sure the shell does not append cursor if editing_mode is set to `vi` in config "--TerminalInteractiveShell.editing_mode=''", "--no-term-title", "-c", - "\"print(app.instance_path, end='')\"", + "print(app.instance_path, end='')", ) ) if result.status_code == 0: diff --git a/invenio_cli/commands/local.py b/invenio_cli/commands/local.py index bf40bed6..29391714 100644 --- a/invenio_cli/commands/local.py +++ b/invenio_cli/commands/local.py @@ -2,7 +2,7 @@ # # Copyright (C) 2026 California Institute of Technology. # Copyright (C) 2020 CERN. -# Copyright (C) 2022 Graz University of Technology. +# Copyright (C) 2022-2026 Graz University of Technology. # # Invenio-Cli is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. @@ -109,22 +109,19 @@ def update_statics_and_assets(self, force, debug=False, log_file=None): # Commands py_pkg_man = self.cli_config.python_package_manager js_pkg_man = self.cli_config.javascript_package_manager - ops = [py_pkg_man.run_command("invenio", "collect", "--verbose")] + ops = [py_pkg_man.send_command("invenio", "collect", "--verbose")] if force: - ops.append(py_pkg_man.run_command("invenio", "webpack", "clean", "create")) + ops.append(py_pkg_man.send_command("invenio", "webpack", "clean", "create")) # We need to copy the js lock files here, since webpack regenerates # package.json and we want the locked version instead. ops.append(self._copy_js_lock_files) - ops.append(py_pkg_man.run_command("invenio", "webpack", "install")) + ops.append(py_pkg_man.send_command("invenio", "webpack", "install")) else: - ops.append(py_pkg_man.run_command("invenio", "webpack", "create")) - # We need to copy the js lock files here, since webpack regenerates - # package.json and we want the locked version instead. + ops.append(py_pkg_man.send_command("invenio", "webpack", "create")) ops.append(self._copy_js_lock_files) ops.append(self._statics) - ops.append(py_pkg_man.run_command("invenio", "webpack", "build")) - + ops.append(py_pkg_man.send_command("invenio", "webpack", "build")) # Keep the same messages for some of the operations for backward compatibility messages = { "build": "Building assets...", diff --git a/invenio_cli/commands/services.py b/invenio_cli/commands/services.py index 254e4f23..a3f9539d 100644 --- a/invenio_cli/commands/services.py +++ b/invenio_cli/commands/services.py @@ -2,7 +2,7 @@ # # Copyright (C) 2020-2024 CERN. # Copyright (C) 2021 Esteban J. G. Gabancho. -# Copyright (C) 2024 Graz University of Technology. +# Copyright (C) 2024-2025 Graz University of Technology. # # Invenio-Cli is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. @@ -136,7 +136,7 @@ def _default_location_path(self): """Build default location path based on file storage selection.""" file_storage = self.cli_config.get_file_storage() if file_storage == "local": - return "{}/data".format(self.cli_config.get_instance_path()) + return "{}/data".format(self.cli_config.get_data_path()) return "{}://default".format(self.cli_config.get_file_storage().lower()) def _setup(self, demo_data=False): @@ -190,7 +190,7 @@ def _setup(self, demo_data=False): ), ] - rdm_version_value = rdm_version() + rdm_version_value = rdm_version(self.cli_config) if rdm_version_value: if rdm_version_value[0] >= 10: steps.extend( diff --git a/invenio_cli/helpers/cli_config.py b/invenio_cli/helpers/cli_config.py index 7373abd3..829ba290 100644 --- a/invenio_cli/helpers/cli_config.py +++ b/invenio_cli/helpers/cli_config.py @@ -3,7 +3,7 @@ # Copyright (C) 2019-2024 CERN. # Copyright (C) 2019-2020 Northwestern University. # Copyright (C) 2021 Esteban J. G. Gabancho. -# Copyright (C) 2024 Graz University of Technology. +# Copyright (C) 2024-2026 Graz University of Technology. # # Invenio-Cli is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. @@ -74,15 +74,17 @@ def __init__(self, project_dir="./"): def python_package_manager(self) -> PythonPackageManager: """Get python packages manager.""" manager_name = self.config[CLIConfig.CLI_SECTION].get("python_package_manager") + use_rpc = self.config[CLIConfig.CLI_SECTION].get("use_rpc", False) + if manager_name == Pipenv.name: - return Pipenv() + return Pipenv(use_rpc=use_rpc) elif manager_name == UV.name: - return UV() + return UV(use_rpc=use_rpc) if (self.project_path / "Pipfile").is_file(): - return Pipenv() + return Pipenv(use_rpc=use_rpc) elif (self.project_path / "pyproject.toml").is_file(): - return UV() + return UV(use_rpc=use_rpc) else: raise RuntimeError( "Could not determine the Python package manager, please configure it." @@ -105,6 +107,14 @@ def get_project_dir(self): """Returns path to project directory.""" return self.config_path.parent.resolve() + def get_data_path(self): + """Return path to data.""" + path = self.private_config[CLIConfig.CLI_SECTION].get("data_path") + if path: + return Path(path) + else: + return self.get_instance_path() + def get_instance_path(self, throw=True): """Returns path to application instance directory. @@ -150,6 +160,10 @@ def get_project_shortname(self): """Returns the project's shortname.""" return self.config[CLIConfig.COOKIECUTTER_SECTION]["project_shortname"] + def get_rpc_server_port(self): + """Returns rpc server port.""" + return self.private_config[CLIConfig.CLI_SECTION].get("rpc_port", "5001") + def get_search_port(self): """Returns the search port.""" return self.private_config[CLIConfig.CLI_SECTION].get("search_port", "9200") @@ -169,6 +183,10 @@ def get_web_host(self): """Returns web host.""" return self.private_config[CLIConfig.CLI_SECTION].get("web_host", "127.0.0.1") + def get_app_rdm_version(self): + """Returns app rdm version.""" + return self.private_config[CLIConfig.CLI_SECTION].get("app_rdm", None) + def get_db_type(self): """Returns the database type (mysql, postgresql).""" return self.config[CLIConfig.COOKIECUTTER_SECTION]["database"] diff --git a/invenio_cli/helpers/package_managers.py b/invenio_cli/helpers/package_managers.py index dd2501ad..8d2cdf57 100644 --- a/invenio_cli/helpers/package_managers.py +++ b/invenio_cli/helpers/package_managers.py @@ -1,22 +1,26 @@ # -*- coding: utf-8 -*- # # Copyright (C) 2025 TU Wien. +# Copyright (C) 2025-2026 Graz University of Technology. # # Invenio-Cli is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. """Wrappers around various package managers to be used under the hood.""" +import atexit import os from abc import ABC from collections.abc import Callable from dataclasses import dataclass from pathlib import Path +from subprocess import Popen from typing import Dict, List, Union from pynpm import NPMPackage, PNPMPackage from ..helpers.process import ProcessResponse +from .process import run_cmd class PythonPackageManager(ABC): @@ -24,6 +28,47 @@ class PythonPackageManager(ABC): name: str = None lock_file_name: str = None + rpc_server_is_running: bool = False + rpc_server: Popen = None + run_prefix: List = [] + + def __init__(self, use_rpc=False): + """Construct.""" + self.use_rpc = use_rpc + + def ensure_rpc_server_is_running(self): + """Ensure rpc server is running.""" + if not self.use_rpc: + return + + if self.rpc_server_is_running: + return + + # first check if a server is already running. so to use long running rpc server + response = run_cmd(self.run_prefix + ["rpc-server", "ping", "--port", "5001"]) + if "pong" in response.output: + self.rpc_server_is_running = True + return + + # open if not + self.rpc_server = Popen( + self.run_prefix + ["invenio", "rpc-server", "start", "--port", "5001"] + ) + + atexit.register(self.cleanup) + # check until the server is up and running + while True: + response = run_cmd( + self.run_prefix + ["rpc-server", "ping", "--port", "5001"] + ) + if "pong" in response.output: + self.rpc_server_is_running = True + break + + def cleanup(self): + """Cleanup.""" + if self.rpc_server: + self.rpc_server.terminate() def run_command(self, *command: str) -> List[str]: """Generate command to run the given command in the managed environment.""" @@ -67,6 +112,26 @@ class Pipenv(PythonPackageManager): name = "pipenv" lock_file_name = "Pipfile.lock" + run_prefix = ["pipenv", "run"] + + def send_command(self, *command): + """Send command to rpc server, default to run_command.""" + self.ensure_rpc_server_is_running() + + if self.rpc_server_is_running: + # [1:] remove "invenio" from commands + return [ + self.name, + "run", + "rpc-server", + "send", + "--port", + "5001", + "--plain", + *command[1:], + ] + else: + self.run_command(*command) def run_command(self, *command): """Generate command to run the given command in the managed environment.""" @@ -124,6 +189,27 @@ class UV(PythonPackageManager): name = "uv" lock_file_name = "uv.lock" + run_prefix = ["uv", "run", "--no-sync"] + + def send_command(self, *command): + """Send command to rpc server, default to run_command.""" + self.ensure_rpc_server_is_running() + + if self.rpc_server_is_running: + # [1:] remove "invenio" from commands + return [ + self.name, + "run", + "--no-sync", + "rpc-server", + "send", + "--port", + "5001", + "--plain", + *command[1:], + ] + else: + return self.run_command(*command) def run_command(self, *command): """Generate command to run the given command in the managed environment.""" diff --git a/invenio_cli/helpers/versions.py b/invenio_cli/helpers/versions.py index 370c6867..500a11ae 100644 --- a/invenio_cli/helpers/versions.py +++ b/invenio_cli/helpers/versions.py @@ -3,6 +3,7 @@ # This file is part of Invenio. # Copyright (C) 2022-2025 CERN. # Copyright (C) 2025 TU Wien. +# Copyright (C) 2025 Graz University of Technology. # # Invenio is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. @@ -58,9 +59,12 @@ def _from_pyproject_toml(dep_name): return _parse_version(v.version) -def rdm_version(): +def rdm_version(cli_config): """Return the latest RDM version.""" - if os.path.isfile("./Pipfile"): + if app_rdm_version := cli_config.get_app_rdm_version(): + return [int(v) for v in app_rdm_version.split(".")] + + elif os.path.isfile("./Pipfile"): return _from_pipfile("invenio-app-rdm") elif os.path.isfile("./pyproject.toml"):