From d0ce8fea03b7d1559eddf4844dba94f2721e29d6 Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Tue, 10 Sep 2024 22:41:59 +0200 Subject: [PATCH 1/2] WIP: make invenio-cli assets build faster --- invenio_cli/cli/assets.py | 2 +- invenio_cli/commands/local.py | 188 ++++++++++++++++++++++++++++------ 2 files changed, 159 insertions(+), 31 deletions(-) diff --git a/invenio_cli/cli/assets.py b/invenio_cli/cli/assets.py index e6f0b2ed..be11022b 100644 --- a/invenio_cli/cli/assets.py +++ b/invenio_cli/cli/assets.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. -# Copyright (C) 2022 Graz University of Technology. +# Copyright (C) 2022-2024 Graz University of Technology. # # Invenio-Cli is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. diff --git a/invenio_cli/commands/local.py b/invenio_cli/commands/local.py index eea8db57..b9e6e3e9 100644 --- a/invenio_cli/commands/local.py +++ b/invenio_cli/commands/local.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # Copyright (C) 2020 CERN. -# Copyright (C) 2022 Graz University of Technology. +# Copyright (C) 2022-2024 Graz University of Technology. # # Invenio-Cli is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. @@ -12,16 +12,125 @@ import signal from distutils.dir_util import copy_tree from os import environ +from os.path import join from pathlib import Path from subprocess import Popen as popen import click +from flask.helpers import get_root_path +from flask_collect import Collect + +# from flask_webpackext import WebpackBundleProject +from invenio_app.factory import create_cli +from pywebpack import WebpackBundleProject as PyWebpackBundleProject +from pywebpack import bundles_from_entry_point +from werkzeug.utils import import_string from ..helpers import env, filesystem from ..helpers.process import ProcessResponse, run_interactive from .commands import Commands +def flask_config(app, project): + """Flask configuration injected in Webpack. + + :return: Dictionary which contains the information Flask-WebpackExt knows + about a Webpack project and the absolute URLs for static files and + assets. The dictionary consists of a key ``build`` with the following + keys inside: + + * ``debug``: Boolean which represents if Flask debug is on. + * ``context``: Absolute path to the generated assets directory. + * ``assetsPath``: Absolute path to the generated static directory. + * ``assetsURL``: URL to access the built files. + * ``staticPath``: Absolute path to the generated static directory. + * ``staticURL``: URL to access the static files.. + """ + assets_url = app.config["WEBPACKEXT_PROJECT_DISTURL"] + if not assets_url.endswith("/"): + assets_url += "/" + static_url = app.static_url_path + if not static_url.endswith("/"): + static_url += "/" + + return { + "build": { + "debug": app.debug, + "context": "/path/to/.venv/var/instance/assets", # TODO: hardcoded + "assetsPath": app.config["WEBPACKEXT_PROJECT_DISTDIR"], + "assetsURL": assets_url, + "staticPath": app.static_folder, + "staticURL": static_url, + } + } + + +class _PathStorageMixin(object): + """Mixin class.""" + + @property + def path(self): + """Get path to project.""" + return self.app.config["WEBPACKEXT_PROJECT_BUILDDIR"] + + @property + def storage_cls(self): + """Get storage class.""" + cls_ = self.app.config["WEBPACKEXT_STORAGE_CLS"] + if isinstance(cls_, str): + return import_string(cls_) + return cls_ + + +class WebpackBundleProject(_PathStorageMixin, PyWebpackBundleProject): + """Flask webpack bundle project.""" + + def __init__( + self, + import_name, + project_folder=None, + bundles=None, + config=None, + config_path=None, + app=None, + ): + """Initialize templated folder. + + :param import_name: Name of the module where the WebpackBundleProject + class is instantiated. It is used to determine the absolute path + to the ``project_folder``. + :param project_folder: Relative path to the Webpack project which is + going to aggregate all the ``bundles``. + :param bundles: List of + :class:`flask_webpackext.bundle.WebpackBundle`. This list can be + statically defined if the bundles are known before hand, or + dinamically generated using + :func:`pywebpack.helpers.bundles_from_entry_point` so the bundles + are discovered from the defined Webpack entrypoints exposed by + other modules. + :param config: Dictionary which overrides the ``config.json`` file + generated by Flask-WebpackExt. Use carefuly and only if you know + what you are doing since ``config.json`` is the file that holds the + key information to integrate Flask with Webpack. + :param config_path: Path where Flask-WebpackExt is going to write the + ``config.json``, this file is generated by + :func:`flask_webpackext.project.flask_config`. + """ + self.app = app + # project_template_dir = join(get_root_path(import_name), project_folder) + project_template_dir = ( + "/path/to/invenio-assets/invenio_assets/assets" # TODO: hardcoded + ) + config = flask_config(self.app, self) + super(WebpackBundleProject, self).__init__( + None, + project_template_dir=project_template_dir, + bundles=bundles, + config=config, + config_path=config_path, + ) + + class LocalCommands(Commands): """Local CLI commands.""" @@ -84,36 +193,55 @@ def update_statics_and_assets(self, force, flask_env="production", log_file=None Needed here (parent) because is used by Assets and Install commands. """ # Commands - prefix = ["pipenv", "run", "invenio"] - - ops = [prefix + ["collect", "--verbose"]] - - if force: - ops.append(prefix + ["webpack", "clean", "create"]) - ops.append(prefix + ["webpack", "install"]) - else: - ops.append(prefix + ["webpack", "create"]) - ops.append(self._statics) - ops.append(prefix + ["webpack", "build"]) - # Keep the same messages for some of the operations for backward compatibility - messages = { - "build": "Building assets...", - "install": "Installing JS dependencies...", - } + # prefix = ["pipenv", "run", "invenio"] + + # ops = [prefix + ["collect", "--verbose"]] - with env(FLASK_ENV=flask_env): - for op in ops: - if callable(op): - response = op() - else: - if op[-1] in messages: - click.secho(messages[op[-1]], fg="green") - response = run_interactive( - op, env={"PIPENV_VERBOSITY": "-1"}, log_file=log_file - ) - if response.status_code != 0: - break - return response + # if force: + # ops.append(prefix + ["webpack", "clean", "create"]) + # ops.append(prefix + ["webpack", "install"]) + # else: + # ops.append(prefix + ["webpack", "create"]) + # ops.append(self._statics) + # ops.append(prefix + ["webpack", "build"]) + # # Keep the same messages for some of the operations for backward compatibility + # messages = { + # "build": "Building assets...", + # "install": "Installing JS dependencies...", + # } + + # with env(FLASK_ENV=flask_env): + # for op in ops: + # if callable(op): + # response = op() + # else: + # if op[-1] in messages: + # click.secho(messages[op[-1]], fg="green") + # response = run_interactive( + # op, env={"PIPENV_VERBOSITY": "-1"}, log_file=log_file + # ) + # if response.status_code != 0: + # break + # return response + + app = create_cli() + collect = Collect(app) + project = WebpackBundleProject( + __name__, + project_folder="assets", + config_path="build/config.json", + config=flask_config, + bundles=bundles_from_entry_point("invenio_assets.webpack"), + app=app, + ) + collect.collect(verbose=True) + + project.clean() + project.create() + project.install() + copied_files = self._copy_statics_and_assets() + self._symlink_assets_templates(copied_files) + project.build() def run(self, host, port, debug=True, services=True, celery_log_file=None): """Run development server and celery queue.""" From 40fa5b3814eb36847ee22b0474979e63057642ca Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Sun, 15 Sep 2024 00:16:55 +0200 Subject: [PATCH 2/2] wip: use flask-webpackext PR --- invenio_cli/commands/local.py | 154 +--------------------------------- 1 file changed, 4 insertions(+), 150 deletions(-) diff --git a/invenio_cli/commands/local.py b/invenio_cli/commands/local.py index b9e6e3e9..3c20702d 100644 --- a/invenio_cli/commands/local.py +++ b/invenio_cli/commands/local.py @@ -12,125 +12,19 @@ import signal from distutils.dir_util import copy_tree from os import environ -from os.path import join from pathlib import Path from subprocess import Popen as popen import click -from flask.helpers import get_root_path from flask_collect import Collect - -# from flask_webpackext import WebpackBundleProject from invenio_app.factory import create_cli -from pywebpack import WebpackBundleProject as PyWebpackBundleProject -from pywebpack import bundles_from_entry_point -from werkzeug.utils import import_string +from invenio_assets.webpack import project -from ..helpers import env, filesystem -from ..helpers.process import ProcessResponse, run_interactive +from ..helpers import filesystem +from ..helpers.process import ProcessResponse from .commands import Commands -def flask_config(app, project): - """Flask configuration injected in Webpack. - - :return: Dictionary which contains the information Flask-WebpackExt knows - about a Webpack project and the absolute URLs for static files and - assets. The dictionary consists of a key ``build`` with the following - keys inside: - - * ``debug``: Boolean which represents if Flask debug is on. - * ``context``: Absolute path to the generated assets directory. - * ``assetsPath``: Absolute path to the generated static directory. - * ``assetsURL``: URL to access the built files. - * ``staticPath``: Absolute path to the generated static directory. - * ``staticURL``: URL to access the static files.. - """ - assets_url = app.config["WEBPACKEXT_PROJECT_DISTURL"] - if not assets_url.endswith("/"): - assets_url += "/" - static_url = app.static_url_path - if not static_url.endswith("/"): - static_url += "/" - - return { - "build": { - "debug": app.debug, - "context": "/path/to/.venv/var/instance/assets", # TODO: hardcoded - "assetsPath": app.config["WEBPACKEXT_PROJECT_DISTDIR"], - "assetsURL": assets_url, - "staticPath": app.static_folder, - "staticURL": static_url, - } - } - - -class _PathStorageMixin(object): - """Mixin class.""" - - @property - def path(self): - """Get path to project.""" - return self.app.config["WEBPACKEXT_PROJECT_BUILDDIR"] - - @property - def storage_cls(self): - """Get storage class.""" - cls_ = self.app.config["WEBPACKEXT_STORAGE_CLS"] - if isinstance(cls_, str): - return import_string(cls_) - return cls_ - - -class WebpackBundleProject(_PathStorageMixin, PyWebpackBundleProject): - """Flask webpack bundle project.""" - - def __init__( - self, - import_name, - project_folder=None, - bundles=None, - config=None, - config_path=None, - app=None, - ): - """Initialize templated folder. - - :param import_name: Name of the module where the WebpackBundleProject - class is instantiated. It is used to determine the absolute path - to the ``project_folder``. - :param project_folder: Relative path to the Webpack project which is - going to aggregate all the ``bundles``. - :param bundles: List of - :class:`flask_webpackext.bundle.WebpackBundle`. This list can be - statically defined if the bundles are known before hand, or - dinamically generated using - :func:`pywebpack.helpers.bundles_from_entry_point` so the bundles - are discovered from the defined Webpack entrypoints exposed by - other modules. - :param config: Dictionary which overrides the ``config.json`` file - generated by Flask-WebpackExt. Use carefuly and only if you know - what you are doing since ``config.json`` is the file that holds the - key information to integrate Flask with Webpack. - :param config_path: Path where Flask-WebpackExt is going to write the - ``config.json``, this file is generated by - :func:`flask_webpackext.project.flask_config`. - """ - self.app = app - # project_template_dir = join(get_root_path(import_name), project_folder) - project_template_dir = ( - "/path/to/invenio-assets/invenio_assets/assets" # TODO: hardcoded - ) - config = flask_config(self.app, self) - super(WebpackBundleProject, self).__init__( - None, - project_template_dir=project_template_dir, - bundles=bundles, - config=config, - config_path=config_path, - ) - - class LocalCommands(Commands): """Local CLI commands.""" @@ -192,50 +86,10 @@ def update_statics_and_assets(self, force, flask_env="production", log_file=None Needed here (parent) because is used by Assets and Install commands. """ - # Commands - # prefix = ["pipenv", "run", "invenio"] - - # ops = [prefix + ["collect", "--verbose"]] - - # if force: - # ops.append(prefix + ["webpack", "clean", "create"]) - # ops.append(prefix + ["webpack", "install"]) - # else: - # ops.append(prefix + ["webpack", "create"]) - # ops.append(self._statics) - # ops.append(prefix + ["webpack", "build"]) - # # Keep the same messages for some of the operations for backward compatibility - # messages = { - # "build": "Building assets...", - # "install": "Installing JS dependencies...", - # } - - # with env(FLASK_ENV=flask_env): - # for op in ops: - # if callable(op): - # response = op() - # else: - # if op[-1] in messages: - # click.secho(messages[op[-1]], fg="green") - # response = run_interactive( - # op, env={"PIPENV_VERBOSITY": "-1"}, log_file=log_file - # ) - # if response.status_code != 0: - # break - # return response - app = create_cli() + project.app = app collect = Collect(app) - project = WebpackBundleProject( - __name__, - project_folder="assets", - config_path="build/config.json", - config=flask_config, - bundles=bundles_from_entry_point("invenio_assets.webpack"), - app=app, - ) collect.collect(verbose=True) - project.clean() project.create() project.install()