diff --git a/bin/stack-config b/bin/stack-config index 66496991..63c82bcc 100755 --- a/bin/stack-config +++ b/bin/stack-config @@ -4,7 +4,7 @@ # dependencies = [ # "jinja2", # "jsonschema", -# "pyYAML", +# "ruamel.yaml >=0.15.1" # ] # /// diff --git a/pyproject.toml b/pyproject.toml index 583f5d46..4a24915b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ requires-python = ">=3.12" dependencies = [ "Jinja2", "jsonschema", - "PyYAML", + "ruamel.yaml >=0.15.1", ] classifiers = [ "Development Status :: 5 - Production/Stable", diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..79d70ed3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +Jinja2 +jsonschema +pytest +ruamel.yaml >=0.15.1 diff --git a/stackinator/builder.py b/stackinator/builder.py index ae4997a1..243c7391 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -9,10 +9,12 @@ from datetime import datetime import jinja2 -import yaml +from ruamel.yaml import YAML from . import VERSION, cache, root_logger, spack_util +yaml = YAML() + def install(src, dst, *, ignore=None, symlinks=False): """Call shutil.copytree or shutil.copy2. copy2 is used if `src` is not a directory. @@ -139,7 +141,7 @@ def environment_meta(self, recipe): "root": /user-environment/env/default, "activate": /user-environment/env/default/activate.sh, "description": "simple devolpment env: compilers, MPI, python, cmake." - "env_vars": { + "recipe_variables": { ... } }, @@ -307,17 +309,17 @@ def generate(self, recipe): # the packages.yaml configuration that will be used when building all environments # - the system packages.yaml with gcc removed # - plus additional packages provided by the recipe - global_packages_yaml = yaml.dump(recipe.packages["global"]) + global_packages_path = config_path / "packages.yaml" with global_packages_path.open("w") as fid: - fid.write(global_packages_yaml) + yaml.dump(recipe.packages["global"], fid) # generate a mirrors.yaml file if build caches have been configured if recipe.mirror: dst = config_path / "mirrors.yaml" self._logger.debug(f"generate the build cache mirror: {dst}") with dst.open("w") as fid: - fid.write(cache.generate_mirrors_yaml(recipe.mirror)) + cache.generate_mirrors_yaml(recipe.mirror, fid) # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to @@ -346,7 +348,7 @@ def generate(self, recipe): if repo_yaml.exists() and repo_yaml.is_file(): # open repos.yaml file and reat the list of repos with repo_yaml.open() as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) P = raw["repos"] self._logger.debug(f"the system configuration has a repo file {repo_yaml} refers to {P}") @@ -438,7 +440,10 @@ def generate(self, recipe): compiler_config_path.mkdir(exist_ok=True) for file, raw in files.items(): with (compiler_config_path / file).open(mode="w") as f: - f.write(raw) + if type(raw) is str: + f.write(raw) + else: + yaml.dump(raw, f) # generate the makefile and spack.yaml files that describe the environments environment_files = recipe.environment_files @@ -477,7 +482,7 @@ def generate(self, recipe): generate_modules_path = self.path / "modules" generate_modules_path.mkdir(exist_ok=True) with (generate_modules_path / "modules.yaml").open("w") as f: - yaml.dump(recipe.modules, f) + yaml.dump(recipe.modules_yaml_data, f) # write the meta data meta_path = store_path / "meta" diff --git a/stackinator/cache.py b/stackinator/cache.py index 24177e33..2adb80ac 100644 --- a/stackinator/cache.py +++ b/stackinator/cache.py @@ -1,15 +1,17 @@ import os import pathlib -import yaml +from ruamel.yaml import YAML from . import schema +yaml = YAML() + def configuration_from_file(file, mount): with file.open() as fid: # load the raw yaml input - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) # validate the yaml schema.CacheValidator.validate(raw) @@ -40,7 +42,7 @@ def configuration_from_file(file, mount): return raw -def generate_mirrors_yaml(config): +def generate_mirrors_yaml(config, out): path = config["path"].as_posix() mirrors = { "mirrors": { @@ -55,4 +57,6 @@ def generate_mirrors_yaml(config): } } - return yaml.dump(mirrors, default_flow_style=False) + yaml = YAML() + yaml.default_flow_style = True + yaml.dump(mirrors, out) diff --git a/stackinator/etc/envvars.py b/stackinator/etc/envvars.py index 0f122057..7c63b6c8 100755 --- a/stackinator/etc/envvars.py +++ b/stackinator/etc/envvars.py @@ -7,8 +7,6 @@ from enum import Enum from typing import List, Optional -import yaml - class EnvVarOp(Enum): PREPEND = 1 @@ -504,12 +502,8 @@ def view_impl(args): envvars.remove_root(args.build_path) if args.compilers is not None: - if not os.path.isfile(args.compilers): - print(f"error - compiler yaml file {args.compilers} does not exist") - exit(1) - with open(args.compilers, "r") as file: - data = yaml.safe_load(file) + data = json.load(file) compilers = [] for p in data["packages"].values(): diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 7850d036..3f9450de 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -3,11 +3,13 @@ import re import jinja2 -import yaml +from ruamel.yaml import YAML from . import cache, root_logger, schema, spack_util from .etc.envvars import EnvVarSet +yaml = YAML() + class Recipe: @property @@ -58,7 +60,7 @@ def __init__(self, args): raise FileNotFoundError(f"The recipe path '{compiler_path}' does not contain compilers.yaml") with compiler_path.open() as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) schema.CompilersValidator.validate(raw) self.generate_compiler_specs(raw) @@ -68,7 +70,7 @@ def __init__(self, args): self._logger.debug(f"opening {modules_path}") if modules_path.is_file(): with modules_path.open() as fid: - self.modules = yaml.load(fid, Loader=yaml.Loader) + self.modules = yaml.load(fid) schema.ModulesValidator.validate(self.modules) # Note: @@ -94,7 +96,7 @@ def __init__(self, args): recipe_packages_path = self.path / "packages.yaml" if recipe_packages_path.is_file(): with recipe_packages_path.open() as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) recipe_packages = raw["packages"] # load system/packages.yaml -> system_packages (if it exists) @@ -103,7 +105,7 @@ def __init__(self, args): if system_packages_path.is_file(): # load system yaml with system_packages_path.open() as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) system_packages = raw["packages"] # extract gcc packages from system packages @@ -123,7 +125,7 @@ def __init__(self, args): if network_path.is_file(): self._logger.debug(f"opening {network_path}") with network_path.open() as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) if "packages" in raw: network_packages = raw["packages"] if "mpi" in raw: @@ -146,7 +148,7 @@ def __init__(self, args): raise FileNotFoundError(f"The recipe path '{environments_path}' does not contain environments.yaml") with environments_path.open() as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) # add a special environment that installs tools required later in the build process. # currently we only need squashfs for creating the squashfs file. raw["uenv_tools"] = { @@ -280,7 +282,7 @@ def config(self, config_path): raise FileNotFoundError(f"The recipe path '{config_path}' does not contain config.yaml") with config_path.open() as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) schema.ConfigValidator.validate(raw) self._config = raw @@ -324,6 +326,13 @@ def environment_view_meta(self): return view_meta + @property + def modules_yaml_data(self): + with self.modules.open() as fid: + raw = yaml.load(fid) + raw["modules"]["default"]["roots"]["tcl"] = (pathlib.Path(self.mount) / "modules").as_posix() + return raw + # creates the self.environments field that describes the full specifications # for all of the environments sets, grouped in environments, from the raw # environments.yaml input. @@ -411,6 +420,7 @@ def generate_environment_specs(self, raw): # ["uenv"]["env_vars"] = {"set": [], "unset": [], "prepend_path": [], "append_path": []} if view_config is None: view_config = {} + view_config.setdefault("link", "roots") view_config.setdefault("uenv", {}) view_config["uenv"].setdefault("add_compilers", True) @@ -525,7 +535,7 @@ def compiler_files(self): files["config"][compiler]["spack.yaml"] = spack_yaml_template.render(config=config) # compilers/gcc/packages.yaml if compiler == "gcc": - files["config"][compiler]["packages.yaml"] = yaml.dump(self.packages["gcc"]) + files["config"][compiler]["packages.yaml"] = self.packages["gcc"] return files diff --git a/stackinator/schema.py b/stackinator/schema.py index 3a2a9842..60335ae7 100644 --- a/stackinator/schema.py +++ b/stackinator/schema.py @@ -3,7 +3,7 @@ from textwrap import dedent import jsonschema -import yaml +from ruamel.yaml import YAML from . import root_logger @@ -11,7 +11,12 @@ def py2yaml(data, indent): - dump = yaml.dump(data) + yaml = YAML() + from io import StringIO + + buffer = StringIO() + yaml.dump(data, buffer) + dump = buffer.getvalue() lines = [ln for ln in dump.split("\n") if ln != ""] res = ("\n" + " " * indent).join(lines) return res diff --git a/stackinator/templates/Make.user b/stackinator/templates/Make.user index cb8acb03..68c8f154 100644 --- a/stackinator/templates/Make.user +++ b/stackinator/templates/Make.user @@ -2,6 +2,8 @@ # Copy this file to Make.user and set some variables. +SHELL := bash + # This is the root of the software stack directory. BUILD_ROOT := {{ build_path }} diff --git a/stackinator/templates/Makefile.environments b/stackinator/templates/Makefile.environments index 5a530232..01929d9c 100644 --- a/stackinator/templates/Makefile.environments +++ b/stackinator/templates/Makefile.environments @@ -34,7 +34,7 @@ all:{% for env in environments %} {{ env }}/generated/build_cache{% endfor %} {{ env }}/generated/view_config: {{ env }}/generated/env {% for view in config.views %} $(SPACK) env activate --with-view {{ view.name }} --sh ./{{ env }} > $(STORE)/env/{{ view.name }}/activate.sh - $(BUILD_ROOT)/envvars.py view {% if view.extra.add_compilers %}--compilers=./{{ env }}/packages.yaml {% endif %} --prefix_paths="{{ view.extra.prefix_string }}" $(STORE)/env/{{ view.name }} $(BUILD_ROOT) + $(BUILD_ROOT)/envvars.py view {% if view.extra.add_compilers %}--compilers=<(yq read -j -p v ./{{ env }}/packages.yaml) {% endif %} --prefix_paths="{{ view.extra.prefix_string }}" $(STORE)/env/{{ view.name }} $(BUILD_ROOT) {% endfor %} touch $@ diff --git a/unittests/test_schema.py b/unittests/test_schema.py index b30f73da..8b85cb57 100644 --- a/unittests/test_schema.py +++ b/unittests/test_schema.py @@ -5,10 +5,12 @@ import jsonschema import pytest -import yaml +from ruamel.yaml import YAML import stackinator.schema as schema +yaml = YAML() + @pytest.fixture def test_path(): @@ -38,7 +40,7 @@ def recipe_paths(test_path, recipes): def test_config_yaml(yaml_path): # test that the defaults are set as expected with open(yaml_path / "config.defaults.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) schema.ConfigValidator.validate(raw) assert raw["store"] == "/user-environment" assert raw["spack"]["commit"] is None @@ -55,10 +57,7 @@ def test_config_yaml(yaml_path): repo: https://github.com/spack/spack.git commit: develop-packages """) - raw = yaml.load( - config, - Loader=yaml.Loader, - ) + raw = yaml.load(config) schema.ConfigValidator.validate(raw) assert raw["spack"]["commit"] is None assert raw["spack"]["packages"]["commit"] is not None @@ -74,10 +73,7 @@ def test_config_yaml(yaml_path): packages: repo: https://github.com/spack/spack.git """) - raw = yaml.load( - config, - Loader=yaml.Loader, - ) + raw = yaml.load(config) schema.ConfigValidator.validate(raw) assert raw["spack"]["commit"] == "develop" assert raw["spack"]["packages"]["commit"] is None @@ -85,7 +81,7 @@ def test_config_yaml(yaml_path): # full config with open(yaml_path / "config.full.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) schema.ConfigValidator.validate(raw) assert raw["store"] == "/alternative-point" assert raw["spack"]["commit"] == "6408b51" @@ -100,7 +96,7 @@ def test_config_yaml(yaml_path): spack: repo: https://github.com/spack/spack.git """) - raw = yaml.load(config, Loader=yaml.Loader) + raw = yaml.load(config) schema.ConfigValidator.validate(raw) @@ -108,20 +104,20 @@ def test_recipe_config_yaml(recipe_paths): # validate the config.yaml in the test recipes for p in recipe_paths: with open(p / "config.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) schema.ConfigValidator.validate(raw) def test_compilers_yaml(yaml_path): # test that the defaults are set as expected with open(yaml_path / "compilers.defaults.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) schema.CompilersValidator.validate(raw) assert raw["gcc"] == {"version": "10.2"} assert raw["llvm"] is None with open(yaml_path / "compilers.full.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) schema.CompilersValidator.validate(raw) assert raw["gcc"] == {"version": "11"} assert raw["llvm"] == {"version": "13"} @@ -132,13 +128,13 @@ def test_recipe_compilers_yaml(recipe_paths): # validate the compilers.yaml in the test recipes for p in recipe_paths: with open(p / "compilers.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) schema.CompilersValidator.validate(raw) def test_environments_yaml(yaml_path): with open(yaml_path / "environments.full.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) schema.EnvironmentsValidator.validate(raw) # the defaults-env does not set fields @@ -181,7 +177,7 @@ def test_environments_yaml(yaml_path): # check that only allowed fields are accepted # from an example that was silently validated with open(yaml_path / "environments.err-providers.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) with pytest.raises( jsonschema.exceptions.ValidationError, match=r"Additional properties are not allowed \('providers' was unexpected", @@ -193,7 +189,7 @@ def test_recipe_environments_yaml(recipe_paths): # validate the environments.yaml in the test recipes for p in recipe_paths: with open(p / "environments.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) schema.EnvironmentsValidator.validate(raw) @@ -248,7 +244,7 @@ def test_recipe_environments_yaml(recipe_paths): ], ) def test_valid_modules_yaml(recipe): - instance = yaml.load(recipe, Loader=yaml.Loader) + instance = yaml.load(recipe) schema.ModulesValidator.validate(instance) assert not instance["modules"]["default"]["arch_folder"] @@ -267,4 +263,18 @@ def test_valid_modules_yaml(recipe): ) def test_invalid_modules_yaml(recipe): with pytest.raises(Exception): - schema.ModulesValidator.validate(yaml.load(recipe, Loader=yaml.Loader)) + schema.ModulesValidator.validate(yaml.load(recipe)) + + +def test_unique_properties(): + invalid_config = dedent( + """ + name: invalid-config + name: duplicate-name + """ + ) + + from ruamel.yaml.constructor import DuplicateKeyError + + with pytest.raises(DuplicateKeyError): + yaml.load(invalid_config)