From d6fdd65894815ed7fe32b68ef8aaf6258272f5e0 Mon Sep 17 00:00:00 2001 From: Alberto Invernizzi Date: Thu, 2 Apr 2026 11:40:41 +0200 Subject: [PATCH 1/8] envvars now depends just on python stdlib we relied on yaml dep installed by the system. since yaml is not part of the python stdlib but json is, here we rely on bash process substitution to pass a temporary file translated by yq from yaml to json. --- stackinator/etc/envvars.py | 8 +------- stackinator/templates/Make.user | 2 ++ stackinator/templates/Makefile.environments | 2 +- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/stackinator/etc/envvars.py b/stackinator/etc/envvars.py index 0f122057..7c63b6c8 100755 --- a/stackinator/etc/envvars.py +++ b/stackinator/etc/envvars.py @@ -7,8 +7,6 @@ from enum import Enum from typing import List, Optional -import yaml - class EnvVarOp(Enum): PREPEND = 1 @@ -504,12 +502,8 @@ def view_impl(args): envvars.remove_root(args.build_path) if args.compilers is not None: - if not os.path.isfile(args.compilers): - print(f"error - compiler yaml file {args.compilers} does not exist") - exit(1) - with open(args.compilers, "r") as file: - data = yaml.safe_load(file) + data = json.load(file) compilers = [] for p in data["packages"].values(): diff --git a/stackinator/templates/Make.user b/stackinator/templates/Make.user index cb8acb03..68c8f154 100644 --- a/stackinator/templates/Make.user +++ b/stackinator/templates/Make.user @@ -2,6 +2,8 @@ # Copy this file to Make.user and set some variables. +SHELL := bash + # This is the root of the software stack directory. BUILD_ROOT := {{ build_path }} diff --git a/stackinator/templates/Makefile.environments b/stackinator/templates/Makefile.environments index 5a530232..01929d9c 100644 --- a/stackinator/templates/Makefile.environments +++ b/stackinator/templates/Makefile.environments @@ -34,7 +34,7 @@ all:{% for env in environments %} {{ env }}/generated/build_cache{% endfor %} {{ env }}/generated/view_config: {{ env }}/generated/env {% for view in config.views %} $(SPACK) env activate --with-view {{ view.name }} --sh ./{{ env }} > $(STORE)/env/{{ view.name }}/activate.sh - $(BUILD_ROOT)/envvars.py view {% if view.extra.add_compilers %}--compilers=./{{ env }}/packages.yaml {% endif %} --prefix_paths="{{ view.extra.prefix_string }}" $(STORE)/env/{{ view.name }} $(BUILD_ROOT) + $(BUILD_ROOT)/envvars.py view {% if view.extra.add_compilers %}--compilers=<(yq read -j -p v ./{{ env }}/packages.yaml) {% endif %} --prefix_paths="{{ view.extra.prefix_string }}" $(STORE)/env/{{ view.name }} $(BUILD_ROOT) {% endfor %} touch $@ From c897d8fa8dc7504f33f137a5e4460337afb1cb70 Mon Sep 17 00:00:00 2001 From: Alberto Invernizzi Date: Wed, 19 Nov 2025 17:26:03 +0100 Subject: [PATCH 2/8] trivial test for duplicate keys --- unittests/test_schema.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/unittests/test_schema.py b/unittests/test_schema.py index b30f73da..0f01600a 100644 --- a/unittests/test_schema.py +++ b/unittests/test_schema.py @@ -268,3 +268,14 @@ def test_valid_modules_yaml(recipe): def test_invalid_modules_yaml(recipe): with pytest.raises(Exception): schema.ModulesValidator.validate(yaml.load(recipe, Loader=yaml.Loader)) + +def test_unique_properties(): + invalid_config = dedent( + """ + name: invalid-config + name: duplicate-name + """ + ) + + with pytest.raises(Exception): + yaml.load(invalid_config, Loader=yaml.Loader) From 318b7e517175530dd05798d00218ee73e8a13fa0 Mon Sep 17 00:00:00 2001 From: Alberto Invernizzi Date: Wed, 1 Apr 2026 11:24:34 +0200 Subject: [PATCH 3/8] migration to ruamel.yaml --- pyproject.toml | 2 +- stackinator/builder.py | 6 ++++-- stackinator/cache.py | 6 ++++-- stackinator/recipe.py | 18 +++++++++------- unittests/test_schema.py | 45 ++++++++++++++++++++-------------------- 5 files changed, 41 insertions(+), 36 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 583f5d46..6fca4b88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ requires-python = ">=3.12" dependencies = [ "Jinja2", "jsonschema", - "PyYAML", + "ruamel.yaml >=0.15.1, <0.18.0", ] classifiers = [ "Development Status :: 5 - Production/Stable", diff --git a/stackinator/builder.py b/stackinator/builder.py index ae4997a1..df338653 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -9,10 +9,12 @@ from datetime import datetime import jinja2 -import yaml +from ruamel.yaml import YAML from . import VERSION, cache, root_logger, spack_util +yaml = YAML() + def install(src, dst, *, ignore=None, symlinks=False): """Call shutil.copytree or shutil.copy2. copy2 is used if `src` is not a directory. @@ -346,7 +348,7 @@ def generate(self, recipe): if repo_yaml.exists() and repo_yaml.is_file(): # open repos.yaml file and reat the list of repos with repo_yaml.open() as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) P = raw["repos"] self._logger.debug(f"the system configuration has a repo file {repo_yaml} refers to {P}") diff --git a/stackinator/cache.py b/stackinator/cache.py index 24177e33..773d0c2b 100644 --- a/stackinator/cache.py +++ b/stackinator/cache.py @@ -1,15 +1,17 @@ import os import pathlib -import yaml +from ruamel.yaml import YAML from . import schema +yaml = YAML() + def configuration_from_file(file, mount): with file.open() as fid: # load the raw yaml input - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) # validate the yaml schema.CacheValidator.validate(raw) diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 7850d036..1dda5a76 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -3,11 +3,13 @@ import re import jinja2 -import yaml +from ruamel.yaml import YAML from . import cache, root_logger, schema, spack_util from .etc.envvars import EnvVarSet +yaml = YAML() + class Recipe: @property @@ -58,7 +60,7 @@ def __init__(self, args): raise FileNotFoundError(f"The recipe path '{compiler_path}' does not contain compilers.yaml") with compiler_path.open() as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) schema.CompilersValidator.validate(raw) self.generate_compiler_specs(raw) @@ -68,7 +70,7 @@ def __init__(self, args): self._logger.debug(f"opening {modules_path}") if modules_path.is_file(): with modules_path.open() as fid: - self.modules = yaml.load(fid, Loader=yaml.Loader) + self.modules = yaml.load(fid) schema.ModulesValidator.validate(self.modules) # Note: @@ -94,7 +96,7 @@ def __init__(self, args): recipe_packages_path = self.path / "packages.yaml" if recipe_packages_path.is_file(): with recipe_packages_path.open() as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) recipe_packages = raw["packages"] # load system/packages.yaml -> system_packages (if it exists) @@ -103,7 +105,7 @@ def __init__(self, args): if system_packages_path.is_file(): # load system yaml with system_packages_path.open() as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) system_packages = raw["packages"] # extract gcc packages from system packages @@ -123,7 +125,7 @@ def __init__(self, args): if network_path.is_file(): self._logger.debug(f"opening {network_path}") with network_path.open() as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) if "packages" in raw: network_packages = raw["packages"] if "mpi" in raw: @@ -146,7 +148,7 @@ def __init__(self, args): raise FileNotFoundError(f"The recipe path '{environments_path}' does not contain environments.yaml") with environments_path.open() as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) # add a special environment that installs tools required later in the build process. # currently we only need squashfs for creating the squashfs file. raw["uenv_tools"] = { @@ -280,7 +282,7 @@ def config(self, config_path): raise FileNotFoundError(f"The recipe path '{config_path}' does not contain config.yaml") with config_path.open() as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) schema.ConfigValidator.validate(raw) self._config = raw diff --git a/unittests/test_schema.py b/unittests/test_schema.py index 0f01600a..8b85cb57 100644 --- a/unittests/test_schema.py +++ b/unittests/test_schema.py @@ -5,10 +5,12 @@ import jsonschema import pytest -import yaml +from ruamel.yaml import YAML import stackinator.schema as schema +yaml = YAML() + @pytest.fixture def test_path(): @@ -38,7 +40,7 @@ def recipe_paths(test_path, recipes): def test_config_yaml(yaml_path): # test that the defaults are set as expected with open(yaml_path / "config.defaults.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) schema.ConfigValidator.validate(raw) assert raw["store"] == "/user-environment" assert raw["spack"]["commit"] is None @@ -55,10 +57,7 @@ def test_config_yaml(yaml_path): repo: https://github.com/spack/spack.git commit: develop-packages """) - raw = yaml.load( - config, - Loader=yaml.Loader, - ) + raw = yaml.load(config) schema.ConfigValidator.validate(raw) assert raw["spack"]["commit"] is None assert raw["spack"]["packages"]["commit"] is not None @@ -74,10 +73,7 @@ def test_config_yaml(yaml_path): packages: repo: https://github.com/spack/spack.git """) - raw = yaml.load( - config, - Loader=yaml.Loader, - ) + raw = yaml.load(config) schema.ConfigValidator.validate(raw) assert raw["spack"]["commit"] == "develop" assert raw["spack"]["packages"]["commit"] is None @@ -85,7 +81,7 @@ def test_config_yaml(yaml_path): # full config with open(yaml_path / "config.full.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) schema.ConfigValidator.validate(raw) assert raw["store"] == "/alternative-point" assert raw["spack"]["commit"] == "6408b51" @@ -100,7 +96,7 @@ def test_config_yaml(yaml_path): spack: repo: https://github.com/spack/spack.git """) - raw = yaml.load(config, Loader=yaml.Loader) + raw = yaml.load(config) schema.ConfigValidator.validate(raw) @@ -108,20 +104,20 @@ def test_recipe_config_yaml(recipe_paths): # validate the config.yaml in the test recipes for p in recipe_paths: with open(p / "config.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) schema.ConfigValidator.validate(raw) def test_compilers_yaml(yaml_path): # test that the defaults are set as expected with open(yaml_path / "compilers.defaults.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) schema.CompilersValidator.validate(raw) assert raw["gcc"] == {"version": "10.2"} assert raw["llvm"] is None with open(yaml_path / "compilers.full.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) schema.CompilersValidator.validate(raw) assert raw["gcc"] == {"version": "11"} assert raw["llvm"] == {"version": "13"} @@ -132,13 +128,13 @@ def test_recipe_compilers_yaml(recipe_paths): # validate the compilers.yaml in the test recipes for p in recipe_paths: with open(p / "compilers.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) schema.CompilersValidator.validate(raw) def test_environments_yaml(yaml_path): with open(yaml_path / "environments.full.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) schema.EnvironmentsValidator.validate(raw) # the defaults-env does not set fields @@ -181,7 +177,7 @@ def test_environments_yaml(yaml_path): # check that only allowed fields are accepted # from an example that was silently validated with open(yaml_path / "environments.err-providers.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) with pytest.raises( jsonschema.exceptions.ValidationError, match=r"Additional properties are not allowed \('providers' was unexpected", @@ -193,7 +189,7 @@ def test_recipe_environments_yaml(recipe_paths): # validate the environments.yaml in the test recipes for p in recipe_paths: with open(p / "environments.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) schema.EnvironmentsValidator.validate(raw) @@ -248,7 +244,7 @@ def test_recipe_environments_yaml(recipe_paths): ], ) def test_valid_modules_yaml(recipe): - instance = yaml.load(recipe, Loader=yaml.Loader) + instance = yaml.load(recipe) schema.ModulesValidator.validate(instance) assert not instance["modules"]["default"]["arch_folder"] @@ -267,7 +263,8 @@ def test_valid_modules_yaml(recipe): ) def test_invalid_modules_yaml(recipe): with pytest.raises(Exception): - schema.ModulesValidator.validate(yaml.load(recipe, Loader=yaml.Loader)) + schema.ModulesValidator.validate(yaml.load(recipe)) + def test_unique_properties(): invalid_config = dedent( @@ -277,5 +274,7 @@ def test_unique_properties(): """ ) - with pytest.raises(Exception): - yaml.load(invalid_config, Loader=yaml.Loader) + from ruamel.yaml.constructor import DuplicateKeyError + + with pytest.raises(DuplicateKeyError): + yaml.load(invalid_config) From afafdbacbc288fd0b6cd9416d1aa5e71c800cce3 Mon Sep 17 00:00:00 2001 From: Alberto Invernizzi Date: Wed, 19 Nov 2025 17:57:22 +0100 Subject: [PATCH 4/8] sneak in a minor fix in doc and formatting --- stackinator/builder.py | 2 +- stackinator/recipe.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index df338653..f50e54c6 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -141,7 +141,7 @@ def environment_meta(self, recipe): "root": /user-environment/env/default, "activate": /user-environment/env/default/activate.sh, "description": "simple devolpment env: compilers, MPI, python, cmake." - "env_vars": { + "recipe_variables": { ... } }, diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 1dda5a76..cf0b6812 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -413,6 +413,7 @@ def generate_environment_specs(self, raw): # ["uenv"]["env_vars"] = {"set": [], "unset": [], "prepend_path": [], "append_path": []} if view_config is None: view_config = {} + view_config.setdefault("link", "roots") view_config.setdefault("uenv", {}) view_config["uenv"].setdefault("add_compilers", True) From 5b6396a774822195ed895938d7d5a783839cc436 Mon Sep 17 00:00:00 2001 From: Alberto Invernizzi Date: Wed, 19 Nov 2025 18:07:11 +0100 Subject: [PATCH 5/8] fix dependency also for stack-config command --- bin/stack-config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/stack-config b/bin/stack-config index 66496991..9491f3a7 100755 --- a/bin/stack-config +++ b/bin/stack-config @@ -4,7 +4,7 @@ # dependencies = [ # "jinja2", # "jsonschema", -# "pyYAML", +# "ruamel.yaml >=0.15.1, <0.18.0", # ] # /// From a0c5a611d7307e693ed2a7d65010e3993e6c1ac2 Mon Sep 17 00:00:00 2001 From: Alberto Invernizzi Date: Wed, 1 Apr 2026 11:28:59 +0200 Subject: [PATCH 6/8] already using the new api --- bin/stack-config | 2 +- requirements.txt | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 requirements.txt diff --git a/bin/stack-config b/bin/stack-config index 9491f3a7..63c82bcc 100755 --- a/bin/stack-config +++ b/bin/stack-config @@ -4,7 +4,7 @@ # dependencies = [ # "jinja2", # "jsonschema", -# "ruamel.yaml >=0.15.1, <0.18.0", +# "ruamel.yaml >=0.15.1" # ] # /// diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..79d70ed3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +Jinja2 +jsonschema +pytest +ruamel.yaml >=0.15.1 From 49d0e1575dbd31ef1c7a00fe67ae94e7303f26a7 Mon Sep 17 00:00:00 2001 From: Alberto Invernizzi Date: Wed, 1 Apr 2026 11:44:10 +0200 Subject: [PATCH 7/8] WIP: dump --- stackinator/builder.py | 13 ++++++++----- stackinator/cache.py | 6 ++++-- stackinator/recipe.py | 9 ++++++++- stackinator/schema.py | 9 +++++++-- 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index f50e54c6..243c7391 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -309,17 +309,17 @@ def generate(self, recipe): # the packages.yaml configuration that will be used when building all environments # - the system packages.yaml with gcc removed # - plus additional packages provided by the recipe - global_packages_yaml = yaml.dump(recipe.packages["global"]) + global_packages_path = config_path / "packages.yaml" with global_packages_path.open("w") as fid: - fid.write(global_packages_yaml) + yaml.dump(recipe.packages["global"], fid) # generate a mirrors.yaml file if build caches have been configured if recipe.mirror: dst = config_path / "mirrors.yaml" self._logger.debug(f"generate the build cache mirror: {dst}") with dst.open("w") as fid: - fid.write(cache.generate_mirrors_yaml(recipe.mirror)) + cache.generate_mirrors_yaml(recipe.mirror, fid) # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to @@ -440,7 +440,10 @@ def generate(self, recipe): compiler_config_path.mkdir(exist_ok=True) for file, raw in files.items(): with (compiler_config_path / file).open(mode="w") as f: - f.write(raw) + if type(raw) is str: + f.write(raw) + else: + yaml.dump(raw, f) # generate the makefile and spack.yaml files that describe the environments environment_files = recipe.environment_files @@ -479,7 +482,7 @@ def generate(self, recipe): generate_modules_path = self.path / "modules" generate_modules_path.mkdir(exist_ok=True) with (generate_modules_path / "modules.yaml").open("w") as f: - yaml.dump(recipe.modules, f) + yaml.dump(recipe.modules_yaml_data, f) # write the meta data meta_path = store_path / "meta" diff --git a/stackinator/cache.py b/stackinator/cache.py index 773d0c2b..2adb80ac 100644 --- a/stackinator/cache.py +++ b/stackinator/cache.py @@ -42,7 +42,7 @@ def configuration_from_file(file, mount): return raw -def generate_mirrors_yaml(config): +def generate_mirrors_yaml(config, out): path = config["path"].as_posix() mirrors = { "mirrors": { @@ -57,4 +57,6 @@ def generate_mirrors_yaml(config): } } - return yaml.dump(mirrors, default_flow_style=False) + yaml = YAML() + yaml.default_flow_style = True + yaml.dump(mirrors, out) diff --git a/stackinator/recipe.py b/stackinator/recipe.py index cf0b6812..3f9450de 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -326,6 +326,13 @@ def environment_view_meta(self): return view_meta + @property + def modules_yaml_data(self): + with self.modules.open() as fid: + raw = yaml.load(fid) + raw["modules"]["default"]["roots"]["tcl"] = (pathlib.Path(self.mount) / "modules").as_posix() + return raw + # creates the self.environments field that describes the full specifications # for all of the environments sets, grouped in environments, from the raw # environments.yaml input. @@ -528,7 +535,7 @@ def compiler_files(self): files["config"][compiler]["spack.yaml"] = spack_yaml_template.render(config=config) # compilers/gcc/packages.yaml if compiler == "gcc": - files["config"][compiler]["packages.yaml"] = yaml.dump(self.packages["gcc"]) + files["config"][compiler]["packages.yaml"] = self.packages["gcc"] return files diff --git a/stackinator/schema.py b/stackinator/schema.py index 3a2a9842..60335ae7 100644 --- a/stackinator/schema.py +++ b/stackinator/schema.py @@ -3,7 +3,7 @@ from textwrap import dedent import jsonschema -import yaml +from ruamel.yaml import YAML from . import root_logger @@ -11,7 +11,12 @@ def py2yaml(data, indent): - dump = yaml.dump(data) + yaml = YAML() + from io import StringIO + + buffer = StringIO() + yaml.dump(data, buffer) + dump = buffer.getvalue() lines = [ln for ln in dump.split("\n") if ln != ""] res = ("\n" + " " * indent).join(lines) return res From f2d43c05f2b4eca43d85acc14ca87eefe55f9580 Mon Sep 17 00:00:00 2001 From: Alberto Invernizzi Date: Wed, 1 Apr 2026 18:01:34 +0200 Subject: [PATCH 8/8] align deps --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6fca4b88..4a24915b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ requires-python = ">=3.12" dependencies = [ "Jinja2", "jsonschema", - "ruamel.yaml >=0.15.1, <0.18.0", + "ruamel.yaml >=0.15.1", ] classifiers = [ "Development Status :: 5 - Production/Stable",