From 6f966f20684529c1493086aaa77830dcc7d6ef0a Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Fri, 15 May 2026 10:58:14 +0200 Subject: [PATCH 01/12] Initial version of multiple package repos in recipe --- stackinator/builder.py | 55 ++++++++++++++++------------ stackinator/etc/envvars.py | 41 ++++++++++++++------- stackinator/schema/config.json | 49 ++++++++++++++++++------- stackinator/templates/Makefile | 2 +- unittests/test_schema.py | 66 ++++++++++++++++++++++++++++++++-- 5 files changed, 161 insertions(+), 52 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index ae4997a..a2e5233 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -191,26 +191,27 @@ def generate(self, recipe): spack_git_commit_result = self._git_clone("spack", spack_repo, spack_commit, spack_path) - # Clone the spack-packages repository and check out commit if one was given - spack_packages = spack["packages"] - spack_packages_repo = spack_packages["repo"] - spack_packages_commit = spack_packages["commit"] - spack_packages_path = self.path / "spack-packages" - - spack_packages_git_commit_result = self._git_clone( - "spack-packages", - spack_packages_repo, - spack_packages_commit, - spack_packages_path, - ) + # Clone the spack package repositories and check out commit if one was given + packages_config = spack["packages"] + packages_resolved = self._resolve_packages(packages_config) + + packages_meta = [] + for name, repo, commit in packages_resolved: + clone_path = self.path / name + git_commit_result = self._git_clone(name, repo, commit, clone_path) + packages_meta.append({ + "name": name, + "url": repo, + "ref": commit, + "commit": git_commit_result, + "path": clone_path, + }) spack_meta = { "url": spack_repo, "ref": spack_commit, "commit": spack_git_commit_result, - "packages_url": spack_packages_repo, - "packages_ref": spack_packages_commit, - "packages_commit": spack_packages_git_commit_result, + "packages": packages_meta, } # load the jinja templating environment @@ -417,14 +418,16 @@ def generate(self, recipe): elif dst.exists(): self._logger.debug(f" NOT installing package {pkg_path}") - # Copy the builtin repo to store, delete if it already exists. - spack_packages_builtin_path = spack_packages_path / "repos" / "spack_repo" / "builtin" - spack_packages_store_path = store_path / "repos" / "spack_repo" / "builtin" - self._logger.debug(f"copying builtin repo from {spack_packages_builtin_path} to {spack_packages_store_path}") - if spack_packages_store_path.exists(): - self._logger.debug(f"{spack_packages_store_path} exists ... deleting") - shutil.rmtree(spack_packages_store_path) - install(spack_packages_builtin_path, spack_packages_store_path) + for idx, pkg_meta in enumerate(spack_meta["packages"]): + clone_path = pkg_meta["path"] + name = pkg_meta["name"] + src_path = clone_path / "repos" / "spack_repo" / name + dst_path = store_path / "repos" / "spack_repo" / name + self._logger.debug(f"copying repo \'{name}\' from {src_path} to {dst_path}") + if dst_path.exists(): + self._logger.debug(f"{dst_path} exists ... deleting") + shutil.rmtree(dst_path) + install(src_path, dst_path) # Generate the makefile and spack.yaml files that describe the compilers compiler_files = recipe.compiler_files @@ -525,6 +528,12 @@ def generate(self, recipe): ) f.write("\n") + @staticmethod + def _resolve_packages(packages): + if isinstance(packages.get("repo"), str): + return [("builtin", packages["repo"], packages.get("commit"))] + return [(name, val["repo"], val.get("commit")) for name, val in packages.items()] + def _git_clone(self, name, repo, commit, path): if not (path / ".git").is_dir(): self._logger.info(f"{name}: clone repository {repo} to {path}") diff --git a/stackinator/etc/envvars.py b/stackinator/etc/envvars.py index 218a77b..ce423ad 100755 --- a/stackinator/etc/envvars.py +++ b/stackinator/etc/envvars.py @@ -624,9 +624,31 @@ def meta_impl(args): spack_packages_url = None spack_packages_ref = None spack_packages_commit = None - if args.spack_packages is not None: - spack_packages_url, spack_packages_ref, spack_packages_commit = args.spack_packages.split(",") + package_repos = [] + if args.spack_package_repo: + for entry in args.spack_package_repo: + parts = entry.split(",") + name, url, ref, commit = parts[0], parts[1], parts[2], parts[3] + package_repos.append({"name": name, "url": url, "ref": ref, "commit": commit}) + if name == "builtin": + spack_packages_url = url + spack_packages_ref = ref + spack_packages_commit = commit spack_path = f"{args.mount}/config".replace("//", "/") + scalar_vars = { + "UENV_SPACK_CONFIG_PATH": spack_path, + "UENV_SPACK_URL": spack_url, + "UENV_SPACK_REF": spack_ref, + "UENV_SPACK_COMMIT": spack_commit, + "UENV_SPACK_PACKAGES_URL": spack_packages_url, + "UENV_SPACK_PACKAGES_REF": spack_packages_ref, + "UENV_SPACK_PACKAGES_COMMIT": spack_packages_commit, + } + for repo in package_repos: + name_upper = repo["name"].upper().replace("-", "_") + scalar_vars[f"UENV_PACKAGE_REPO_{name_upper}_URL"] = repo["url"] + scalar_vars[f"UENV_PACKAGE_REPO_{name_upper}_REF"] = repo["ref"] + scalar_vars[f"UENV_PACKAGE_REPO_{name_upper}_COMMIT"] = repo["commit"] meta["views"]["spack"] = { "activate": "/dev/null", "description": "configure spack upstream", @@ -636,15 +658,7 @@ def meta_impl(args): "type": "augment", "values": { "list": {}, - "scalar": { - "UENV_SPACK_CONFIG_PATH": spack_path, - "UENV_SPACK_URL": spack_url, - "UENV_SPACK_REF": spack_ref, - "UENV_SPACK_COMMIT": spack_commit, - "UENV_SPACK_PACKAGES_URL": spack_packages_url, - "UENV_SPACK_PACKAGES_REF": spack_packages_ref, - "UENV_SPACK_PACKAGES_COMMIT": spack_packages_commit, - }, + "scalar": scalar_vars, }, }, } @@ -686,9 +700,10 @@ def meta_impl(args): default=None, ) uenv_parser.add_argument( - "--spack-packages", - help='configure spack-packages repository metadata. Format is "spack_url,git_ref,git_commit"', + "--spack-package-repo", + help='configure spack package repository metadata. Format is "name,spack_url,git_ref,git_commit". Can be repeated.', type=str, + action="append", default=None, ) diff --git a/stackinator/schema/config.json b/stackinator/schema/config.json index d9de503..5636965 100644 --- a/stackinator/schema/config.json +++ b/stackinator/schema/config.json @@ -28,21 +28,44 @@ "default": null }, "packages": { - "type": "object", - "additionalProperties": false, - "required": ["repo"], - "properties" : { - "repo": { - "type": "string" + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["repo"], + "properties": { + "repo": { + "type": "string" + }, + "commit": { + "oneOf": [ + {"type" : "string"}, + {"type" : "null"} + ] + } + } }, - "commit": { - "oneOf": [ - {"type" : "string"}, - {"type" : "null"} - ], - "default": null + { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "object", + "additionalProperties": false, + "required": ["repo"], + "properties": { + "repo": { + "type": "string" + }, + "commit": { + "oneOf": [ + {"type" : "string"}, + {"type" : "null"} + ] + } + } + } } - } + ] } } }, diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 10a0ea5..41b9d5b 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -59,7 +59,7 @@ modules-done: environments generate-config env-meta: generate-config environments{% if modules %} modules-done{% endif %} - $(SANDBOX) $(BUILD_ROOT)/envvars.py uenv {% if modules %}--modules{% endif %} --spack='{{ spack_meta.url }},{{ spack_meta.ref }},{{ spack_meta.commit }}' --spack-packages='{{ spack_meta.packages_url }},{{ spack_meta.packages_ref }},{{ spack_meta.packages_commit }}' $(STORE) + $(SANDBOX) $(BUILD_ROOT)/envvars.py uenv {% if modules %}--modules{% endif %} --spack='{{ spack_meta.url }},{{ spack_meta.ref }},{{ spack_meta.commit }}'{% for pkg in spack_meta.packages %} --spack-package-repo='{{ pkg.name }},{{ pkg.url }},{{ pkg.ref }},{{ pkg.commit }}'{% endfor %} $(STORE) touch env-meta post-install: env-meta diff --git a/unittests/test_schema.py b/unittests/test_schema.py index b30f73d..c2e83cf 100644 --- a/unittests/test_schema.py +++ b/unittests/test_schema.py @@ -27,6 +27,7 @@ def recipes(): "base-nvgpu", "cache", "with-repo", + "with-multi-repos", ] @@ -42,7 +43,7 @@ def test_config_yaml(yaml_path): schema.ConfigValidator.validate(raw) assert raw["store"] == "/user-environment" assert raw["spack"]["commit"] is None - assert raw["spack"]["packages"]["commit"] is None + assert raw["spack"]["packages"].get("commit") is None assert raw["description"] is None # no spack:commit @@ -80,7 +81,7 @@ def test_config_yaml(yaml_path): ) schema.ConfigValidator.validate(raw) assert raw["spack"]["commit"] == "develop" - assert raw["spack"]["packages"]["commit"] is None + assert raw["spack"]["packages"].get("commit") is None assert raw["description"] is None # full config @@ -103,6 +104,67 @@ def test_config_yaml(yaml_path): raw = yaml.load(config, Loader=yaml.Loader) schema.ConfigValidator.validate(raw) + # map format: single entry + config = dedent(""" + version: 2 + name: map-single + spack: + repo: https://github.com/spack/spack.git + packages: + my-packages: + repo: https://github.com/example/spack-packages.git + """) + raw = yaml.load(config, Loader=yaml.Loader) + schema.ConfigValidator.validate(raw) + assert "my-packages" in raw["spack"]["packages"] + assert raw["spack"]["packages"]["my-packages"]["repo"] == "https://github.com/example/spack-packages.git" + assert raw["spack"]["packages"]["my-packages"].get("commit") is None + + # map format: multiple entries with commits + config = dedent(""" + version: 2 + name: map-multi + spack: + repo: https://github.com/spack/spack.git + packages: + my-packages: + repo: https://github.com/example/spack-packages.git + commit: v1.0 + other-packages: + repo: https://github.com/example/other-packages.git + commit: v2.0 + """) + raw = yaml.load(config, Loader=yaml.Loader) + schema.ConfigValidator.validate(raw) + assert raw["spack"]["packages"]["my-packages"]["commit"] == "v1.0" + assert raw["spack"]["packages"]["other-packages"]["commit"] == "v2.0" + + # map format: empty map should fail + with pytest.raises(Exception): + config = dedent(""" + version: 2 + name: map-empty + spack: + repo: https://github.com/spack/spack.git + packages: {} + """) + raw = yaml.load(config, Loader=yaml.Loader) + schema.ConfigValidator.validate(raw) + + # map format: entry missing repo should fail + with pytest.raises(Exception): + config = dedent(""" + version: 2 + name: map-no-repo + spack: + repo: https://github.com/spack/spack.git + packages: + my-packages: + commit: v1.0 + """) + raw = yaml.load(config, Loader=yaml.Loader) + schema.ConfigValidator.validate(raw) + def test_recipe_config_yaml(recipe_paths): # validate the config.yaml in the test recipes From 7dd9d6590142239422ebf65e793afc957223ff3a Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Fri, 15 May 2026 11:45:43 +0200 Subject: [PATCH 02/12] Separate alps-cluster-config and recipe package repos --- stackinator/builder.py | 64 +++++++++++++++++++++++--------- stackinator/schema/config.json | 3 ++ stackinator/templates/repos.yaml | 3 ++ unittests/test_schema.py | 30 +++++++++++++++ 4 files changed, 83 insertions(+), 17 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index a2e5233..9a4c9c9 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -196,7 +196,7 @@ def generate(self, recipe): packages_resolved = self._resolve_packages(packages_config) packages_meta = [] - for name, repo, commit in packages_resolved: + for name, repo, commit, repo_path in packages_resolved: clone_path = self.path / name git_commit_result = self._git_clone(name, repo, commit, clone_path) packages_meta.append({ @@ -205,6 +205,7 @@ def generate(self, recipe): "ref": commit, "commit": git_commit_result, "path": clone_path, + "repo_path": repo_path, }) spack_meta = { @@ -334,13 +335,11 @@ def generate(self, recipe): # to search for package # 1. builtin repo - # Build a list of repos with packages to install. - repos = [] + # Determine whether the recipe provides its own package repo + has_recipe_repo = recipe.spack_repo is not None - # check for a repo in the recipe - if recipe.spack_repo is not None: - self._logger.debug(f"adding recipe spack package repo: {recipe.spack_repo}") - repos.append(recipe.spack_repo) + # Build a list of system repos with packages to install. + system_repos = [] # look for repos.yaml file in the system configuration repo_yaml = recipe.system_config_path / "repos.yaml" @@ -356,13 +355,13 @@ def generate(self, recipe): for rel_path in P: repo_path = (recipe.system_config_path / rel_path).resolve() if spack_util.is_repo(repo_path): - repos.append(repo_path) + system_repos.append(repo_path) self._logger.debug(f"adding site spack package repo: {repo_path}") else: self._logger.error(f"{repo_path} from {repo_yaml} is not a spack package repository") raise RuntimeError("invalid system-provided package repository") - self._logger.debug(f"full list of spack package repo: {repos}") + self._logger.debug(f"full list of system spack package repos: {system_repos}") # Delete the store/repo path, if it already exists. # Do this so that incremental builds (though not officially supported) won't break if a repo is updated. @@ -389,25 +388,56 @@ def generate(self, recipe): """ ) + # If the recipe provides a package repo, install it as a separate + # "recipe" repo in the store with highest precedence. + if has_recipe_repo: + recipe_dst = repos_path / "recipe" + self._logger.debug(f"creating the recipe spack repo in {recipe_dst}") + if recipe_dst.exists(): + self._logger.debug(f"{recipe_dst} exists ... deleting") + shutil.rmtree(recipe_dst) + + recipe_pkg_dst = recipe_dst / "packages" + recipe_pkg_dst.mkdir(mode=0o755, parents=True) + + with (recipe_dst / "repo.yaml").open("w") as f: + f.write( + """\ +repo: + namespace: recipe + api: v2.0 +""" + ) + + packages_path = recipe.spack_repo / "packages" + for pkg_path in packages_path.iterdir(): + dst = recipe_pkg_dst / pkg_path.name + if pkg_path.is_dir(): + self._logger.debug(f" installing recipe package {pkg_path} to {recipe_pkg_dst}") + install(pkg_path, dst) + # create the repository step 2: create the repos.yaml file in build_path/config repos_yaml_template = jinja_env.get_template("repos.yaml") with (config_path / "repos.yaml").open("w") as f: repo_path = recipe.mount / "repos" / "spack_repo" / "alps" builtin_repo_path = recipe.mount / "repos" / "spack_repo" / "builtin" + recipe_repo_path = recipe.mount / "repos" / "spack_repo" / "recipe" f.write( repos_yaml_template.render( repo_path=repo_path.as_posix(), builtin_repo_path=builtin_repo_path.as_posix(), + recipe_repo_path=recipe_repo_path.as_posix(), + has_recipe_repo=has_recipe_repo, verbose=False, ) ) f.write("\n") - # Iterate over the source repositories copying their contents to the consolidated repo in the uenv. - # Do overwrite packages that have been copied from an earlier source repo, enforcing a descending - # order of precidence. - if len(repos) > 0: - for repo_src in repos: + # Iterate over the system source repositories copying their contents to the + # consolidated alps repo in the uenv. Do not overwrite packages that have been + # copied from an earlier source repo, enforcing a descending order of precidence. + if len(system_repos) > 0: + for repo_src in system_repos: self._logger.debug(f"installing repo {repo_src}") packages_path = repo_src / "packages" for pkg_path in packages_path.iterdir(): @@ -421,7 +451,7 @@ def generate(self, recipe): for idx, pkg_meta in enumerate(spack_meta["packages"]): clone_path = pkg_meta["path"] name = pkg_meta["name"] - src_path = clone_path / "repos" / "spack_repo" / name + src_path = clone_path / pkg_meta["repo_path"] dst_path = store_path / "repos" / "spack_repo" / name self._logger.debug(f"copying repo \'{name}\' from {src_path} to {dst_path}") if dst_path.exists(): @@ -531,8 +561,8 @@ def generate(self, recipe): @staticmethod def _resolve_packages(packages): if isinstance(packages.get("repo"), str): - return [("builtin", packages["repo"], packages.get("commit"))] - return [(name, val["repo"], val.get("commit")) for name, val in packages.items()] + return [("builtin", packages["repo"], packages.get("commit"), "repos/spack_repo/builtin")] + return [(name, val["repo"], val.get("commit"), val.get("path", f"repos/spack_repo/{name}")) for name, val in packages.items()] def _git_clone(self, name, repo, commit, path): if not (path / ".git").is_dir(): diff --git a/stackinator/schema/config.json b/stackinator/schema/config.json index 5636965..3bba630 100644 --- a/stackinator/schema/config.json +++ b/stackinator/schema/config.json @@ -61,6 +61,9 @@ {"type" : "string"}, {"type" : "null"} ] + }, + "path": { + "type": "string" } } } diff --git a/stackinator/templates/repos.yaml b/stackinator/templates/repos.yaml index 3cf34f2..32f9dc0 100644 --- a/stackinator/templates/repos.yaml +++ b/stackinator/templates/repos.yaml @@ -1,3 +1,6 @@ repos: +{% if has_recipe_repo %} + recipe: {{ recipe_repo_path }} +{% endif %} alps: {{ repo_path }} builtin: {{ builtin_repo_path }} diff --git a/unittests/test_schema.py b/unittests/test_schema.py index c2e83cf..3d77f62 100644 --- a/unittests/test_schema.py +++ b/unittests/test_schema.py @@ -165,6 +165,36 @@ def test_config_yaml(yaml_path): raw = yaml.load(config, Loader=yaml.Loader) schema.ConfigValidator.validate(raw) + # map format: custom path + config = dedent(""" + version: 2 + name: map-custom-path + spack: + repo: https://github.com/spack/spack.git + packages: + my-packages: + repo: https://github.com/example/spack-packages.git + path: custom/repo/location + """) + raw = yaml.load(config, Loader=yaml.Loader) + schema.ConfigValidator.validate(raw) + assert raw["spack"]["packages"]["my-packages"]["path"] == "custom/repo/location" + + # map format: no path (default behavior) + config = dedent(""" + version: 2 + name: map-no-path + spack: + repo: https://github.com/spack/spack.git + packages: + my-packages: + repo: https://github.com/example/spack-packages.git + commit: v2.0 + """) + raw = yaml.load(config, Loader=yaml.Loader) + schema.ConfigValidator.validate(raw) + assert "path" not in raw["spack"]["packages"]["my-packages"] + def test_recipe_config_yaml(recipe_paths): # validate the config.yaml in the test recipes From f9e0ee5c3af73d3b63bcbd7d199dd0c66f60410b Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Fri, 15 May 2026 11:53:40 +0200 Subject: [PATCH 03/12] Put package repos under repos in build root --- stackinator/builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 9a4c9c9..16d032f 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -197,7 +197,7 @@ def generate(self, recipe): packages_meta = [] for name, repo, commit, repo_path in packages_resolved: - clone_path = self.path / name + clone_path = self.path / "repos" / name git_commit_result = self._git_clone(name, repo, commit, clone_path) packages_meta.append({ "name": name, From fe85aace38e0e9f4182f95a1dbf57f3af9e00326 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Fri, 15 May 2026 15:35:16 +0200 Subject: [PATCH 04/12] Fix custom repos in repos.yaml --- stackinator/builder.py | 7 +++++-- stackinator/templates/repos.yaml | 4 +++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 16d032f..e2e9342 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -420,12 +420,15 @@ def generate(self, recipe): repos_yaml_template = jinja_env.get_template("repos.yaml") with (config_path / "repos.yaml").open("w") as f: repo_path = recipe.mount / "repos" / "spack_repo" / "alps" - builtin_repo_path = recipe.mount / "repos" / "spack_repo" / "builtin" recipe_repo_path = recipe.mount / "repos" / "spack_repo" / "recipe" + package_repos = [ + {"name": pkg["name"], "path": (recipe.mount / "repos" / "spack_repo" / pkg["name"]).as_posix()} + for pkg in spack_meta["packages"] + ] f.write( repos_yaml_template.render( repo_path=repo_path.as_posix(), - builtin_repo_path=builtin_repo_path.as_posix(), + package_repos=package_repos, recipe_repo_path=recipe_repo_path.as_posix(), has_recipe_repo=has_recipe_repo, verbose=False, diff --git a/stackinator/templates/repos.yaml b/stackinator/templates/repos.yaml index 32f9dc0..18f2e31 100644 --- a/stackinator/templates/repos.yaml +++ b/stackinator/templates/repos.yaml @@ -3,4 +3,6 @@ repos: recipe: {{ recipe_repo_path }} {% endif %} alps: {{ repo_path }} - builtin: {{ builtin_repo_path }} +{% for repo in package_repos %} + {{ repo.name }}: {{ repo.path }} +{% endfor %} From cfb03899ead3b7c0744314551106f32038b3a4ef Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Mon, 18 May 2026 14:38:11 +0200 Subject: [PATCH 05/12] Clean up package repo config --- stackinator/builder.py | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index e2e9342..e573257 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -191,22 +191,9 @@ def generate(self, recipe): spack_git_commit_result = self._git_clone("spack", spack_repo, spack_commit, spack_path) - # Clone the spack package repositories and check out commit if one was given - packages_config = spack["packages"] - packages_resolved = self._resolve_packages(packages_config) - - packages_meta = [] - for name, repo, commit, repo_path in packages_resolved: - clone_path = self.path / "repos" / name - git_commit_result = self._git_clone(name, repo, commit, clone_path) - packages_meta.append({ - "name": name, - "url": repo, - "ref": commit, - "commit": git_commit_result, - "path": clone_path, - "repo_path": repo_path, - }) + packages_meta = self._resolve_packages(spack["packages"]) + for pkg in packages_meta: + pkg["commit"] = self._git_clone(pkg["name"], pkg["url"], pkg["ref"], pkg["path"]) spack_meta = { "url": spack_repo, @@ -561,11 +548,14 @@ def generate(self, recipe): ) f.write("\n") - @staticmethod - def _resolve_packages(packages): + def _resolve_packages(self, packages): + base = self.path / "repos" if isinstance(packages.get("repo"), str): - return [("builtin", packages["repo"], packages.get("commit"), "repos/spack_repo/builtin")] - return [(name, val["repo"], val.get("commit"), val.get("path", f"repos/spack_repo/{name}")) for name, val in packages.items()] + return [{"name": "builtin", "url": packages["repo"], "ref": packages.get("commit"), + "path": base / "builtin", "repo_path": "repos/spack_repo/builtin"}] + return [{"name": name, "url": val["repo"], "ref": val.get("commit"), + "path": base / name, "repo_path": val.get("path", f"repos/spack_repo/{name}")} + for name, val in packages.items()] def _git_clone(self, name, repo, commit, path): if not (path / ".git").is_dir(): From c7bbb8838985d490f748ffe1e11fba183eb09a77 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Mon, 18 May 2026 14:57:19 +0200 Subject: [PATCH 06/12] Small cleanup --- stackinator/builder.py | 46 +++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index e573257..60fc838 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -320,13 +320,11 @@ def generate(self, recipe): # 2. cluster-config/repos.yaml # - if the repos.yaml file exists it will contain a list of relative paths # to search for package - # 1. builtin repo + # 1. package repos from config.yaml in the order specified (typically + # only spack-packages builtin repo) - # Determine whether the recipe provides its own package repo - has_recipe_repo = recipe.spack_repo is not None - - # Build a list of system repos with packages to install. - system_repos = [] + # Build a list of repos with packages to install from system config and recipe. + repos = [] # look for repos.yaml file in the system configuration repo_yaml = recipe.system_config_path / "repos.yaml" @@ -342,13 +340,13 @@ def generate(self, recipe): for rel_path in P: repo_path = (recipe.system_config_path / rel_path).resolve() if spack_util.is_repo(repo_path): - system_repos.append(repo_path) + repos.append(repo_path) self._logger.debug(f"adding site spack package repo: {repo_path}") else: self._logger.error(f"{repo_path} from {repo_yaml} is not a spack package repository") raise RuntimeError("invalid system-provided package repository") - self._logger.debug(f"full list of system spack package repos: {system_repos}") + self._logger.debug(f"full list of system spack package repos: {repos}") # Delete the store/repo path, if it already exists. # Do this so that incremental builds (though not officially supported) won't break if a repo is updated. @@ -365,7 +363,7 @@ def generate(self, recipe): self._logger.debug(f"created the repo packages path {pkg_dst}") # create the repository step 2: create the repo.yaml file that - # configures alps and builtin repos + # configures the alps repo with (repo_dst / "repo.yaml").open("w") as f: f.write( """\ @@ -377,6 +375,7 @@ def generate(self, recipe): # If the recipe provides a package repo, install it as a separate # "recipe" repo in the store with highest precedence. + has_recipe_repo = recipe.spack_repo is not None if has_recipe_repo: recipe_dst = repos_path / "recipe" self._logger.debug(f"creating the recipe spack repo in {recipe_dst}") @@ -423,21 +422,22 @@ def generate(self, recipe): ) f.write("\n") - # Iterate over the system source repositories copying their contents to the - # consolidated alps repo in the uenv. Do not overwrite packages that have been - # copied from an earlier source repo, enforcing a descending order of precidence. - if len(system_repos) > 0: - for repo_src in system_repos: - self._logger.debug(f"installing repo {repo_src}") - packages_path = repo_src / "packages" - for pkg_path in packages_path.iterdir(): - dst = pkg_dst / pkg_path.name - if pkg_path.is_dir() and not dst.exists(): - self._logger.debug(f" installing package {pkg_path} to {pkg_dst}") - install(pkg_path, dst) - elif dst.exists(): - self._logger.debug(f" NOT installing package {pkg_path}") + # Iterate over the alps and recipe repositories copying their contents + # to the final repo locations. Because of the order of repos in the + # repos.yaml config file, recipe packages have precedence. + for repo_src in repos: + self._logger.debug(f"installing repo {repo_src}") + packages_path = repo_src / "packages" + for pkg_path in packages_path.iterdir(): + dst = pkg_dst / pkg_path.name + if pkg_path.is_dir() and not dst.exists(): + self._logger.debug(f" installing package {pkg_path} to {pkg_dst}") + install(pkg_path, dst) + elif dst.exists(): + self._logger.debug(f" NOT installing package {pkg_path}") + # Copy all package repos defined in config.yaml to their final repo + # locations. for idx, pkg_meta in enumerate(spack_meta["packages"]): clone_path = pkg_meta["path"] name = pkg_meta["name"] From db34c249e3c05ee3f0f244d0b67b40d09d248ccd Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Mon, 18 May 2026 15:27:01 +0200 Subject: [PATCH 07/12] A bit of cleanup --- stackinator/etc/envvars.py | 35 ++++++++++++++--------------------- unittests/test-envvars.sh | 37 ++++++++++++++++++++++++++++++++++++- 2 files changed, 50 insertions(+), 22 deletions(-) diff --git a/stackinator/etc/envvars.py b/stackinator/etc/envvars.py index ce423ad..6bf4e72 100755 --- a/stackinator/etc/envvars.py +++ b/stackinator/etc/envvars.py @@ -621,34 +621,27 @@ def meta_impl(args): if args.spack is not None: spack_url, spack_ref, spack_commit = args.spack.split(",") - spack_packages_url = None - spack_packages_ref = None - spack_packages_commit = None - package_repos = [] - if args.spack_package_repo: - for entry in args.spack_package_repo: - parts = entry.split(",") - name, url, ref, commit = parts[0], parts[1], parts[2], parts[3] - package_repos.append({"name": name, "url": url, "ref": ref, "commit": commit}) - if name == "builtin": - spack_packages_url = url - spack_packages_ref = ref - spack_packages_commit = commit spack_path = f"{args.mount}/config".replace("//", "/") scalar_vars = { "UENV_SPACK_CONFIG_PATH": spack_path, "UENV_SPACK_URL": spack_url, "UENV_SPACK_REF": spack_ref, "UENV_SPACK_COMMIT": spack_commit, - "UENV_SPACK_PACKAGES_URL": spack_packages_url, - "UENV_SPACK_PACKAGES_REF": spack_packages_ref, - "UENV_SPACK_PACKAGES_COMMIT": spack_packages_commit, } - for repo in package_repos: - name_upper = repo["name"].upper().replace("-", "_") - scalar_vars[f"UENV_PACKAGE_REPO_{name_upper}_URL"] = repo["url"] - scalar_vars[f"UENV_PACKAGE_REPO_{name_upper}_REF"] = repo["ref"] - scalar_vars[f"UENV_PACKAGE_REPO_{name_upper}_COMMIT"] = repo["commit"] + if args.spack_package_repo: + repo_names = [] + for entry in args.spack_package_repo: + name, url, ref, commit = entry.split(",") + repo_names.append(name) + name_upper = name.upper().replace("-", "_") + scalar_vars[f"UENV_PACKAGE_REPO_{name_upper}_URL"] = url + scalar_vars[f"UENV_PACKAGE_REPO_{name_upper}_REF"] = ref + scalar_vars[f"UENV_PACKAGE_REPO_{name_upper}_COMMIT"] = commit + if name == "builtin": + scalar_vars["UENV_SPACK_PACKAGES_URL"] = url + scalar_vars["UENV_SPACK_PACKAGES_REF"] = ref + scalar_vars["UENV_SPACK_PACKAGES_COMMIT"] = commit + scalar_vars["UENV_PACKAGE_REPOS"] = ",".join(repo_names) meta["views"]["spack"] = { "activate": "/dev/null", "description": "configure spack upstream", diff --git a/unittests/test-envvars.sh b/unittests/test-envvars.sh index 79e98fb..e88eb6d 100755 --- a/unittests/test-envvars.sh +++ b/unittests/test-envvars.sh @@ -36,7 +36,7 @@ find $scratch_path -name env.json echo "===== running final meta data stage ${mount_path}" -../stackinator/etc/envvars.py uenv ${mount_path}/ --modules --spack="https://github.com/spack/spack.git,releases/v0.20" --spack-packages="https://github.com/spack/spack.git,develop" +../stackinator/etc/envvars.py uenv ${mount_path}/ --modules --spack="https://github.com/spack/spack.git,releases/v0.20,abc123" --spack-package-repo="builtin,https://github.com/spack/spack-packages.git,develop,abc123" echo echo "===== develop" @@ -58,3 +58,38 @@ echo "===== modules view" echo cat ${meta_path} | jq .views.modules +echo +echo "===== test UENV_PACKAGE_REPOS with multiple package repos" +rm -rf ${mount_path} +mkdir -p ${scratch_path} +cp -R ${input_path} ${mount_path} +if [[ "$OSTYPE" == "linux-gnu"* ]]; then + sed -i "s|@@mount@@|${mount_path}|g" ${meta_in_path} +else + sed -i '' "s|@@mount@@|${mount_path}|g" ${meta_in_path} +fi + +../stackinator/etc/envvars.py view ${mount_path}/env/arbor /dev/shm/bcumming/arbor +../stackinator/etc/envvars.py view --prefix_paths="LD_LIBRARY_PATH=lib:lib64" ${mount_path}/env/develop /dev/shm/bcumming/arbor + +../stackinator/etc/envvars.py uenv ${mount_path}/ \ + --spack="https://github.com/spack/spack.git,releases/v0.21,abc123def" \ + --spack-package-repo="my-packages,https://github.com/example/spack-packages.git,v1.0,abc123" \ + --spack-package-repo="other-packages,https://github.com/example/other-packages.git,main,def456" + +UENV_PACKAGE_REPOS=$(cat ${meta_path} | jq -r '.views.spack.env.values.scalar.UENV_PACKAGE_REPOS') +echo "UENV_PACKAGE_REPOS=$UENV_PACKAGE_REPOS" +if [[ "$UENV_PACKAGE_REPOS" != "my-packages,other-packages" ]]; then + echo "FAIL: expected my-packages,other-packages, got '${UENV_PACKAGE_REPOS}'" + exit 1 +fi + +UENV_PACKAGE_REPO_MY_PACKAGES_URL=$(cat ${meta_path} | jq -r '.views.spack.env.values.scalar.UENV_PACKAGE_REPO_MY_PACKAGES_URL') +echo "UENV_PACKAGE_REPO_MY_PACKAGES_URL=$UENV_PACKAGE_REPO_MY_PACKAGES_URL" +if [[ "$UENV_PACKAGE_REPO_MY_PACKAGES_URL" != "https://github.com/example/spack-packages.git" ]]; then + echo "FAIL: expected https://github.com/example/spack-packages.git, got '${UENV_PACKAGE_REPO_MY_PACKAGES_URL}'" + exit 1 +fi + +echo "PASSED" + From 15383038b41b57d07df3bf5c13af5a12e38d9a67 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Mon, 18 May 2026 16:34:03 +0200 Subject: [PATCH 08/12] Add multi-package-repo test --- unittests/recipes/with-multi-repos/compilers.yaml | 2 ++ unittests/recipes/with-multi-repos/config.yaml | 13 +++++++++++++ .../recipes/with-multi-repos/environments.yaml | 13 +++++++++++++ 3 files changed, 28 insertions(+) create mode 100644 unittests/recipes/with-multi-repos/compilers.yaml create mode 100644 unittests/recipes/with-multi-repos/config.yaml create mode 100644 unittests/recipes/with-multi-repos/environments.yaml diff --git a/unittests/recipes/with-multi-repos/compilers.yaml b/unittests/recipes/with-multi-repos/compilers.yaml new file mode 100644 index 0000000..bce7184 --- /dev/null +++ b/unittests/recipes/with-multi-repos/compilers.yaml @@ -0,0 +1,2 @@ +gcc: + version: "11" diff --git a/unittests/recipes/with-multi-repos/config.yaml b/unittests/recipes/with-multi-repos/config.yaml new file mode 100644 index 0000000..a84f6f7 --- /dev/null +++ b/unittests/recipes/with-multi-repos/config.yaml @@ -0,0 +1,13 @@ +name: with-multi-repos +store: '/user-environment' +spack: + repo: https://github.com/spack/spack.git + commit: v21.0 + packages: + my-packages: + repo: https://github.com/example/spack-packages.git + commit: v1.0 + path: custom/path/to/packages + other-packages: + repo: https://github.com/example/other-packages.git +version: 2 diff --git a/unittests/recipes/with-multi-repos/environments.yaml b/unittests/recipes/with-multi-repos/environments.yaml new file mode 100644 index 0000000..33735ee --- /dev/null +++ b/unittests/recipes/with-multi-repos/environments.yaml @@ -0,0 +1,13 @@ +gcc-env: + compiler: [gcc] + unify: true + specs: + - osu-micro-benchmarks@5.9 + - hdf5 +mpi + network: + mpi: cray-mpich +tools: + compiler: [gcc] + unify: true + specs: + - cmake From d6491561036d0ef63afa926e328c0f5f2cf6f9a8 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Mon, 18 May 2026 16:38:59 +0200 Subject: [PATCH 09/12] Use parameterized test fixtures --- unittests/test_schema.py | 45 ++++++++++++++++------------------------ 1 file changed, 18 insertions(+), 27 deletions(-) diff --git a/unittests/test_schema.py b/unittests/test_schema.py index 3d77f62..88a63cc 100644 --- a/unittests/test_schema.py +++ b/unittests/test_schema.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 import pathlib from textwrap import dedent @@ -20,20 +20,14 @@ def yaml_path(test_path): return test_path / "yaml" -@pytest.fixture -def recipes(): - return [ - "host-recipe", - "base-nvgpu", - "cache", - "with-repo", - "with-multi-repos", - ] +@pytest.fixture(params=["host-recipe", "base-nvgpu", "cache", "with-repo", "with-multi-repos"]) +def recipe(request): + return request.param @pytest.fixture -def recipe_paths(test_path, recipes): - return [test_path / "recipes" / r for r in recipes] +def recipe_path(test_path, recipe): + return test_path / "recipes" / recipe def test_config_yaml(yaml_path): @@ -196,12 +190,11 @@ def test_config_yaml(yaml_path): assert "path" not in raw["spack"]["packages"]["my-packages"] -def test_recipe_config_yaml(recipe_paths): +def test_recipe_config_yaml(recipe_path): # validate the config.yaml in the test recipes - for p in recipe_paths: - with open(p / "config.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) - schema.ConfigValidator.validate(raw) + with open(recipe_path / "config.yaml") as fid: + raw = yaml.load(fid, Loader=yaml.Loader) + schema.ConfigValidator.validate(raw) def test_compilers_yaml(yaml_path): @@ -220,12 +213,11 @@ def test_compilers_yaml(yaml_path): assert raw["nvhpc"] == {"version": "25.1"} -def test_recipe_compilers_yaml(recipe_paths): +def test_recipe_compilers_yaml(recipe_path): # validate the compilers.yaml in the test recipes - for p in recipe_paths: - with open(p / "compilers.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) - schema.CompilersValidator.validate(raw) + with open(recipe_path / "compilers.yaml") as fid: + raw = yaml.load(fid, Loader=yaml.Loader) + schema.CompilersValidator.validate(raw) def test_environments_yaml(yaml_path): @@ -281,12 +273,11 @@ def test_environments_yaml(yaml_path): schema.EnvironmentsValidator.validate(raw) -def test_recipe_environments_yaml(recipe_paths): +def test_recipe_environments_yaml(recipe_path): # validate the environments.yaml in the test recipes - for p in recipe_paths: - with open(p / "environments.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) - schema.EnvironmentsValidator.validate(raw) + with open(recipe_path / "environments.yaml") as fid: + raw = yaml.load(fid, Loader=yaml.Loader) + schema.EnvironmentsValidator.validate(raw) @pytest.mark.parametrize( From 92ab8e6d5fae26c1c72160814628c1b839e065f6 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Tue, 19 May 2026 09:32:24 +0200 Subject: [PATCH 10/12] Add test-envvars.sh to CI --- .github/workflows/main.yaml | 5 +++++ unittests/data/arbor-uenv/meta/env.json.in | 2 ++ 2 files changed, 7 insertions(+) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index cb2fd33..d4d8ee4 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -26,3 +26,8 @@ jobs: run: uv sync --group dev - name: Run Unit Tests run: uv run pytest + - name: Run Envvars Test + working-directory: unittests + run: | + sudo apt-get install -y jq + ./test-envvars.sh diff --git a/unittests/data/arbor-uenv/meta/env.json.in b/unittests/data/arbor-uenv/meta/env.json.in index 25086e8..fa9e9b4 100644 --- a/unittests/data/arbor-uenv/meta/env.json.in +++ b/unittests/data/arbor-uenv/meta/env.json.in @@ -9,11 +9,13 @@ "arbor": { "activate": "@@mount@@/env/arbor/activate.sh", "description": "", + "recipe_variables": {"scalar": {}, "list": {}}, "root": "@@mount@@/env/arbor" }, "develop": { "activate": "@@mount@@/env/develop/activate.sh", "description": "", + "recipe_variables": {"scalar": {}, "list": {}}, "root": "@@mount@@/env/develop" } } From d38d855b6fd46b24ca92b2a2f72db61d9c6fd03f Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 20 May 2026 09:46:19 +0200 Subject: [PATCH 11/12] Formatting --- stackinator/builder.py | 31 ++++++++++++++++++++++++------- stackinator/etc/envvars.py | 3 ++- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 60fc838..d181df4 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -408,7 +408,10 @@ def generate(self, recipe): repo_path = recipe.mount / "repos" / "spack_repo" / "alps" recipe_repo_path = recipe.mount / "repos" / "spack_repo" / "recipe" package_repos = [ - {"name": pkg["name"], "path": (recipe.mount / "repos" / "spack_repo" / pkg["name"]).as_posix()} + { + "name": pkg["name"], + "path": (recipe.mount / "repos" / "spack_repo" / pkg["name"]).as_posix(), + } for pkg in spack_meta["packages"] ] f.write( @@ -443,7 +446,7 @@ def generate(self, recipe): name = pkg_meta["name"] src_path = clone_path / pkg_meta["repo_path"] dst_path = store_path / "repos" / "spack_repo" / name - self._logger.debug(f"copying repo \'{name}\' from {src_path} to {dst_path}") + self._logger.debug(f"copying repo '{name}' from {src_path} to {dst_path}") if dst_path.exists(): self._logger.debug(f"{dst_path} exists ... deleting") shutil.rmtree(dst_path) @@ -551,11 +554,25 @@ def generate(self, recipe): def _resolve_packages(self, packages): base = self.path / "repos" if isinstance(packages.get("repo"), str): - return [{"name": "builtin", "url": packages["repo"], "ref": packages.get("commit"), - "path": base / "builtin", "repo_path": "repos/spack_repo/builtin"}] - return [{"name": name, "url": val["repo"], "ref": val.get("commit"), - "path": base / name, "repo_path": val.get("path", f"repos/spack_repo/{name}")} - for name, val in packages.items()] + return [ + { + "name": "builtin", + "url": packages["repo"], + "ref": packages.get("commit"), + "path": base / "builtin", + "repo_path": "repos/spack_repo/builtin", + } + ] + return [ + { + "name": name, + "url": val["repo"], + "ref": val.get("commit"), + "path": base / name, + "repo_path": val.get("path", f"repos/spack_repo/{name}"), + } + for name, val in packages.items() + ] def _git_clone(self, name, repo, commit, path): if not (path / ".git").is_dir(): diff --git a/stackinator/etc/envvars.py b/stackinator/etc/envvars.py index d74c0ef..fb4c17e 100755 --- a/stackinator/etc/envvars.py +++ b/stackinator/etc/envvars.py @@ -694,7 +694,8 @@ def meta_impl(args): ) uenv_parser.add_argument( "--spack-package-repo", - help='configure spack package repository metadata. Format is "name,spack_url,git_ref,git_commit". Can be repeated.', + help="configure spack package repository metadata. " + 'Format is "name,spack_url,git_ref,git_commit". Can be repeated.', type=str, action="append", default=None, From 3cb3aff2cde4923459d2a4b79483f6677e6709a5 Mon Sep 17 00:00:00 2001 From: Mikael Simberg Date: Wed, 20 May 2026 09:47:41 +0200 Subject: [PATCH 12/12] Run test-envvars.sh with uv venv --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index d4d8ee4..3793da6 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -30,4 +30,4 @@ jobs: working-directory: unittests run: | sudo apt-get install -y jq - ./test-envvars.sh + uv run bash ./test-envvars.sh