diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index cb2fd33..3793da6 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -26,3 +26,8 @@ jobs: run: uv sync --group dev - name: Run Unit Tests run: uv run pytest + - name: Run Envvars Test + working-directory: unittests + run: | + sudo apt-get install -y jq + uv run bash ./test-envvars.sh diff --git a/stackinator/builder.py b/stackinator/builder.py index ae4997a..d181df4 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -191,26 +191,15 @@ def generate(self, recipe): spack_git_commit_result = self._git_clone("spack", spack_repo, spack_commit, spack_path) - # Clone the spack-packages repository and check out commit if one was given - spack_packages = spack["packages"] - spack_packages_repo = spack_packages["repo"] - spack_packages_commit = spack_packages["commit"] - spack_packages_path = self.path / "spack-packages" - - spack_packages_git_commit_result = self._git_clone( - "spack-packages", - spack_packages_repo, - spack_packages_commit, - spack_packages_path, - ) + packages_meta = self._resolve_packages(spack["packages"]) + for pkg in packages_meta: + pkg["commit"] = self._git_clone(pkg["name"], pkg["url"], pkg["ref"], pkg["path"]) spack_meta = { "url": spack_repo, "ref": spack_commit, "commit": spack_git_commit_result, - "packages_url": spack_packages_repo, - "packages_ref": spack_packages_commit, - "packages_commit": spack_packages_git_commit_result, + "packages": packages_meta, } # load the jinja templating environment @@ -331,16 +320,12 @@ def generate(self, recipe): # 2. cluster-config/repos.yaml # - if the repos.yaml file exists it will contain a list of relative paths # to search for package - # 1. builtin repo + # 1. package repos from config.yaml in the order specified (typically + # only spack-packages builtin repo) - # Build a list of repos with packages to install. + # Build a list of repos with packages to install from system config and recipe. repos = [] - # check for a repo in the recipe - if recipe.spack_repo is not None: - self._logger.debug(f"adding recipe spack package repo: {recipe.spack_repo}") - repos.append(recipe.spack_repo) - # look for repos.yaml file in the system configuration repo_yaml = recipe.system_config_path / "repos.yaml" if repo_yaml.exists() and repo_yaml.is_file(): @@ -361,7 +346,7 @@ def generate(self, recipe): self._logger.error(f"{repo_path} from {repo_yaml} is not a spack package repository") raise RuntimeError("invalid system-provided package repository") - self._logger.debug(f"full list of spack package repo: {repos}") + self._logger.debug(f"full list of system spack package repos: {repos}") # Delete the store/repo path, if it already exists. # Do this so that incremental builds (though not officially supported) won't break if a repo is updated. @@ -378,7 +363,7 @@ def generate(self, recipe): self._logger.debug(f"created the repo packages path {pkg_dst}") # create the repository step 2: create the repo.yaml file that - # configures alps and builtin repos + # configures the alps repo with (repo_dst / "repo.yaml").open("w") as f: f.write( """\ @@ -388,43 +373,84 @@ def generate(self, recipe): """ ) + # If the recipe provides a package repo, install it as a separate + # "recipe" repo in the store with highest precedence. + has_recipe_repo = recipe.spack_repo is not None + if has_recipe_repo: + recipe_dst = repos_path / "recipe" + self._logger.debug(f"creating the recipe spack repo in {recipe_dst}") + if recipe_dst.exists(): + self._logger.debug(f"{recipe_dst} exists ... deleting") + shutil.rmtree(recipe_dst) + + recipe_pkg_dst = recipe_dst / "packages" + recipe_pkg_dst.mkdir(mode=0o755, parents=True) + + with (recipe_dst / "repo.yaml").open("w") as f: + f.write( + """\ +repo: + namespace: recipe + api: v2.0 +""" + ) + + packages_path = recipe.spack_repo / "packages" + for pkg_path in packages_path.iterdir(): + dst = recipe_pkg_dst / pkg_path.name + if pkg_path.is_dir(): + self._logger.debug(f" installing recipe package {pkg_path} to {recipe_pkg_dst}") + install(pkg_path, dst) + # create the repository step 2: create the repos.yaml file in build_path/config repos_yaml_template = jinja_env.get_template("repos.yaml") with (config_path / "repos.yaml").open("w") as f: repo_path = recipe.mount / "repos" / "spack_repo" / "alps" - builtin_repo_path = recipe.mount / "repos" / "spack_repo" / "builtin" + recipe_repo_path = recipe.mount / "repos" / "spack_repo" / "recipe" + package_repos = [ + { + "name": pkg["name"], + "path": (recipe.mount / "repos" / "spack_repo" / pkg["name"]).as_posix(), + } + for pkg in spack_meta["packages"] + ] f.write( repos_yaml_template.render( repo_path=repo_path.as_posix(), - builtin_repo_path=builtin_repo_path.as_posix(), + package_repos=package_repos, + recipe_repo_path=recipe_repo_path.as_posix(), + has_recipe_repo=has_recipe_repo, verbose=False, ) ) f.write("\n") - # Iterate over the source repositories copying their contents to the consolidated repo in the uenv. - # Do overwrite packages that have been copied from an earlier source repo, enforcing a descending - # order of precidence. - if len(repos) > 0: - for repo_src in repos: - self._logger.debug(f"installing repo {repo_src}") - packages_path = repo_src / "packages" - for pkg_path in packages_path.iterdir(): - dst = pkg_dst / pkg_path.name - if pkg_path.is_dir() and not dst.exists(): - self._logger.debug(f" installing package {pkg_path} to {pkg_dst}") - install(pkg_path, dst) - elif dst.exists(): - self._logger.debug(f" NOT installing package {pkg_path}") - - # Copy the builtin repo to store, delete if it already exists. - spack_packages_builtin_path = spack_packages_path / "repos" / "spack_repo" / "builtin" - spack_packages_store_path = store_path / "repos" / "spack_repo" / "builtin" - self._logger.debug(f"copying builtin repo from {spack_packages_builtin_path} to {spack_packages_store_path}") - if spack_packages_store_path.exists(): - self._logger.debug(f"{spack_packages_store_path} exists ... deleting") - shutil.rmtree(spack_packages_store_path) - install(spack_packages_builtin_path, spack_packages_store_path) + # Iterate over the alps and recipe repositories copying their contents + # to the final repo locations. Because of the order of repos in the + # repos.yaml config file, recipe packages have precedence. + for repo_src in repos: + self._logger.debug(f"installing repo {repo_src}") + packages_path = repo_src / "packages" + for pkg_path in packages_path.iterdir(): + dst = pkg_dst / pkg_path.name + if pkg_path.is_dir() and not dst.exists(): + self._logger.debug(f" installing package {pkg_path} to {pkg_dst}") + install(pkg_path, dst) + elif dst.exists(): + self._logger.debug(f" NOT installing package {pkg_path}") + + # Copy all package repos defined in config.yaml to their final repo + # locations. + for idx, pkg_meta in enumerate(spack_meta["packages"]): + clone_path = pkg_meta["path"] + name = pkg_meta["name"] + src_path = clone_path / pkg_meta["repo_path"] + dst_path = store_path / "repos" / "spack_repo" / name + self._logger.debug(f"copying repo '{name}' from {src_path} to {dst_path}") + if dst_path.exists(): + self._logger.debug(f"{dst_path} exists ... deleting") + shutil.rmtree(dst_path) + install(src_path, dst_path) # Generate the makefile and spack.yaml files that describe the compilers compiler_files = recipe.compiler_files @@ -525,6 +551,29 @@ def generate(self, recipe): ) f.write("\n") + def _resolve_packages(self, packages): + base = self.path / "repos" + if isinstance(packages.get("repo"), str): + return [ + { + "name": "builtin", + "url": packages["repo"], + "ref": packages.get("commit"), + "path": base / "builtin", + "repo_path": "repos/spack_repo/builtin", + } + ] + return [ + { + "name": name, + "url": val["repo"], + "ref": val.get("commit"), + "path": base / name, + "repo_path": val.get("path", f"repos/spack_repo/{name}"), + } + for name, val in packages.items() + ] + def _git_clone(self, name, repo, commit, path): if not (path / ".git").is_dir(): self._logger.info(f"{name}: clone repository {repo} to {path}") diff --git a/stackinator/etc/envvars.py b/stackinator/etc/envvars.py index a20fabc..fb4c17e 100755 --- a/stackinator/etc/envvars.py +++ b/stackinator/etc/envvars.py @@ -621,12 +621,27 @@ def meta_impl(args): if args.spack is not None: spack_url, spack_ref, spack_commit = args.spack.split(",") - spack_packages_url = None - spack_packages_ref = None - spack_packages_commit = None - if args.spack_packages is not None: - spack_packages_url, spack_packages_ref, spack_packages_commit = args.spack_packages.split(",") spack_path = f"{args.mount}/config".replace("//", "/") + scalar_vars = { + "UENV_SPACK_CONFIG_PATH": spack_path, + "UENV_SPACK_URL": spack_url, + "UENV_SPACK_REF": spack_ref, + "UENV_SPACK_COMMIT": spack_commit, + } + if args.spack_package_repo: + repo_names = [] + for entry in args.spack_package_repo: + name, url, ref, commit = entry.split(",") + repo_names.append(name) + name_upper = name.upper().replace("-", "_") + scalar_vars[f"UENV_PACKAGE_REPO_{name_upper}_URL"] = url + scalar_vars[f"UENV_PACKAGE_REPO_{name_upper}_REF"] = ref + scalar_vars[f"UENV_PACKAGE_REPO_{name_upper}_COMMIT"] = commit + if name == "builtin": + scalar_vars["UENV_SPACK_PACKAGES_URL"] = url + scalar_vars["UENV_SPACK_PACKAGES_REF"] = ref + scalar_vars["UENV_SPACK_PACKAGES_COMMIT"] = commit + scalar_vars["UENV_PACKAGE_REPOS"] = ",".join(repo_names) meta["views"]["spack"] = { "activate": "/dev/null", "description": "configure spack upstream", @@ -636,15 +651,7 @@ def meta_impl(args): "type": "augment", "values": { "list": {}, - "scalar": { - "UENV_SPACK_CONFIG_PATH": spack_path, - "UENV_SPACK_URL": spack_url, - "UENV_SPACK_REF": spack_ref, - "UENV_SPACK_COMMIT": spack_commit, - "UENV_SPACK_PACKAGES_URL": spack_packages_url, - "UENV_SPACK_PACKAGES_REF": spack_packages_ref, - "UENV_SPACK_PACKAGES_COMMIT": spack_packages_commit, - }, + "scalar": scalar_vars, }, }, } @@ -686,9 +693,11 @@ def meta_impl(args): default=None, ) uenv_parser.add_argument( - "--spack-packages", - help='configure spack-packages repository metadata. Format is "spack_url,git_ref,git_commit"', + "--spack-package-repo", + help="configure spack package repository metadata. " + 'Format is "name,spack_url,git_ref,git_commit". Can be repeated.', type=str, + action="append", default=None, ) diff --git a/stackinator/schema/config.json b/stackinator/schema/config.json index d9de503..3bba630 100644 --- a/stackinator/schema/config.json +++ b/stackinator/schema/config.json @@ -28,21 +28,47 @@ "default": null }, "packages": { - "type": "object", - "additionalProperties": false, - "required": ["repo"], - "properties" : { - "repo": { - "type": "string" + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["repo"], + "properties": { + "repo": { + "type": "string" + }, + "commit": { + "oneOf": [ + {"type" : "string"}, + {"type" : "null"} + ] + } + } }, - "commit": { - "oneOf": [ - {"type" : "string"}, - {"type" : "null"} - ], - "default": null + { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "object", + "additionalProperties": false, + "required": ["repo"], + "properties": { + "repo": { + "type": "string" + }, + "commit": { + "oneOf": [ + {"type" : "string"}, + {"type" : "null"} + ] + }, + "path": { + "type": "string" + } + } + } } - } + ] } } }, diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 10a0ea5..41b9d5b 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -59,7 +59,7 @@ modules-done: environments generate-config env-meta: generate-config environments{% if modules %} modules-done{% endif %} - $(SANDBOX) $(BUILD_ROOT)/envvars.py uenv {% if modules %}--modules{% endif %} --spack='{{ spack_meta.url }},{{ spack_meta.ref }},{{ spack_meta.commit }}' --spack-packages='{{ spack_meta.packages_url }},{{ spack_meta.packages_ref }},{{ spack_meta.packages_commit }}' $(STORE) + $(SANDBOX) $(BUILD_ROOT)/envvars.py uenv {% if modules %}--modules{% endif %} --spack='{{ spack_meta.url }},{{ spack_meta.ref }},{{ spack_meta.commit }}'{% for pkg in spack_meta.packages %} --spack-package-repo='{{ pkg.name }},{{ pkg.url }},{{ pkg.ref }},{{ pkg.commit }}'{% endfor %} $(STORE) touch env-meta post-install: env-meta diff --git a/stackinator/templates/repos.yaml b/stackinator/templates/repos.yaml index 3cf34f2..18f2e31 100644 --- a/stackinator/templates/repos.yaml +++ b/stackinator/templates/repos.yaml @@ -1,3 +1,8 @@ repos: +{% if has_recipe_repo %} + recipe: {{ recipe_repo_path }} +{% endif %} alps: {{ repo_path }} - builtin: {{ builtin_repo_path }} +{% for repo in package_repos %} + {{ repo.name }}: {{ repo.path }} +{% endfor %} diff --git a/unittests/data/arbor-uenv/meta/env.json.in b/unittests/data/arbor-uenv/meta/env.json.in index 25086e8..fa9e9b4 100644 --- a/unittests/data/arbor-uenv/meta/env.json.in +++ b/unittests/data/arbor-uenv/meta/env.json.in @@ -9,11 +9,13 @@ "arbor": { "activate": "@@mount@@/env/arbor/activate.sh", "description": "", + "recipe_variables": {"scalar": {}, "list": {}}, "root": "@@mount@@/env/arbor" }, "develop": { "activate": "@@mount@@/env/develop/activate.sh", "description": "", + "recipe_variables": {"scalar": {}, "list": {}}, "root": "@@mount@@/env/develop" } } diff --git a/unittests/recipes/with-multi-repos/compilers.yaml b/unittests/recipes/with-multi-repos/compilers.yaml new file mode 100644 index 0000000..bce7184 --- /dev/null +++ b/unittests/recipes/with-multi-repos/compilers.yaml @@ -0,0 +1,2 @@ +gcc: + version: "11" diff --git a/unittests/recipes/with-multi-repos/config.yaml b/unittests/recipes/with-multi-repos/config.yaml new file mode 100644 index 0000000..a84f6f7 --- /dev/null +++ b/unittests/recipes/with-multi-repos/config.yaml @@ -0,0 +1,13 @@ +name: with-multi-repos +store: '/user-environment' +spack: + repo: https://github.com/spack/spack.git + commit: v21.0 + packages: + my-packages: + repo: https://github.com/example/spack-packages.git + commit: v1.0 + path: custom/path/to/packages + other-packages: + repo: https://github.com/example/other-packages.git +version: 2 diff --git a/unittests/recipes/with-multi-repos/environments.yaml b/unittests/recipes/with-multi-repos/environments.yaml new file mode 100644 index 0000000..33735ee --- /dev/null +++ b/unittests/recipes/with-multi-repos/environments.yaml @@ -0,0 +1,13 @@ +gcc-env: + compiler: [gcc] + unify: true + specs: + - osu-micro-benchmarks@5.9 + - hdf5 +mpi + network: + mpi: cray-mpich +tools: + compiler: [gcc] + unify: true + specs: + - cmake diff --git a/unittests/test-envvars.sh b/unittests/test-envvars.sh index 97cba66..fe66c59 100755 --- a/unittests/test-envvars.sh +++ b/unittests/test-envvars.sh @@ -36,7 +36,7 @@ find $scratch_path -name env.json echo "===== running final meta data stage ${mount_path}" -../stackinator/etc/envvars.py uenv ${mount_path}/ --modules --spack="https://github.com/spack/spack.git,releases/v0.20" --spack-packages="https://github.com/spack/spack.git,develop" +../stackinator/etc/envvars.py uenv ${mount_path}/ --modules --spack="https://github.com/spack/spack.git,releases/v0.20,abc123" --spack-package-repo="builtin,https://github.com/spack/spack-packages.git,develop,abc123" echo echo "===== develop" @@ -58,3 +58,38 @@ echo "===== modules view" echo cat ${meta_path} | jq .views.modules +echo +echo "===== test UENV_PACKAGE_REPOS with multiple package repos" +rm -rf ${mount_path} +mkdir -p ${scratch_path} +cp -R ${input_path} ${mount_path} +if [[ "$OSTYPE" == "linux-gnu"* ]]; then + sed -i "s|@@mount@@|${mount_path}|g" ${meta_in_path} +else + sed -i '' "s|@@mount@@|${mount_path}|g" ${meta_in_path} +fi + +../stackinator/etc/envvars.py view ${mount_path}/env/arbor /dev/shm/bcumming/arbor +../stackinator/etc/envvars.py view --prefix_paths="LD_LIBRARY_PATH=lib:lib64" ${mount_path}/env/develop /dev/shm/bcumming/arbor + +../stackinator/etc/envvars.py uenv ${mount_path}/ \ + --spack="https://github.com/spack/spack.git,releases/v0.21,abc123def" \ + --spack-package-repo="my-packages,https://github.com/example/spack-packages.git,v1.0,abc123" \ + --spack-package-repo="other-packages,https://github.com/example/other-packages.git,main,def456" + +UENV_PACKAGE_REPOS=$(cat ${meta_path} | jq -r '.views.spack.env.values.scalar.UENV_PACKAGE_REPOS') +echo "UENV_PACKAGE_REPOS=$UENV_PACKAGE_REPOS" +if [[ "$UENV_PACKAGE_REPOS" != "my-packages,other-packages" ]]; then + echo "FAIL: expected my-packages,other-packages, got '${UENV_PACKAGE_REPOS}'" + exit 1 +fi + +UENV_PACKAGE_REPO_MY_PACKAGES_URL=$(cat ${meta_path} | jq -r '.views.spack.env.values.scalar.UENV_PACKAGE_REPO_MY_PACKAGES_URL') +echo "UENV_PACKAGE_REPO_MY_PACKAGES_URL=$UENV_PACKAGE_REPO_MY_PACKAGES_URL" +if [[ "$UENV_PACKAGE_REPO_MY_PACKAGES_URL" != "https://github.com/example/spack-packages.git" ]]; then + echo "FAIL: expected https://github.com/example/spack-packages.git, got '${UENV_PACKAGE_REPO_MY_PACKAGES_URL}'" + exit 1 +fi + +echo "PASSED" + diff --git a/unittests/test_schema.py b/unittests/test_schema.py index 607f80e..88a63cc 100644 --- a/unittests/test_schema.py +++ b/unittests/test_schema.py @@ -20,7 +20,7 @@ def yaml_path(test_path): return test_path / "yaml" -@pytest.fixture(params=["host-recipe", "base-nvgpu", "cache", "with-repo"]) +@pytest.fixture(params=["host-recipe", "base-nvgpu", "cache", "with-repo", "with-multi-repos"]) def recipe(request): return request.param @@ -37,7 +37,7 @@ def test_config_yaml(yaml_path): schema.ConfigValidator.validate(raw) assert raw["store"] == "/user-environment" assert raw["spack"]["commit"] is None - assert raw["spack"]["packages"]["commit"] is None + assert raw["spack"]["packages"].get("commit") is None assert raw["description"] is None # no spack:commit @@ -75,7 +75,7 @@ def test_config_yaml(yaml_path): ) schema.ConfigValidator.validate(raw) assert raw["spack"]["commit"] == "develop" - assert raw["spack"]["packages"]["commit"] is None + assert raw["spack"]["packages"].get("commit") is None assert raw["description"] is None # full config @@ -98,6 +98,97 @@ def test_config_yaml(yaml_path): raw = yaml.load(config, Loader=yaml.Loader) schema.ConfigValidator.validate(raw) + # map format: single entry + config = dedent(""" + version: 2 + name: map-single + spack: + repo: https://github.com/spack/spack.git + packages: + my-packages: + repo: https://github.com/example/spack-packages.git + """) + raw = yaml.load(config, Loader=yaml.Loader) + schema.ConfigValidator.validate(raw) + assert "my-packages" in raw["spack"]["packages"] + assert raw["spack"]["packages"]["my-packages"]["repo"] == "https://github.com/example/spack-packages.git" + assert raw["spack"]["packages"]["my-packages"].get("commit") is None + + # map format: multiple entries with commits + config = dedent(""" + version: 2 + name: map-multi + spack: + repo: https://github.com/spack/spack.git + packages: + my-packages: + repo: https://github.com/example/spack-packages.git + commit: v1.0 + other-packages: + repo: https://github.com/example/other-packages.git + commit: v2.0 + """) + raw = yaml.load(config, Loader=yaml.Loader) + schema.ConfigValidator.validate(raw) + assert raw["spack"]["packages"]["my-packages"]["commit"] == "v1.0" + assert raw["spack"]["packages"]["other-packages"]["commit"] == "v2.0" + + # map format: empty map should fail + with pytest.raises(Exception): + config = dedent(""" + version: 2 + name: map-empty + spack: + repo: https://github.com/spack/spack.git + packages: {} + """) + raw = yaml.load(config, Loader=yaml.Loader) + schema.ConfigValidator.validate(raw) + + # map format: entry missing repo should fail + with pytest.raises(Exception): + config = dedent(""" + version: 2 + name: map-no-repo + spack: + repo: https://github.com/spack/spack.git + packages: + my-packages: + commit: v1.0 + """) + raw = yaml.load(config, Loader=yaml.Loader) + schema.ConfigValidator.validate(raw) + + # map format: custom path + config = dedent(""" + version: 2 + name: map-custom-path + spack: + repo: https://github.com/spack/spack.git + packages: + my-packages: + repo: https://github.com/example/spack-packages.git + path: custom/repo/location + """) + raw = yaml.load(config, Loader=yaml.Loader) + schema.ConfigValidator.validate(raw) + assert raw["spack"]["packages"]["my-packages"]["path"] == "custom/repo/location" + + # map format: no path (default behavior) + config = dedent(""" + version: 2 + name: map-no-path + spack: + repo: https://github.com/spack/spack.git + packages: + my-packages: + repo: https://github.com/example/spack-packages.git + commit: v2.0 + """) + raw = yaml.load(config, Loader=yaml.Loader) + schema.ConfigValidator.validate(raw) + assert "path" not in raw["spack"]["packages"]["my-packages"] + def test_recipe_config_yaml(recipe_path): # validate the config.yaml in the test recipes