From 687f2789f1cf8ee74d4ef76ba41112573cfc509c Mon Sep 17 00:00:00 2001 From: cvanelteren Date: Thu, 19 Mar 2026 15:28:01 +1000 Subject: [PATCH] Publish Zenodo releases via API --- .github/workflows/publish-pypi.yml | 32 +++ .zenodo.json | 39 --- docs/contributing.rst | 21 +- tools/release/publish_zenodo.py | 303 +++++++++++++++++++++++ ultraplot/tests/test_release_metadata.py | 47 +++- 5 files changed, 388 insertions(+), 54 deletions(-) delete mode 100644 .zenodo.json create mode 100644 tools/release/publish_zenodo.py diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index 80ff7fd0f..ef44f9994 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -129,3 +129,35 @@ jobs: uses: softprops/action-gh-release@v2 with: generate_release_notes: true + + publish-zenodo: + name: Publish Zenodo release + needs: publish-github-release + runs-on: ubuntu-latest + if: github.event_name == 'push' + permissions: + contents: read + env: + ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }} + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install release tooling + run: | + python -m pip install --upgrade pip PyYAML + shell: bash + + - name: Download artifacts + uses: actions/download-artifact@v8 + with: + name: dist-${{ github.sha }}-${{ github.run_id }}-${{ github.run_number }} + path: dist + + - name: Publish to Zenodo + run: | + python tools/release/publish_zenodo.py --dist-dir dist + shell: bash diff --git a/.zenodo.json b/.zenodo.json deleted file mode 100644 index fb302d4b0..000000000 --- a/.zenodo.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "title": "UltraPlot: A succinct wrapper for Matplotlib", - "upload_type": "software", - "description": "UltraPlot provides a compact and extensible API on top of Matplotlib, inspired by ProPlot. It simplifies the creation of scientific plots with consistent layout, colorbars, and shared axes.", - "creators": [ - { - "name": "van Elteren, Casper", - "orcid": "0000-0001-9862-8936", - "affiliation": "University of Amsterdam, Polder Center, Institute for Advanced Study Amsterdam" - }, - { - "name": "Becker, Matthew R.", - "orcid": "0000-0001-7774-2246", - "affiliation": "Argonne National Laboratory, Lemont, IL USA" - } - ], - "license": "MIT", - "keywords": [ - "matplotlib", - "scientific visualization", - "plotting", - "wrapper", - "python" - ], - "related_identifiers": [ - { - "relation": "isDerivedFrom", - "identifier": "https://github.com/lukelbd/proplot", - "scheme": "url" - }, - { - "relation": "isDerivedFrom", - "identifier": "https://matplotlib.org/", - "scheme": "url" - } - ], - "version": "2.1.3", - "publication_date": "2026-03-11" -} diff --git a/docs/contributing.rst b/docs/contributing.rst index 93828557d..8f0961416 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -256,12 +256,12 @@ be carried out as follows: #. Create a new branch ``release-vX.Y.Z`` with the version for the release. #. Make sure to update ``CHANGELOG.rst`` and that all new changes are reflected - in the documentation. Before tagging, also sync ``CITATION.cff`` and - ``.zenodo.json`` to the release version and date: + in the documentation. Before tagging, sync ``CITATION.cff`` to the release + version and date: .. code-block:: bash - git add CHANGELOG.rst CITATION.cff .zenodo.json + git add CHANGELOG.rst CITATION.cff git commit -m 'Prepare release metadata' #. Open a new pull request for this branch targeting ``main``. @@ -284,11 +284,16 @@ be carried out as follows: git push origin main --tags Pushing a ``vX.Y.Z`` tag triggers the release workflow, which publishes the - package and creates the corresponding GitHub release. Zenodo archives GitHub - releases, not bare git tags. + package, creates the corresponding GitHub release, and uploads the same + ``dist/`` artifacts to Zenodo through the Zenodo deposit API. #. After the workflow completes, confirm that the repository "Cite this repository" panel reflects ``CITATION.cff``, that the release is available - on TestPyPI and PyPI, and that Zenodo created a new release record. If - Zenodo does not create a new version, reconnect the repository in Zenodo - and re-run the GitHub release workflow. + on TestPyPI and PyPI, and that Zenodo created a new release record. + + The Zenodo release job uses ``CITATION.cff`` as the maintained metadata + source and requires a GitHub Actions secret named + ``ZENODO_ACCESS_TOKEN`` with the Zenodo scopes ``deposit:write`` and + ``deposit:actions``. To avoid duplicate Zenodo records, disable the + repository's Zenodo GitHub auto-archiving integration once the API-based + workflow is enabled. diff --git a/tools/release/publish_zenodo.py b/tools/release/publish_zenodo.py new file mode 100644 index 000000000..56dd80bb7 --- /dev/null +++ b/tools/release/publish_zenodo.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import mimetypes +import os +import sys +import tomllib +from pathlib import Path +from urllib import error, parse, request + +try: + import yaml +except ImportError as exc: # pragma: no cover - exercised in release workflow + raise SystemExit( + "PyYAML is required to publish Zenodo releases. Install it before " + "running tools/release/publish_zenodo.py." + ) from exc + + +DEFAULT_API_URL = "https://zenodo.org/api" +DOI_PREFIX = "10.5281/zenodo." + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Publish the current UltraPlot release artifacts to Zenodo." + ) + parser.add_argument( + "--dist-dir", + type=Path, + default=Path("dist"), + help="Directory containing the built release artifacts.", + ) + parser.add_argument( + "--citation", + type=Path, + default=Path("CITATION.cff"), + help="Path to the repository CITATION.cff file.", + ) + parser.add_argument( + "--pyproject", + type=Path, + default=Path("pyproject.toml"), + help="Path to the repository pyproject.toml file.", + ) + parser.add_argument( + "--api-url", + default=os.environ.get("ZENODO_API_URL", DEFAULT_API_URL), + help="Zenodo API base URL.", + ) + parser.add_argument( + "--access-token", + default=os.environ.get("ZENODO_ACCESS_TOKEN"), + help="Zenodo personal access token.", + ) + return parser.parse_args() + + +def load_citation(path: Path) -> dict: + with path.open("r", encoding="utf-8") as handle: + data = yaml.safe_load(handle) + if not isinstance(data, dict): + raise ValueError(f"{path} did not parse to a mapping") + return data + + +def load_pyproject(path: Path) -> dict: + with path.open("rb") as handle: + return tomllib.load(handle) + + +def author_to_creator(author: dict) -> dict: + family = author["family-names"].strip() + given = author["given-names"].strip() + creator = {"name": f"{family}, {given}"} + orcid = author.get("orcid") + if orcid: + creator["orcid"] = normalize_orcid(orcid) + return creator + + +def normalize_orcid(orcid: str) -> str: + return orcid.removeprefix("https://orcid.org/").rstrip("/") + + +def build_related_identifiers(citation: dict) -> list[dict]: + related = [] + repository = citation.get("repository-code", "").rstrip("/") + version = citation["version"] + if repository: + related.append( + { + "relation": "isSupplementTo", + "identifier": f"{repository}/tree/v{version}", + "scheme": "url", + "resource_type": "software", + } + ) + for reference in citation.get("references", []): + url = reference.get("url") + if not url: + continue + related.append( + { + "relation": "isDerivedFrom", + "identifier": url, + "scheme": "url", + } + ) + return related + + +def build_metadata(citation: dict, pyproject: dict) -> dict: + project = pyproject["project"] + creators = [author_to_creator(author) for author in citation["authors"]] + description = project["description"].strip() + repository = citation.get("repository-code") + if repository: + description = f"{description}\n\nSource code: {repository}" + metadata = { + "title": citation["title"], + "upload_type": "software", + "description": description, + "creators": creators, + "access_right": "open", + "license": citation.get("license"), + "keywords": citation.get("keywords", []), + "version": citation["version"], + "publication_date": citation["date-released"], + } + related = build_related_identifiers(citation) + if related: + metadata["related_identifiers"] = related + return metadata + + +def doi_record_id(doi: str) -> str: + value = doi.removeprefix("https://doi.org/").strip() + if not value.startswith(DOI_PREFIX): + raise ValueError( + f"Unsupported Zenodo DOI {doi!r}. Expected prefix {DOI_PREFIX!r}." + ) + return value.removeprefix(DOI_PREFIX) + + +def api_request( + method: str, + url: str, + *, + token: str | None = None, + json_data: dict | None = None, + data: bytes | None = None, + content_type: str | None = None, + expect_json: bool = True, +): + headers = {"Accept": "application/json"} + if token: + headers["Authorization"] = f"Bearer {token}" + body = data + if json_data is not None: + body = json.dumps(json_data).encode("utf-8") + headers["Content-Type"] = "application/json" + elif content_type: + headers["Content-Type"] = content_type + req = request.Request(url, data=body, headers=headers, method=method) + try: + with request.urlopen(req) as response: + payload = response.read() + except error.HTTPError as exc: + details = exc.read().decode("utf-8", errors="replace") + raise RuntimeError(f"{method} {url} failed with {exc.code}: {details}") from exc + if not expect_json: + return None + if not payload: + return None + return json.loads(payload) + + +def resolve_concept_recid(api_url: str, doi: str) -> str: + recid = doi_record_id(doi) + record = api_request("GET", f"{api_url}/records/{recid}") + return str(record.get("conceptrecid") or record.get("id") or recid) + + +def latest_record_id(api_url: str, conceptrecid: str) -> int: + query = parse.urlencode( + { + "q": f"conceptrecid:{conceptrecid}", + "all_versions": 1, + "sort": "mostrecent", + "size": 1, + } + ) + payload = api_request("GET", f"{api_url}/records?{query}") + hits = payload.get("hits", {}).get("hits", []) + if not hits: + raise RuntimeError( + f"Could not find any Zenodo records for conceptrecid {conceptrecid}." + ) + return int(hits[0]["id"]) + + +def create_new_version(api_url: str, token: str, record_id: int) -> dict: + response = api_request( + "POST", + f"{api_url}/deposit/depositions/{record_id}/actions/newversion", + token=token, + ) + latest_draft = response.get("links", {}).get("latest_draft") + if not latest_draft: + raise RuntimeError( + "Zenodo did not return links.latest_draft after requesting a new version." + ) + return api_request("GET", latest_draft, token=token) + + +def clear_draft_files(draft: dict, token: str) -> None: + files_url = draft.get("links", {}).get("files") + deposition_id = draft["id"] + if not files_url: + return + files = api_request("GET", files_url, token=token) or [] + for file_info in files: + file_id = file_info["id"] + api_request( + "DELETE", + f"{files_url}/{file_id}", + token=token, + expect_json=False, + ) + print(f"Deleted inherited Zenodo file {file_id} from draft {deposition_id}.") + + +def upload_dist_files(draft: dict, token: str, dist_dir: Path) -> None: + bucket_url = draft.get("links", {}).get("bucket") + if not bucket_url: + raise RuntimeError("Zenodo draft is missing the upload bucket URL.") + for path in sorted(dist_dir.iterdir()): + if not path.is_file(): + continue + content_type = mimetypes.guess_type(path.name)[0] or "application/octet-stream" + with path.open("rb") as handle: + api_request( + "PUT", + f"{bucket_url}/{parse.quote(path.name)}", + token=token, + data=handle.read(), + content_type=content_type, + ) + print(f"Uploaded {path.name} to Zenodo draft {draft['id']}.") + + +def update_metadata(draft: dict, token: str, metadata: dict) -> dict: + return api_request( + "PUT", + draft["links"]["self"], + token=token, + json_data={"metadata": metadata}, + ) + + +def publish_draft(draft: dict, token: str) -> dict: + return api_request("POST", draft["links"]["publish"], token=token) + + +def validate_inputs(dist_dir: Path, access_token: str | None) -> None: + if not access_token: + raise SystemExit( + "Missing Zenodo access token. Set ZENODO_ACCESS_TOKEN or pass " + "--access-token." + ) + if not dist_dir.is_dir(): + raise SystemExit(f"Distribution directory {dist_dir} does not exist.") + files = [path for path in dist_dir.iterdir() if path.is_file()] + if not files: + raise SystemExit(f"Distribution directory {dist_dir} does not contain files.") + + +def main() -> int: + args = parse_args() + validate_inputs(args.dist_dir, args.access_token) + citation = load_citation(args.citation) + pyproject = load_pyproject(args.pyproject) + metadata = build_metadata(citation, pyproject) + conceptrecid = resolve_concept_recid(args.api_url, citation["doi"]) + record_id = latest_record_id(args.api_url, conceptrecid) + draft = create_new_version(args.api_url, args.access_token, record_id) + clear_draft_files(draft, args.access_token) + upload_dist_files(draft, args.access_token, args.dist_dir) + draft = update_metadata(draft, args.access_token, metadata) + published = publish_draft(draft, args.access_token) + doi = published.get("doi") or published.get("metadata", {}).get("doi") + print( + f"Published Zenodo release record {published['id']} for " + f"version {metadata['version']} ({doi})." + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ultraplot/tests/test_release_metadata.py b/ultraplot/tests/test_release_metadata.py index 8e91a63dd..784fd14ed 100644 --- a/ultraplot/tests/test_release_metadata.py +++ b/ultraplot/tests/test_release_metadata.py @@ -1,17 +1,20 @@ from __future__ import annotations -import json +import importlib.util import re import subprocess +import tomllib from pathlib import Path import pytest +import yaml ROOT = Path(__file__).resolve().parents[2] CITATION_CFF = ROOT / "CITATION.cff" -ZENODO_JSON = ROOT / ".zenodo.json" README = ROOT / "README.rst" PUBLISH_WORKFLOW = ROOT / ".github" / "workflows" / "publish-pypi.yml" +PYPROJECT = ROOT / "pyproject.toml" +ZENODO_SCRIPT = ROOT / "tools" / "release" / "publish_zenodo.py" def _citation_scalar(key): @@ -57,6 +60,18 @@ def _latest_release_tag(): return tag.removeprefix("v"), date_result.stdout.strip() +def _load_publish_zenodo(): + """ + Import the Zenodo release helper directly from the repo checkout. + """ + spec = importlib.util.spec_from_file_location("publish_zenodo", ZENODO_SCRIPT) + if spec is None or spec.loader is None: + raise ImportError(f"Could not load publish_zenodo from {ZENODO_SCRIPT}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + def test_release_metadata_matches_latest_git_tag(): """ Citation metadata should track the latest tagged release. @@ -66,13 +81,28 @@ def test_release_metadata_matches_latest_git_tag(): assert _citation_scalar("date-released") == release_date -def test_zenodo_metadata_is_valid_and_synced(): +def test_zenodo_release_metadata_is_built_from_repository_sources(): """ - Zenodo metadata should parse as JSON and match the citation file. + Zenodo metadata should be derived from the maintained repository metadata. """ - metadata = json.loads(ZENODO_JSON.read_text(encoding="utf-8")) + publish_zenodo = _load_publish_zenodo() + citation = yaml.safe_load(CITATION_CFF.read_text(encoding="utf-8")) + with PYPROJECT.open("rb") as handle: + pyproject = tomllib.load(handle) + metadata = publish_zenodo.build_metadata(citation, pyproject) + assert metadata["title"] == citation["title"] + assert metadata["upload_type"] == "software" assert metadata["version"] == _citation_scalar("version") assert metadata["publication_date"] == _citation_scalar("date-released") + assert metadata["creators"][0]["name"] == "van Elteren, Casper" + assert metadata["creators"][0]["orcid"] == "0000-0001-9862-8936" + + +def test_zenodo_json_is_not_committed(): + """ + Zenodo metadata should no longer be duplicated in a separate committed file. + """ + assert not (ROOT / ".zenodo.json").exists() def test_readme_citation_section_uses_repository_metadata(): @@ -84,10 +114,13 @@ def test_readme_citation_section_uses_repository_metadata(): assert "@software{" not in text -def test_publish_workflow_creates_github_release_for_tags(): +def test_publish_workflow_creates_github_release_and_pushes_to_zenodo(): """ - Release tags should create a GitHub release so Zenodo can archive it. + Release tags should create a GitHub release and publish the same dist to Zenodo. """ text = PUBLISH_WORKFLOW.read_text(encoding="utf-8") assert 'tags: ["v*"]' in text assert "softprops/action-gh-release@v2" in text + assert "publish-zenodo:" in text + assert "ZENODO_ACCESS_TOKEN" in text + assert "tools/release/publish_zenodo.py --dist-dir dist" in text