From af551a4e4187bbbd96428bef4c72cc6a43865145 Mon Sep 17 00:00:00 2001 From: Keigh Rim Date: Tue, 24 Mar 2026 17:51:54 -0400 Subject: [PATCH 1/3] docs pipeline standardization and minor doc fixes --- .github/workflows/publish.yml | 3 -- build-tools/docs.py | 7 ++++ clams/appmetadata/__init__.py | 19 +++++++++ documentation/appmetadata.rst | 16 +++---- documentation/cli.rst | 9 ++-- documentation/conf.py | 76 ++++++++++++++++++++-------------- documentation/index.rst | 7 ++-- documentation/introduction.rst | 2 +- 8 files changed, 88 insertions(+), 51 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 80d0bd5..4862332 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -48,7 +48,4 @@ jobs: source_repo: clamsproject/clams-python source_ref: ${{ needs.check-pypi.outputs.version }} project_name: clams-python - build_command: 'echo "${{ needs.check-pypi.outputs.version }}" > VERSION && python3 build-tools/docs.py --output-dir docs' - docs_output_dir: 'docs' - python_version: '3.11' secrets: inherit diff --git a/build-tools/docs.py b/build-tools/docs.py index 3ef8cce..82927dc 100644 --- a/build-tools/docs.py +++ b/build-tools/docs.py @@ -76,6 +76,13 @@ def main(): parser = argparse.ArgumentParser( description="Build documentation for the clams-python project." ) + parser.add_argument( + '--build-ver', + metavar='', + default=None, + help='Accepted for CLI compatibility with other SDK repos. ' + 'Ignored by this script (clams-python uses ' + 'unversioned documentation).') parser.add_argument( '--output-dir', type=Path, default=None, help='Output directory for built docs (default: docs-test)') diff --git a/clams/appmetadata/__init__.py b/clams/appmetadata/__init__.py index 74ff712..6a2bc92 100644 --- a/clams/appmetadata/__init__.py +++ b/clams/appmetadata/__init__.py @@ -438,6 +438,18 @@ def add_input(self, at_type: Union[str, vocabulary.ThingTypesBase], required: bo return new def add_input_oneof(self, *inputs: Union[str, Input, vocabulary.ThingTypesBase]): + """ + Helper method to add a ``oneOf`` (disjunctive) group to + the ``input`` list. When a single type is given, it is + added as a regular (conjunctive) input. When multiple + types are given, they are wrapped in a nested list to + indicate that exactly one of them is required. + + :param inputs: one or more input types (as URI strings, + :class:`Input` objects, or vocabulary types) + :raises ValueError: if any input in a ``oneOf`` group is + optional, or if a duplicate input is detected + """ newinputs = [] if len(inputs) == 1: if isinstance(inputs[0], Input): @@ -520,6 +532,13 @@ def add_more(self, key: str, value: str): raise ValueError("Key and value should not be empty!") def jsonify(self, pretty=False): + """ + Serialize the app metadata to a JSON string. + + :param pretty: if True, indent the output with 2 spaces + :returns: JSON string of the metadata + :rtype: str + """ return self.model_dump_json(exclude_defaults=True, by_alias=True, indent=2 if pretty else None) diff --git a/documentation/appmetadata.rst b/documentation/appmetadata.rst index 6b008bc..d0ba20a 100644 --- a/documentation/appmetadata.rst +++ b/documentation/appmetadata.rst @@ -13,7 +13,7 @@ Format A CLAMS App Metadata should be able to be serialized into a JSON string. -Input/Output type specification +Input/Output Type Specification =============================== Essentially, all CLAMS apps are designed to take one MMIF file as input and produce another MMIF file as output. In this @@ -29,7 +29,7 @@ how that information should be formatted in terms of the App Metadata syntax, co additional information about submission. Visit the `CLAMS app directory `_ to see how the app metadata is rendered. -Annotation types in MMIF +Annotation Types in MMIF ------------------------ As described in the `MMIF documentation `_, MMIF files can contain annotations of various types. @@ -56,8 +56,8 @@ needs to add additional information to the type definition, they can do so by ad definition in action. In such a case, the app developer is expected to provide the explanation of the extended type in the app metadata. See below for the syntax of I/O specification in the app metadata. -Syntax for I/O specification in App Metadata --------------------------------------------- +Syntax for I/O Specification in App Metadata +--------------------------------------------- In the App Metadata, the input and output types are specified as lists of objects. Each object in the list should have the following fields: @@ -69,7 +69,7 @@ the following fields: defaults to ``true``. Not applicable for output types. -Simple case - using types as defined in the vocabularies +Simple Case - Using Types as Defined in the Vocabularies ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In the simplest case, where a developer merely re-uses an annotation type definition and pre-defined properties, an @@ -195,14 +195,14 @@ Note that in the actual output MMIF, more properties can be stored in the ``Time specification in the app metadata is a subset of the properties to be produced that are useful for type checking in the downstream apps, as well as for human readers to understand the output. -Extended case - adding custom properties to the type definition +Extended Case - Adding Custom Properties to the Type Definition ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ When the type definition is extended on the fly, developers are expected to provide the extended specification in the form of key-value pairs in the ``properties`` field. The grammar of the JSON object does not change, but developers are expected to provide a verbose description of the type extension in the ``description`` field. -Runtime parameter specification +Runtime Parameter Specification =============================== CLAMS apps designed to be run as HTTP servers, preferably as `stateless `_. @@ -219,7 +219,7 @@ can be specified as ``multivalued=True`` to accept multiple values as a list. Fo parameter value parsing works, please refer to the App Metadata json scheme (in the `below <#clams-app-runtime-parameter>`_ section). -Syntax for parameter specification in App Metadata +Syntax for Parameter Specification in App Metadata -------------------------------------------------- Metadata Schema diff --git a/documentation/cli.rst b/documentation/cli.rst index 6e189bd..c15f535 100644 --- a/documentation/cli.rst +++ b/documentation/cli.rst @@ -3,13 +3,12 @@ ``clams`` shell command ======================= -``clams-python`` comes with a command line interface (CLI) that allows you to +``clams-python`` comes with a command line interface (CLI) for creating a new CLAMS app from a template (``develop`` subcommand). -#. create a new CLAMS app from a template -#. create a new MMIF file with selected source documents and an empty view +The CLI is installed as the ``clams`` shell command. For backward compatibility, it also exposes all ``mmif`` subcommands (``source``, ``rewind``, ``describe``, ``summarize``). See the `mmif-python CLI documentation `_ for details on those commands. -The CLI is installed as ``clams`` shell command. To see the available commands, run +To see the available commands, run -.. code-block:: bash +.. code-block:: bash clams --help diff --git a/documentation/conf.py b/documentation/conf.py index e4c9989..3ad6d11 100644 --- a/documentation/conf.py +++ b/documentation/conf.py @@ -11,11 +11,14 @@ # documentation root, use os.path.abspath to make it absolute, like shown here. import datetime -from pathlib import Path -import shutil -import sys import inspect +import json import os +import re +import shutil +import subprocess +import sys +from pathlib import Path import mmif @@ -134,40 +137,50 @@ def linkcode_resolve(domain, info): def generate_whatsnew_rst(app): - import subprocess + """ + Generate whatsnew.md by fetching the latest release PR body + from GitHub via ``gh pr list``. + + Falls back gracefully if ``gh`` is unavailable (local builds). + """ output_path = (proj_root_dir / 'documentation' / 'whatsnew.md') - content = None + repo = f'clamsproject/{project}' + try: - jq_expr = ( - 'sort_by(.mergedAt)' - ' | if length > 0 then .[-1].body' - ' else "" end' - ) result = subprocess.run( ['gh', 'pr', 'list', - '-R', f'clamsproject/{project}', - '-s', 'merged', - '--search', - f'releasing {version} in:title', - '--json', 'body,mergedAt', - '--jq', jq_expr], - capture_output=True, text=True, timeout=10 + '-s', 'merged', '-B', 'main', + '-L', '100', + '--json', 'title,body', + '--repo', repo], + capture_output=True, text=True, timeout=15, ) - if result.returncode == 0: - content = result.stdout.strip() or None - except (FileNotFoundError, subprocess.TimeoutExpired): - pass + if result.returncode != 0: + raise RuntimeError(result.stderr) + + prs = json.loads(result.stdout) + pr = next( + (p for p in prs + if p['title'].startswith('releasing ')), + None, + ) + if pr is None: + raise RuntimeError("No release PR found") + title = pr['title'] + body = pr.get('body', '') - with open(output_path, 'w') as f: - if content: - f.write( - f"## What's New in {version}\n\n" - f"(Full changelog available in the " - f"[CHANGELOG.md]({blob_base_url}" - f"/main/CHANGELOG.md))\n" - ) - f.write(content) + with open(output_path, 'w') as f: + f.write(f"## {title}\n\n") + f.write(f"(Full changelog: " + f"[CHANGELOG.md]" + f"({blob_base_url}/main/CHANGELOG.md))\n\n") + if body: + f.write(body) + + except Exception as e: + with open(output_path, 'w') as f: + f.write("") def generate_jsonschema(app): @@ -186,6 +199,9 @@ def update_target_spec(app): target_vers_csv = Path(__file__).parent / 'target-versions.csv' with open(proj_root_dir / "VERSION", 'r') as version_f: version = version_f.read().strip() + # Skip dev/dummy versions to avoid dirtying the git-tracked CSV + if 'dev' in version or not re.match(r'^\d+\.\d+\.\d+$', version): + return mmifver = mmif.__version__ specver = mmif.__specver__ with open(target_vers_csv) as in_f, open(f'{target_vers_csv}.new', 'w') as out_f: diff --git a/documentation/index.rst b/documentation/index.rst index 109647d..a03c587 100644 --- a/documentation/index.rst +++ b/documentation/index.rst @@ -1,5 +1,5 @@ -Welcome to CLAMS Python SDK documentation! -========================================== +CLAMS Python SDK +================ .. mdinclude:: ../README.md @@ -28,9 +28,8 @@ Welcome to CLAMS Python SDK documentation! modules -Indices and tables +Indices and Tables ================== * :ref:`genindex` * :ref:`modindex` -* :ref:`search` diff --git a/documentation/introduction.rst b/documentation/introduction.rst index fe3c9f6..ce907e8 100644 --- a/documentation/introduction.rst +++ b/documentation/introduction.rst @@ -1,6 +1,6 @@ .. _introduction: -Getting started +Getting Started =============== Overview From 80a2860f7b3a594fc7f6de39005fbfe9fb8f4069 Mon Sep 17 00:00:00 2001 From: Keigh Rim Date: Wed, 25 Mar 2026 17:03:14 -0400 Subject: [PATCH 2/3] migrated from setup.py, now build process is based on pyproject --- .github/workflows/publish.yml | 2 +- .gitignore | 4 +- CONTRIBUTING.md | 91 ++++++++++++++ MANIFEST.in | 3 - Makefile | 102 --------------- build-tools/build.py | 54 ++++++++ build-tools/clean.py | 70 +++++++++++ build-tools/docs.py | 93 +++++++------- build-tools/publish.py | 202 ++++++++++++++++++++++++++++++ build-tools/requirements.docs.txt | 4 - build-tools/test.py | 72 +++++++++++ clams/ver/__init__.py | 2 + documentation/conf.py | 10 +- pyproject.toml | 52 ++++++++ requirements.dev | 12 -- requirements.txt | 8 -- setup.py | 58 --------- 17 files changed, 598 insertions(+), 241 deletions(-) create mode 100644 CONTRIBUTING.md delete mode 100644 MANIFEST.in delete mode 100644 Makefile create mode 100644 build-tools/build.py create mode 100644 build-tools/clean.py create mode 100644 build-tools/publish.py delete mode 100644 build-tools/requirements.docs.txt create mode 100644 build-tools/test.py create mode 100644 clams/ver/__init__.py create mode 100644 pyproject.toml delete mode 100644 requirements.dev delete mode 100644 requirements.txt delete mode 100644 setup.py diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4862332..28d21a4 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -36,7 +36,7 @@ jobs: name: "📦 Build and upload to PyPI" needs: check-pypi if: needs.check-pypi.outputs.exists == 'false' - uses: clamsproject/.github/.github/workflows/sdk-publish.yml@main + uses: clamsproject/.github/.github/workflows/sdk-publish-pyproj.yml@main secrets: inherit publish-docs: diff --git a/.gitignore b/.gitignore index 2582f0a..2618793 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,6 @@ # build time temp files VERSION* -clams/ver # linux .*.sw? @@ -67,6 +66,7 @@ hs_err_pid* # virtual machine crash logs, see http://www.java.com/en/download/he build/ dist/ *.egg-info +clams_python-*/ coverage.xml # shared folders @@ -84,3 +84,5 @@ tags # sphinx docs-test/ +documentation/appmetadata.jsonschema +documentation/whatsnew.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..e8939f1 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,91 @@ +# Contributing to clams-python + +## Prerequisites + +- Python 3.10+ +- `gh` CLI (for changelog generation) + +## Setup + +```bash +pip install -e ".[dev]" +``` + +Unlike the old `setup.py`-based workflow, an editable install +(`pip install -e .`) is now required before running tests or building +docs. The package uses `importlib.metadata` for version resolution at +runtime, which only works when the package is registered in the +environment. You can no longer run `pytest` or `pytype` directly +against the source tree without installing first. If you want to avoid +pulling in all dependencies, `pip install -e . --no-deps` is sufficient +to register the package metadata. + +## Local Development + +All build tasks are handled by scripts in `build-tools/`. Each script +is self-contained and installs its own dependencies as needed. + +| Task | Command | +|------|---------| +| Build (sdist + wheel) | `python build-tools/build.py` | +| Run tests | `python build-tools/test.py` | +| Build docs | `python build-tools/docs.py` | +| Clean artifacts | `python build-tools/clean.py` | +| Publish | `python build-tools/publish.py` | + +All scripts support `--help` for full usage details. + +### Build + +```bash +python build-tools/build.py +``` + +Produces sdist and wheel in `dist/`. + +### Test + +```bash +python build-tools/test.py +``` + +Runs pytest with coverage. Use `--skip-install` if you already have the +package installed in editable mode. + +### Documentation + +```bash +python build-tools/docs.py +``` + +Builds Sphinx HTML docs into `docs-test/` (override with `--output-dir`). +The `--build-ver` flag is accepted for CI compatibility but has no effect +— clams-python uses unversioned documentation. + +### Versioning + +Versions are derived automatically from git tags via `setuptools-scm`. +There is no `VERSION` file to manage. At runtime, the version is +accessed through `importlib.metadata`: + +```python +from clams.ver import __version__ +``` + +For a dev install without a matching tag, `setuptools-scm` generates a +version like `1.4.1.dev20+gaf551a4e4.d20260325`. + +## Migration from Makefile + +The old `Makefile` and `setup.py` have been removed. If you are +accustomed to the old workflow, here is a mapping: + +| Old command | New equivalent | +|-------------|----------------| +| `make package` / `python setup.py sdist` | `python build-tools/build.py` | +| `make develop` / `python setup.py develop` | `pip install -e ".[dev]"` | +| `make test` | `python build-tools/test.py` | +| `make doc` | `python build-tools/docs.py` | +| `make version` / `make devversion` | Automatic via `setuptools-scm` (tag-based) | +| `make clean` | `python build-tools/clean.py` | +| `make publish` | `python build-tools/publish.py` | diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 44e8bd8..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,3 +0,0 @@ -include ./requirements.txt -include ./VERSION - diff --git a/Makefile b/Makefile deleted file mode 100644 index 5e53823..0000000 --- a/Makefile +++ /dev/null @@ -1,102 +0,0 @@ -# check for dependencies -SHELL := /bin/bash -deps = curl jq git python3 -check_deps := $(foreach dep,$(deps), $(if $(shell which $(dep)),some string,$(error "No $(dep) in PATH!"))) - -# constants -packagename = clams -generatedcode = $(packagename)/ver -distname = $(packagename)_python -artifact = build/lib/$(packagename) -buildcaches = build/bdist* $(distname).egg-info __pycache__ -testcaches = .hypothesis .pytest_cache .pytype coverage.xml htmlcov .coverage - -.PHONY: all -.PHONY: clean -.PHONY: test -.PHONY: develop -.PHONY: publish -.PHONY: docs -.PHONY: package -.PHONY: devversion - -all: version test build - -develop: devversion package test - python3 setup.py develop --uninstall - python3 setup.py develop - -publish: distclean version package test - test `git branch --show-current` = "master" - @git tag `cat VERSION` - @git push origin `cat VERSION` - -$(generatedcode): VERSION - # this will generate the version subpackage inside clams package - python3 setup.py --help 2>/dev/null || echo "Ignore setuptools import error for now" - ls $(generatedcode)* - -# generating jsonschema depends on mmif-python and pydantic -docs: - @echo "WARNING: The 'docs' target is deprecated and will be removed." - @echo "The 'docs' directory is no longer used. Documentation is now hosted in the central CLAMS documentation hub." - @echo "Use 'make doc' for local builds." - @echo "Nothing is done." - -doc: VERSION - python3 build-tools/docs.py - -package: VERSION - pip install --upgrade -r requirements.dev - python3 setup.py sdist - -build: $(artifact) -$(artifact): - python3 setup.py build - -# invoking `test` without a VERSION file will generated a dev version - this ensures `make test` runs unmanned -test: devversion $(generatedcode) - pip install --upgrade -r requirements.dev - pip install -r requirements.txt - pytype --config .pytype.cfg $(packagename) - python3 -m pytest --cov=$(packagename) --cov-report=xml - -# helper functions -e := -space := $(e) $(e) -## handling version numbers -macro = $(word 1,$(subst .,$(space),$(1))) -micro = $(word 2,$(subst .,$(space),$(1))) -patch = $(word 3,$(subst .,$(space),$(1))) -increase_patch = $(call macro,$(1)).$(call micro,$(1)).$$(($(call patch,$(1))+1)) -## handling versioning for dev version -add_dev = $(call macro,$(1)).$(call micro,$(1)).$(call patch,$(1)).dev1 -split_dev = $(word 2,$(subst .dev,$(space),$(1))) -increase_dev = $(call macro,$(1)).$(call micro,$(1)).$(call patch,$(1)).dev$$(($(call split_dev,$(1))+1)) - -devversion: VERSION.dev VERSION; cat VERSION -version: VERSION; cat VERSION - -VERSION.dev: devver := $(shell curl --silent "https://api.github.com/repos/clamsproject/clams-python/git/refs/tags" | grep '"ref":' | sed -E 's/.+refs\/tags\/([0-9.]+)",/\1/g' | sort | tail -n 1) -VERSION.dev: - @if [ -z "$(devver)" ]; then if [ -e VERSION ] ; then cp VERSION{"",".dev"}; else echo "0.0.0.dev1" > VERSION.dev ; fi \ - else if [[ "$(devver)" == *.dev* ]]; then echo $(call increase_dev,$(devver)); else echo $(call add_dev,$(call increase_patch, $(devver))); fi > VERSION.dev; \ - fi - -VERSION: version := $(shell git tag | sort -r | head -n 1) -VERSION: - @if [ -e VERSION.dev ] ; \ - then cp VERSION.dev VERSION; \ - else (read -p "Current version is ${version}, please enter a new version (default: increase *patch* level by 1): " new_ver; \ - [ -z $$new_ver ] && echo $(call increase_patch,$(version)) || echo $$new_ver) > VERSION; \ - fi - -distclean: - @rm -rf dist $(artifact) build/bdist* -clean: distclean - @rm -rf VERSION VERSION.dev $(testcaches) $(buildcaches) $(generatedcode) - @rm -rf docs - @rm -rf .*cache - @rm -rf .hypothesis tests/.hypothesis -cleandocs: - @git checkout -- docs && git clean -fx docs diff --git a/build-tools/build.py b/build-tools/build.py new file mode 100644 index 0000000..be8871b --- /dev/null +++ b/build-tools/build.py @@ -0,0 +1,54 @@ +""" +Build the clams-python package. + +Installs dependencies and runs `python -m build` to produce sdist + wheel. +""" +import argparse +import subprocess +import sys +from pathlib import Path + +SCRIPT_DIR = Path(__file__).parent + + +def run_command(command, cwd=None, check=True): + """Helper to run a shell command.""" + print(f"Running: {' '.join(str(c) for c in command)}") + result = subprocess.run(command, cwd=cwd) + if check and result.returncode != 0: + print( + f"Error: Command failed with exit code " + f"{result.returncode}" + ) + sys.exit(result.returncode) + return result + + +def main(): + parser = argparse.ArgumentParser( + description="Build the clams-python package." + ) + parser.parse_args() + + project_root = SCRIPT_DIR.parent + + # Install dev + build dependencies + print("--- Installing dependencies ---") + run_command( + [sys.executable, "-m", "pip", + "install", "-e", ".[dev]", "build"], + cwd=project_root, + ) + + # Build sdist + wheel + print("\n--- Building sdist + wheel ---") + run_command( + [sys.executable, "-m", "build"], + cwd=project_root, + ) + + print("\nBuild complete. Output in: dist/") + + +if __name__ == "__main__": + main() diff --git a/build-tools/clean.py b/build-tools/clean.py new file mode 100644 index 0000000..784dfda --- /dev/null +++ b/build-tools/clean.py @@ -0,0 +1,70 @@ +""" +Clean build artifacts, caches, and generated files. + +Replaces ``make clean`` / ``make distclean`` from the old Makefile. +""" +import argparse +import shutil +from pathlib import Path + +SCRIPT_DIR = Path(__file__).parent +PROJECT_ROOT = SCRIPT_DIR.parent + +# Directories to remove +CLEAN_DIRS = [ + "build", "dist", "*.egg-info", "clams_python-*", + ".pytest_cache", ".pytype", ".hypothesis", + "tests/.hypothesis", "htmlcov", + "docs-test", +] + +# Files to remove +CLEAN_FILES = [ + "coverage.xml", ".coverage", +] + +# Glob patterns for recursive removal +CLEAN_GLOBS = [ + "**/__pycache__", +] + + +def clean(root: Path): + removed = [] + + for pattern in CLEAN_DIRS: + for p in root.glob(pattern): + if p.is_dir(): + shutil.rmtree(p) + removed.append(str(p.relative_to(root))) + + for name in CLEAN_FILES: + p = root / name + if p.exists(): + p.unlink() + removed.append(str(p.relative_to(root))) + + for pattern in CLEAN_GLOBS: + for p in root.glob(pattern): + if p.is_dir(): + shutil.rmtree(p) + removed.append(str(p.relative_to(root))) + + if removed: + print(f"Removed {len(removed)} items:") + for item in sorted(removed): + print(f" {item}") + else: + print("Nothing to clean.") + + +def main(): + parser = argparse.ArgumentParser( + description="Clean build artifacts and caches." + ) + parser.parse_args() + clean(PROJECT_ROOT) + + +if __name__ == "__main__": + main() diff --git a/build-tools/docs.py b/build-tools/docs.py index 82927dc..e27a426 100644 --- a/build-tools/docs.py +++ b/build-tools/docs.py @@ -1,94 +1,93 @@ +""" +Build documentation for the clams-python project. + +This script is equivalent to: + 1. pip install -e .[docs] + 2. sphinx-build -b html -a -E documentation +""" import argparse +import shutil import subprocess import sys -import os -import shutil from pathlib import Path -def run_command(command, cwd=None, check=True, env=None): + +def run_command(command, cwd=None, check=True): """Helper to run a shell command.""" print(f"Running: {' '.join(str(c) for c in command)}") - result = subprocess.run(command, cwd=cwd, env=env) + result = subprocess.run(command, cwd=cwd) if check and result.returncode != 0: print(f"Error: Command failed with exit code {result.returncode}") sys.exit(result.returncode) return result -def build_docs_local(source_dir: Path, output_dir: Path = None): + +def build_docs_local(source_dir: Path, output_dir: Path): """ Builds documentation for the provided source directory. - Assumes it's running in an environment with necessary tools. + + :param source_dir: Path to the source directory containing the project. + :param output_dir: Path to the output directory for built documentation. """ - if output_dir is None: - output_dir = source_dir / "docs-test" - print("--- Running in Local Build Mode ---") - - # 1. Generate source code and install in editable mode. - print("\n--- Step 1: Installing in editable mode ---") - try: - run_command([sys.executable, "-m", "pip", "install", "-e", "."], cwd=source_dir) - except SystemExit: - print("Warning: 'pip install -e .' failed. This might be due to an externally managed environment.") - print("Attempting to proceed with documentation build assuming dependencies are met...") + print("--- Building clams-python documentation ---") - # 2. Install documentation-specific dependencies. - print("\n--- Step 2: Installing documentation dependencies ---") - doc_reqs = source_dir / "build-tools" / "requirements.docs.txt" - if not doc_reqs.exists(): - print(f"Error: Documentation requirements not found at {doc_reqs}") - sys.exit(1) + # Install package with docs dependencies in editable mode. + print("\n--- Step 1: Installing package with docs dependencies ---") try: - run_command([sys.executable, "-m", "pip", "install", "-r", str(doc_reqs)]) + run_command( + [sys.executable, "-m", "pip", "install", "-e", ".[docs]"], + cwd=source_dir, + ) except SystemExit: - print("Warning: Failed to install documentation dependencies.") - # Check if sphinx-build is available + print("Warning: 'pip install -e .[docs]' failed.") if shutil.which("sphinx-build") is None: print("Error: 'sphinx-build' not found and installation failed.") - print("Please install dependencies manually or run this script inside a virtual environment.") sys.exit(1) print("Assuming dependencies are already installed...") - # 3. Build the documentation using Sphinx. - print("\n--- Step 3: Building Sphinx documentation ---") + # Build the documentation using Sphinx. + print("\n--- Step 2: Building Sphinx documentation ---") docs_source_dir = source_dir / "documentation" - docs_build_dir = output_dir - - # Schema generation is now handled in conf.py - # schema_src = source_dir / "clams" / "appmetadata.jsonschema" - # schema_dst = docs_source_dir / "appmetadata.jsonschema" - # if schema_src.exists(): - # shutil.copy(schema_src, schema_dst) sphinx_command = [ sys.executable, "-m", "sphinx.cmd.build", str(docs_source_dir), - str(docs_build_dir), + str(output_dir), "-b", "html", # build html "-a", # write all files (rebuild everything) "-E", # don't use a saved environment, reread all files ] run_command(sphinx_command) - print(f"\nDocumentation build complete. Output in: {docs_build_dir}") - return docs_build_dir + print(f"\nDocumentation build complete. Output in: {output_dir}") + return output_dir + def main(): parser = argparse.ArgumentParser( description="Build documentation for the clams-python project." ) parser.add_argument( - '--build-ver', - metavar='', + "--build-ver", + metavar="", default=None, - help='Accepted for CLI compatibility with other SDK repos. ' - 'Ignored by this script (clams-python uses ' - 'unversioned documentation).') + help="Accepted for CLI compatibility with other SDK repos. " + "Ignored by this script (clams-python uses " + "unversioned documentation)." + ) parser.add_argument( - '--output-dir', type=Path, default=None, - help='Output directory for built docs (default: docs-test)') + "--output-dir", + metavar="", + default="docs-test", + help="The directory for documentation output " + "(default: docs-test)." + ) args = parser.parse_args() - build_docs_local(Path.cwd(), output_dir=args.output_dir) + output_dir = Path(args.output_dir) + output_dir.mkdir(exist_ok=True) + build_docs_local(Path.cwd(), output_dir) + if __name__ == "__main__": main() diff --git a/build-tools/publish.py b/build-tools/publish.py new file mode 100644 index 0000000..b1b59eb --- /dev/null +++ b/build-tools/publish.py @@ -0,0 +1,202 @@ +""" +Publish the clams-python package. + +This script is equivalent to `make publish` in the old Makefile build: + 1. Generate CHANGELOG.md from merged release PRs (requires `gh` CLI) + 2. Upload dist/ to PyPI via twine (requires TWINE_PASSWORD env var) + +Credentials are passed via environment variables (twine reads them +natively): + TWINE_USERNAME — defaults to __token__ for API tokens + TWINE_PASSWORD — the PyPI/TestPyPI API token + TWINE_REPOSITORY_URL — override for TestPyPI (or use --testpypi) +""" +import argparse +import json +import os +import subprocess +import sys +from pathlib import Path + +SCRIPT_DIR = Path(__file__).parent +TESTPYPI_URL = "https://test.pypi.org/legacy/" + + +def run_command(command, cwd=None, capture=False, check=False): + """Helper to run a shell command.""" + print(f"Running: {' '.join(str(c) for c in command)}") + result = subprocess.run( + command, + cwd=cwd, + capture_output=capture, + text=capture, + ) + if check and result.returncode != 0: + print( + f"Error: Command failed with exit code " + f"{result.returncode}" + ) + sys.exit(result.returncode) + return result + + +def check_gh_available(): + """Check if gh CLI is installed and authenticated.""" + result = run_command( + ["gh", "auth", "status"], + capture=True, + ) + return result.returncode == 0 + + +def generate_changelog(repo=None): + """ + Generate CHANGELOG.md from merged release PRs. + + :param repo: GitHub repo in owner/name format. + If None, uses the repo from the current git remote. + """ + project_root = SCRIPT_DIR.parent + + repo_args = ["--repo", repo] if repo else [] + + # Query merged PRs with "releas" in title + result = run_command( + ["gh", "pr", "list", + "-L", "1000", + "-s", "merged", + "--json", "number,title,body,mergedAt"] + + repo_args, + cwd=project_root, + capture=True, + ) + if result.returncode != 0: + print(f"Error querying PRs: {result.stderr}") + return False + + prs = json.loads(result.stdout) + # Filter to release PRs + release_prs = [ + pr for pr in prs + if pr["title"].lower().startswith("releas") + ] + + if not release_prs: + print("No release PRs found.") + return False + + # Sort by merge date (newest first) + release_prs.sort( + key=lambda pr: pr["mergedAt"], reverse=True + ) + + # Format changelog + lines = [] + for pr in release_prs: + merged_date = pr["mergedAt"][:10] # YYYY-MM-DD + lines.append(f"\n## {pr['title']} ({merged_date})") + if pr["body"]: + lines.append(pr["body"]) + lines.append("") + + changelog_path = project_root / "CHANGELOG.md" + changelog_path.write_text("\n".join(lines)) + print( + f"CHANGELOG.md written with {len(release_prs)} entries." + ) + return True + + +def upload_to_pypi(testpypi=False): + """ + Upload dist/ to PyPI via twine. + + Auth via env vars: TWINE_USERNAME (default: __token__), + TWINE_PASSWORD (required). + """ + project_root = SCRIPT_DIR.parent + dist_dir = project_root / "dist" + + tarballs = list(dist_dir.glob("*.tar.gz")) + if not tarballs: + print("No tarball found in dist/. Run build.py first.") + sys.exit(1) + + if not os.environ.get("TWINE_PASSWORD"): + print( + "Warning: TWINE_PASSWORD not set. " + "Skipping PyPI upload." + ) + print( + "Set TWINE_PASSWORD to your PyPI API token " + "to enable upload." + ) + return + + # Set default username for token auth + if not os.environ.get("TWINE_USERNAME"): + os.environ["TWINE_USERNAME"] = "__token__" + + # Install twine + run_command( + [sys.executable, "-m", "pip", "install", "twine"], + cwd=project_root, + ) + + # Build upload command + cmd = [sys.executable, "-m", "twine", "upload"] + if testpypi: + cmd.extend(["--repository-url", TESTPYPI_URL]) + cmd.extend(str(t) for t in tarballs) + + run_command(cmd, cwd=project_root, check=True) + + +def main(): + parser = argparse.ArgumentParser( + description="Publish: generate CHANGELOG and upload to PyPI." + ) + parser.add_argument( + "--repo", + default=None, + help="GitHub repo in owner/name format " + "(default: infer from git remote)." + ) + parser.add_argument( + "--skip-changelog", + action="store_true", + help="Skip changelog generation." + ) + parser.add_argument( + "--skip-upload", + action="store_true", + help="Skip PyPI upload (changelog only)." + ) + parser.add_argument( + "--testpypi", + action="store_true", + help="Upload to TestPyPI instead of PyPI." + ) + args = parser.parse_args() + + # Changelog + if args.skip_changelog: + print("Skipping changelog generation.") + elif not check_gh_available(): + print( + "Warning: gh CLI not available or not authenticated. " + "Skipping changelog generation." + ) + else: + if not generate_changelog(repo=args.repo): + sys.exit(1) + + # Upload + if args.skip_upload: + print("Skipping PyPI upload.") + else: + upload_to_pypi(testpypi=args.testpypi) + + +if __name__ == "__main__": + main() diff --git a/build-tools/requirements.docs.txt b/build-tools/requirements.docs.txt deleted file mode 100644 index 43e71f2..0000000 --- a/build-tools/requirements.docs.txt +++ /dev/null @@ -1,4 +0,0 @@ -sphinx>=7.0,<8.0 -furo -m2r2 -sphinx-jsonschema diff --git a/build-tools/test.py b/build-tools/test.py new file mode 100644 index 0000000..1a93227 --- /dev/null +++ b/build-tools/test.py @@ -0,0 +1,72 @@ +""" +Run tests for the clams-python package. + +This script is equivalent to ``make test`` in the Makefile-based repos: + pip install -e ".[test]" + pytype --config .pytype.cfg clams + python -m pytest --cov=clams --cov-report=xml +""" +import argparse +import subprocess +import sys +from pathlib import Path + +SCRIPT_DIR = Path(__file__).parent + + +def run_command(command, cwd=None, check=True): + """Helper to run a shell command.""" + print(f"Running: {' '.join(str(c) for c in command)}") + result = subprocess.run(command, cwd=cwd) + if check and result.returncode != 0: + print( + f"Error: Command failed with exit code " + f"{result.returncode}" + ) + sys.exit(result.returncode) + return result + + +def main(): + parser = argparse.ArgumentParser( + description="Run tests for the clams-python package." + ) + parser.add_argument( + "--skip-install", + action="store_true", + help="Skip pip install step " + "(useful if already installed)." + ) + args = parser.parse_args() + + project_root = SCRIPT_DIR.parent + + # Install package with test dependencies + if not args.skip_install: + print("--- Installing package with test dependencies ---") + run_command( + [sys.executable, "-m", "pip", + "install", "-e", ".[test]"], + cwd=project_root, + ) + + # Run pytype static analysis + print("\n--- Running pytype ---") + run_command( + ["pytype", "--config", ".pytype.cfg", "clams"], + cwd=project_root, + ) + + # Run pytest with coverage + print("\n--- Running pytest ---") + run_command( + [sys.executable, "-m", "pytest", + "--cov=clams", "--cov-report=xml"], + cwd=project_root, + ) + + print("\nAll tests passed.") + + +if __name__ == "__main__": + main() diff --git a/clams/ver/__init__.py b/clams/ver/__init__.py new file mode 100644 index 0000000..38a483d --- /dev/null +++ b/clams/ver/__init__.py @@ -0,0 +1,2 @@ +from importlib.metadata import version +__version__ = version("clams-python") diff --git a/documentation/conf.py b/documentation/conf.py index 3ad6d11..e056cd2 100644 --- a/documentation/conf.py +++ b/documentation/conf.py @@ -33,9 +33,10 @@ copyright = f'{datetime.date.today().year}, Brandeis LLC' author = 'Brandeis LLC' try: - version = open(proj_root_dir / 'VERSION').read().strip() -except FileNotFoundError: - print("WARNING: VERSION file not found, using 'dev' as version.") + from importlib.metadata import version as _get_version + version = _get_version('clams-python') +except Exception: + print("WARNING: could not read package version, using 'dev'.") version = 'dev' root_doc = 'index' @@ -197,8 +198,7 @@ def generate_jsonschema(app): def update_target_spec(app): target_vers_csv = Path(__file__).parent / 'target-versions.csv' - with open(proj_root_dir / "VERSION", 'r') as version_f: - version = version_f.read().strip() + version = _get_version('clams-python') # Skip dev/dummy versions to avoid dirtying the git-tracked CSV if 'dev' in version or not re.match(r'^\d+\.\d+\.\d+$', version): return diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..d8835d6 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,52 @@ +[build-system] +requires = ["setuptools>=61.0", "setuptools-scm>=8.0"] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] + +[project] +name = "clams-python" +dynamic = ["version"] +description = "A collection of APIs to develop CLAMS app for python" +readme = "README.md" +license = "Apache-2.0" +requires-python = ">=3.10" +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Framework :: Flask", + "Framework :: Pytest", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3 :: Only", +] +dependencies = [ + "mmif-python==1.3.1", + "Flask>=2", + "Flask-RESTful>=0.3.9", + "gunicorn>=20", + "lapps>=0.0.2", + "pydantic>=2", + "jsonschema>=3", +] + +[project.scripts] +clams = "clams:cli" + +[project.urls] +homepage = "https://clams.ai" +source = "https://github.com/clamsproject/clams-python" + +[project.optional-dependencies] +dev = ["pytype", "pytest", "pytest-cov", "setuptools"] +docs = ["sphinx>=7.0,<8.0", "furo", "m2r2", "sphinx-jsonschema"] +test = ["pytest", "pytest-cov"] + +[tool.setuptools.packages.find] +where = ["."] +include = ["clams*"] + +[tool.setuptools.package-data] +clams = ["develop/templates/**/*", "develop/templates/**/.*"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "--cov=clams --cov-report=xml" diff --git a/requirements.dev b/requirements.dev deleted file mode 100644 index e5d9f9d..0000000 --- a/requirements.dev +++ /dev/null @@ -1,12 +0,0 @@ -pytype -pytest -pytest-cov -twine -sphinx -sphinx-rtd-theme -sphinx-jsonschema -sphinx-autobuild -autodoc -m2r2 -pillow -setuptools diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 6cd63d0..0000000 --- a/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -mmif-python==1.3.1 - -Flask>=2 -Flask-RESTful>=0.3.9 -gunicorn>=20 -lapps>=0.0.2 -pydantic>=2 -jsonschema>=3 diff --git a/setup.py b/setup.py deleted file mode 100644 index b96a0ea..0000000 --- a/setup.py +++ /dev/null @@ -1,58 +0,0 @@ -#! /usr/bin/env python3 -import os -from os import path -import shutil - -name = "clams-python" -cmdclass = {} - -with open("VERSION", 'r') as version_f: - version = version_f.read().strip() - -with open('README.md') as readme: - long_desc = readme.read() - -with open('requirements.txt') as requirements: - requires = requirements.readlines() - -ver_pack_dir = path.join('clams', 'ver') -shutil.rmtree(ver_pack_dir, ignore_errors=True) -os.makedirs(ver_pack_dir, exist_ok=True) -init_mod = open(path.join(ver_pack_dir, '__init__.py'), 'w') -init_mod.write(f'__version__ = "{version}"') -init_mod.close() - -import setuptools - -setuptools.setup( - name=name, - version=version, - author="Brandeis Lab for Linguistics and Computation", - author_email="admin@clams.ai", - description="A collection of APIs to develop CLAMS app for python", - long_description=long_desc, - long_description_content_type="text/markdown", - url="https://clams.ai", - license="Apache-2.0", - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Framework :: Flask', - 'Framework :: Pytest', - 'Intended Audience :: Developers', - 'Programming Language :: Python :: 3 :: Only', - ], - cmdclass=cmdclass, - # this is for *building*, building (build, bdist_*) doesn't get along with MANIFEST.in - # so using this param explicitly is much safer implementation - package_data={ - 'clams': ['develop/templates/**/*', 'develop/templates/**/.*'] - }, - install_requires=requires, - python_requires='>=3.10', - packages=setuptools.find_packages(), - entry_points={ - 'console_scripts': [ - 'clams = clams.__init__:cli', - ], - }, -) From b5e20e2d60fbeb9ed232825f0430cc489c77c38e Mon Sep 17 00:00:00 2001 From: Keigh Rim Date: Fri, 27 Mar 2026 05:24:40 -0400 Subject: [PATCH 3/3] updated test CI path --- .github/workflows/codecov.yml | 2 +- pyproject.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index e9f12f3..68bee90 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -12,7 +12,7 @@ on: jobs: test-and-codecov: name: "🤙 Call SDK test workflow" - uses: clamsproject/.github/.github/workflows/sdk-codecov.yml@main + uses: clamsproject/.github/.github/workflows/sdk-codecov-pyproj.yml@main secrets: CC_REPO_UPLOAD_TOKEN: ${{ secrets.CODECOV_UPLOAD_TOKEN_CLAMS_PYTHON }} diff --git a/pyproject.toml b/pyproject.toml index d8835d6..1f97d59 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,9 +36,9 @@ homepage = "https://clams.ai" source = "https://github.com/clamsproject/clams-python" [project.optional-dependencies] -dev = ["pytype", "pytest", "pytest-cov", "setuptools"] +dev = ["pytype", "pytest", "pytest-cov", "pillow", "setuptools"] docs = ["sphinx>=7.0,<8.0", "furo", "m2r2", "sphinx-jsonschema"] -test = ["pytest", "pytest-cov"] +test = ["pytype", "pytest", "pytest-cov", "pillow"] [tool.setuptools.packages.find] where = ["."]