Spec and client generation

.dprint.jsonc

@@ -3,6 +3,12 @@
   },
   "json": {
   },
+  // Match the repo-wide 120 line-length set in .editorconfig and ruff.toml,
+  // otherwise dprint's bundled ruff would reformat Python files to its
+  // default and fight with `mise run ruff-fmt`.
+  "ruff": {
+    "lineLength": 120,
+  },
   "malva": {
   },
   "markdown": {

.editorconfig

@@ -30,3 +30,28 @@ indent_size = unset
 indent_style = unset
 trim_trailing_whitespace = unset
 max_line_length = unset
+
+# Upstream WIT packages fetched verbatim — their formatting is whatever the
+# upstream repos use and shouldn't be normalised on our side.
+[generated/specs/wit/deps/wasi-*/**]
+charset = unset
+end_of_line = unset
+insert_final_newline = unset
+indent_size = unset
+indent_style = unset
+trim_trailing_whitespace = unset
+max_line_length = unset
+
+# openapi-python-client emits framework-boilerplate docstrings (e.g. the
+# "errors.UnexpectedStatus: If the server returns an undocumented status code
+# …" line in every operation) that exceed 120 chars. ruff format doesn't
+# reflow plain-text docstrings, so we can't fix this short of patching the
+# generator's templates — drop the line-length check for this tree.
+[generated/python-rest/**]
+max_line_length = unset
+
+# Same story for openapi2zig: a handful of helpers (SSE parsing, query-string
+# encoding) emit lines that exceed 120 chars and `zig fmt` won't reflow them.
+# The generator is the source of truth; we don't edit its output by hand.
+[generated/zig-rest/**]
+max_line_length = unset

.github/workflows/check.yml

@@ -38,6 +38,10 @@ jobs:
           mise install
         env:
           GITHUB_TOKEN: ${{ github.token }}
+          # GitHub release downloads occasionally take longer than mise's
+          # default 30s HTTP timeout; bump it so transient network slowness
+          # doesn't fail the whole `mise install` step.
+          MISE_HTTP_TIMEOUT: "120"
 
       - name: Run checkers
         run: |

.github/workflows/test.yml

@@ -36,6 +36,9 @@ jobs:
           mise install
         env:
           GITHUB_TOKEN: ${{ github.token }}
+          # Match check.yml: bump mise's 30s HTTP timeout so GitHub-release
+          # downloads don't fail the install on transient slowness.
+          MISE_HTTP_TIMEOUT: "120"
 
       - name: Fetch MNIST model
         run: mise run fetch-mnist-rclone

.mise.toml

@@ -1,35 +1,53 @@
 [tools]
 action-validator = "latest"
-"cargo:ast-grep" = "latest"
 cargo-binstall = "latest"
+"cargo:ast-grep" = "latest"
 "cargo:aube" = "latest"
+"cargo:dart-typegen" = "latest"
 "cargo:taplo-cli" = "latest"
 "cargo:wasm-pack" = "latest"
 "chromedriver" = "146"
-claude = "latest"
 cmake = "latest"
-codex = "latest"
-dart = { version = "latest", url = "https://storage.googleapis.com/dart-archive/channels/stable/release/{{ version }}/sdk/dartsdk-{{ os() }}-{{ arch() }}-release.zip", version_expr = 'fromJSON(body).prefixes | filter({ # matches "^channels/stable/release/(\\d+\\.\\d+\\.\\d+)/$" }) | map({split(#, "/")[3]}) | sortVersions()', version_list_url = "https://storage.googleapis.com/storage/v1/b/dart-archive/o?prefix=channels/stable/release/&delimiter=/" }
+"conda:lld" = "latest"
+# Major-version pin: `dart format`'s output drifts on minor releases, which
+# turns dart-fmt-check into a flaky CI gate when `version = "latest"` resolves
+# differently on each machine. "3" still drifts within the major (re-resolved
+# per `mise install`), but lets us bump intentionally when 4.x ships.
+dart = { version = "3", url = "https://storage.googleapis.com/dart-archive/channels/stable/release/{{ version }}/sdk/dartsdk-{{ os() }}-{{ arch() }}-release.zip", version_expr = 'fromJSON(body).prefixes | filter({ # matches "^channels/stable/release/(\\d+\\.\\d+\\.\\d+)/$" }) | map({split(#, "/")[3]}) | sortVersions()', version_list_url = "https://storage.googleapis.com/storage/v1/b/dart-archive/o?prefix=channels/stable/release/&delimiter=/" }
 dotnet = "latest"
 dotnet-core = "latest"
 "dotnet:roslynator.dotnet.cli" = "latest"
 dprint = "latest"
 editorconfig-checker = "latest"
 gemini-cli = "latest"
-"github:block/goose" = "latest"
+# openapi2zig v0.2.0 doesn't support YAML input yet — `gen-zig-rest` feeds it
+# the JSON form via `yq`. Its `Client` ships `std.http.Client` which doesn't
+# compile for `wasm32-unknown-unknown`; `et-int-gen` post-processes the
+# generated source via tree-sitter-zig to swap in a WASM-friendly extern.
+"github:christianhelle/openapi2zig" = "latest"
 "github:grok-rs/waitup" = "latest"
 "github:wasm-bindgen/wasm-bindgen" = "0.2.114"
 java = "latest"
 maven = "latest"
 mprocs = "latest"
 node = "22"
 "npm:onnxruntime-web" = "latest"
-"npm:pyodide" = "0.29.3"
-ollama = "latest"
+# Use the GitHub release tarball, not `npm:pyodide`. The npm package is only
+# the runtime (pyodide.js, .wasm, stdlib, micropip) ~5 MB — it's designed for
+# JS apps that fetch the rest from JsDelivr's Pyodide custom-CDN namespace
+# (https://www.jsdelivr.com/oss-cdn) at runtime. We need the ~200 MB "full"
+# distribution served locally so browser modules work offline; that's only
+# published as a tarball on GitHub releases. The extracted tarball contains a
+# top-level `pyodide/` directory the modules service picks up — see
+# `default_modules_folders` in libs/edge-toolkit/src/config.rs.
+"http:pyodide" = { version = "0.29.3", url = "https://github.com/pyodide/pyodide/releases/download/{{ version }}/pyodide-{{ version }}.tar.bz2" }
 osv-scanner = "latest"
 pipx = "latest"
+python = "3.13"
 "pipx:cmake" = "latest"
 "pipx:componentize-py" = "latest"
+"pipx:datamodel-code-generator" = "latest"
+"pipx:openapi-python-client" = "latest"
 "pipx:pytest" = "latest"
 protoc = "latest"
 rclone = "latest"
@@ -44,13 +62,27 @@ wasm-tools = "latest"
 yq = "latest"
 zig = "latest"
 
+# Recent Apple `ld` versions trip an assertion on long mangled symbols emitted
+# by some deep-dep crates (notably dart-typegen, installed below). Routing the
+# link step through LLVM's `ld64.lld` avoids the bug — `lld` is on PATH via the
+# `conda:lld` mise tool. We set this here so `mise install cargo:dart-typegen`
+# also picks it up (cargo install runs in a tempdir and doesn't read project
+# `.cargo/config.toml`).
+[env]
+CARGO_TARGET_AARCH64_APPLE_DARWIN_RUSTFLAGS = "-C link-arg=-fuse-ld=lld"
+CARGO_TARGET_X86_64_APPLE_DARWIN_RUSTFLAGS = "-C link-arg=-fuse-ld=lld"
+
 [tasks]
 ec = "ec"
 editorconfig-check = "ec"
-ruff-check = "ruff check services/ws-modules/"
-ruff-fmt = "ruff format services/ws-modules/"
-zig-check = "zig fmt --check services/ws-modules/"
-zig-fmt = "zig fmt services/ws-modules/"
+# `wit_world` and `componentize_py_async_support` are componentize-py outputs
+# (gitignored, regenerated each build) under wasi-graphics-info — ruff would
+# flag 200+ issues in code we don't own and don't ship.
+ruff-check = "ruff check --exclude wit_world --exclude componentize_py_async_support services/ws-modules/ generated/python-ws/ generated/python-rest/"
+ruff-fmt = "ruff format --exclude wit_world --exclude componentize_py_async_support services/ws-modules/ generated/python-ws/ generated/python-rest/"
+ruff-fmt-check = "ruff format --check --exclude wit_world --exclude componentize_py_async_support services/ws-modules/ generated/python-ws/ generated/python-rest/"
+zig-check = "zig fmt --check services/ws-modules/ generated/zig-rest/"
+zig-fmt = "zig fmt services/ws-modules/ generated/zig-rest/"
 
 [tasks.dotnet-fmt]
 description = "Format dotnet C# code"
@@ -70,14 +102,22 @@ DPRINT_CACHE_DIR = "/tmp/dprint-cache"
 run = "dprint fmt"
 
 [tasks.fmt]
-depends = ["cargo-clippy-fix", "cargo-fmt", "dart-fmt", "dotnet-fmt", "dprint-fmt", "taplo-fmt", "zig-fmt"]
+depends = ["cargo-clippy-fix", "cargo-fmt", "dart-fmt", "dotnet-fmt", "dprint-fmt", "ruff-fmt", "taplo-fmt", "zig-fmt"]
 description = "Run repository formatters"
 
 [tasks.dart-check]
-run = "dart analyze services/ws-modules/dart-comm1/"
+run = """
+dart pub get --directory services/ws-modules/dart-comm1
+dart analyze services/ws-modules/dart-comm1/ generated/dart-ws/
+"""
 
 [tasks.dart-fmt]
-run = "dart format services/ws-modules/dart-comm1/"
+run = "dart format services/ws-modules/dart-comm1/ generated/dart-ws/"
+
+[tasks.dart-fmt-check]
+# Parallel of cargo-fmt-check: fail if any tracked Dart source would be
+# reformatted, so CI enforces the same formatting `dart-fmt` applies.
+run = "dart format --output=none --set-exit-if-changed services/ws-modules/dart-comm1/ generated/dart-ws/"
 
 [tasks.check]
 depends = [
@@ -86,10 +126,14 @@ depends = [
   "cargo-clippy",
   "cargo-fmt-check",
   "dart-check",
+  "dart-fmt-check",
   "dotnet-check",
   "dprint-check",
   "editorconfig-check",
+  "gen-specs-check",
   "osv-scanner",
+  "ruff-check",
+  "ruff-fmt-check",
   "taplo-check",
   "typos",
   "verification-check",
@@ -139,6 +183,103 @@ depends = ["regenerate-verification"]
 description = "Fail if generated scenario output files are stale"
 run = "git diff --exit-code -- verification"
 
+[tasks.fetch-wit-deps]
+description = "Fetch upstream WASI WIT packages into generated/specs/wit/ at pinned versions. Run by hand when bumping a package version; gen-specs-check catches drift."
+run = "cargo run -q -p et-int-gen --bin et-int-gen -- fetch-deps"
+
+[tasks.gen-ws-spec]
+description = "Run et-int-gen: emit the WS protocol's AsyncAPI YAML, OpenAPI YAML, WIT package, KDL schema, and Rust REST client from edge_toolkit::ws::WsMessage and the service handlers"
+# et-int-gen runs progenitor + prettyplease to produce
+# generated/rust-rest/src/lib.rs. prettyplease's style disagrees with our
+# nightly rustfmt (different import grouping + line width), so the workspace
+# `mise run fmt` would reformat it and `mise run gen-specs-check` would see
+# drift on the next regen. Make rustfmt the canonical formatter by running
+# it here as a post-process — the committed file is then a fixed point of
+# both tools.
+run = """
+cargo run -q -p et-int-gen --bin et-int-gen
+cargo +nightly fmt -p et-rest-client
+zig fmt generated/zig-rest/src/et_rest_client.zig
+"""
+
+[tasks.gen-dart-ws]
+depends = ["gen-ws-spec"]
+description = "Emit Dart sealed-class client for the WS protocol via dart-typegen (consumes target/int-gen/ws.kdl emitted by gen-ws-spec)"
+run = """
+mkdir -p generated/dart-ws/lib
+dart-typegen generate -i target/int-gen/ws.kdl -o generated/dart-ws/lib/ws_messages.dart
+dart format generated/dart-ws/lib/ws_messages.dart
+"""
+
+[tasks.gen-python-rest]
+depends = ["gen-ws-spec"]
+description = "Emit typed Python REST client for the ws-server API via openapi-python-client (consumes generated/specs/rest.yaml)"
+run = """
+mkdir -p generated/python-rest/et_rest_client
+openapi-python-client generate \
+  --path generated/specs/rest.yaml \
+  --meta none \
+  --overwrite \
+  --output-path generated/python-rest/et_rest_client
+# openapi-python-client drops a .ruff_cache next to the generated source; we
+# don't want that committed.
+rm -rf generated/python-rest/et_rest_client/.ruff_cache
+"""
+
+[tasks.gen-python-ws]
+depends = ["gen-ws-spec"]
+description = "Emit Pydantic models for the WS protocol via datamodel-code-generator"
+run = """
+mkdir -p generated/python-ws/et_ws
+# `--custom-file-header "#"` is the only way to suppress datamodel-codegen's
+# "generated by datamodel-codegen / filename: …" banner (an empty string is
+# treated as "no override" and the default banner returns). Strip the
+# remaining lone `#` line below so the file starts cleanly.
+datamodel-codegen \
+  --input target/int-gen/ws.schema.json \
+  --input-file-type jsonschema \
+  --output generated/python-ws/et_ws/messages.py \
+  --output-model-type pydantic_v2.BaseModel \
+  --target-python-version 3.10 \
+  --use-schema-description \
+  --use-title-as-name \
+  --use-double-quotes \
+  --use-union-operator \
+  --field-constraints \
+  --disable-timestamp \
+  --custom-file-header '#'
+sed -i.bak -e '1{/^#$/d;}' -e '2{/^$/d;}' generated/python-ws/et_ws/messages.py
+rm generated/python-ws/et_ws/messages.py.bak
+# datamodel-codegen formats via black at line-length 88; our repo-wide
+# ruff.toml sets 120. Re-run ruff to match — otherwise gen-specs-check
+# diffs single-line vs wrapped fields on every regen.
+ruff format generated/python-ws/et_ws/messages.py
+"""
+
+[tasks.gen-specs]
+depends = ["gen-dart-ws", "gen-python-rest", "gen-python-ws", "gen-ws-spec"]
+description = "Regenerate every spec artifact under generated/ (AsyncAPI YAML, OpenAPI YAML, WIT, Dart, Python, Rust)"
+
+[tasks.gen-specs-check]
+depends = ["gen-specs"]
+description = "Fail if any checked-in artifact under generated/ is stale"
+# All et-int-gen outputs land under generated/. The runner's
+# wit/deps/et-ws-messages is a symlink into that tree, so a single diff
+# against generated/ covers everything.
+run = "git diff --exit-code -- generated"
+
+[tasks.build-et-ws-wheel]
+depends = ["gen-python-ws"]
+description = "Build the generated et-ws Python package as a wheel"
+dir = "generated/python-ws"
+run = "uv build --wheel --out-dir dist"
+
+[tasks.build-et-rest-client-wheel]
+depends = ["gen-python-rest"]
+description = "Build the generated et-rest-client Python package as a wheel"
+dir = "generated/python-rest"
+run = "uv build --wheel --out-dir dist"
+
 [tasks.ws-server]
 description = "Run the WebSocket server"
 dir = "services/ws-server"
@@ -237,7 +378,10 @@ run = "wasm-pack build . --target web"
 [tasks.build-ws-dart-comm1-module]
 description = "Build the dart-comm1 workflow module"
 dir = "services/ws-modules/dart-comm1"
-run = "dart compile js lib/dart_comm1.dart -o pkg/et_ws_dart_comm1_compiled.js --no-source-maps"
+run = """
+dart pub get
+dart compile js lib/dart_comm1.dart -o pkg/et_ws_dart_comm1_compiled.js --no-source-maps
+"""
 
 [tasks.build-ws-dotnet-data1-module]
 description = "Build the dotnet-data1 C# WASM workflow module"
@@ -253,19 +397,19 @@ cp "$PUBLISH"/*.js "$PUBLISH"/*.wasm "$PUBLISH"/*.dat pkg/
 [tasks.build-ws-wasi-graphics-info-module]
 description = "Build the WASI graphics-info Python module as a WASI Preview 2 component"
 dir = "services/ws-modules/wasi-graphics-info"
-# WIT lives once in services/ws-wasi-runner/wit/ - the runner's `runner`
-# world is what its bindgen consumes; the `module` world (which `include`s
-# `runner` and adds wasi-nn) is what guest modules target. `mnist-12.onnx`
-# is fetched into `pkg/` by `mise run download-models` (see
+# WIT lives once at generated/specs/wit/ — the runner's `runner` world is
+# what its bindgen consumes; the `module` world (which `include`s `runner`
+# and adds wasi-nn) is what guest modules target. `mnist-12.onnx` is
+# fetched into `pkg/` by `mise run download-models` (see
 # `fetch-mnist-rclone`); componentize-py 0.23 doesn't bundle arbitrary
 # data files inside the component image, so the guest fetches it via the
 # `wasi:keyvalue/store` host import from the ws-server at runtime.
 # componentize-py 0.23's `bindings` subcommand refuses to overwrite an
 # existing output dir, so we wipe it first.
 run = """
 rm -rf wit_world
-componentize-py -d ../../ws-wasi-runner/wit -w module bindings .
-componentize-py -d ../../ws-wasi-runner/wit -w module componentize wasi_graphics_info -o pkg/et_ws_wasi_graphics_info.wasm
+componentize-py -d ../../../generated/specs/wit -w module bindings .
+componentize-py -d ../../../generated/specs/wit -w module componentize wasi_graphics_info -o pkg/et_ws_wasi_graphics_info.wasm
 cargo run -p et-cli -- module-package-json
 """
 
@@ -293,18 +437,22 @@ cargo run -p et-cli -- module-package-json --module-dir services/ws-modules/wasi
 """
 
 [tasks.build-ws-pydata1-module]
+depends = ["build-et-rest-client-wheel"]
 description = "Build the pydata1 Python workflow module"
 dir = "services/ws-modules/pydata1"
 run = """
 uv build --wheel --out-dir pkg
+cp ../../../generated/python-rest/dist/et_rest_client-*.whl pkg/
 cargo run -p et-cli -- module-package-json
 """
 
 [tasks.build-ws-pyface1-module]
+depends = ["build-et-ws-wheel"]
 description = "Build the pyface1 Python face detection workflow module"
 dir = "services/ws-modules/pyface1"
 run = """
 uv build --wheel --out-dir pkg
+cp ../../../generated/python-ws/dist/et_ws-*.whl pkg/
 cargo run -p et-cli -- module-package-json
 """
 
@@ -399,13 +547,32 @@ run = """
 [tasks.download-models]
 depends = ["fetch-face1-rclone", "fetch-har-motion1-rclone", "fetch-mnist-rclone"]
 
+[tasks.prefetch]
+description = "Pre-download all dependencies and models (Rust crates, Dart packages, Python envs, Java/Maven, .NET, Node, WIT)"
+depends = ["download-models"]
+run = """
+cargo check --workspace
+dart pub get --directory services/ws-modules/dart-comm1
+uv sync --directory services/ws-modules/pyface1
+uv sync --directory services/ws-modules/pydata1
+mvn dependency:resolve --quiet
+dotnet restore
+npm install --prefix services/ws-server/static
+"""
+
+[tasks.prefetch.env]
+UV_PYTHON = "cpython"
+
 [tasks.cargo-test]
 run = "cargo test --workspace"
 
 [tasks.test-pyface1]
 dir = "services/ws-modules/pyface1"
 run = "uv run pytest"
 
+[tasks.test-pyface1.env]
+UV_PYTHON = "{{exec(command='mise where python')}}/bin/python3"
+
 [tasks.test]
 depends = ["cargo-test", "test-pyface1"]
 description = "Run all tests"