translation worker

.github/workflows/linting.yml

@@ -9,7 +9,7 @@ on:
       - 'datashare-python/**.py'
       - 'worker-template/**.py'
       - 'asr-worker/**.py'
-      - '.github/workflows/linting.yml'
+      - 'translation-worker/**.py'
 
 # TODO: leverage some caching here
 jobs:
@@ -37,26 +37,41 @@ jobs:
       - name: Lint test
         run: ruff check --config qa/ruff.toml worker-template
 
-  doc:
+  asr-worker:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: astral-sh/ruff-action@v3
         with:
           args: "--version" # skips test by displaying the version
       - name: Check formatting
-        run: ruff format --config qa/ruff.toml --check docs
+        run: ruff format --config qa/ruff.toml --check asr-worker
       - name: Lint test
-        run: ruff check --config qa/ruff.toml docs
+        run: ruff check --config qa/ruff.toml asr-worker
 
-  asr-worker:
+  translation-worker:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/pylint
-        name: "Lint ASR worker"
+      - uses: astral-sh/ruff-action@v3
         with:
-          path: asr-worker
+          args: "--version" # skips test by displaying the version
+      - name: Check formatting
+        run: ruff format --config qa/ruff.toml --check translation-worker
+      - name: Lint test
+        run: ruff check --config qa/ruff.toml translation-worker
+
+  doc:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/ruff-action@v3
+        with:
+          args: "--version" # skips test by displaying the version
+      - name: Check formatting
+        run: ruff format --config qa/ruff.toml --check docs
+      - name: Lint test
+        run: ruff check --config qa/ruff.toml docs
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

.github/workflows/publish_datashare_python.yml

@@ -0,0 +1,42 @@
+name: Publish datashare-python
+
+on:
+  workflow_dispatch:
+  push:
+    tags:
+      - 'datashare-python-*'
+
+jobs:
+  create-release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Create GH release
+        run: gh release create "$tag" --generate-notes
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          tag: ${{ github.ref_name }}
+
+  publish-datashare-python-to-pypi:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+    environment:
+      name: pypi
+      url: https://pypi.org/p/datashare-python
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          version: "0.10.8"
+      - name: Install Python 3.11
+        run: uv python install 3.11
+      - name: Build
+        run: cd datashare-python && uv build
+      - name: Publish
+        run: cd datashare-python && uv publish
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false

.github/workflows/publish_worker_template.yml

@@ -0,0 +1,42 @@
+name: Publish worker-template
+
+on:
+  workflow_dispatch:
+  push:
+    tags:
+      - 'worker-template-*'
+
+jobs:
+  create-release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Create GH release
+        run: gh release create "$tag" --generate-notes
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          tag: ${{ github.ref_name }}
+
+  publish-datashare-python-to-pypi:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+    environment:
+      name: pypi
+      url: https://pypi.org/p/datashare-worker-template
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          version: "0.10.8"
+      - name: Install Python 3.11
+        run: uv python install 3.11
+      - name: Build
+        run: cd worker-template && uv build
+      - name: Publish
+        run: cd worker-template && uv publish
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false

.github/workflows/tests.yml

@@ -71,6 +71,60 @@ jobs:
           uv sync --frozen --all-extras
           uv run --frozen python -m pytest --timeout=180 -vvv --cache-clear --show-capture=all -r A
 
+  test-asr-worker:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Python project
+        uses: actions/setup-python@v6
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Cache Docker images
+        uses: ScribeMD/docker-cache@0.5.0
+        with:
+          key: docker-${{ runner.os }}-${{ hashFiles('docker-compose.yml')  }}
+      - name: Start test services
+        run: docker compose up -d datashare temporal-post-init elasticsearch
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          version: ${{ env.ASTRAL_VERSION }}
+          python-version: ${{ env.PYTHON_VERSION }}
+          enable-cache: true
+          working-directory: asr-worker
+      - name: Run tests
+        run: |
+          cd asr-worker
+          uv sync --frozen --all-extras
+          uv run --frozen python -m pytest --timeout=180 -vvv --cache-clear --show-capture=all -r A
+
+  test-translation-worker:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Python project
+        uses: actions/setup-python@v6
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Cache Docker images
+        uses: ScribeMD/docker-cache@0.5.0
+        with:
+          key: docker-${{ runner.os }}-${{ hashFiles('docker-compose.yml')  }}
+      - name: Start test services
+        run: docker compose up -d datashare temporal-post-init elasticsearch
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          version: ${{ env.ASTRAL_VERSION }}
+          python-version: ${{ env.PYTHON_VERSION }}
+          enable-cache: true
+          working-directory: translation-worker
+      - name: Run tests
+        run: |
+          cd translation-worker
+          uv sync --frozen --all-extras
+          uv run --frozen python -m pytest --timeout=180 -vvv --cache-clear --show-capture=all -r A
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true

.pre-commit-config.yaml

@@ -1,6 +1,9 @@
 repos:
-  - repo: https://github.com/psf/black
-    rev: 26.1.0
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.15.5
     hooks:
-      - id: black
-        language_version: python3.10
+      - id: ruff-check
+        args: [--fix]
+        types_or: [python, pyi]
+      - id: ruff-format
+        types_or: [python, pyi]

Dockerfile

@@ -16,10 +16,11 @@ WORKDIR /app
 
 # add task cli
 ADD datashare-python/ ./datashare-python/
+ADD worker-template/ ./worker-template/
 
 # install python deps
-RUN --mount=type=cache,target=~/.cache/uv \
-    uv pip install --system datashare-python/
+RUN --mount=type=cache,target=~/.cache/uv uv pip install --system worker-template/
+RUN --mount=type=cache,target=~/.cache/uv uv pip install --system datashare-python/
 
 # copy build-independant files
 ADD scripts scripts

asr-worker/asr_worker/constants.py

@@ -8,8 +8,4 @@
 
 PARAKEET = "parakeet"
 
-DEFAULT_TEMPORAL_ADDRESS = "temporal:7233"
-
-RESPONSE_SUCCESS = "success"
-
-RESPONSE_ERROR = "error"
+ASR_WORKFLOW_NAME = "asr-workflow"