From a4d28180d47c60937502c52df5a092716c3f53bf Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 27 Oct 2025 18:34:04 +0000 Subject: [PATCH 01/10] Initial plan From 032b83b2688cf40c2cd793ee20116a8e23d2a44a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 27 Oct 2025 18:46:11 +0000 Subject: [PATCH 02/10] Add HaRP support for Nextcloud 32+ Co-authored-by: R0Wi <19730957+R0Wi@users.noreply.github.com> --- Dockerfile | 26 ++++++++++++++++++++-- README.md | 39 +++++++++++++++++++++++++++++++- start.sh | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 127 insertions(+), 3 deletions(-) create mode 100644 start.sh diff --git a/Dockerfile b/Dockerfile index da0c810..1fe179a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,9 +4,25 @@ ARG USER=serviceuser ENV HOME=/home/$USER RUN apk update && \ - apk add --no-cache sudo ocrmypdf $(apk search tesseract-ocr-data- | sed 's/-[0-9].*//') && \ + apk add --no-cache sudo ocrmypdf $(apk search tesseract-ocr-data- | sed 's/-[0-9].*//') curl bash && \ adduser -D $USER +# Download and install FRP client +RUN set -ex; \ + ARCH=$(uname -m); \ + if [ "$ARCH" = "aarch64" ]; then \ + FRP_URL="https://raw.githubusercontent.com/nextcloud/HaRP/main/exapps_dev/frp_0.61.1_linux_arm64.tar.gz"; \ + else \ + FRP_URL="https://raw.githubusercontent.com/nextcloud/HaRP/main/exapps_dev/frp_0.61.1_linux_amd64.tar.gz"; \ + fi; \ + echo "Downloading FRP client from $FRP_URL"; \ + curl -L "$FRP_URL" -o /tmp/frp.tar.gz; \ + tar -C /tmp -xzf /tmp/frp.tar.gz; \ + mv /tmp/frp_0.61.1_linux_* /tmp/frp; \ + cp /tmp/frp/frpc /usr/local/bin/frpc; \ + chmod +x /usr/local/bin/frpc; \ + rm -rf /tmp/frp /tmp/frp.tar.gz + USER $USER WORKDIR /app @@ -14,10 +30,16 @@ WORKDIR /app COPY --chown=$USER:$USER requirements.txt requirements.txt COPY --chown=$USER:$USER main.py . COPY --chown=$USER:$USER workflow_ocr_backend/ ./workflow_ocr_backend +COPY --chown=$USER:$USER start.sh /start.sh RUN pip install -r requirements.txt -ENTRYPOINT ["python3", "-u", "main.py"] +# Make start.sh executable +USER root +RUN chmod +x /start.sh +USER $USER + +ENTRYPOINT ["/start.sh", "python3", "-u", "main.py"] FROM app AS devcontainer diff --git a/README.md b/README.md index 31bfd55..257a506 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ It's written in Python and provides a simple REST API for [ocrmypdf](https://ocr - [Prerequisites](#prerequisites) - [Installation](#installation) - [`docker-compose` Example](#docker-compose-example) + - [HaRP Support (Nextcloud 32+)](#harp-support-nextcloud-32) ## Prerequisites @@ -17,7 +18,9 @@ It will take care of all the heavy lifting like installation, orchestration, con 1. Install [`docker`](https://docs.docker.com/engine/install/ubuntu/) on the host where the app should be installed. 2. Install the [`AppApi`](https://docs.nextcloud.com/server/latest/admin_manual/exapps_management/AppAPIAndExternalApps.html#installing-appapi) app. It will take care of the installation and orchestration of the backend as Docker Container. -3. Setup a [Deploy Daemon](https://docs.nextcloud.com/server/latest/admin_manual/exapps_management/AppAPIAndExternalApps.html#setup-deploy-daemon). It's recommended to use the [Docker Socket Proxy](https://github.com/nextcloud/docker-socket-proxy#readme) to communicate with the docker daemon. +3. Setup a [Deploy Daemon](https://docs.nextcloud.com/server/latest/admin_manual/exapps_management/AppAPIAndExternalApps.html#setup-deploy-daemon): + - **For Nextcloud 32+**: It's recommended to use [HaRP (AppAPI HaProxy Reversed Proxy)](https://github.com/nextcloud/HaRP) for better performance and simplified deployment. + - **For older versions**: Use the [Docker Socket Proxy](https://github.com/nextcloud/docker-socket-proxy#readme) to communicate with the docker daemon. ## Installation @@ -164,3 +167,37 @@ nc_py_api._exceptions.NextcloudException: [400] Bad Request :warning: Make sure to create the docker network `nextcloud` before starting the stack. If you don't declare the network as > `external`, `docker-compose` will create the network with some [project/directory prefix](https://docs.docker.com/compose/how-tos/networking/), which will cause the Deploy Daemon to fail because it doesn't find the network. + +## HaRP Support (Nextcloud 32+) + +Since Nextcloud 32, [HaRP (AppAPI HaProxy Reversed Proxy)](https://github.com/nextcloud/HaRP) is the recommended deployment method for ExApps. This app now supports HaRP out of the box. + +### What is HaRP? + +HaRP is a reverse proxy system that simplifies the deployment workflow for Nextcloud's AppAPI. It enables direct communication between clients and ExApps, bypassing the Nextcloud instance to improve performance and reduce complexity compared to the Docker Socket Proxy (DSP) setup. + +### Key Benefits + +- **Simplified Deployment**: Replaces more complex setups with an easy-to-use container +- **Better Performance**: Routes requests directly to ExApps +- **Enhanced Security**: Uses brute-force protection and basic authentication +- **Flexible**: Supports both HTTP and HTTPS for ExApps and Nextcloud control + +### Installation + +Follow the [HaRP installation guide](https://github.com/nextcloud/HaRP#how-to-install-it) to deploy HaRP on your system. Once HaRP is running, you can install this ExApp through the Nextcloud AppStore or via the `occ` command as described in the [Installation](#installation) section. + +### Migration from Docker Socket Proxy + +If you're upgrading to Nextcloud 32 and want to migrate from DSP to HaRP: + +1. Install HaRP on the same Docker Engine that you were using for DSP +2. Test deployment on HaRP using the usual "TestDeploy" button +3. Set HaRP as the default deployment daemon for ExApps +4. Remove the ExApp **without** deleting its data volumes: + - From terminal: Do not use the `--rm-data` option + - From UI: Do not check "Delete data when removing" +5. Reinstall the ExApp - it will now use HaRP +6. Remove DSP once all ExApps have been migrated + +For more details, see the [HaRP migration guide](https://github.com/nextcloud/HaRP#nextcloud-32-migrating-existing-exapps-from-dsp-to-harp). diff --git a/start.sh b/start.sh new file mode 100644 index 0000000..82eaeb3 --- /dev/null +++ b/start.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +set -e + +# Only create a config file if HP_SHARED_KEY is set. +if [ -n "$HP_SHARED_KEY" ]; then + echo "HP_SHARED_KEY is set, creating /frpc.toml configuration file..." + if [ -d "/certs/frp" ]; then + echo "Found /certs/frp directory. Creating configuration with TLS certificates." + cat < /frpc.toml +serverAddr = "$HP_FRP_ADDRESS" +serverPort = $HP_FRP_PORT +loginFailExit = false + +transport.tls.enable = true +transport.tls.certFile = "/certs/frp/client.crt" +transport.tls.keyFile = "/certs/frp/client.key" +transport.tls.trustedCaFile = "/certs/frp/ca.crt" +transport.tls.serverName = "harp.nc" + +metadatas.token = "$HP_SHARED_KEY" + +[[proxies]] +remotePort = $APP_PORT +type = "tcp" +name = "$APP_ID" +[proxies.plugin] +type = "unix_domain_socket" +unixPath = "/tmp/exapp.sock" +EOF + else + echo "Directory /certs/frp not found. Creating configuration without TLS certificates." + cat < /frpc.toml +serverAddr = "$HP_FRP_ADDRESS" +serverPort = $HP_FRP_PORT +loginFailExit = false + +transport.tls.enable = false + +metadatas.token = "$HP_SHARED_KEY" + +[[proxies]] +remotePort = $APP_PORT +type = "tcp" +name = "$APP_ID" +[proxies.plugin] +type = "unix_domain_socket" +unixPath = "/tmp/exapp.sock" +EOF + fi +else + echo "HP_SHARED_KEY is not set. Skipping FRP configuration." +fi + +# If we have a configuration file and the shared key is present, start the FRP client +if [ -f /frpc.toml ] && [ -n "$HP_SHARED_KEY" ]; then + echo "Starting frpc in the background..." + frpc -c /frpc.toml & +fi + +# Start the main application (launch cmd for ExApp is an argument for this script) +echo "Starting application: $@" +exec "$@" From 5c475061c8791f401917876e16bc69af77a6d5d9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 27 Oct 2025 18:49:00 +0000 Subject: [PATCH 03/10] Make FRP version configurable via build argument Co-authored-by: R0Wi <19730957+R0Wi@users.noreply.github.com> --- Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1fe179a..c95c7ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ FROM python:3.12-alpine3.21 AS app ARG USER=serviceuser +ARG FRP_VERSION=0.61.1 ENV HOME=/home/$USER RUN apk update && \ @@ -11,14 +12,14 @@ RUN apk update && \ RUN set -ex; \ ARCH=$(uname -m); \ if [ "$ARCH" = "aarch64" ]; then \ - FRP_URL="https://raw.githubusercontent.com/nextcloud/HaRP/main/exapps_dev/frp_0.61.1_linux_arm64.tar.gz"; \ + FRP_URL="https://raw.githubusercontent.com/nextcloud/HaRP/main/exapps_dev/frp_${FRP_VERSION}_linux_arm64.tar.gz"; \ else \ - FRP_URL="https://raw.githubusercontent.com/nextcloud/HaRP/main/exapps_dev/frp_0.61.1_linux_amd64.tar.gz"; \ + FRP_URL="https://raw.githubusercontent.com/nextcloud/HaRP/main/exapps_dev/frp_${FRP_VERSION}_linux_amd64.tar.gz"; \ fi; \ echo "Downloading FRP client from $FRP_URL"; \ curl -L "$FRP_URL" -o /tmp/frp.tar.gz; \ tar -C /tmp -xzf /tmp/frp.tar.gz; \ - mv /tmp/frp_0.61.1_linux_* /tmp/frp; \ + mv /tmp/frp_${FRP_VERSION}_linux_* /tmp/frp; \ cp /tmp/frp/frpc /usr/local/bin/frpc; \ chmod +x /usr/local/bin/frpc; \ rm -rf /tmp/frp /tmp/frp.tar.gz From b4c8bc536da64e774a39a8d53458333d756c0362 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 27 Oct 2025 19:29:10 +0000 Subject: [PATCH 04/10] Simplify FRP installation and condense documentation Co-authored-by: R0Wi <19730957+R0Wi@users.noreply.github.com> --- Dockerfile | 19 +------------------ README.md | 32 +++----------------------------- 2 files changed, 4 insertions(+), 47 deletions(-) diff --git a/Dockerfile b/Dockerfile index c95c7ae..6556dbe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,29 +1,12 @@ FROM python:3.12-alpine3.21 AS app ARG USER=serviceuser -ARG FRP_VERSION=0.61.1 ENV HOME=/home/$USER RUN apk update && \ - apk add --no-cache sudo ocrmypdf $(apk search tesseract-ocr-data- | sed 's/-[0-9].*//') curl bash && \ + apk add --no-cache sudo ocrmypdf $(apk search tesseract-ocr-data- | sed 's/-[0-9].*//') curl bash frp && \ adduser -D $USER -# Download and install FRP client -RUN set -ex; \ - ARCH=$(uname -m); \ - if [ "$ARCH" = "aarch64" ]; then \ - FRP_URL="https://raw.githubusercontent.com/nextcloud/HaRP/main/exapps_dev/frp_${FRP_VERSION}_linux_arm64.tar.gz"; \ - else \ - FRP_URL="https://raw.githubusercontent.com/nextcloud/HaRP/main/exapps_dev/frp_${FRP_VERSION}_linux_amd64.tar.gz"; \ - fi; \ - echo "Downloading FRP client from $FRP_URL"; \ - curl -L "$FRP_URL" -o /tmp/frp.tar.gz; \ - tar -C /tmp -xzf /tmp/frp.tar.gz; \ - mv /tmp/frp_${FRP_VERSION}_linux_* /tmp/frp; \ - cp /tmp/frp/frpc /usr/local/bin/frpc; \ - chmod +x /usr/local/bin/frpc; \ - rm -rf /tmp/frp /tmp/frp.tar.gz - USER $USER WORKDIR /app diff --git a/README.md b/README.md index 257a506..600b0f2 100644 --- a/README.md +++ b/README.md @@ -170,34 +170,8 @@ nc_py_api._exceptions.NextcloudException: [400] Bad Request Date: Tue, 28 Oct 2025 12:09:14 +0000 Subject: [PATCH 05/10] Devcontainer adjustments * Add docker socket to devcontainer * Consolidate chmod +x for start.sh --- .devcontainer/devcontainer.json | 8 ++++++-- Dockerfile | 13 +++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 1881bac..2c7f3c4 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -13,5 +13,9 @@ "pamaron.pytest-runner" ] } - } -} + }, + "mounts": [ + "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind" + ], + "postStartCommand": "sudo chown $USER:$USER /var/run/docker.sock" +} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 6556dbe..a862d27 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,26 +16,23 @@ COPY --chown=$USER:$USER main.py . COPY --chown=$USER:$USER workflow_ocr_backend/ ./workflow_ocr_backend COPY --chown=$USER:$USER start.sh /start.sh -RUN pip install -r requirements.txt - -# Make start.sh executable -USER root -RUN chmod +x /start.sh -USER $USER +RUN chmod +x /start.sh && \ + pip install -r requirements.txt ENTRYPOINT ["/start.sh", "python3", "-u", "main.py"] FROM app AS devcontainer +ENV USER=${USER} COPY --chown=$USER:$USER requirements-dev.txt requirements-dev.txt # Install dev dependencies and set up sudo USER root -RUN apk add --no-cache git curl make gnupg && \ +RUN apk add --no-cache git docker-cli make gnupg && \ echo "$USER ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USER && \ chmod 0440 /etc/sudoers.d/$USER USER $USER -RUN pip install -r requirements-dev.txt +RUN pip install -r requirements-dev.txt FROM devcontainer AS test From 282a7564ee1f399839e68616cbbcc3b1314390e2 Mon Sep 17 00:00:00 2001 From: Robin Windey Date: Thu, 30 Oct 2025 16:31:15 +0000 Subject: [PATCH 06/10] Fix /frpc.toml permission issue * Ensure containers "serviceuser" is able to write /frpc.toml * Write python integration tests to ensure container is able to start properly with frpc enabled --- Dockerfile | 5 +- test/test_docker_integration.py | 143 ++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 test/test_docker_integration.py diff --git a/Dockerfile b/Dockerfile index a862d27..704996f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,10 @@ ENV HOME=/home/$USER RUN apk update && \ apk add --no-cache sudo ocrmypdf $(apk search tesseract-ocr-data- | sed 's/-[0-9].*//') curl bash frp && \ - adduser -D $USER + adduser -D $USER && \ + touch /frpc.toml && \ + chown $USER:$USER /frpc.toml && \ + chmod 600 /frpc.toml USER $USER diff --git a/test/test_docker_integration.py b/test/test_docker_integration.py new file mode 100644 index 0000000..1cb1369 --- /dev/null +++ b/test/test_docker_integration.py @@ -0,0 +1,143 @@ +import os +import shutil +import subprocess +import time +import uuid +from pathlib import Path + +import pytest +from dotenv import dotenv_values + +@pytest.mark.skipif(shutil.which("docker") is None, reason="Docker CLI not available") +def test_docker_container_starts_and_creates_frpc_config(): + repo_root = Path(__file__).resolve().parent.parent + image_tag = f"workflow-ocr-backend-test:{uuid.uuid4().hex}" + container_name = f"workflow-ocr-backend-test-{uuid.uuid4().hex}" + + build_proc = run_command( + [ + "docker", + "build", + "--target", + "app", + "-t", + image_tag, + str(repo_root), + ], + capture_output=True, + text=True, + cwd=repo_root, + ) + if build_proc.returncode != 0: + pytest.fail(f"docker build failed: {build_proc.stderr}\n{build_proc.stdout}") + + # Get basic env vars from ".env" file + env_vars = dotenv_values() + # Add additional required env vars for the test + env_vars.update({ + "HP_SHARED_KEY": "docker_test_key", + "HP_FRP_ADDRESS": "frp.example.com", + "HP_FRP_PORT": "7000" + }) + + run_cmd: list[str] = ["docker", "run", "-d", "--name", container_name] + for key, value in env_vars.items(): + run_cmd.extend(["-e", f"{key}={value}"]) + run_cmd.append(image_tag) + + run_proc = run_command(run_cmd, capture_output=True, text=True) + if run_proc.returncode != 0: + cleanup_docker(container_name, image_tag) + pytest.fail(f"docker run failed: {run_proc.stderr}\n{run_proc.stdout}") + + container_id = run_proc.stdout.strip() + if not container_id: + cleanup_docker(container_name, image_tag) + pytest.fail("docker run did not return a container ID") + + try: + deadline = time.time() + 45 + while time.time() < deadline: + status_proc = run_command( + ["docker", "inspect", "-f", "{{.State.Running}}", container_name], + capture_output=True, + text=True, + ) + if status_proc.returncode == 0 and status_proc.stdout.strip() == "true": + break + time.sleep(1) + else: + logs = docker_logs(container_name) + pytest.fail(f"Container did not reach running state in time. Logs:\n{logs}") + + logs = "" + lower_logs = "" + wait_deadline = time.time() + 60 + while time.time() < wait_deadline: + logs = docker_logs(container_name) + lower_logs = logs.lower() + if "permission denied" in lower_logs: + pytest.fail(f"Permission issue encountered in logs:\n{logs}") + has_config_message = "creating /frpc.toml configuration file" in lower_logs + has_app_start_message = ( + "starting application: python3 -u main.py" in lower_logs + or "started server process" in lower_logs + ) + if has_config_message and has_app_start_message: + break + time.sleep(1) + else: + pytest.fail( + "Expected startup confirmation messages not found in container logs:\n" + f"{logs}" + ) + + config_proc = run_command( + ["docker", "exec", container_name, "cat", "/frpc.toml"], + capture_output=True, + text=True, + ) + if config_proc.returncode != 0: + pytest.fail( + "Unable to read /frpc.toml inside container:\n" + f"stdout: {config_proc.stdout}\n" + f"stderr: {config_proc.stderr}" + ) + config_contents = config_proc.stdout + assert "remotePort = " + env_vars["APP_PORT"] in config_contents + assert 'metadatas.token = "docker_test_key"' in config_contents + assert 'transport.tls.enable = false' in config_contents + finally: + cleanup_docker(container_name, image_tag) + + +def docker_logs(container_name: str) -> str: + # Combine stdout/stderr so container error logs are not dropped. + logs_proc = run_command( + ["docker", "logs", container_name], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + output = logs_proc.stdout or "" + if logs_proc.returncode != 0: + return "\n" f"output: {output}" + return output + + +def cleanup_docker(container_name: str, image_tag: str) -> None: + run_command( + ["docker", "rm", "-f", container_name], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + run_command( + ["docker", "rmi", "-f", image_tag], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + +def run_command(args: list[str], **kwargs) -> subprocess.CompletedProcess: + kwargs.setdefault("timeout", 180) + return subprocess.run(args, **kwargs) From 8f092d873b2516c655af9613f335b0af9966e5ba Mon Sep 17 00:00:00 2001 From: Robin Windey Date: Thu, 30 Oct 2025 17:01:49 +0000 Subject: [PATCH 07/10] Fix python test setup --- .github/actions/docker-setup/action.yml | 6 ++++-- .github/workflows/test.yml | 28 ++++++++++++++++++++++--- Makefile | 6 +++++- pytest.ini | 3 +++ test/test_docker_integration.py | 1 + 5 files changed, 38 insertions(+), 6 deletions(-) create mode 100644 pytest.ini diff --git a/.github/actions/docker-setup/action.yml b/.github/actions/docker-setup/action.yml index e56eee7..b739577 100644 --- a/.github/actions/docker-setup/action.yml +++ b/.github/actions/docker-setup/action.yml @@ -2,8 +2,9 @@ name: Setup Docker workflow description: Shared steps for Docker builds and tests. inputs: registry-token: - description: Token used to authenticate against the container registry. - required: true + description: Token used to authenticate against the container registry. Optional — if not provided the login step will be skipped. + required: false + default: '' outputs: image_base: description: Base image name used for GHCR publishes. @@ -32,6 +33,7 @@ runs: images: ${{ steps.image_base.outputs.IMAGE_BASE }} - name: Login to GitHub Container Registry + if: ${{ inputs.registry-token != '' }} uses: docker/login-action@v3 with: registry: ghcr.io diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d9980e1..22bf42a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,8 +15,6 @@ jobs: - name: Setup Docker workflow id: docker_setup uses: ./.github/actions/docker-setup - with: - registry-token: ${{ secrets.GITHUB_TOKEN }} - name: Build Docker test image uses: docker/build-push-action@v6 @@ -57,4 +55,28 @@ jobs: name: test-results path: | coverage - code-coverage-results.md \ No newline at end of file + code-coverage-results.md + + integration-test: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Docker workflow + id: docker_setup + uses: ./.github/actions/docker-setup + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install dev dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements-dev.txt + + - name: Run Docker Integration Tests + run: make docker-integrationtest \ No newline at end of file diff --git a/Makefile b/Makefile index 692579d..030bec7 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,11 @@ deps: .PHONY: test test: - python -m pytest --cov-report html:coverage --cov-report xml:coverage/coverage.xml --cov=workflow_ocr_backend test + python -m pytest --cov-report html:coverage --cov-report xml:coverage/coverage.xml --cov=workflow_ocr_backend -m "not docker_integration" test + +.PHONY: docker-integrationtest +docker-integrationtest: + python -m pytest -m "docker_integration" test .PHONY: build build: diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..781b066 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +markers = + docker_integration: mark test as docker integration. Need to run on a Docker host \ No newline at end of file diff --git a/test/test_docker_integration.py b/test/test_docker_integration.py index 1cb1369..6dff4f8 100644 --- a/test/test_docker_integration.py +++ b/test/test_docker_integration.py @@ -9,6 +9,7 @@ from dotenv import dotenv_values @pytest.mark.skipif(shutil.which("docker") is None, reason="Docker CLI not available") +@pytest.mark.docker_integration def test_docker_container_starts_and_creates_frpc_config(): repo_root = Path(__file__).resolve().parent.parent image_tag = f"workflow-ocr-backend-test:{uuid.uuid4().hex}" From 6176871c86c5a81f6f8c0bca73905820a445959f Mon Sep 17 00:00:00 2001 From: Robin Windey Date: Sun, 2 Nov 2025 20:11:22 +0000 Subject: [PATCH 08/10] Fix NC environment permission issues * Switch to "gosu" to start the main process as "serviceuser" instead of using docker "USER" directive * Default user for docker commands will be root now (so NC docker commands are able to copy certs etc) * Let serviceuser own /certs directory so that frpc process is able to use the certs from there --- Dockerfile | 44 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 704996f..30f0f3c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,42 @@ FROM python:3.12-alpine3.21 AS app ARG USER=serviceuser + +ENV USER=$USER ENV HOME=/home/$USER +ENV GOSU_VERSION=1.19 RUN apk update && \ - apk add --no-cache sudo ocrmypdf $(apk search tesseract-ocr-data- | sed 's/-[0-9].*//') curl bash frp && \ - adduser -D $USER && \ - touch /frpc.toml && \ - chown $USER:$USER /frpc.toml && \ + apk add --no-cache ocrmypdf $(apk search tesseract-ocr-data- | sed 's/-[0-9].*//') curl bash frp ca-certificates && \ + adduser -D $USER + + # TODO :: put together after testing +RUN touch /frpc.toml && \ + mkdir -p /certs && \ + chown -R $USER:$USER /frpc.toml /certs && \ chmod 600 /frpc.toml -USER $USER +RUN set -eux; \ + \ + apk add --no-cache --virtual .gosu-deps \ + ca-certificates \ + dpkg \ + gnupg \ + ; \ + \ + dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \ + wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch"; \ + wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch.asc"; \ + \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \ + \ + apk del --no-network .gosu-deps; \ + \ + chmod +x /usr/local/bin/gosu WORKDIR /app @@ -18,20 +44,18 @@ COPY --chown=$USER:$USER requirements.txt requirements.txt COPY --chown=$USER:$USER main.py . COPY --chown=$USER:$USER workflow_ocr_backend/ ./workflow_ocr_backend COPY --chown=$USER:$USER start.sh /start.sh - RUN chmod +x /start.sh && \ - pip install -r requirements.txt + pip install -r requirements.txt -ENTRYPOINT ["/start.sh", "python3", "-u", "main.py"] +ENTRYPOINT ["/bin/sh", "-c", "exec gosu \"$USER\" /start.sh python3 -u main.py"] FROM app AS devcontainer -ENV USER=${USER} COPY --chown=$USER:$USER requirements-dev.txt requirements-dev.txt # Install dev dependencies and set up sudo USER root -RUN apk add --no-cache git docker-cli make gnupg && \ +RUN apk add --no-cache sudo git docker-cli make gnupg && \ echo "$USER ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USER && \ chmod 0440 /etc/sudoers.d/$USER USER $USER From 725ea770505b3442aa6953d425e11ea0aacaeb47 Mon Sep 17 00:00:00 2001 From: Robin Windey Date: Sun, 16 Nov 2025 18:36:36 +0000 Subject: [PATCH 09/10] Add HaRP integration test --- .github/workflows/test.yml | 6 +- .vscode/mcp.json | 11 + Dockerfile | 9 +- Makefile | 6 +- pytest.ini | 4 +- test/test_docker_integration.py | 144 -------------- test/test_harp_integration.py | 342 ++++++++++++++++++++++++++++++++ 7 files changed, 366 insertions(+), 156 deletions(-) create mode 100644 .vscode/mcp.json delete mode 100644 test/test_docker_integration.py create mode 100644 test/test_harp_integration.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 22bf42a..f050a6f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -57,7 +57,7 @@ jobs: coverage code-coverage-results.md - integration-test: + harp-integration-test: runs-on: ubuntu-latest steps: @@ -78,5 +78,5 @@ jobs: python -m pip install --upgrade pip pip install -r requirements-dev.txt - - name: Run Docker Integration Tests - run: make docker-integrationtest \ No newline at end of file + - name: Run HaRP Docker Integration Tests + run: make harp-integrationtest \ No newline at end of file diff --git a/.vscode/mcp.json b/.vscode/mcp.json new file mode 100644 index 0000000..0d68a6f --- /dev/null +++ b/.vscode/mcp.json @@ -0,0 +1,11 @@ +{ + "servers": { + "github/github-mcp-server": { + "type": "http", + "url": "https://api.githubcopilot.com/mcp/", + "gallery": "https://api.mcp.github.com", + "version": "0.13.0" + } + }, + "inputs": [] +} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 30f0f3c..82bc629 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-alpine3.21 AS app +FROM python:3.12-alpine AS app ARG USER=serviceuser @@ -8,14 +8,13 @@ ENV GOSU_VERSION=1.19 RUN apk update && \ apk add --no-cache ocrmypdf $(apk search tesseract-ocr-data- | sed 's/-[0-9].*//') curl bash frp ca-certificates && \ - adduser -D $USER - - # TODO :: put together after testing -RUN touch /frpc.toml && \ + adduser -D $USER && \ + touch /frpc.toml && \ mkdir -p /certs && \ chown -R $USER:$USER /frpc.toml /certs && \ chmod 600 /frpc.toml +# Install GOSU RUN set -eux; \ \ apk add --no-cache --virtual .gosu-deps \ diff --git a/Makefile b/Makefile index 030bec7..897def4 100644 --- a/Makefile +++ b/Makefile @@ -8,9 +8,9 @@ deps: test: python -m pytest --cov-report html:coverage --cov-report xml:coverage/coverage.xml --cov=workflow_ocr_backend -m "not docker_integration" test -.PHONY: docker-integrationtest -docker-integrationtest: - python -m pytest -m "docker_integration" test +.PHONY: harp-integrationtest +harp-integrationtest: + python -m pytest -m "harp_integration" test .PHONY: build build: diff --git a/pytest.ini b/pytest.ini index 781b066..6a37091 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,5 @@ [pytest] +log_cli = true +log_cli_level = INFO markers = - docker_integration: mark test as docker integration. Need to run on a Docker host \ No newline at end of file + harp_integration: mark test as full HaRP integration. Spins up HaRP container and manages ExApp lifecycle \ No newline at end of file diff --git a/test/test_docker_integration.py b/test/test_docker_integration.py deleted file mode 100644 index 6dff4f8..0000000 --- a/test/test_docker_integration.py +++ /dev/null @@ -1,144 +0,0 @@ -import os -import shutil -import subprocess -import time -import uuid -from pathlib import Path - -import pytest -from dotenv import dotenv_values - -@pytest.mark.skipif(shutil.which("docker") is None, reason="Docker CLI not available") -@pytest.mark.docker_integration -def test_docker_container_starts_and_creates_frpc_config(): - repo_root = Path(__file__).resolve().parent.parent - image_tag = f"workflow-ocr-backend-test:{uuid.uuid4().hex}" - container_name = f"workflow-ocr-backend-test-{uuid.uuid4().hex}" - - build_proc = run_command( - [ - "docker", - "build", - "--target", - "app", - "-t", - image_tag, - str(repo_root), - ], - capture_output=True, - text=True, - cwd=repo_root, - ) - if build_proc.returncode != 0: - pytest.fail(f"docker build failed: {build_proc.stderr}\n{build_proc.stdout}") - - # Get basic env vars from ".env" file - env_vars = dotenv_values() - # Add additional required env vars for the test - env_vars.update({ - "HP_SHARED_KEY": "docker_test_key", - "HP_FRP_ADDRESS": "frp.example.com", - "HP_FRP_PORT": "7000" - }) - - run_cmd: list[str] = ["docker", "run", "-d", "--name", container_name] - for key, value in env_vars.items(): - run_cmd.extend(["-e", f"{key}={value}"]) - run_cmd.append(image_tag) - - run_proc = run_command(run_cmd, capture_output=True, text=True) - if run_proc.returncode != 0: - cleanup_docker(container_name, image_tag) - pytest.fail(f"docker run failed: {run_proc.stderr}\n{run_proc.stdout}") - - container_id = run_proc.stdout.strip() - if not container_id: - cleanup_docker(container_name, image_tag) - pytest.fail("docker run did not return a container ID") - - try: - deadline = time.time() + 45 - while time.time() < deadline: - status_proc = run_command( - ["docker", "inspect", "-f", "{{.State.Running}}", container_name], - capture_output=True, - text=True, - ) - if status_proc.returncode == 0 and status_proc.stdout.strip() == "true": - break - time.sleep(1) - else: - logs = docker_logs(container_name) - pytest.fail(f"Container did not reach running state in time. Logs:\n{logs}") - - logs = "" - lower_logs = "" - wait_deadline = time.time() + 60 - while time.time() < wait_deadline: - logs = docker_logs(container_name) - lower_logs = logs.lower() - if "permission denied" in lower_logs: - pytest.fail(f"Permission issue encountered in logs:\n{logs}") - has_config_message = "creating /frpc.toml configuration file" in lower_logs - has_app_start_message = ( - "starting application: python3 -u main.py" in lower_logs - or "started server process" in lower_logs - ) - if has_config_message and has_app_start_message: - break - time.sleep(1) - else: - pytest.fail( - "Expected startup confirmation messages not found in container logs:\n" - f"{logs}" - ) - - config_proc = run_command( - ["docker", "exec", container_name, "cat", "/frpc.toml"], - capture_output=True, - text=True, - ) - if config_proc.returncode != 0: - pytest.fail( - "Unable to read /frpc.toml inside container:\n" - f"stdout: {config_proc.stdout}\n" - f"stderr: {config_proc.stderr}" - ) - config_contents = config_proc.stdout - assert "remotePort = " + env_vars["APP_PORT"] in config_contents - assert 'metadatas.token = "docker_test_key"' in config_contents - assert 'transport.tls.enable = false' in config_contents - finally: - cleanup_docker(container_name, image_tag) - - -def docker_logs(container_name: str) -> str: - # Combine stdout/stderr so container error logs are not dropped. - logs_proc = run_command( - ["docker", "logs", container_name], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - ) - output = logs_proc.stdout or "" - if logs_proc.returncode != 0: - return "\n" f"output: {output}" - return output - - -def cleanup_docker(container_name: str, image_tag: str) -> None: - run_command( - ["docker", "rm", "-f", container_name], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - run_command( - ["docker", "rmi", "-f", image_tag], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - - -def run_command(args: list[str], **kwargs) -> subprocess.CompletedProcess: - kwargs.setdefault("timeout", 180) - return subprocess.run(args, **kwargs) diff --git a/test/test_harp_integration.py b/test/test_harp_integration.py new file mode 100644 index 0000000..e107679 --- /dev/null +++ b/test/test_harp_integration.py @@ -0,0 +1,342 @@ + +import os +import shutil +import socket +import subprocess +import time +import uuid +from pathlib import Path +from typing import Iterable + +import json +import httpx +import pytest +from dotenv import dotenv_values +import logging + +pytestmark = [ + pytest.mark.skipif(shutil.which("docker") is None, reason="Docker CLI not available"), + pytest.mark.harp_integration, +] + +HARP_IMAGE = os.environ.get("HARP_TEST_IMAGE", "ghcr.io/nextcloud/nextcloud-appapi-harp:release") +LOCAL_DOCKER_ENGINE_PORT = 24000 +AGENT_TIMEOUT = 180 +CONTAINER_TIMEOUT = 180 + + +def test_harp_deploys_and_configures_frp(tmp_path): + repo_root = Path(__file__).resolve().parent.parent + env_vars = dotenv_values() + required_keys = ["APP_ID", "APP_PORT", "APP_VERSION", "APP_SECRET", "AA_VERSION"] + missing = [key for key in required_keys if not env_vars.get(key)] + if missing: + pytest.skip(f"Missing required keys in .env for HaRP test: {', '.join(missing)}") + + shared_key = f"harp-key-{uuid.uuid4().hex}" + harp_container = f"harp-integration-{uuid.uuid4().hex[:8]}" + network_name = f"harp-net-{uuid.uuid4().hex[:8]}" + image_tag = f"workflow-ocr-backend-harp:{uuid.uuid4().hex}" + instance_id = f"it{uuid.uuid4().hex[:8]}" + exapp_name = env_vars["APP_ID"] + exapp_container = f"nc_app_{instance_id}_{exapp_name}" + volume_name = f"{exapp_container}_data" + exapps_port = _reserve_host_port() + frp_port = _reserve_host_port() + cert_dir = tmp_path / "harp-certs" + cert_dir.mkdir(parents=True, exist_ok=True) + + app_env = {key: value for key, value in env_vars.items() if value is not None} + app_env.update( + { + "HP_SHARED_KEY": shared_key, + "HP_FRP_ADDRESS": f"{harp_container}:8782", + "HP_FRP_PORT": "8782", + "HP_FRP_DISABLE_TLS": "false", + } + ) + + network_created = False + harp_started = False + http_client: httpx.Client | None = None + try: + _build_app_image(repo_root, image_tag) + _run_command(["docker", "network", "create", network_name], check=True) + network_created = True + + # Create HaRP container + harp_run_cmd = [ + "docker", + "run", + "-d", + "--name", + harp_container, + "--hostname", + harp_container, + "--network", + network_name, + "-v", + "/var/run/docker.sock:/var/run/docker.sock", + "-v", + f"{cert_dir}:/certs", + "-e", + f"HP_SHARED_KEY={shared_key}", + "-e", + "NC_INSTANCE_URL=http://nextcloud.local", + "-e", + "HP_LOG_LEVEL=debug", + "-e", + "HP_VERBOSE_START=1", + "-p", + f"{exapps_port}:8780", + "-p", + f"{frp_port}:8782", + HARP_IMAGE, + ] + _run_command(harp_run_cmd, check=True) + harp_started = True + + http_client = httpx.Client(timeout=15.0, trust_env=False) + exapps_base = f"http://127.0.0.1:{exapps_port}" + agent_headers = { + "harp-shared-key": shared_key, + "EX-APP-ID": exapp_name, + "AUTHORIZATION-APP-API": "test-token", + "AA-VERSION": env_vars.get("AA_VERSION", "32"), + } + + # Wait for HaRP agent to be ready via ExApps app_api + _wait_for_agent_ready_via_exapps(http_client, exapps_base, agent_headers) + + docker_headers = agent_headers | {"docker-engine-port": str(LOCAL_DOCKER_ENGINE_PORT)} + metadata_payload = { + "exapp_token": "test-token", + "exapp_version": env_vars["APP_VERSION"], + "host": "127.0.0.1", + "port": int(env_vars["APP_PORT"]), + "routes": [], + } + + # Create ExApp storage + resp = _call_exapps( + http_client, + exapps_base, + "POST", + f"/exapps/app_api/exapp_storage/{exapp_name}", + agent_headers, + json_payload=metadata_payload, + ) + if resp.status_code != 204: + pytest.fail(f"Failed to seed ExApp metadata: {resp.status_code} {resp.text}") + + env_list = _format_env(app_env) + create_payload = { + "name": exapp_name, + "instance_id": instance_id, + "image_id": image_tag, + "network_mode": network_name, + "environment_variables": env_list, + "mount_points": [], + "resource_limits": {}, + "restart_policy": "no", + } + + # Create ExApp container + resp = _call_exapps(http_client, exapps_base, "POST", "/exapps/app_api/docker/exapp/create", docker_headers, json_payload=create_payload, timeout=60.0) + if resp.status_code != 201: + harp_logs = docker_logs(harp_container) + pytest.fail( + "HaRP failed to create ExApp: " + f"{resp.status_code} {resp.text}\nHaRP logs:\n{harp_logs}" + ) + + install_payload = { + "name": exapp_name, + "instance_id": instance_id, + "system_certs_bundle": None, + "install_frp_certs": True, + } + + # Install certificates in ExApp container + resp = _call_exapps(http_client, exapps_base, "POST", "/exapps/app_api/docker/exapp/install_certificates", docker_headers, json_payload=install_payload) + if resp.status_code != 204: + pytest.fail(f"Failed to install certificates: {resp.status_code} {resp.text}") + + # Start ExApp container + resp = _call_exapps(http_client, exapps_base, "POST", "/exapps/app_api/docker/exapp/start", docker_headers, json_payload={"name": exapp_name, "instance_id": instance_id}) + if resp.status_code not in (200, 204): + pytest.fail(f"Failed to start ExApp container: {resp.status_code} {resp.text}") + + # Wait for ExApp to be running + _wait_for_exapp_start_via_exapps(http_client, exapps_base, docker_headers, exapp_name, instance_id) + + # ASSERTIONS + logs = docker_logs(exapp_container) + if "permission denied" in logs.lower(): + pytest.fail(f"Detected permission issue in ExApp logs:\n{logs}") + assert "Found /certs/frp directory." in logs, f"FRP certificates were not detected in logs:\n{logs}" + assert "Creating configuration with TLS certificates" in logs + + _assert_cert_files_present(exapp_container) + config_contents = docker_exec(exapp_container, ["cat", "/frpc.toml"]) + assert 'transport.tls.enable = true' in config_contents + assert 'transport.tls.certFile = "/certs/frp/client.crt"' in config_contents + assert f'remotePort = {env_vars["APP_PORT"]}' in config_contents + assert f'metadatas.token = "{shared_key}"' in config_contents + + # Cleanup ExApp container + resp = _call_exapps(http_client, exapps_base, "POST", "/exapps/app_api/docker/exapp/remove", docker_headers, json_payload={"name": exapp_name, "instance_id": instance_id, "remove_data": True}) + if resp.status_code != 204: + pytest.fail(f"Failed to remove ExApp container: {resp.status_code} {resp.text}") + + resp = _call_exapps(http_client, exapps_base, "POST", "/exapps/app_api/docker/exapp/exists", docker_headers, json_payload={"name": exapp_name, "instance_id": instance_id}) + if resp.status_code != 200 or resp.json().get("exists"): + pytest.fail("ExApp container still exists after removal") + + _run_command(["docker", "volume", "rm", "-f", volume_name]) + + finally: + if http_client is not None: + http_client.close() + container_exists = _run_command( + ["docker", "ps", "-a", "-q", "-f", f"name={exapp_container}"], + capture_output=True, + text=True, + ).stdout.strip() + if container_exists: + app_logs = docker_logs(exapp_container) + if app_logs: + _log(f"\n\n#### App Container Logs #### \n\n{app_logs}") + _run_command(["docker", "rm", "-f", exapp_container]) + if harp_started: + _run_command(["docker", "rm", "-f", harp_container]) + if network_created: + _run_command(["docker", "network", "rm", network_name]) + + +def _build_app_image(repo_root: Path, image_tag: str) -> None: + build_cmd = [ + "docker", + "build", + "--target", + "app", + "-t", + image_tag, + str(repo_root), + ] + result = _run_command(build_cmd, capture_output=True, check=False) + if result.returncode != 0: + pytest.fail(f"docker build failed: {result.stderr}\n{result.stdout}") + + +def _run_command( + args: list[str], *, check: bool = False, capture_output: bool = False, text: bool = True +) -> subprocess.CompletedProcess: + return subprocess.run( + args, + check=check, + capture_output=capture_output, + text=text, + ) + +def _reserve_host_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("", 0)) + return sock.getsockname()[1] + +def _call_exapps(http_client: httpx.Client, exapps_base: str, method: str, path: str, headers: dict, json_payload=None, timeout: float = 15.0): + if not path.startswith("/"): + path = "/" + path + url = f"{exapps_base}{path}" + try: + if method.upper() == "GET": + return http_client.get(url, headers=headers, timeout=timeout) + if method.upper() == "POST": + return http_client.post(url, headers=headers, json=json_payload, timeout=timeout) + raise ValueError(f"Unsupported method: {method}") + except httpx.ConnectError: + raise + +def _wait_for_agent_ready_via_exapps(http_client: httpx.Client, exapps_base: str, headers: dict) -> None: + probe_headers = headers.copy() + # Ask HaRP to use the local (built-in) Docker Engine remote port. + probe_headers.setdefault("docker-engine-port", str(LOCAL_DOCKER_ENGINE_PORT)) + # AA-VERSION isn't relevant for app_api ping but keep a sane default + probe_headers.setdefault("AA-VERSION", "32") + + ping_path = "/exapps/app_api/v1.41/_ping" + deadline = time.time() + AGENT_TIMEOUT + while time.time() < deadline: + try: + resp = _call_exapps(http_client, exapps_base, "GET", ping_path, probe_headers) + if resp.status_code == 200: + # Expect the body to contain 'OK' + try: + body = resp.text.strip() + except Exception: + body = "" + if body == "OK": + return + except httpx.ConnectError as ce: + _log(f"ConnectError while probing HaRP readiness via ExApps: {ce}") + except httpx.ReadError as re: + _log(f"ReadError while probing HaRP readiness via ExApps: {re}") + except httpx.RemoteProtocolError as rpe: + _log(f"RemoteProtocolError while probing HaRP readiness via ExApps: {rpe}") + time.sleep(1) + pytest.fail("Timed out waiting for HaRP readiness via ExApps app_api _ping") + +def _wait_for_exapp_start_via_exapps(http_client: httpx.Client, exapps_base: str, headers: dict, app_name: str, instance_id: str) -> None: + payload = {"name": app_name, "instance_id": instance_id} + probe_headers = headers.copy() + probe_headers.setdefault("EX-APP-ID", app_name) + probe_headers.setdefault("AUTHORIZATION-APP-API", "test-token") + deadline = time.time() + CONTAINER_TIMEOUT + while time.time() < deadline: + try: + resp = _call_exapps(http_client, exapps_base, "POST", "/exapps/app_api/docker/exapp/wait_for_start", probe_headers, json_payload=payload) + if resp.status_code == 200: + try: + j = resp.json() + if j.get("started"): + return + except Exception: + pass + except httpx.ConnectError as ce: + _log(f"ConnectError while waiting for ExApp start via ExApps: {ce}") + except httpx.RemoteProtocolError as rpe: + _log(f"RemoteProtocolError while waiting for ExApp start via ExApps: {rpe}") + time.sleep(2) + pytest.fail("ExApp container did not reach running state in time (via ex_apps)") + +def _format_env(items: dict[str, str]) -> list[str]: + formatted: list[str] = [] + for key, value in sorted(items.items()): + formatted.append(f"{key}={value}") + return formatted + +def docker_logs(container_name: str) -> str: + proc = _run_command(["docker", "logs", container_name], capture_output=True) + return proc.stdout + proc.stderr if proc.stderr else proc.stdout + +def docker_exec(container_name: str, cmd: Iterable[str]) -> str: + proc = _run_command(["docker", "exec", container_name, *cmd], capture_output=True) + if proc.returncode != 0: + pytest.fail( + f"docker exec {' '.join(cmd)} failed with rc={proc.returncode}:\nstdout:{proc.stdout}\nstderr:{proc.stderr}" + ) + return proc.stdout + +def _assert_cert_files_present(container_name: str) -> None: + for filename in ("ca.crt", "client.crt", "client.key"): + proc = _run_command( + ["docker", "exec", container_name, "test", "-f", f"/certs/frp/{filename}"], capture_output=True + ) + if proc.returncode != 0: + pytest.fail( + f"Expected certificate {filename} is missing in container {container_name}: {proc.stdout} {proc.stderr}" + ) + +def _log(string: str) -> None: + logger = logging.getLogger(__name__) + logger.info(string) From 7316c81955b6bac1fc1ca87afaa6aee862145f29 Mon Sep 17 00:00:00 2001 From: Robin Windey Date: Sun, 16 Nov 2025 20:00:26 +0000 Subject: [PATCH 10/10] Fix Unittests --- .github/workflows/test.yml | 5 +++-- Dockerfile | 1 + Makefile | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f050a6f..a100ef1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,7 +5,7 @@ on: pull_request: jobs: - test: + unit-test: runs-on: ubuntu-latest steps: @@ -56,7 +56,8 @@ jobs: path: | coverage code-coverage-results.md - + + # Note: harp tests need to run on a host where docker is available harp-integration-test: runs-on: ubuntu-latest diff --git a/Dockerfile b/Dockerfile index 82bc629..bf79f68 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,6 +44,7 @@ COPY --chown=$USER:$USER main.py . COPY --chown=$USER:$USER workflow_ocr_backend/ ./workflow_ocr_backend COPY --chown=$USER:$USER start.sh /start.sh RUN chmod +x /start.sh && \ + chown -R $USER:$USER /app && \ pip install -r requirements.txt ENTRYPOINT ["/bin/sh", "-c", "exec gosu \"$USER\" /start.sh python3 -u main.py"] diff --git a/Makefile b/Makefile index 897def4..0fc76ed 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ deps: .PHONY: test test: - python -m pytest --cov-report html:coverage --cov-report xml:coverage/coverage.xml --cov=workflow_ocr_backend -m "not docker_integration" test + python -m pytest --cov-report html:coverage --cov-report xml:coverage/coverage.xml --cov=workflow_ocr_backend -m "not harp_integration" test .PHONY: harp-integrationtest harp-integrationtest: