diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 1881bac..2c7f3c4 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -13,5 +13,9 @@ "pamaron.pytest-runner" ] } - } -} + }, + "mounts": [ + "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind" + ], + "postStartCommand": "sudo chown $USER:$USER /var/run/docker.sock" +} \ No newline at end of file diff --git a/.github/actions/docker-setup/action.yml b/.github/actions/docker-setup/action.yml index e56eee7..b739577 100644 --- a/.github/actions/docker-setup/action.yml +++ b/.github/actions/docker-setup/action.yml @@ -2,8 +2,9 @@ name: Setup Docker workflow description: Shared steps for Docker builds and tests. inputs: registry-token: - description: Token used to authenticate against the container registry. - required: true + description: Token used to authenticate against the container registry. Optional — if not provided the login step will be skipped. + required: false + default: '' outputs: image_base: description: Base image name used for GHCR publishes. @@ -32,6 +33,7 @@ runs: images: ${{ steps.image_base.outputs.IMAGE_BASE }} - name: Login to GitHub Container Registry + if: ${{ inputs.registry-token != '' }} uses: docker/login-action@v3 with: registry: ghcr.io diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d9980e1..a100ef1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,7 +5,7 @@ on: pull_request: jobs: - test: + unit-test: runs-on: ubuntu-latest steps: @@ -15,8 +15,6 @@ jobs: - name: Setup Docker workflow id: docker_setup uses: ./.github/actions/docker-setup - with: - registry-token: ${{ secrets.GITHUB_TOKEN }} - name: Build Docker test image uses: docker/build-push-action@v6 @@ -57,4 +55,29 @@ jobs: name: test-results path: | coverage - code-coverage-results.md \ No newline at end of file + code-coverage-results.md + + # Note: harp tests need to run on a host where docker is available + harp-integration-test: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Docker workflow + id: docker_setup + uses: ./.github/actions/docker-setup + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install dev dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements-dev.txt + + - name: Run HaRP Docker Integration Tests + run: make harp-integrationtest \ No newline at end of file diff --git a/.vscode/mcp.json b/.vscode/mcp.json new file mode 100644 index 0000000..0d68a6f --- /dev/null +++ b/.vscode/mcp.json @@ -0,0 +1,11 @@ +{ + "servers": { + "github/github-mcp-server": { + "type": "http", + "url": "https://api.githubcopilot.com/mcp/", + "gallery": "https://api.mcp.github.com", + "version": "0.13.0" + } + }, + "inputs": [] +} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index da0c810..bf79f68 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,23 +1,53 @@ -FROM python:3.12-alpine3.21 AS app +FROM python:3.12-alpine AS app ARG USER=serviceuser + +ENV USER=$USER ENV HOME=/home/$USER +ENV GOSU_VERSION=1.19 RUN apk update && \ - apk add --no-cache sudo ocrmypdf $(apk search tesseract-ocr-data- | sed 's/-[0-9].*//') && \ - adduser -D $USER + apk add --no-cache ocrmypdf $(apk search tesseract-ocr-data- | sed 's/-[0-9].*//') curl bash frp ca-certificates && \ + adduser -D $USER && \ + touch /frpc.toml && \ + mkdir -p /certs && \ + chown -R $USER:$USER /frpc.toml /certs && \ + chmod 600 /frpc.toml -USER $USER +# Install GOSU +RUN set -eux; \ + \ + apk add --no-cache --virtual .gosu-deps \ + ca-certificates \ + dpkg \ + gnupg \ + ; \ + \ + dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \ + wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch"; \ + wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch.asc"; \ + \ + export GNUPGHOME="$(mktemp -d)"; \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \ + \ + apk del --no-network .gosu-deps; \ + \ + chmod +x /usr/local/bin/gosu WORKDIR /app COPY --chown=$USER:$USER requirements.txt requirements.txt COPY --chown=$USER:$USER main.py . COPY --chown=$USER:$USER workflow_ocr_backend/ ./workflow_ocr_backend +COPY --chown=$USER:$USER start.sh /start.sh +RUN chmod +x /start.sh && \ + chown -R $USER:$USER /app && \ + pip install -r requirements.txt -RUN pip install -r requirements.txt - -ENTRYPOINT ["python3", "-u", "main.py"] +ENTRYPOINT ["/bin/sh", "-c", "exec gosu \"$USER\" /start.sh python3 -u main.py"] FROM app AS devcontainer @@ -25,11 +55,11 @@ COPY --chown=$USER:$USER requirements-dev.txt requirements-dev.txt # Install dev dependencies and set up sudo USER root -RUN apk add --no-cache git curl make gnupg && \ +RUN apk add --no-cache sudo git docker-cli make gnupg && \ echo "$USER ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USER && \ chmod 0440 /etc/sudoers.d/$USER USER $USER -RUN pip install -r requirements-dev.txt +RUN pip install -r requirements-dev.txt FROM devcontainer AS test diff --git a/Makefile b/Makefile index 692579d..0fc76ed 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,11 @@ deps: .PHONY: test test: - python -m pytest --cov-report html:coverage --cov-report xml:coverage/coverage.xml --cov=workflow_ocr_backend test + python -m pytest --cov-report html:coverage --cov-report xml:coverage/coverage.xml --cov=workflow_ocr_backend -m "not harp_integration" test + +.PHONY: harp-integrationtest +harp-integrationtest: + python -m pytest -m "harp_integration" test .PHONY: build build: diff --git a/README.md b/README.md index 31bfd55..600b0f2 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ It's written in Python and provides a simple REST API for [ocrmypdf](https://ocr - [Prerequisites](#prerequisites) - [Installation](#installation) - [`docker-compose` Example](#docker-compose-example) + - [HaRP Support (Nextcloud 32+)](#harp-support-nextcloud-32) ## Prerequisites @@ -17,7 +18,9 @@ It will take care of all the heavy lifting like installation, orchestration, con 1. Install [`docker`](https://docs.docker.com/engine/install/ubuntu/) on the host where the app should be installed. 2. Install the [`AppApi`](https://docs.nextcloud.com/server/latest/admin_manual/exapps_management/AppAPIAndExternalApps.html#installing-appapi) app. It will take care of the installation and orchestration of the backend as Docker Container. -3. Setup a [Deploy Daemon](https://docs.nextcloud.com/server/latest/admin_manual/exapps_management/AppAPIAndExternalApps.html#setup-deploy-daemon). It's recommended to use the [Docker Socket Proxy](https://github.com/nextcloud/docker-socket-proxy#readme) to communicate with the docker daemon. +3. Setup a [Deploy Daemon](https://docs.nextcloud.com/server/latest/admin_manual/exapps_management/AppAPIAndExternalApps.html#setup-deploy-daemon): + - **For Nextcloud 32+**: It's recommended to use [HaRP (AppAPI HaProxy Reversed Proxy)](https://github.com/nextcloud/HaRP) for better performance and simplified deployment. + - **For older versions**: Use the [Docker Socket Proxy](https://github.com/nextcloud/docker-socket-proxy#readme) to communicate with the docker daemon. ## Installation @@ -164,3 +167,11 @@ nc_py_api._exceptions.NextcloudException: [400] Bad Request :warning: Make sure to create the docker network `nextcloud` before starting the stack. If you don't declare the network as > `external`, `docker-compose` will create the network with some [project/directory prefix](https://docs.docker.com/compose/how-tos/networking/), which will cause the Deploy Daemon to fail because it doesn't find the network. + +## HaRP Support (Nextcloud 32+) + +Since Nextcloud 32, [HaRP (AppAPI HaProxy Reversed Proxy)](https://github.com/nextcloud/HaRP) is the recommended deployment method for ExApps, replacing Docker Socket Proxy. This app now supports HaRP out of the box. + +HaRP simplifies deployment and improves performance by enabling direct communication between clients and ExApps. The implementation is fully backward compatible with Docker Socket Proxy deployments. + +For installation and migration instructions, see the [HaRP documentation](https://github.com/nextcloud/HaRP#readme). diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..6a37091 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +log_cli = true +log_cli_level = INFO +markers = + harp_integration: mark test as full HaRP integration. Spins up HaRP container and manages ExApp lifecycle \ No newline at end of file diff --git a/start.sh b/start.sh new file mode 100644 index 0000000..82eaeb3 --- /dev/null +++ b/start.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +set -e + +# Only create a config file if HP_SHARED_KEY is set. +if [ -n "$HP_SHARED_KEY" ]; then + echo "HP_SHARED_KEY is set, creating /frpc.toml configuration file..." + if [ -d "/certs/frp" ]; then + echo "Found /certs/frp directory. Creating configuration with TLS certificates." + cat < /frpc.toml +serverAddr = "$HP_FRP_ADDRESS" +serverPort = $HP_FRP_PORT +loginFailExit = false + +transport.tls.enable = true +transport.tls.certFile = "/certs/frp/client.crt" +transport.tls.keyFile = "/certs/frp/client.key" +transport.tls.trustedCaFile = "/certs/frp/ca.crt" +transport.tls.serverName = "harp.nc" + +metadatas.token = "$HP_SHARED_KEY" + +[[proxies]] +remotePort = $APP_PORT +type = "tcp" +name = "$APP_ID" +[proxies.plugin] +type = "unix_domain_socket" +unixPath = "/tmp/exapp.sock" +EOF + else + echo "Directory /certs/frp not found. Creating configuration without TLS certificates." + cat < /frpc.toml +serverAddr = "$HP_FRP_ADDRESS" +serverPort = $HP_FRP_PORT +loginFailExit = false + +transport.tls.enable = false + +metadatas.token = "$HP_SHARED_KEY" + +[[proxies]] +remotePort = $APP_PORT +type = "tcp" +name = "$APP_ID" +[proxies.plugin] +type = "unix_domain_socket" +unixPath = "/tmp/exapp.sock" +EOF + fi +else + echo "HP_SHARED_KEY is not set. Skipping FRP configuration." +fi + +# If we have a configuration file and the shared key is present, start the FRP client +if [ -f /frpc.toml ] && [ -n "$HP_SHARED_KEY" ]; then + echo "Starting frpc in the background..." + frpc -c /frpc.toml & +fi + +# Start the main application (launch cmd for ExApp is an argument for this script) +echo "Starting application: $@" +exec "$@" diff --git a/test/test_harp_integration.py b/test/test_harp_integration.py new file mode 100644 index 0000000..e107679 --- /dev/null +++ b/test/test_harp_integration.py @@ -0,0 +1,342 @@ + +import os +import shutil +import socket +import subprocess +import time +import uuid +from pathlib import Path +from typing import Iterable + +import json +import httpx +import pytest +from dotenv import dotenv_values +import logging + +pytestmark = [ + pytest.mark.skipif(shutil.which("docker") is None, reason="Docker CLI not available"), + pytest.mark.harp_integration, +] + +HARP_IMAGE = os.environ.get("HARP_TEST_IMAGE", "ghcr.io/nextcloud/nextcloud-appapi-harp:release") +LOCAL_DOCKER_ENGINE_PORT = 24000 +AGENT_TIMEOUT = 180 +CONTAINER_TIMEOUT = 180 + + +def test_harp_deploys_and_configures_frp(tmp_path): + repo_root = Path(__file__).resolve().parent.parent + env_vars = dotenv_values() + required_keys = ["APP_ID", "APP_PORT", "APP_VERSION", "APP_SECRET", "AA_VERSION"] + missing = [key for key in required_keys if not env_vars.get(key)] + if missing: + pytest.skip(f"Missing required keys in .env for HaRP test: {', '.join(missing)}") + + shared_key = f"harp-key-{uuid.uuid4().hex}" + harp_container = f"harp-integration-{uuid.uuid4().hex[:8]}" + network_name = f"harp-net-{uuid.uuid4().hex[:8]}" + image_tag = f"workflow-ocr-backend-harp:{uuid.uuid4().hex}" + instance_id = f"it{uuid.uuid4().hex[:8]}" + exapp_name = env_vars["APP_ID"] + exapp_container = f"nc_app_{instance_id}_{exapp_name}" + volume_name = f"{exapp_container}_data" + exapps_port = _reserve_host_port() + frp_port = _reserve_host_port() + cert_dir = tmp_path / "harp-certs" + cert_dir.mkdir(parents=True, exist_ok=True) + + app_env = {key: value for key, value in env_vars.items() if value is not None} + app_env.update( + { + "HP_SHARED_KEY": shared_key, + "HP_FRP_ADDRESS": f"{harp_container}:8782", + "HP_FRP_PORT": "8782", + "HP_FRP_DISABLE_TLS": "false", + } + ) + + network_created = False + harp_started = False + http_client: httpx.Client | None = None + try: + _build_app_image(repo_root, image_tag) + _run_command(["docker", "network", "create", network_name], check=True) + network_created = True + + # Create HaRP container + harp_run_cmd = [ + "docker", + "run", + "-d", + "--name", + harp_container, + "--hostname", + harp_container, + "--network", + network_name, + "-v", + "/var/run/docker.sock:/var/run/docker.sock", + "-v", + f"{cert_dir}:/certs", + "-e", + f"HP_SHARED_KEY={shared_key}", + "-e", + "NC_INSTANCE_URL=http://nextcloud.local", + "-e", + "HP_LOG_LEVEL=debug", + "-e", + "HP_VERBOSE_START=1", + "-p", + f"{exapps_port}:8780", + "-p", + f"{frp_port}:8782", + HARP_IMAGE, + ] + _run_command(harp_run_cmd, check=True) + harp_started = True + + http_client = httpx.Client(timeout=15.0, trust_env=False) + exapps_base = f"http://127.0.0.1:{exapps_port}" + agent_headers = { + "harp-shared-key": shared_key, + "EX-APP-ID": exapp_name, + "AUTHORIZATION-APP-API": "test-token", + "AA-VERSION": env_vars.get("AA_VERSION", "32"), + } + + # Wait for HaRP agent to be ready via ExApps app_api + _wait_for_agent_ready_via_exapps(http_client, exapps_base, agent_headers) + + docker_headers = agent_headers | {"docker-engine-port": str(LOCAL_DOCKER_ENGINE_PORT)} + metadata_payload = { + "exapp_token": "test-token", + "exapp_version": env_vars["APP_VERSION"], + "host": "127.0.0.1", + "port": int(env_vars["APP_PORT"]), + "routes": [], + } + + # Create ExApp storage + resp = _call_exapps( + http_client, + exapps_base, + "POST", + f"/exapps/app_api/exapp_storage/{exapp_name}", + agent_headers, + json_payload=metadata_payload, + ) + if resp.status_code != 204: + pytest.fail(f"Failed to seed ExApp metadata: {resp.status_code} {resp.text}") + + env_list = _format_env(app_env) + create_payload = { + "name": exapp_name, + "instance_id": instance_id, + "image_id": image_tag, + "network_mode": network_name, + "environment_variables": env_list, + "mount_points": [], + "resource_limits": {}, + "restart_policy": "no", + } + + # Create ExApp container + resp = _call_exapps(http_client, exapps_base, "POST", "/exapps/app_api/docker/exapp/create", docker_headers, json_payload=create_payload, timeout=60.0) + if resp.status_code != 201: + harp_logs = docker_logs(harp_container) + pytest.fail( + "HaRP failed to create ExApp: " + f"{resp.status_code} {resp.text}\nHaRP logs:\n{harp_logs}" + ) + + install_payload = { + "name": exapp_name, + "instance_id": instance_id, + "system_certs_bundle": None, + "install_frp_certs": True, + } + + # Install certificates in ExApp container + resp = _call_exapps(http_client, exapps_base, "POST", "/exapps/app_api/docker/exapp/install_certificates", docker_headers, json_payload=install_payload) + if resp.status_code != 204: + pytest.fail(f"Failed to install certificates: {resp.status_code} {resp.text}") + + # Start ExApp container + resp = _call_exapps(http_client, exapps_base, "POST", "/exapps/app_api/docker/exapp/start", docker_headers, json_payload={"name": exapp_name, "instance_id": instance_id}) + if resp.status_code not in (200, 204): + pytest.fail(f"Failed to start ExApp container: {resp.status_code} {resp.text}") + + # Wait for ExApp to be running + _wait_for_exapp_start_via_exapps(http_client, exapps_base, docker_headers, exapp_name, instance_id) + + # ASSERTIONS + logs = docker_logs(exapp_container) + if "permission denied" in logs.lower(): + pytest.fail(f"Detected permission issue in ExApp logs:\n{logs}") + assert "Found /certs/frp directory." in logs, f"FRP certificates were not detected in logs:\n{logs}" + assert "Creating configuration with TLS certificates" in logs + + _assert_cert_files_present(exapp_container) + config_contents = docker_exec(exapp_container, ["cat", "/frpc.toml"]) + assert 'transport.tls.enable = true' in config_contents + assert 'transport.tls.certFile = "/certs/frp/client.crt"' in config_contents + assert f'remotePort = {env_vars["APP_PORT"]}' in config_contents + assert f'metadatas.token = "{shared_key}"' in config_contents + + # Cleanup ExApp container + resp = _call_exapps(http_client, exapps_base, "POST", "/exapps/app_api/docker/exapp/remove", docker_headers, json_payload={"name": exapp_name, "instance_id": instance_id, "remove_data": True}) + if resp.status_code != 204: + pytest.fail(f"Failed to remove ExApp container: {resp.status_code} {resp.text}") + + resp = _call_exapps(http_client, exapps_base, "POST", "/exapps/app_api/docker/exapp/exists", docker_headers, json_payload={"name": exapp_name, "instance_id": instance_id}) + if resp.status_code != 200 or resp.json().get("exists"): + pytest.fail("ExApp container still exists after removal") + + _run_command(["docker", "volume", "rm", "-f", volume_name]) + + finally: + if http_client is not None: + http_client.close() + container_exists = _run_command( + ["docker", "ps", "-a", "-q", "-f", f"name={exapp_container}"], + capture_output=True, + text=True, + ).stdout.strip() + if container_exists: + app_logs = docker_logs(exapp_container) + if app_logs: + _log(f"\n\n#### App Container Logs #### \n\n{app_logs}") + _run_command(["docker", "rm", "-f", exapp_container]) + if harp_started: + _run_command(["docker", "rm", "-f", harp_container]) + if network_created: + _run_command(["docker", "network", "rm", network_name]) + + +def _build_app_image(repo_root: Path, image_tag: str) -> None: + build_cmd = [ + "docker", + "build", + "--target", + "app", + "-t", + image_tag, + str(repo_root), + ] + result = _run_command(build_cmd, capture_output=True, check=False) + if result.returncode != 0: + pytest.fail(f"docker build failed: {result.stderr}\n{result.stdout}") + + +def _run_command( + args: list[str], *, check: bool = False, capture_output: bool = False, text: bool = True +) -> subprocess.CompletedProcess: + return subprocess.run( + args, + check=check, + capture_output=capture_output, + text=text, + ) + +def _reserve_host_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("", 0)) + return sock.getsockname()[1] + +def _call_exapps(http_client: httpx.Client, exapps_base: str, method: str, path: str, headers: dict, json_payload=None, timeout: float = 15.0): + if not path.startswith("/"): + path = "/" + path + url = f"{exapps_base}{path}" + try: + if method.upper() == "GET": + return http_client.get(url, headers=headers, timeout=timeout) + if method.upper() == "POST": + return http_client.post(url, headers=headers, json=json_payload, timeout=timeout) + raise ValueError(f"Unsupported method: {method}") + except httpx.ConnectError: + raise + +def _wait_for_agent_ready_via_exapps(http_client: httpx.Client, exapps_base: str, headers: dict) -> None: + probe_headers = headers.copy() + # Ask HaRP to use the local (built-in) Docker Engine remote port. + probe_headers.setdefault("docker-engine-port", str(LOCAL_DOCKER_ENGINE_PORT)) + # AA-VERSION isn't relevant for app_api ping but keep a sane default + probe_headers.setdefault("AA-VERSION", "32") + + ping_path = "/exapps/app_api/v1.41/_ping" + deadline = time.time() + AGENT_TIMEOUT + while time.time() < deadline: + try: + resp = _call_exapps(http_client, exapps_base, "GET", ping_path, probe_headers) + if resp.status_code == 200: + # Expect the body to contain 'OK' + try: + body = resp.text.strip() + except Exception: + body = "" + if body == "OK": + return + except httpx.ConnectError as ce: + _log(f"ConnectError while probing HaRP readiness via ExApps: {ce}") + except httpx.ReadError as re: + _log(f"ReadError while probing HaRP readiness via ExApps: {re}") + except httpx.RemoteProtocolError as rpe: + _log(f"RemoteProtocolError while probing HaRP readiness via ExApps: {rpe}") + time.sleep(1) + pytest.fail("Timed out waiting for HaRP readiness via ExApps app_api _ping") + +def _wait_for_exapp_start_via_exapps(http_client: httpx.Client, exapps_base: str, headers: dict, app_name: str, instance_id: str) -> None: + payload = {"name": app_name, "instance_id": instance_id} + probe_headers = headers.copy() + probe_headers.setdefault("EX-APP-ID", app_name) + probe_headers.setdefault("AUTHORIZATION-APP-API", "test-token") + deadline = time.time() + CONTAINER_TIMEOUT + while time.time() < deadline: + try: + resp = _call_exapps(http_client, exapps_base, "POST", "/exapps/app_api/docker/exapp/wait_for_start", probe_headers, json_payload=payload) + if resp.status_code == 200: + try: + j = resp.json() + if j.get("started"): + return + except Exception: + pass + except httpx.ConnectError as ce: + _log(f"ConnectError while waiting for ExApp start via ExApps: {ce}") + except httpx.RemoteProtocolError as rpe: + _log(f"RemoteProtocolError while waiting for ExApp start via ExApps: {rpe}") + time.sleep(2) + pytest.fail("ExApp container did not reach running state in time (via ex_apps)") + +def _format_env(items: dict[str, str]) -> list[str]: + formatted: list[str] = [] + for key, value in sorted(items.items()): + formatted.append(f"{key}={value}") + return formatted + +def docker_logs(container_name: str) -> str: + proc = _run_command(["docker", "logs", container_name], capture_output=True) + return proc.stdout + proc.stderr if proc.stderr else proc.stdout + +def docker_exec(container_name: str, cmd: Iterable[str]) -> str: + proc = _run_command(["docker", "exec", container_name, *cmd], capture_output=True) + if proc.returncode != 0: + pytest.fail( + f"docker exec {' '.join(cmd)} failed with rc={proc.returncode}:\nstdout:{proc.stdout}\nstderr:{proc.stderr}" + ) + return proc.stdout + +def _assert_cert_files_present(container_name: str) -> None: + for filename in ("ca.crt", "client.crt", "client.key"): + proc = _run_command( + ["docker", "exec", container_name, "test", "-f", f"/certs/frp/{filename}"], capture_output=True + ) + if proc.returncode != 0: + pytest.fail( + f"Expected certificate {filename} is missing in container {container_name}: {proc.stdout} {proc.stderr}" + ) + +def _log(string: str) -> None: + logger = logging.getLogger(__name__) + logger.info(string)