Skip to content

Commit a41b273

Browse files
oleksandr-ncoleksandr-nc
committed
test: add K8s integration tests and CI workflow
Add integration tests for the Kubernetes deployment support: - tests/test_occ_commands_k8s.py: Python test script with 4 groups: Group A: K8s daemon registration validation (--k8s flag, expose types, nodeport range, manual upstream host, traffic policy, etc.) Group B: Single-role K8s deploy lifecycle (deploy, enable/disable, unregister with/without --rm-data) Group C: Multi-role K8s deploy lifecycle (2 roles, expose only one, enable/disable scales both, unregister cleans all) Group D: Failure & edge cases (bad image rollback, --force unregister, --silent nonexistent app) - .github/workflows/tests-deploy-k8s.yml: CI workflow with 2 jobs: k8s-daemon-validation: fast (~3min), no k3s needed k8s-deploy-lifecycle: full e2e with k3s + HaRP K8s backend Temporarily builds HaRP from dev branch (has K8s support). After merge, switch to ghcr.io/nextcloud/nextcloud-appapi-harp:latest. Signed-off-by: Oleksander Piskun <oleksandr2088@icloud.com>
1 parent 8404b81 commit a41b273

2 files changed

Lines changed: 921 additions & 0 deletions

File tree

‎.github/workflows/tests-deploy-k8s.yml‎

Lines changed: 306 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,306 @@
1+
# SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
2+
# SPDX-License-Identifier: MIT
3+
name: Tests - K8s Deploy
4+
5+
on:
6+
pull_request:
7+
branches: [main]
8+
push:
9+
branches: [main]
10+
workflow_dispatch:
11+
inputs:
12+
harp_ref:
13+
description: 'HaRP branch to build from'
14+
default: 'dev' # TEMPORARY: change to 'main' after K8s merge
15+
16+
permissions:
17+
contents: read
18+
19+
concurrency:
20+
group: tests-deploy-k8s-${{ github.head_ref || github.run_id }}
21+
cancel-in-progress: true
22+
23+
env:
24+
HARP_REF: ${{ github.event.inputs.harp_ref || 'dev' }} # TEMPORARY: change to 'main' after K8s merge
25+
HP_SHARED_KEY: 'test_shared_key_12345'
26+
27+
jobs:
28+
k8s-daemon-validation:
29+
runs-on: ubuntu-22.04
30+
name: K8s Daemon Validation
31+
32+
services:
33+
postgres:
34+
image: ghcr.io/nextcloud/continuous-integration-postgres-14:latest # zizmor: ignore[unpinned-images]
35+
ports:
36+
- 4444:5432/tcp
37+
env:
38+
POSTGRES_USER: root
39+
POSTGRES_PASSWORD: rootpassword
40+
POSTGRES_DB: nextcloud
41+
options: --health-cmd pg_isready --health-interval 5s --health-timeout 2s --health-retries 5
42+
43+
steps:
44+
- name: Set app env
45+
run: echo "APP_NAME=${GITHUB_REPOSITORY##*/}" >> $GITHUB_ENV
46+
47+
- name: Checkout server
48+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
49+
with:
50+
persist-credentials: false
51+
submodules: true
52+
repository: nextcloud/server
53+
ref: master
54+
55+
- name: Checkout AppAPI
56+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
57+
with:
58+
persist-credentials: false
59+
path: apps/${{ env.APP_NAME }}
60+
61+
- name: Set up php
62+
uses: shivammathur/setup-php@44454db4f0199b8b9685a5d763dc37cbf79108e1 # v2
63+
with:
64+
php-version: '8.3'
65+
extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, session, simplexml, xmlreader, xmlwriter, zip, zlib, pgsql, pdo_pgsql
66+
coverage: none
67+
ini-file: development
68+
env:
69+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
70+
71+
- name: Check composer file existence
72+
id: check_composer
73+
uses: andstor/file-existence-action@076e0072799f4942c8bc574a82233e1e4d13e9d6 # v2
74+
with:
75+
files: apps/${{ env.APP_NAME }}/composer.json
76+
77+
- name: Set up dependencies
78+
if: steps.check_composer.outputs.files_exists == 'true'
79+
working-directory: apps/${{ env.APP_NAME }}
80+
run: composer i
81+
82+
- name: Set up Nextcloud
83+
env:
84+
DB_PORT: 4444
85+
run: |
86+
mkdir data
87+
./occ maintenance:install --verbose --database=pgsql --database-name=nextcloud --database-host=127.0.0.1 \
88+
--database-port=$DB_PORT --database-user=root --database-pass=rootpassword \
89+
--admin-user admin --admin-pass admin
90+
./occ config:system:set loglevel --value=0 --type=integer
91+
./occ config:system:set debug --value=true --type=boolean
92+
./occ app:enable --force ${{ env.APP_NAME }}
93+
94+
- name: Run K8s daemon validation tests
95+
run: |
96+
PHP_CLI_SERVER_WORKERS=2 php -S 127.0.0.1:8080 &
97+
python3 apps/${{ env.APP_NAME }}/tests/test_occ_commands_k8s.py --validation-only
98+
99+
- name: Upload NC logs
100+
if: always()
101+
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
102+
with:
103+
name: k8s_daemon_validation_nextcloud.log
104+
path: data/nextcloud.log
105+
if-no-files-found: warn
106+
107+
k8s-deploy-lifecycle:
108+
runs-on: ubuntu-22.04
109+
name: K8s Deploy Lifecycle (k3s + HaRP)
110+
111+
services:
112+
postgres:
113+
image: ghcr.io/nextcloud/continuous-integration-postgres-14:latest # zizmor: ignore[unpinned-images]
114+
ports:
115+
- 4444:5432/tcp
116+
env:
117+
POSTGRES_USER: root
118+
POSTGRES_PASSWORD: rootpassword
119+
POSTGRES_DB: nextcloud
120+
options: --health-cmd pg_isready --health-interval 5s --health-timeout 2s --health-retries 5
121+
122+
steps:
123+
- name: Set app env
124+
run: echo "APP_NAME=${GITHUB_REPOSITORY##*/}" >> $GITHUB_ENV
125+
126+
- name: Checkout server
127+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
128+
with:
129+
persist-credentials: false
130+
submodules: true
131+
repository: nextcloud/server
132+
ref: master
133+
134+
- name: Checkout AppAPI
135+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
136+
with:
137+
persist-credentials: false
138+
path: apps/${{ env.APP_NAME }}
139+
140+
- name: Set up php
141+
uses: shivammathur/setup-php@44454db4f0199b8b9685a5d763dc37cbf79108e1 # v2
142+
with:
143+
php-version: '8.3'
144+
extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, session, simplexml, xmlreader, xmlwriter, zip, zlib, pgsql, pdo_pgsql
145+
coverage: none
146+
ini-file: development
147+
env:
148+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
149+
150+
- name: Check composer file existence
151+
id: check_composer
152+
uses: andstor/file-existence-action@076e0072799f4942c8bc574a82233e1e4d13e9d6 # v2
153+
with:
154+
files: apps/${{ env.APP_NAME }}/composer.json
155+
156+
- name: Set up dependencies
157+
if: steps.check_composer.outputs.files_exists == 'true'
158+
working-directory: apps/${{ env.APP_NAME }}
159+
run: composer i
160+
161+
- name: Set up Nextcloud
162+
env:
163+
DB_PORT: 4444
164+
run: |
165+
mkdir data
166+
./occ maintenance:install --verbose --database=pgsql --database-name=nextcloud --database-host=127.0.0.1 \
167+
--database-port=$DB_PORT --database-user=root --database-pass=rootpassword \
168+
--admin-user admin --admin-pass admin
169+
./occ config:system:set loglevel --value=0 --type=integer
170+
./occ config:system:set debug --value=true --type=boolean
171+
./occ config:system:set overwrite.cli.url --value http://127.0.0.1 --type=string
172+
./occ app:enable --force ${{ env.APP_NAME }}
173+
174+
- name: Install k3s
175+
run: |
176+
curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--disable traefik --disable servicelb" sh -
177+
sudo chmod 644 /etc/rancher/k3s/k3s.yaml
178+
echo "KUBECONFIG=/etc/rancher/k3s/k3s.yaml" >> $GITHUB_ENV
179+
180+
- name: Wait for k3s and create namespace
181+
run: |
182+
kubectl wait --for=condition=Ready node --all --timeout=120s
183+
kubectl create namespace nextcloud-exapps
184+
185+
- name: Create K8s service account for HaRP
186+
run: |
187+
kubectl -n nextcloud-exapps create serviceaccount harp-sa
188+
kubectl create clusterrolebinding harp-admin \
189+
--clusterrole=cluster-admin \
190+
--serviceaccount=nextcloud-exapps:harp-sa
191+
K3S_TOKEN=$(kubectl -n nextcloud-exapps create token harp-sa --duration=2h)
192+
echo "K3S_TOKEN=${K3S_TOKEN}" >> $GITHUB_ENV
193+
194+
- name: Pre-pull ExApp image into k3s
195+
run: sudo k3s ctr images pull ghcr.io/nextcloud/app-skeleton-python:latest
196+
197+
# TEMPORARY: Build HaRP from dev branch. After K8s merge, change to:
198+
# docker pull ghcr.io/nextcloud/nextcloud-appapi-harp:latest
199+
# docker tag ghcr.io/nextcloud/nextcloud-appapi-harp:latest harp:test
200+
- name: Build HaRP from dev branch
201+
run: |
202+
git clone --branch ${{ env.HARP_REF }} --depth 1 https://github.com/nextcloud/HaRP.git /tmp/HaRP
203+
cd /tmp/HaRP && docker build -t harp:test .
204+
205+
- name: Start HaRP with K8s backend
206+
run: |
207+
docker run --net host --name appapi-harp \
208+
-e HP_SHARED_KEY="${{ env.HP_SHARED_KEY }}" \
209+
-e NC_INSTANCE_URL="http://127.0.0.1" \
210+
-e HP_LOG_LEVEL="debug" \
211+
-e HP_K8S_ENABLED="true" \
212+
-e HP_K8S_API_SERVER="https://127.0.0.1:6443" \
213+
-e HP_K8S_BEARER_TOKEN="${{ env.K3S_TOKEN }}" \
214+
-e HP_K8S_NAMESPACE="nextcloud-exapps" \
215+
-e HP_K8S_VERIFY_SSL="false" \
216+
--restart unless-stopped \
217+
-d harp:test
218+
219+
- name: Start nginx proxy
220+
run: |
221+
docker run --net host --name nextcloud --rm \
222+
-v $(pwd)/apps/${{ env.APP_NAME }}/tests/simple-nginx-NOT-FOR-PRODUCTION.conf:/etc/nginx/conf.d/default.conf:ro \
223+
-d nginx
224+
225+
- name: Start Nextcloud
226+
run: PHP_CLI_SERVER_WORKERS=2 php -S 127.0.0.1:8080 &
227+
228+
- name: Wait for HaRP K8s readiness
229+
run: |
230+
for i in $(seq 1 30); do
231+
if curl -sf http://127.0.0.1:8780/exapps/app_api/info \
232+
-H "harp-shared-key: ${{ env.HP_SHARED_KEY }}" 2>/dev/null | grep -q '"kubernetes"'; then
233+
echo "HaRP is ready with K8s backend"
234+
exit 0
235+
fi
236+
echo "Waiting for HaRP... ($i/30)"
237+
sleep 2
238+
done
239+
echo "HaRP K8s readiness check failed"
240+
docker logs appapi-harp
241+
exit 1
242+
243+
- name: Register K8s daemon
244+
run: |
245+
./occ app_api:daemon:register \
246+
k8s_test "K8s Test" "kubernetes-install" "http" "127.0.0.1:8780" "http://127.0.0.1" \
247+
--harp --harp_shared_key "${{ env.HP_SHARED_KEY }}" --harp_frp_address "127.0.0.1:8782" \
248+
--k8s --k8s_expose_type=nodeport --set-default
249+
./occ app_api:daemon:list
250+
251+
- name: Run K8s integration tests
252+
run: python3 apps/${{ env.APP_NAME }}/tests/test_occ_commands_k8s.py --full
253+
254+
- name: Collect HaRP logs
255+
if: always()
256+
run: docker logs appapi-harp > harp.log 2>&1
257+
258+
- name: Collect K8s resources
259+
if: always()
260+
run: |
261+
kubectl -n nextcloud-exapps get all -o wide > k8s-resources.txt 2>&1 || true
262+
kubectl -n nextcloud-exapps describe pods > k8s-pods-describe.txt 2>&1 || true
263+
kubectl -n nextcloud-exapps get pvc -o wide >> k8s-resources.txt 2>&1 || true
264+
265+
- name: Show all logs
266+
if: always()
267+
run: |
268+
echo "=== HaRP logs ===" && cat harp.log || true
269+
echo "=== K8s resources ===" && cat k8s-resources.txt || true
270+
echo "=== K8s pods ===" && cat k8s-pods-describe.txt || true
271+
echo "=== Nextcloud log (last 100 lines) ===" && tail -100 data/nextcloud.log || true
272+
273+
- name: Upload HaRP logs
274+
if: always()
275+
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
276+
with:
277+
name: k8s_deploy_harp.log
278+
path: harp.log
279+
if-no-files-found: warn
280+
281+
- name: Upload K8s resources
282+
if: always()
283+
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
284+
with:
285+
name: k8s_deploy_resources.txt
286+
path: |
287+
k8s-resources.txt
288+
k8s-pods-describe.txt
289+
if-no-files-found: warn
290+
291+
- name: Upload NC logs
292+
if: always()
293+
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
294+
with:
295+
name: k8s_deploy_nextcloud.log
296+
path: data/nextcloud.log
297+
if-no-files-found: warn
298+
299+
tests-success:
300+
permissions:
301+
contents: none
302+
runs-on: ubuntu-22.04
303+
needs: [k8s-daemon-validation, k8s-deploy-lifecycle]
304+
name: K8s-Tests-OK
305+
steps:
306+
- run: echo "K8s tests passed successfully"

0 commit comments

Comments
 (0)