Skip to content

Commit 22aed22

Browse files
oleksandr-ncoleksandr-nc
committed
ci: add K8s LoadBalancer expose type CI tests
Signed-off-by: Alexander Piskun <bigcat88@icloud.com>
1 parent 8700311 commit 22aed22

1 file changed

Lines changed: 240 additions & 0 deletions

File tree

‎.github/workflows/tests-deploy-k8s-loadbalancer.yml‎

Lines changed: 240 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,240 @@
1+
# SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
2+
# SPDX-License-Identifier: MIT
3+
#
4+
# K8s LoadBalancer expose type tests.
5+
#
6+
# Uses k3s with its built-in Klipper ServiceLB (enabled by default).
7+
# Klipper assigns the node IP as the LoadBalancer external IP, which
8+
# is sufficient for single-node CI testing.
9+
#
10+
name: Tests - K8s Deploy (LoadBalancer)
11+
12+
on:
13+
pull_request:
14+
branches: [main]
15+
push:
16+
branches: [main]
17+
workflow_dispatch:
18+
19+
permissions:
20+
contents: read
21+
22+
concurrency:
23+
group: tests-deploy-k8s-loadbalancer-${{ github.head_ref || github.run_id }}
24+
cancel-in-progress: true
25+
26+
env:
27+
HP_SHARED_KEY: 'test_shared_key_12345'
28+
29+
jobs:
30+
k8s-deploy-loadbalancer:
31+
runs-on: ubuntu-22.04
32+
name: K8s Deploy Lifecycle (LoadBalancer)
33+
34+
services:
35+
postgres:
36+
image: ghcr.io/nextcloud/continuous-integration-postgres-14:latest # zizmor: ignore[unpinned-images]
37+
ports:
38+
- 4444:5432/tcp
39+
env:
40+
POSTGRES_USER: root
41+
POSTGRES_PASSWORD: rootpassword
42+
POSTGRES_DB: nextcloud
43+
options: --health-cmd pg_isready --health-interval 5s --health-timeout 2s --health-retries 5
44+
45+
steps:
46+
- name: Set app env
47+
run: echo "APP_NAME=${GITHUB_REPOSITORY##*/}" >> $GITHUB_ENV
48+
49+
- name: Checkout server
50+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
51+
with:
52+
persist-credentials: false
53+
submodules: true
54+
repository: nextcloud/server
55+
ref: master
56+
57+
- name: Checkout AppAPI
58+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
59+
with:
60+
persist-credentials: false
61+
path: apps/${{ env.APP_NAME }}
62+
63+
- name: Set up php
64+
uses: shivammathur/setup-php@44454db4f0199b8b9685a5d763dc37cbf79108e1 # v2
65+
with:
66+
php-version: '8.3'
67+
extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, session, simplexml, xmlreader, xmlwriter, zip, zlib, pgsql, pdo_pgsql
68+
coverage: none
69+
ini-file: development
70+
env:
71+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
72+
73+
- name: Check composer file existence
74+
id: check_composer
75+
uses: andstor/file-existence-action@076e0072799f4942c8bc574a82233e1e4d13e9d6 # v2
76+
with:
77+
files: apps/${{ env.APP_NAME }}/composer.json
78+
79+
- name: Set up dependencies
80+
if: steps.check_composer.outputs.files_exists == 'true'
81+
working-directory: apps/${{ env.APP_NAME }}
82+
run: composer i
83+
84+
- name: Set up Nextcloud
85+
env:
86+
DB_PORT: 4444
87+
run: |
88+
mkdir data
89+
./occ maintenance:install --verbose --database=pgsql --database-name=nextcloud --database-host=127.0.0.1 \
90+
--database-port=$DB_PORT --database-user=root --database-pass=rootpassword \
91+
--admin-user admin --admin-pass admin
92+
./occ config:system:set loglevel --value=0 --type=integer
93+
./occ config:system:set debug --value=true --type=boolean
94+
./occ app:enable --force ${{ env.APP_NAME }}
95+
96+
- name: Install k3s
97+
run: |
98+
curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--disable traefik" sh -
99+
sudo chmod 644 /etc/rancher/k3s/k3s.yaml
100+
echo "KUBECONFIG=/etc/rancher/k3s/k3s.yaml" >> $GITHUB_ENV
101+
102+
- name: Wait for k3s and create namespace
103+
run: |
104+
kubectl wait --for=condition=Ready node --all --timeout=120s
105+
kubectl create namespace nextcloud-exapps
106+
NODE_IP=$(kubectl get node -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}')
107+
echo "NODE_IP=${NODE_IP}" >> $GITHUB_ENV
108+
echo "k3s node IP: $NODE_IP"
109+
110+
- name: Wait for Klipper ServiceLB to be ready
111+
run: |
112+
echo "Waiting for svclb pods in kube-system..."
113+
kubectl wait --for=condition=Ready pods -l app=svclb-traefik -n kube-system --timeout=60s 2>/dev/null || true
114+
kubectl get pods -n kube-system | grep -E 'svclb|svc' || echo "No svclb pods yet (will start when first LB Service is created)"
115+
116+
- name: Configure Nextcloud for k3s networking
117+
run: |
118+
./occ config:system:set overwrite.cli.url --value "http://${{ env.NODE_IP }}" --type=string
119+
./occ config:system:set trusted_domains 1 --value "${{ env.NODE_IP }}"
120+
121+
- name: Create K8s service account for HaRP
122+
run: |
123+
kubectl -n nextcloud-exapps create serviceaccount harp-sa
124+
kubectl create clusterrolebinding harp-admin \
125+
--clusterrole=cluster-admin \
126+
--serviceaccount=nextcloud-exapps:harp-sa
127+
K3S_TOKEN=$(kubectl -n nextcloud-exapps create token harp-sa --duration=2h)
128+
echo "K3S_TOKEN=${K3S_TOKEN}" >> $GITHUB_ENV
129+
130+
- name: Pre-pull ExApp image into k3s
131+
run: sudo k3s ctr images pull ghcr.io/nextcloud/app-skeleton-python:latest
132+
133+
- name: Pull HaRP image
134+
run: docker pull ghcr.io/nextcloud/nextcloud-appapi-harp:latest
135+
136+
- name: Start HaRP with K8s backend
137+
run: |
138+
docker run --net host --name appapi-harp \
139+
-e HP_SHARED_KEY="${{ env.HP_SHARED_KEY }}" \
140+
-e NC_INSTANCE_URL="http://${{ env.NODE_IP }}" \
141+
-e HP_LOG_LEVEL="debug" \
142+
-e HP_K8S_ENABLED="true" \
143+
-e HP_K8S_API_SERVER="https://127.0.0.1:6443" \
144+
-e HP_K8S_BEARER_TOKEN="${{ env.K3S_TOKEN }}" \
145+
-e HP_K8S_NAMESPACE="nextcloud-exapps" \
146+
-e HP_K8S_VERIFY_SSL="false" \
147+
--restart unless-stopped \
148+
-d ghcr.io/nextcloud/nextcloud-appapi-harp:latest
149+
150+
- name: Start nginx proxy
151+
run: |
152+
docker run --net host --name nextcloud --rm \
153+
-v $(pwd)/apps/${{ env.APP_NAME }}/tests/simple-nginx-NOT-FOR-PRODUCTION.conf:/etc/nginx/conf.d/default.conf:ro \
154+
-d nginx
155+
156+
- name: Start Nextcloud
157+
run: PHP_CLI_SERVER_WORKERS=2 php -S 0.0.0.0:8080 &
158+
159+
- name: Wait for HaRP K8s readiness
160+
run: |
161+
for i in $(seq 1 30); do
162+
if curl -sf http://${{ env.NODE_IP }}:8780/exapps/app_api/info \
163+
-H "harp-shared-key: ${{ env.HP_SHARED_KEY }}" 2>/dev/null | grep -q '"kubernetes"'; then
164+
echo "HaRP is ready with K8s backend"
165+
exit 0
166+
fi
167+
echo "Waiting for HaRP... ($i/30)"
168+
sleep 2
169+
done
170+
echo "HaRP K8s readiness check failed"
171+
docker logs appapi-harp
172+
exit 1
173+
174+
- name: Register K8s daemon (LoadBalancer)
175+
run: |
176+
./occ app_api:daemon:register \
177+
k8s_test "K8s Test" "kubernetes-install" "http" "${{ env.NODE_IP }}:8780" "http://${{ env.NODE_IP }}" \
178+
--harp --harp_shared_key "${{ env.HP_SHARED_KEY }}" --harp_frp_address "${{ env.NODE_IP }}:8782" \
179+
--k8s --k8s_expose_type=loadbalancer --set-default
180+
./occ app_api:daemon:list
181+
182+
- name: Run K8s integration tests (LoadBalancer)
183+
env:
184+
K8S_EXPOSE_TYPE: loadbalancer
185+
run: python3 apps/${{ env.APP_NAME }}/tests/test_occ_commands_k8s.py
186+
187+
- name: Collect HaRP logs
188+
if: always()
189+
run: docker logs appapi-harp > harp.log 2>&1
190+
191+
- name: Collect K8s resources
192+
if: always()
193+
run: |
194+
kubectl -n nextcloud-exapps get all -o wide > k8s-resources.txt 2>&1 || true
195+
kubectl -n nextcloud-exapps describe pods > k8s-pods-describe.txt 2>&1 || true
196+
kubectl -n nextcloud-exapps get pvc -o wide >> k8s-resources.txt 2>&1 || true
197+
kubectl -n nextcloud-exapps get svc -o yaml >> k8s-resources.txt 2>&1 || true
198+
199+
- name: Show all logs
200+
if: always()
201+
run: |
202+
echo "=== HaRP logs ===" && cat harp.log || true
203+
echo "=== K8s resources ===" && cat k8s-resources.txt || true
204+
echo "=== K8s pods ===" && cat k8s-pods-describe.txt || true
205+
echo "=== Nextcloud log (last 100 lines) ===" && tail -100 data/nextcloud.log || true
206+
207+
- name: Upload HaRP logs
208+
if: always()
209+
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
210+
with:
211+
name: k8s_deploy_loadbalancer_harp.log
212+
path: harp.log
213+
if-no-files-found: warn
214+
215+
- name: Upload K8s resources
216+
if: always()
217+
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
218+
with:
219+
name: k8s_deploy_loadbalancer_resources.txt
220+
path: |
221+
k8s-resources.txt
222+
k8s-pods-describe.txt
223+
if-no-files-found: warn
224+
225+
- name: Upload NC logs
226+
if: always()
227+
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
228+
with:
229+
name: k8s_deploy_loadbalancer_nextcloud.log
230+
path: data/nextcloud.log
231+
if-no-files-found: warn
232+
233+
tests-success:
234+
permissions:
235+
contents: none
236+
runs-on: ubuntu-22.04
237+
needs: [k8s-deploy-loadbalancer]
238+
name: K8s-LoadBalancer-Tests-OK
239+
steps:
240+
- run: echo "K8s LoadBalancer tests passed successfully"

0 commit comments

Comments
 (0)