chore: update

Author: cukhoaimon

apps/spark-job/job-streaming-satellite/.dockerignore

@@ -0,0 +1,7 @@
+.venv/
+__pycache__/
+*.pyc
+*.pyo
+.python-version
+README.md
+build-and-push.sh

apps/spark-job/job-streaming-satellite/build-and-push.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+JOB_NAME="streaming-satellite"
+GITHUB_USERNAME="cukhoaimon"
+REGISTRY="ghcr.io/${GITHUB_USERNAME}/spark-job-${JOB_NAME}"
+TAG="${1:-latest}"
+
+echo "==> Building ${REGISTRY}:${TAG} (linux/amd64)"
+
+docker build \
+  --platform linux/amd64 \
+  -t "${REGISTRY}:${TAG}" \
+  "$(dirname "$0")"
+
+echo "==> Logging in to ghcr.io"
+gh auth token | docker login ghcr.io -u "${GITHUB_USERNAME}" --password-stdin
+
+echo "==> Pushing ${REGISTRY}:${TAG}"
+docker push "${REGISTRY}:${TAG}"
+echo "==> Done: ${REGISTRY}:${TAG}"

spark-jobs/job-streaming-satellite.yaml

@@ -13,7 +13,7 @@ spec:
     - ghcr-pull-secret
   mainApplicationFile: local:///opt/spark/work-dir/main.py 
 
-  sparkVersion: "3.5.3"
+  sparkVersion: "4.0.0"
   sparkConfigMap: spark-defaults
 
   sparkConf:
@@ -54,7 +54,7 @@ spec:
         value: "latest"
       # ── Streaming tuning ───────────────────────────────────────────────────
       - name: CHECKPOINT_LOCATION
-        value: "s3a://warehouse/checkpoints/satellite-telemetry/"
+        value: "s3a://warehouse/checkpoints/satellite-telemetry/v2/"
       - name: TRIGGER_INTERVAL_SECONDS
         value: "30"