From 5842d91a445462434c32bbc2b50ccdbee44a42f6 Mon Sep 17 00:00:00 2001 From: Jake Faulkner Date: Tue, 17 Mar 2026 17:03:52 +1300 Subject: [PATCH 1/9] update container definition to use two-stage design --- .github/workflows/container.yml | 92 ++++++++++++++------------------- container/runner.def | 59 +++++---------------- 2 files changed, 52 insertions(+), 99 deletions(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index bb53abdf..7eb3b254 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -7,60 +7,46 @@ on: - pegasus workflow_dispatch: +env: + REGISTRY: docker.io + IMAGE_NAME: earthquakesuc/workflow-boostrap + jobs: build-test-upload: runs-on: ubuntu-latest - - env: - COMMIT_HASH: ${{ github.sha }} - CONTAINER_NAME: runner-${{ github.sha }}.sif - steps: - - name: Configure rclone for Dropbox - run: | - mkdir -p ~/.config/rclone - cat < ~/.config/rclone/rclone.conf - [dropbox] - type = dropbox - client_id = - client_secret = - token = ${{ secrets.RCLONE_CONFIG }} - EOF - - - name: Set up SSH for deploy key - run: | - mkdir -p ~/.ssh - echo "${{ secrets.DEPLOY_KEY }}" > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - ssh-keyscan github.com >> ~/.ssh/known_hosts - - - name: Clone private repo - env: - TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - git clone git@github.com:ucgmsim/EMOD3D.git /tmp/EMOD3D - - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Install dependencies - run: | - sudo apt update - sudo apt install -y squashfuse e2fsprogs runc rclone - wget --quiet https://github.com/apptainer/apptainer/releases/download/v1.4.0/apptainer_1.4.0_amd64.deb -O /tmp/apptainer_1.4.0_amd64.deb - sudo apt install -y /tmp/apptainer_1.4.0_amd64.deb - - - name: Build Apptainer container - run: | - apptainer build $CONTAINER_NAME container/runner.def - - - name: Run pytest inside container - run: | - apptainer exec \ - --bind $PWD:/mnt \ - $CONTAINER_NAME \ - bash -c "cd /mnt && pytest tests/" - - - name: Upload container to Dropbox - run: | - rclone copy $CONTAINER_NAME dropbox:/QuakeCoRE/Containers + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: earthquakesuc + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.IMAGE_NAME }} + tags: | + type=sha,format=long + type=ref,event=branch + latest + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + file: ./container/Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + - name: Run pytest + run: | + # Use the long SHA tag generated by the meta step + docker run --rm \ + -v ${{ github.workspace }}:/mnt \ + ${{ env.IMAGE_NAME }}:sha-${{ github.sha }} \ + bash -c "pytest /mnt/tests" diff --git a/container/runner.def b/container/runner.def index ff72a860..6922cfdd 100644 --- a/container/runner.def +++ b/container/runner.def @@ -1,56 +1,23 @@ Bootstrap: docker -From: ubuntu:latest - -%setup - [ -d /tmp/EMOD3D ] || git clone --depth 1 git@github.com:ucgmsim/EMOD3D.git /tmp/EMOD3D +From: earthquakesuc/workflow-bootstrap:latest %files /tmp/EMOD3D /EMOD3D -%post - apt-get update && apt-get install -y \ - build-essential \ - cmake \ - gdal-bin \ - gfortran \ - ghostscript \ - git \ - gmt \ - libgmt-dev \ - gmt-dcw \ - gmt-gshhg \ - graphicsmagick \ - libfftw3-dev \ - libopenmpi-dev \ - libpapi-dev \ - python-is-python3 \ - python3-pip \ - sqlite3 \ - wget +%setup + rm -rf /tmp/EMOD3D && git clone --depth=1 git@github.com:ucgmsim/EMOD3D.git /tmp/EMOD3D - wget "https://www.dropbox.com/scl/fi/b4m9g32x1j70ti69zyybq/genslip_velocity_model.vmod?rlkey=yxurhrdn8uvqh33hsb0inhsq0&st=o77aaccp&dl=0" -O /genslip_velocity_model.vmod - wget "https://www.dropbox.com/scl/fi/qakchhg6qu23aclcb02ye/Cant1D_v2-midQ_leer.1d?rlkey=wn1a6nnol2eaf5n6qsy5t1nym&st=8yahw7id&dl=0" -O /Cant1D_v2-midQ_leer.1d - wget "https://www.dropbox.com/scl/fi/tw1m15owloi0eo8h3u1kw/geoNet_stats-2023-06-28.vs30?rlkey=plu90zg0ti4lc2nvf51zu9xmg&st=iz8c2ls3&dl=0" -O /stations.vs30 - wget "https://www.dropbox.com/scl/fi/bb852b1f0rly6cfvs9p9b/geoNet_stats-2023-06-28.ll?rlkey=3y8k5qviy52nbksfdkm1n92yh&st=605h60u6&dl=0" -O /stations.ll +%post cd /EMOD3D && \ - mkdir build && \ - cd build && \ - cmake .. && \ - make -j $(nproc) genslip_v5.6.2 srf2stoch hb_high_binmod_v6.0.3 hb_high_binmod_v5.4.5 - - # Remove all folders in EMOD3D except the tools folder - find /EMOD3D -mindepth 1 -maxdepth 1 ! -name "tools" -exec rm -rf \{\} \; - - - git clone --depth 1 https://github.com/ucgmsim/Velocity-Model.git /Velocity-Model && \ - cd /Velocity-Model && \ - mkdir build && \ - cd build && \ - cmake ../ -DCMAKE_C_FLAGS="-fopenmp" -DCMAKE_CXX_FLAGS="-fopenmp" && \ - make -j $(nproc) - - pip install "git+https://github.com/ucgmsim/workflow@pegasus" --break-system-packages + mkdir build && cd build && \ + cmake -DHF_TIME_WINDOW=off -DUSE_FFTW=on -DCMAKE_C_FLAGS="-march=x86-64 -mtune=generic" .. && \ + make -j $(nproc) genslip_v5.6.2 srf2stoch generic_slip2srf \ + hb_high_binmod_v6.0.3 %environment - export PATH="$PATH:/EMOD3D/tools:/Velocity-Model" + export RUSTUP_HOME=/sw/rust + export CARGO_HOME=/sw/cargo + export PATH="/sw/venv/bin:/EMOD3D/tools:/sw/cargo/bin:/sw/python/bin:/sw/uv:$PATH" + export VIRTUAL_ENV="/sw/venv" + export LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" From c544085ff60285fa379f17ddcb562c213476dc15 Mon Sep 17 00:00:00 2001 From: Jake Faulkner Date: Tue, 17 Mar 2026 17:08:45 +1300 Subject: [PATCH 2/9] add new dockerfile --- container/Dockerfile | 78 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 container/Dockerfile diff --git a/container/Dockerfile b/container/Dockerfile new file mode 100644 index 00000000..913f5e61 --- /dev/null +++ b/container/Dockerfile @@ -0,0 +1,78 @@ +# Bootstrap Dockerfile +FROM ubuntu:latest + +# Set noninteractive mode for apt +ENV DEBIAN_FRONTEND=noninteractive + +# Update package lists and install build dependencies +RUN apt-get update && apt-get install -y \ + build-essential \ + cmake \ + curl \ + gfortran \ + git \ + pkg-config \ + libfftw3-dev \ + libpapi-dev \ + libopenmpi-dev \ + openmpi-bin \ + openmpi-common \ + libopenblas-openmp-dev \ + libhdf5-dev \ + libssl-dev \ + zlib1g-dev \ + libbz2-dev \ + libreadline-dev \ + libsqlite3-dev \ + libffi-dev \ + liblzma-dev \ + libexpat1-dev \ + libgdbm-dev \ + libgdbm-compat-dev \ + libncurses5-dev \ + libncursesw5-dev \ + uuid-dev \ + libnss3-dev \ + libedit-dev \ + libuuid1 \ + libdb5.3-dev \ + lzma \ + tcl-dev \ + gdb \ + lcov \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Create software directories (removed uv directory) +RUN mkdir -p /sw/cargo /sw/rust /sw/python /sw/venv + +# Python installation +ARG PYTHON_VERSION=3.13.1 +RUN curl -o /tmp/Python-${PYTHON_VERSION}.tar.xz \ + https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tar.xz && \ + cd /tmp && \ + tar xf Python-${PYTHON_VERSION}.tar.xz && \ + cd Python-${PYTHON_VERSION} && \ + ./configure --prefix=/sw/python --enable-optimizations && \ + make -j$(nproc) && \ + make install && \ + cd /tmp && \ + rm -rf Python-${PYTHON_VERSION}* + +# Set Environment Paths +ENV RUSTUP_HOME=/sw/rust +ENV CARGO_HOME=/sw/cargo +ENV VIRTUAL_ENV="/sw/venv" +# Added /sw/python/bin to path early to ensure we use the compiled python +ENV PATH="$VIRTUAL_ENV/bin:/sw/python/bin:/sw/cargo/bin:/EMOD3D/tools:$PATH" +ENV LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" + +# Install Rust +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + +# Setup Virtual Environment and Install via pip +ARG WORKFLOW_BRANCH=pegasus +RUN python3 -m venv $VIRTUAL_ENV && \ + pip install --no-cache-dir --upgrade pip setuptools wheel && \ + pip install --no-cache-dir "git+https://github.com/ucgmsim/workflow@${WORKFLOW_BRANCH}" --no-binary=h5py && \ + chmod -R a+rX /sw From 39da56e0b272174eaaf94c99e9c7b7e4b9793a11 Mon Sep 17 00:00:00 2001 From: Jake Faulkner Date: Tue, 17 Mar 2026 17:10:52 +1300 Subject: [PATCH 3/9] Simplify even further --- container/runner.def | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/container/runner.def b/container/runner.def index 6922cfdd..356a2c99 100644 --- a/container/runner.def +++ b/container/runner.def @@ -2,11 +2,7 @@ Bootstrap: docker From: earthquakesuc/workflow-bootstrap:latest %files - /tmp/EMOD3D /EMOD3D - -%setup - rm -rf /tmp/EMOD3D && git clone --depth=1 git@github.com:ucgmsim/EMOD3D.git /tmp/EMOD3D - + EMOD3D /EMOD3D %post cd /EMOD3D && \ @@ -15,6 +11,10 @@ From: earthquakesuc/workflow-bootstrap:latest make -j $(nproc) genslip_v5.6.2 srf2stoch generic_slip2srf \ hb_high_binmod_v6.0.3 + cd / + # Keep only the EMOD3D tools folder and compiled binaries + find /EMOD3D -mindepth 1 -maxdepth 1 ! -name "tools" -exec rm -rf {} \; + %environment export RUSTUP_HOME=/sw/rust export CARGO_HOME=/sw/cargo From 8062c2c03e61fc45da2288b0cb9b291a97d3bd90 Mon Sep 17 00:00:00 2001 From: Jake Faulkner Date: Tue, 17 Mar 2026 17:14:53 +1300 Subject: [PATCH 4/9] wrong version of python --- container/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/Dockerfile b/container/Dockerfile index 913f5e61..66be2382 100644 --- a/container/Dockerfile +++ b/container/Dockerfile @@ -47,7 +47,7 @@ RUN apt-get update && apt-get install -y \ RUN mkdir -p /sw/cargo /sw/rust /sw/python /sw/venv # Python installation -ARG PYTHON_VERSION=3.13.1 +ARG PYTHON_VERSION=3.13.12 RUN curl -o /tmp/Python-${PYTHON_VERSION}.tar.xz \ https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tar.xz && \ cd /tmp && \ From 6fae1bf9a0f395306256894bfa91a2c58e52a434 Mon Sep 17 00:00:00 2001 From: Jake Faulkner Date: Tue, 17 Mar 2026 17:16:56 +1300 Subject: [PATCH 5/9] remove uv path --- container/runner.def | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/runner.def b/container/runner.def index 356a2c99..652a50e7 100644 --- a/container/runner.def +++ b/container/runner.def @@ -18,6 +18,6 @@ From: earthquakesuc/workflow-bootstrap:latest %environment export RUSTUP_HOME=/sw/rust export CARGO_HOME=/sw/cargo - export PATH="/sw/venv/bin:/EMOD3D/tools:/sw/cargo/bin:/sw/python/bin:/sw/uv:$PATH" + export PATH="/sw/venv/bin:/EMOD3D/tools:/sw/cargo/bin:/sw/python/bin:$PATH" export VIRTUAL_ENV="/sw/venv" export LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" From e2745c4a03c5cec91749c5f54428f82fc6dd989d Mon Sep 17 00:00:00 2001 From: Jake Faulkner Date: Mon, 23 Mar 2026 09:37:21 +1300 Subject: [PATCH 6/9] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- container/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/Dockerfile b/container/Dockerfile index 66be2382..84bd47ad 100644 --- a/container/Dockerfile +++ b/container/Dockerfile @@ -64,7 +64,7 @@ ENV RUSTUP_HOME=/sw/rust ENV CARGO_HOME=/sw/cargo ENV VIRTUAL_ENV="/sw/venv" # Added /sw/python/bin to path early to ensure we use the compiled python -ENV PATH="$VIRTUAL_ENV/bin:/sw/python/bin:/sw/cargo/bin:/EMOD3D/tools:$PATH" +ENV PATH="$VIRTUAL_ENV/bin:/sw/python/bin:/sw/cargo/bin:$PATH" ENV LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" # Install Rust From 1c4679ee1eeda0e5fb842f655b5e20142c584345 Mon Sep 17 00:00:00 2001 From: Jake Faulkner Date: Mon, 23 Mar 2026 09:37:42 +1300 Subject: [PATCH 7/9] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .github/workflows/container.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 7eb3b254..eb209bf3 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -28,7 +28,7 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: ${{ env.IMAGE_NAME }} + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | type=sha,format=long type=ref,event=branch @@ -48,5 +48,5 @@ jobs: # Use the long SHA tag generated by the meta step docker run --rm \ -v ${{ github.workspace }}:/mnt \ - ${{ env.IMAGE_NAME }}:sha-${{ github.sha }} \ + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:sha-${{ github.sha }} \ bash -c "pytest /mnt/tests" From 811725398e5925411ab04fa81ece7726de5426dc Mon Sep 17 00:00:00 2001 From: Jake Faulkner Date: Mon, 23 Mar 2026 09:37:59 +1300 Subject: [PATCH 8/9] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .github/workflows/container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index eb209bf3..1bb46a9d 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -9,7 +9,7 @@ on: env: REGISTRY: docker.io - IMAGE_NAME: earthquakesuc/workflow-boostrap + IMAGE_NAME: earthquakesuc/workflow-bootstrap jobs: build-test-upload: From cf9cd6b0d1dc333b428f9b33e1c94c1f1ddde914 Mon Sep 17 00:00:00 2001 From: Jake Faulkner Date: Mon, 23 Mar 2026 09:38:14 +1300 Subject: [PATCH 9/9] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .github/workflows/container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 1bb46a9d..c34121a5 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -32,7 +32,7 @@ jobs: tags: | type=sha,format=long type=ref,event=branch - latest + type=raw,value=latest - name: Build and push uses: docker/build-push-action@v5