OSPOOL-158: Add NRP OSPool EP image builds

.github/workflows/build-containers.yml

@@ -30,7 +30,7 @@ jobs:
 
       - id: image-list
         run: |
-          ORG_DIR=opensciencegrid
+          ORG_DIR_REGEX="^(opensciencegrid|osg-htc)/"
           # Get the list of files changed based on the type of event
           # kicking off the GHA:
           # 1. For the main branch, diff the previous state of main vs
@@ -54,7 +54,7 @@ jobs:
                images=$(git diff --name-only \
                                  "$BASE" \
                                  "$GITHUB_SHA" |
-                        egrep "^$ORG_DIR/" |
+                        grep -E "$ORG_DIR_REGEX" |
                         cut -d/ -f -2 |
                         sort |
                         uniq |
@@ -64,10 +64,11 @@ jobs:
           else
                # List all image root dirs. Example value:
                # "opensciencegrid/vo-frontend opensciencegrid/ospool-cm"
-               images=$(find $ORG_DIR -mindepth 1 \
-                                      -maxdepth 1 \
-                                      -type d \
-                                      -printf "$ORG_DIR/%P\n")
+               images=$(find -mindepth 2 \
+                             -maxdepth 2 \
+                             -type d \
+                             -printf "%P\n" |
+                        grep -E "${ORG_DIR_REGEX}")
           fi
 
             image_json=$(echo -n "${images:-dummy}" | jq -Rcs '.|split("\n") | map(select(. != ""))')
@@ -124,7 +125,7 @@ jobs:
           BASE_OS=$(echo $CONFIG | awk -F'-' '{print $1}')
           OSG_SERIES=$(echo $CONFIG | awk -F'-' '{print $2}')
           BASE_REPO=$(echo $CONFIG | awk -F'-' '{print $3}')
-          CONTEXT="opensciencegrid/${{ matrix.name }}"
+          CONTEXT="${{ matrix.name }}"
           echo "BASE_OS=${BASE_OS}" >> $GITHUB_ENV
           echo "OSG_SERIES=${OSG_SERIES}" >> $GITHUB_ENV
           echo "BASE_REPO=${BASE_REPO}" >> $GITHUB_ENV
@@ -176,7 +177,7 @@ jobs:
           BASE_OS=$(echo $CONFIG | awk -F'-' '{print $1}')
           OSG_SERIES=$(echo $CONFIG | awk -F'-' '{print $2}')
           BASE_REPO=$(echo $CONFIG | awk -F'-' '{print $3}')
-          CONTEXT="opensciencegrid/${{ matrix.name }}"
+          CONTEXT="${{ matrix.name }}"
           echo "BASE_OS=${BASE_OS}" >> $GITHUB_ENV
           echo "OSG_SERIES=${OSG_SERIES}" >> $GITHUB_ENV
           echo "BASE_REPO=${BASE_REPO}" >> $GITHUB_ENV

osg-htc/nrp-ospool-ep/Dockerfile

@@ -0,0 +1,77 @@
+ARG BASE_OSG_SERIES=25
+ARG BASE_YUM_REPO=release
+
+FROM hub.osg-htc.org/osg-htc/ospool-ep:${BASE_OSG_SERIES}-${BASE_YUM_REPO}
+
+# Previous args have gone out of scope
+ARG BASE_OSG_SERIES=25
+ARG BASE_YUM_REPO=release
+
+# Enable OpenCL
+# As suggested by https://github.com/WIPACrepo/pyglidein/blob/master/Dockerfile
+RUN mkdir -p /etc/OpenCL/vendors && \
+    echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd
+
+# Some helper OpenCL tools
+RUN yum install -y clinfo && \
+    yum clean all
+
+# Fix python vs python3
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10
+
+# Add kubectl, to be able to interact with the k8s cluster
+RUN curl -L "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" -o /usr/sbin/kubectl && \
+    chmod u+x /usr/sbin/kubectl
+
+# Tweaked singularity for container environments
+# Remove default singularity from path, too
+COPY scripts/singularity_npid.sh /usr/bin/singularity_npid.sh
+RUN mv /usr/bin/singularity /usr/bin/singularity.org && \
+    chmod a+x /usr/bin/singularity_npid.sh && \
+    ln -s /usr/bin/singularity_npid.sh /usr/bin/singularity
+
+COPY scripts/apptainer_npid.sh /usr/bin/apptainer_npid.sh
+RUN mv /usr/bin/apptainer /usr/bin/apptainer.org && \
+    chmod a+x /usr/bin/apptainer_npid.sh && \
+    ln -s /usr/bin/apptainer_npid.sh /usr/bin/apptainer
+
+# We want to be able to add system-level changes
+# and run without explicit privilege drop
+RUN mv /usr/local/sbin/entrypoint.sh /usr/local/sbin/entrypoint.osg.sh
+COPY scripts/entrypoint.sh /usr/local/sbin/entrypoint.sh
+RUN chmod a+rx /usr/local/sbin/entrypoint.sh
+
+COPY scripts/check_master.sh /bin/check_master.sh
+RUN chmod a+rx /bin/check_master.sh
+
+# system level scripts
+COPY scripts/01_token.sh /etc/entrypoint/image-config.d/01_token.sh
+COPY scripts/01_no_condor_host.sh /etc/entrypoint/image-config.d/01_no_condor_host.sh
+COPY scripts/11_set_OSGInstitutionID.sh /etc/entrypoint/image-config.d/11_set_OSGInstitutionID.sh
+
+# pilot level scripts
+COPY scripts/02_validate_singularity.sh /etc/osg/image-init.d/02_validate_singularity.sh 
+COPY scripts/02_validate_apptainer.sh /etc/osg/image-init.d/02_validate_apptainer.sh 
+COPY scripts/19_set_resources.sh /etc/osg/image-init.d/19_set_resources.sh
+COPY scripts/20_advertise_glidein.sh /etc/osg/image-init.d/20_advertise_glidein.sh
+COPY scripts/20_advertise_k8s_domain.sh /etc/osg/image-init.d/20_advertise_k8s_domain.sh
+COPY scripts/21_advertise_k8s_provisioner.sh  /etc/osg/image-init.d/21_advertise_k8s_provisioner.sh
+COPY scripts/22_set_requirements.sh /etc/osg/image-init.d/22_set_requirements.sh
+
+# keep default low, as we expect to be running in opportunistic mode
+env ACCEPT_JOBS_FOR_HOURS=24
+
+# keep default idle time low, as we may over-provision certain kinds of resources
+# but others may be waiting
+env ACCEPT_IDLE_MINUTES=20
+
+#
+# In addition to osgvo-docker-pilot envs,
+# it accepts also the following env variables
+# NUM_CPUS MEMORY DISK NUM_GPUS
+# PHYSICAL_HOSTNAME
+# K8S_NAMESPACE K8S_DOMAIN
+# K8S_PROVISIONER_NAME K8S_PROVISIONER_TYPE
+# FORCE_K8SNAMESPACE_MATCHING ADDITIONAL_REQUIREMENTS
+#
+

osg-htc/nrp-ospool-ep/build-config.json

@@ -0,0 +1,7 @@
+{
+    "standard_build": true,
+    "repo_build": false,
+    "base_os": ["el9"],
+    "osg_series": ["24", "25"],
+    "base_repo": ["release"]
+  }

osg-htc/nrp-ospool-ep/scripts/01_no_condor_host.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+#
+# osgvo pilot does not like is CONDOR_HOST is set
+#
+
+export CONDOR_HOST=
+unset CONDOR_HOST
+

osg-htc/nrp-ospool-ep/scripts/01_token.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+#
+# osgvo-pilot expects the token as an env variable
+#
+
+export TOKEN=`cat /etc/condor/tokens.d/prp-wn.token`
+

osg-htc/nrp-ospool-ep/scripts/02_validate_apptainer.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# check if we need to test nvidia
+nvf=
+ls -l /dev/nvidia*
+if [ $? -eq 0 ]; then
+  nvf=--nv
+fi
+
+# else do nothing, let Condor figure it out
+
+if [ -f "/usr/bin/apptainer" ]; then
+  # only test for apptainer functionality if singularity is present
+  # may not be in all pods
+
+  /usr/bin/apptainer exec $nvf --contain --ipc --pid --bind /cvmfs /cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-el7:latest /usr/bin/dc -e "3 5 + p"
+  rc=$?
+
+  if [ $rc -ne 0 ]; then
+    echo "Apptainer test execution failed!"
+    sleep 30
+    exit 1
+  fi
+
+fi

osg-htc/nrp-ospool-ep/scripts/02_validate_singularity.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# check if we need to test nvidia
+nvf=
+ls -l /dev/nvidia*
+if [ $? -eq 0 ]; then
+  nvf=--nv
+fi
+
+# else do nothing, let Condor figure it out
+
+if [ -f "/usr/bin/singularity" ]; then
+  # only test for singularity functionality if singularity is present
+  # may not be in all pods
+
+  /usr/bin/singularity exec $nvf --contain --ipc --pid --bind /cvmfs /cvmfs/singularity.opensciencegrid.org/opensciencegrid/osgvo-el8:latest /usr/bin/dc -e "3 5 + p"
+  rc=$?
+
+  if [ $rc -ne 0 ]; then
+    echo "Singularity test execution failed!"
+    sleep 30
+    exit 1
+  fi
+
+fi

osg-htc/nrp-ospool-ep/scripts/11_set_OSGInstitutionID.sh

@@ -0,0 +1,10 @@
+#
+# Fetch OSGInstitutionID from k8s and set it as an env variable
+# unless it is already set
+#
+if [ "x${OSG_INSTITUTION_ID}" == "x" ]; then
+   OSG_INSTITUTION_ID=`/usr/sbin/kubectl get node ${PHYSICAL_HOSTNAME} -L nautilus.io/OSGInstitutionID | tail -1 | awk '{print $6}'`
+   if [ "x${OSG_INSTITUTION_ID}" != "x" ]; then
+      export OSG_INSTITUTION_ID
+   fi
+fi

osg-htc/nrp-ospool-ep/scripts/19_set_resources.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+
+#
+# Advertise the k8s namespace and physical hostname
+#
+
+#
+# NUM_CPUS and MEMORY are also handled by the osgvo-pilot
+# but we re-set it here
+#
+
+full_num_cpus="${NUM_CPUS:-1}"
+full_memory="${MEMORY:-1024}"
+full_disk="${DISK:-100000}"
+full_num_gpus="${NUM_GPUS:-0}"
+
+echo "NUM_CPUS = ${full_num_cpus}" >> "${PILOT_CONFIG_FILE}"
+echo "MEMORY = ${full_memory}" >> "${PILOT_CONFIG_FILE}"
+echo "DISK = ${full_disk}" >> "${PILOT_CONFIG_FILE}"
+
+# single slot using all the requested resources
+echo "NUM_SLOTS_TYPE_1 = 1" >> "${PILOT_CONFIG_FILE}"
+echo "SLOT_TYPE_1_PARTITIONABLE = FALSE" >> "${PILOT_CONFIG_FILE}"
+
+if [ "x${full_num_gpus}" != "x0" ]; then
+   # we cannot really set the number of GPUs, just enable auto-detect
+   echo "use feature : GPUs" >> "${PILOT_CONFIG_FILE}"
+   echo "SLOT_TYPE_1 = cpu=${full_num_cpus},mem=${full_memory},disk=auto,swap=auto,gpus=${full_num_gpus}" \
+      >> "${PILOT_CONFIG_FILE}"
+else
+   echo "SLOT_TYPE_1 = cpu=${full_num_cpus},mem=${full_memory},disk=auto,swap=auto" \
+      >> "${PILOT_CONFIG_FILE}"
+fi
+

osg-htc/nrp-ospool-ep/scripts/20_advertise_glidein.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+
+#
+# Advertise that this is a glidein
+#
+
+if [ "x${ADVERTISE_IS_GLIDEIN}" != "xN" ]; then
+  echo "IS_GLIDEIN = true" >> "${PILOT_CONFIG_FILE}"
+  echo 'STARTD_EXPRS = $(STARTD_EXPRS) IS_GLIDEIN'  >> "${PILOT_CONFIG_FILE}"
+fi
+
+

osg-htc/nrp-ospool-ep/scripts/20_advertise_k8s_domain.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+
+#
+# Advertise the k8s namespace and physical hostname
+#
+
+echo "# K8S params" >> "${PILOT_CONFIG_FILE}"
+
+if [ "x${HOSTNAME}" != "x" ]; then
+  echo "K8SPodName=\"${HOSTNAME}\"" >> "${PILOT_CONFIG_FILE}"
+  echo 'STARTD_EXPRS = $(STARTD_EXPRS) K8SPodName'  >> "${PILOT_CONFIG_FILE}"
+fi
+
+if [ "x${K8S_DOMAIN}" != "x" ]; then
+  echo "K8SDomain=\"${K8S_DOMAIN}\"" >> "${PILOT_CONFIG_FILE}"
+  echo 'STARTD_EXPRS = $(STARTD_EXPRS) K8SDomain'  >> "${PILOT_CONFIG_FILE}"
+fi
+
+if [ "x${K8S_NAMESPACE}" != "x" ]; then
+  echo "K8SNamespace=\"${K8S_NAMESPACE}\"" >> "${PILOT_CONFIG_FILE}"
+  echo 'STARTD_EXPRS = $(STARTD_EXPRS) K8SNamespace'  >> "${PILOT_CONFIG_FILE}"
+fi
+
+if [ "x${PHYSICAL_HOSTNAME}" != "x" ]; then
+  echo "K8SPhysicalHostName=\"${PHYSICAL_HOSTNAME}\"" >> "${PILOT_CONFIG_FILE}"
+  echo 'STARTD_EXPRS = $(STARTD_EXPRS) K8SPhysicalHostName'  >> "${PILOT_CONFIG_FILE}"
+fi
+

osg-htc/nrp-ospool-ep/scripts/21_advertise_k8s_provisioner.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+
+#
+# Advertise the k8s provisioner
+#
+
+if [ "x${K8S_PROVISIONER_TYPE}" != "x" ]; then
+  echo "K8SProvisionerType=\"${K8S_PROVISIONER_TYPE}\"" >> "${PILOT_CONFIG_FILE}"
+  echo 'STARTD_EXPRS = $(STARTD_EXPRS) K8SProvisionerType'  >> "${PILOT_CONFIG_FILE}"
+fi
+
+if [ "x${K8S_PROVISIONER_NAME}" != "x" ]; then
+  echo "K8SProvisionerName=\"${K8S_PROVISIONER_NAME}\"" >> "${PILOT_CONFIG_FILE}"
+  echo 'STARTD_EXPRS = $(STARTD_EXPRS) K8SProvisionerName'  >> "${PILOT_CONFIG_FILE}"
+fi
+

osg-htc/nrp-ospool-ep/scripts/22_set_requirements.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+cat >>  "${PILOT_CONFIG_FILE}" << EOF
+#
+# Force matching on K8SNamespace
+# unless FORCE_K8SNAMESPACE_MATCHING=="no"
+#
+
+FORCE_K8SNAMESPACE_MATCHING = "${FORCE_K8SNAMESPACE_MATCHING:-no}"
+STARTD_EXPRS = \$(STARTD_EXPRS) FORCE_K8SNAMESPACE_MATCHING
+
+MATCHING_START = ( (FORCE_K8SNAMESPACE_MATCHING=?="no") || regexp(TARGET.RequestK8SNamespace,K8SNamespace) )
+
+#
+# Force matching on provisioned resources
+#
+
+PROVISIONING_START = ifthenelse(TARGET.RequestCPUs=!=undefined, CPUs=?=TARGET.RequestCPUs, CPUs=?=1)
+
+# Pretend small memory and disk requests are the equivalent of not set
+# Note: 4096 is special in the provisioner
+PROVISIONING_START = \$(PROVISIONING_START) && \\
+                     ifthenelse(TARGET.RequestMemory=!=undefined, \\
+                           ifthenelse(TARGET.RequestMemory<4096, \\
+                                 Memory=?=4096, \\
+                                 ((Memory>=TARGET.RequestMemory) && ((TARGET.RequestMemory+1536)>Memory))), \\
+                           Memory=?=4096)
+
+# Note: 8000000 is special in the provisioner
+PROVISIONING_START = \$(PROVISIONING_START) && \\
+                     ifthenelse(TARGET.RequestDisk=!=undefined, \\
+                           ifthenelse(TARGET.RequestDisk<8000000, \\
+                                 Disk=?=8000000, \\
+                                 ((Disk>=TARGET.RequestDisk) && ((TARGET.RequestDisk+4500000)>Disk))), \\
+                           Disk=?=8000000)
+
+# GPUs will not be defined if there are no GPUs
+PROVISIONING_START = \$(PROVISIONING_START) && \\
+                     ifthenelse(TARGET.RequestGPUs=!=undefined, \\
+                                ifthenelse(GPUs=!=undefined, GPUs=?=TARGET.RequestGPUs, TARGET.RequestGPUs=?=0), \\
+                                (GPUs=?=undefined) || (GPUs=?=0))
+
+START = ( \$(START) ) && ( \$(PROVISIONING_START) ) && ( \$(MATCHING_START) )
+
+EOF
+
+
+if [ "x${ADDITIONAL_REQUIREMENTS}" != "x" ]; then
+  echo "# Additional requirements added at runtime " >> "${PILOT_CONFIG_FILE}"
+  echo "MATCHING_START = ( \$(MATCHING_START) ) && ( ${ADDITIONAL_REQUIREMENTS} )" >> "${PILOT_CONFIG_FILE}"
+fi