check for file existance

Author: memalhot

Dockerfile.dev

@@ -0,0 +1,92 @@
+# Ultimately to be consistent with the Red Hat Open AI (RHOAI) environment
+# (and attendant HW) that is avaiable to us in the NERC MOC we base our
+# images on the RHOAI provided images.  The src for these are maintained
+# here:
+#  https://github.com/red-hat-data-services/notebooks
+# the images are published here:
+#  https://quay.io/organization/modh
+#
+
+# The image we have choosen is
+#  quay.io/modh/cuda-notebooks:cuda-jupyter-minimal-ubi9-python-3.11-20250808
+
+# NERC maintains a drived based image that added various packages we need for
+# our development environment see distro packages below
+# src: https://github.com/nerc-images/csw-base-cuda-minimal
+# img: quay.io/nerc-images/csw-base-cuda-minimal:latest
+FROM quay.io/nerc-images/csw-base-cuda-minimal:latest
+
+USER root
+
+# CUDA
+# the following repo is already available from the base so we don't re add but here
+# for documentation purposes
+# as per https://docs.nvidia.com/cuda/cuda-installation-guide-linux/#network-repo-installation-for-fedora
+# we do this so that additional repositories are available even non-NVIDIA ones
+
+# add nvidia nsight-systems cli : https://docs.nvidia.com/nsight-systems/InstallationGuide/index.html
+RUN rpm --import https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub && \
+    dnf config-manager --add-repo "https://developer.download.nvidia.com/devtools/repos/rhel$(source /etc/os-release; echo ${VERSION_ID%%.*})/$(rpm --eval '%{_arch}' | sed s/aarch/arm/)/"
+    
+# add cmake so that nvidia samples can be built    
+RUN dnf install -y \
+ nsight-systems-cli && \
+ yum clean all \
+ && rm -rf /var/cache/yum/*
+
+# DISTRO PACKAGES
+#  nerc minimal has all the based distro packages we need see
+#    https://github.com/nerc-images/csw-base-cuda-minimal/blob/main/Containerfile
+# cmake, emacs, bc, jq, texlive, man-pages 
+
+# PYTHON PACKAGES
+#  python stuff for ope compatability
+RUN pip install \
+    nbstripout \
+    jupyter-book \
+    ghp-import \
+    jupytext \
+    jupyter_nbextensions_configurator \
+    jupyter_contrib_nbextensions \
+    jupyterlab-spellchecker \
+    nbconvert \
+    pyppeteer \
+    jupyterlab-myst \
+    nbgitpuller \
+    jupyterlab_rise \
+    pyright \
+    python-lsp-server
+
+# bash kernel to make it easier bash based juypter notebooks
+RUN pip install bash_kernel && python -m bash_kernel.install
+
+# as per the nbstripout readme we setup nbstripout be always be used for the joyvan user for all repos
+RUN nbstripout --install --system 
+
+# OPE TOOLS
+RUN mkdir /home/ope && \
+    cd /home/ope && \
+    git clone https://github.com/OPEFFORT/tools.git . && \
+    ./install.sh && \
+    fix-permissions /home/ope
+
+# CUDA SUDA WUDA TOOLS
+RUN mkdir /home/csw && \
+    cd /home/csw && \
+    git clone https://github.com/jappavoo/batchtools.git && \
+    for f in /home/csw/batchtools/b*; do \
+       ln -s $f /usr/local/bin/$(basename $f); \
+    done
+
+COPY . /tools
+RUN pwd && ls -la && ls -la /tools
+WORKDIR /tools
+RUN pip install -e .
+
+# Cats and dogs python packages
+RUN pip install \
+    numpy \
+    matplotlib \
+    pandas
+    
+USER 1001

batchtools/br.py

@@ -17,6 +17,7 @@
 from .helpers import oc_delete
 from .helpers import fmt
 from .file_setup import prepare_context
+from pathlib import Path
 from .prom_metrics import (
     PROMETHEUS_INSTANCE,
     IN_PROGRESS,
@@ -229,8 +230,27 @@ def run(args: argparse.Namespace):
             sys.exit(f"Error occurred while creating job: {e}")
 
         if args.job_delete and args.wait:
-            subprocess.run(["cat", f"jobs/{job_name}/{job_name}.log"], check=True)
-            print(f"RUNDIR: jobs/{job_name}")
+            job_dir = Path("jobs") / job_name
+            log_file = job_dir / f"{job_name}.log"
+            
+            if result_phase == "succeeded":
+                # Wait for the log file to appear (rsync may take a moment)
+                max_wait = 10  # seconds
+                wait_interval = 0.5
+                elapsed = 0
+                
+                while not log_file.exists() and elapsed < max_wait:
+                    time.sleep(wait_interval)
+                    elapsed += wait_interval
+                
+                if log_file.exists():
+                    subprocess.run(["cat", f"jobs/{job_name}/{job_name}.log"], check=True)
+                    print(f"RUNDIR: jobs/{job_name}")
+                else:
+                    print(f"Warning: Log file not found after {max_wait}s. Check jobs/{job_name}/")
+            else:
+                print("Something went wrong with running your job. Check over your code and please try again.")
+            
             oc_delete("job", job_name)
         else:
             print(

batchtools/prom_metrics.py

@@ -158,3 +158,4 @@ def push_registry_text(grouping_key: dict[str, str] | None = None) -> None:
     except Exception as e:
         pass
         # print(f"PROM: failed to push metrics to pushgateway {PUSHGATEWAY_ADDR}: {e}")
+

push.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+export ORG="${ORG:-memalhot}"
+export IMAGE_NAME="${IMAGE_NAME:-csw-dev-test}"
+export TAG="${TAG:-latest}"
+
+IMAGE="quay.io/${ORG}/${IMAGE_NAME}:${TAG}"
+
+# Jupyter defaults
+CONTAINER_PORT="${CONTAINER_PORT:-8888}"
+HOST_PORT="${HOST_PORT:-8888}"
+
+cmd="${1:-}"
+
+case "$cmd" in
+  build|"")
+    docker build -f Dockerfile.dev -t "$IMAGE" .
+    ;;
+  push)
+    docker push "$IMAGE"
+    ;;
+  buildpush)
+    docker build -f Dockerfile.dev -t "$IMAGE" .
+    docker push "$IMAGE"
+    ;;
+  run)
+    docker run --rm -it \
+      -p "${HOST_PORT}:${CONTAINER_PORT}" \
+      "$IMAGE"
+    ;;
+  run-hostnet)
+    docker run --rm -it --network host "$IMAGE"
+    ;;
+  *)
+    echo "Usage: $0 {build|push|buildpush|run|run-hostnet}"
+    echo "Env: ORG IMAGE_NAME TAG HOST_PORT CONTAINER_PORT"
+    exit 2
+    ;;
+esac
+
+
+#docker run --rm -it -p 8888:8888 a2969a46eebd