run.sh.in

#!/usr/bin/env bash

# THIS IS A TEMPLATE FILE (it will be used globally to define
# runs for the bench-runner system).
# BE CAREFUL WHAT YOU CHANGE HERE!

# As this is gear toward SaC based programs and the compiler,
# there are some strictly non-generic things happening here.
# For example we do a profiling run.

# SBATCH RELATED
#SBATCH --exclusive
#SBATCH --partition specialq
#SBATCH --exclude gpu01
#SBATCH --job-name @NAME@
#SBATCH --time @TIMELIMIT@
#SBATCH --mail-type=FAIL
#SBATCH --mail-user=hans@viess.mn
#SBATCH --gres=gpu:k20:1

# SETUP MODULES
module purge
module load shared
module load gcc/4.8.5
module load cuda90/toolkit/9.0.176
module load hwloc/1.11.6
# we want to use the production version as the compiled
# code is probably a bit faster - and it matchs normal
# user experience (most users will never use the develop
# version of the compiler).
module load sac/amd-prod-git

# BASH SETTINGS
shopt -s nullglob
set -bm

# activate ramdisk mounting
MOUNT=true

# because of HDD space limits, we need to mount a ramdrive
# this has to happen first otherwise things could go fubar.
if [ "$MOUNT" = true ]; then
    if ! mount /tmp; then
        error "unable to mount /tmp"
        exit 10
    fi
fi

# SETUP SCRIPT
WORKDIR="$(mktemp -d)"
STAGE="NULL"
declare -A SKIP
LOCALDIR="@PWD@"
# shellcheck disable=SC2034
LOGFILE="${WORKDIR}/output.log"
# shellcheck disable=SC2034
VERBOSITY=4 # info

# PROFILE
@PROFILE@

# STDLIB
for p in @PWD@/modules/*.bash; do
    # shellcheck source=/dev/null
    source "${p}"
done

# REDIRECTS
exec 2>&1 >&3 # redirect stdout and stderr to fd 3 (logger)

log_dir()
{
    local -a content=( "${1}"/* )
    printf '        ........................ LOCAL FILES .....................\n'
    for i in "${content[@]}"; do
        printf '        %s\n' "$(basename "$i")"
    done
    printf '        ..........................................................\n'
}

state()
{
    arraylog "STAGE = ${STAGE}" \
        "COMPILER = ${COMPILER}" "WORKDIR = ${WORKDIR}" \
        "LOCALDIR = ${LOCALDIR}" "TARGET = ${target:-unset}" \
        "VARIENT = ${varient:-unset}" \
        "BUILDFLAGS = ${buildflags:-unset}"
    log_dir "${WORKDIR}"
}

saveit()
{
    inf "saving work directory..."

    pushd "/tmp" || exit 10

    # we don't always copy over the sources, so we need to make sure we do this
    # eventually.
    # FIXME perhaps we should do this always...
    for s in "${SOURCES[@]}"; do
        [ -e "$(basename "${s}")" ] || cp -r "${LOCALDIR}/${s}" "${WORKDIR}/"
    done

    if ! tar -czf "${BENCHSUITE}-${BENCHNAME}-result.tar.gz" "${WORKDIR}/"; then
        error_handling
    fi
    [ -d "${LOCALDIR}/results/${BENCHSUITE}-${BENCHNAME}" ] || mkdir -p "${LOCALDIR}/results/${BENCHSUITE}-${BENCHNAME}"
    if ! mv "${BENCHSUITE}-${BENCHNAME}-result.tar.gz" "${LOCALDIR}/results/${BENCHSUITE}-${BENCHNAME}/"; then
        error_handling
    fi
    [ -e "${BENCHSUITE}-${BENCHNAME}-result.tar.gz" ] && rm "${BENCHSUITE}-${BENCHNAME}-result.tar.gz"
    popd || exit 10
}

timeit() {
    # params: binary runflags logfile stdin
    local -a runflags
    read -r -a runflags <<< "${2}"
    local timerformat='elapsed: %e s'
    local stdin="${4:-/dev/null}"
    debug "RUN: ./${1} ${runflags[*]} < ${stdin} &>> ${3}"
    if ! /usr/bin/time -p -o "timerfile" -f "${timerformat}" ./"${1}" "${runflags[@]}" < "${stdin}" &>> "${3}"; then
        error_handling
    fi
    __log_noredirect 4 "INFO" "  Time with profiler" >> "${3}"
    cat "timerfile" >> "${3}"
}

timeit_cuda() {
    # params: binary runflags logfile stdin
    local -a runflags
    read -r -a runflags <<< "${2}"
    local timerformat='elapsed: %e s'
    local stdin="${4:-/dev/null}"
    inf "Running with NVPROF"
    debug "RUN: ./${1} ${runflags[*]} < ${stdin} &> /dev/null"
    __log_noredirect 4 "INFO" "  CUDA profiler output:" >> "${3}"
    if ! nvprof -f --profile-api-trace all --unified-memory-profiling per-process-device -u ms --csv --log-file "${3}" ./"${1}" "${runflags[@]}" < "${stdin}" &> /dev/null; then
        error_handling
    fi
}

finish()
{
    inf "Removing NVPROF tmp dir, if it exists"
    test -d /tmp/.nvprof && rm -rf /tmp/.nvprof
    inf "Deleting workdir..."
    rm -rf "${WORKDIR}"

    # if we mounted our ramdrive we need to unmount it now
    # XXX this should only be safe on GPU nodes
    if [ "$MOUNT" = true ] && grep -q '/tmp' /etc/mtab; then
        if ! umount /tmp; then
            error "unable to umount /tmp"
            exit 10
        fi
    fi
}

error_handling()
{
    error "An error was detected, getting state information"
    state
    if [ "x${STAGE}" = "xBUILD" ]; then
        warn "will skip running \`${target}-${varient}-${btype}' later..."
        SKIP["${target}-${varient}-${btype}"]=1
    fi
    inf "Trying to continue..."
}

handle_child()
{
    if [[ $? -eq 139 ]]; then
        error "current run has segfaulted, no output could be saved"
        error_handling
    fi
}

# TRAPS

trap finish EXIT
#trap error_handling ERR
trap handle_child CHLD

# we move to the workdir
pushd "${WORKDIR}" || exit 10

# GET SYSTEM INFO

STAGE='INFO'
inf "Running \`${BENCHNAME}' from \`${BENCHSUITE}'"
inf "Gathering system Information"
sysinfo

# RUN SPECIFICATIONs
MAXREP=5 # number of repatitions to do
MAXTHREADS=$(( $(get_physical_core_count) - 1 )) # we don't want to oversubscribe the system
inf "Max number repetitions for each run is ${MAXREP}"
inf "Max number of threads for multi-threaded run is ${MAXTHREADS}"

# COMPILE BENCHMARK

STAGE="BUILD"

if [ "x${MODE}" = "xMANUAL" ]; then
    for source in "${SOURCES[@]}"; do
        cp -r "${LOCALDIR}/${source}" "${WORKDIR}/"
    done

    # call build function
    capture build
else
    for source in "${SOURCES[@]}"; do
        count=0
        for target in "${TARGETS[@]}"; do
            for varient in "${VARIENTS[@]}"; do
                read -r -a buildflags <<< "$(eval echo "\${BUILDFLAGS_${varient}[${count}]}")"
                binary_name="${BENCHSUITE}-${BENCHNAME}-${target}-${varient}.out"
                build_log="${WORKDIR}/sac2c-build-${target}-${varient}.log"

                if [[ ! "${target}" =~ "mt" ]]; then
                    btype='p'
                    inf "compiling Profiling-Version target '$target' variant '$varient' with \`${buildflags[*]}'"
                    if ! $COMPILER "${buildflags[@]}" -profile m -t "${target}" -o "profile-${binary_name}" "${LOCALDIR}/${source}" &>> "${build_log}"; then
                        error_handling
                    fi
                fi
                btype='t'
                inf "compiling Timing-Version target '$target' variant '$varient' with \`${buildflags[*]}'"
                if ! $COMPILER "${buildflags[@]}" -t "${target}" -o "${binary_name}" "${LOCALDIR}/${source}" &>> "${build_log}"; then
                    error_handling
                fi
                unset btype
            done
            (( count += 1 ))
        done
    done
fi

# we should unset the counter from before
unset count

# HANDLE INPUT SAC_P

STAGE="INPUT"

if [ ${#INPUTS[@]} -ne 0 ]; then
    for input in "${INPUTS[@]}"; do
        inf "copying $input to working directory ${WORKDIR}"
        cp -r "${LOCALDIR}/${input}" "${WORKDIR}/"
    done
fi

# RUN BENCHMARK

STAGE="RUN"

if [ "x${MODE}" = "xMANUAL" ]; then
    # call run function
    capture run
else
    for target in "${TARGETS[@]}"; do
        for varient in "${VARIENTS[@]}"; do
            binary_name="${BENCHSUITE}-${BENCHNAME}-${target}-${varient}.out"
            if [ ! -e "${binary_name}" ]; then
                error "Timing binary missing! SKIPPING target '$target' and variant '$varient'"
                state
                continue
            fi
            binary_log="${WORKDIR}/profile-${BENCHSUITE}-${BENCHNAME}-${target}-${varient}.log"
            # we skip running profiling for MT as its incompatible at the moment
            if [[ ! "${target}" =~ "mt" ]]; then
                if [ ${SKIP[${target}-${varient}-p]-0} -gt 0 ]; then
                    inf "SKIPPING profiling target '$target' and variant '$varient'"
                else
                    inf "PROFILING target '$target' with variant '$varient'"
                    if [ ! -e "profile-${binary_name}" ]; then
                        error "Profile binary missing! SKIPPING target '$target' and variant '$varient'"
                        state
                        continue
                    fi
                    # shellcheck disable=SC2153
                    timeit "profile-${binary_name}" "${RUNFLAGS}" "${binary_log}" "${STDINS}"

                    if [[ "${target}" =~ "cuda" ]]; then
                        # we need to clear out the nvprof dir as it can get really full...
                        debug "Clearing the NVPROF tmp directory"
                        [ -d '/tmp/.nvprof' ] && rm -rf '/tmp/.nvprof'
                        cuda_log="${WORKDIR}/profile-nvprof-${target}-${varient}.csv.log"
                        timeit_cuda "${binary_name}" "${RUNFLAGS}" "${cuda_log}" "${STDINS}"
                        unset cuda_log
                    fi
                fi
            fi
            if [ ${SKIP[${target}-${varient}-t]-0} -gt 0 ]; then
                inf "SKIPPING timing target '$target' and variant '$varient'"
            else
                inf "TIMING target '$target' with variant '$varient'"
                inf "Doing ${MAXREP} repetitions of this run"
                binary_log="${WORKDIR}/${BENCHSUITE}-${BENCHNAME}-${target}-${varient}.log"
                for ((count=1; count <= MAXREP; count++)); do
                    inf "REPETITION: ${count}/${MAXREP}"
                    __log_noredirect 4 "INFO" "REPETITION: ${count}/${MAXREP}" >> "${binary_log}"
                    if [[ "${target}" =~ "mt" ]]; then
                        for ((thread = 1; thread <= MAXTHREADS; thread+=1)); do
                            # we only want even numbers, and one: 1 2 4 6 8 ...
                            if [ "x${thread}" = "x1" ] || [ "$(( thread % 2 ))" = "0" ]; then
                                inf "  MT MODE: ${thread}/${MAXTHREADS} threads"
                                __log_noredirect 4 "INFO" "  MT MODE: ${thread}/${MAXTHREADS} threads" >> "${binary_log}"
                                export SAC_PARALLEL=${thread}
                                # shellcheck disable=SC2153
                                timeit "${binary_name}" "${RUNFLAGS}" "${binary_log}" "${STDINS}"
                                unset SAC_PARALLEL
                            fi
                        done
                    else # sequentional targets
                        timeit "${binary_name}" "${RUNFLAGS}" "${binary_log}" "${STDINS}"
                    fi
                done
            fi
            unset binary_name binary_log count
        done
    done
fi

# we leave the workdir
popd || exit 10

inf "done running"

# SAVE LOGS

STAGE="SAVE"

saveit