From f90a20f0124244c917ec5e2060859599ca001f39 Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Thu, 16 Apr 2026 12:05:51 -0500 Subject: [PATCH 1/3] Added a --random-subset option to dunedaq_integtest_bundle.sh to pick a specified number of random integtests to run. --- scripts/dunedaq_integtest_bundle.sh | 68 +++++++++++++++++------------ 1 file changed, 40 insertions(+), 28 deletions(-) diff --git a/scripts/dunedaq_integtest_bundle.sh b/scripts/dunedaq_integtest_bundle.sh index f58a543..b6aa027 100755 --- a/scripts/dunedaq_integtest_bundle.sh +++ b/scripts/dunedaq_integtest_bundle.sh @@ -1,7 +1,7 @@ #!/bin/bash # 10-Oct-2023, KAB -integtest_list=() +initial_integtest_list=() usage() { declare -r script_name=$(basename "$0") @@ -18,16 +18,17 @@ Options: - it can have the special value of \"local\" - integtests in locally-cloned repos will be run -k, --include -x, --exclude - -n - -N - --stop-on-failure : causes the script to stop when one of the integtests reports a failure + --random-subset : randomly picks the specified number of tests from the results of -r/-k/-x + --list-only : list the tests that match the requested patterns without running them --verbosity : requested level of console messages, in range 1-6, where 1 is least, 6 is DRUNC debug + --stop-on-failure : causes the script to stop when one of the integtests reports a failure + --tmpdir : specifies a root directory to use for test output, e.g. a directory instead of '/tmp' --trigger-full-rc-output - the phrase can be a Python regex, which can be useful in handling colorized text --concise-output : suppresses run control and DAQApp messages in order to focus on test results - - this is equivalent to \"--verbosity 1\" - --tmpdir : specifies a root directory to use for test output, e.g. a directory instead of '/tmp' - --list-only : list the tests that match the requested patterns without running them + - this is equivalent to \"--verbosity 1\", and this option may be removed at some point in time + -n + -N --pytest-options : string with one or more dunedaq-specific command-line options to pass to Pytest - available options include the following: --dunerc-path : Path to DUNE run control. Default is to search in \$PATH @@ -54,7 +55,7 @@ CaptureOutput() { tee -a $1 } -GETOPT_TEMP=`getopt -o hr:k:x:n:N: --long help,stop-on-failure,concise-output,include:,exclude:,tmpdir:,verbosity:,trigger-full-rc-output:,list-only,pytest-options: -- "$@"` +GETOPT_TEMP=`getopt -o hr:k:x:n:N: --long help,stop-on-failure,concise-output,include:,exclude:,tmpdir:,verbosity:,trigger-full-rc-output:,random-subset:,list-only,pytest-options: -- "$@"` if [ $? -ne 0 ]; then usage exit 1 @@ -66,6 +67,7 @@ let full_set_requested_interations=1 let stop_on_failure=0 requested_test_names= excluded_test_names= +let random_subset_count=0 only_list_tests="" PYTEST_COMMAND="pytest -s --tb=short" # our core pytest command, with DAQ printout included and short pytest traceback PYTEST_OPTIONS="" @@ -80,8 +82,8 @@ while true; do if [[ "$2" == "all" ]]; then echo "" echo "Building the list of _all_ integtests..." - integtest_list=(`list_available_integtests.sh 2>/dev/null`) - if [[ ${#integtest_list[@]} -eq 0 ]]; then + initial_integtest_list=(`list_available_integtests.sh 2>/dev/null`) + if [[ ${#initial_integtest_list[@]} -eq 0 ]]; then echo "" echo "*** No integtests were found!" echo "" @@ -90,8 +92,8 @@ while true; do elif [[ "$2" == "local" ]]; then echo "" echo "Building the list of _local_ integtests..." - integtest_list=(`list_available_integtests.sh local 2>/dev/null`) - if [[ ${#integtest_list[@]} -eq 0 ]]; then + initial_integtest_list=(`list_available_integtests.sh local 2>/dev/null`) + if [[ ${#initial_integtest_list[@]} -eq 0 ]]; then echo "" echo "*** No integtests were found in local repositories!" echo "" @@ -99,8 +101,8 @@ while true; do fi else repo_list_string=`echo $2 | sed 's/|/ /g'` - integtest_list=(`list_available_integtests.sh ${repo_list_string} 2>/dev/null`) - if [[ ${#integtest_list[@]} -eq 0 ]]; then + initial_integtest_list=(`list_available_integtests.sh ${repo_list_string} 2>/dev/null`) + if [[ ${#initial_integtest_list[@]} -eq 0 ]]; then echo "" echo "*** No integtests were found in the \"${repo_list_string}\" repo(s)." echo "" @@ -152,6 +154,10 @@ while true; do PYTEST_OPTIONS="$PYTEST_OPTIONS --dunerc-fullprint-watch-string $watch_string" shift 2 ;; + --random-subset) + let random_subset_count=$2 + shift 2 + ;; --pytest-options) PYTEST_OPTIONS="$PYTEST_OPTIONS $2" shift 2 @@ -171,8 +177,8 @@ if [[ "${PYTEST_OPTIONS}" != "" ]]; then fi # run the integtests from the daqsystemtest repo if no repo was specified -if [[ "${integtest_list}" == "" ]]; then - integtest_list=(`list_available_integtests.sh daqsystemtest 2>/dev/null`) +if [[ "${initial_integtest_list}" == "" ]]; then + initial_integtest_list=(`list_available_integtests.sh daqsystemtest 2>/dev/null`) echo "" echo "Integtests from the _daqsystemtest_ repo will be run..." fi @@ -208,35 +214,41 @@ mkdir -p ${pytest_user_dir} ITGRUNNER_LOG_FILE="${pytest_user_dir}/dunedaq_integtest_bundle_${INITIAL_TIMESTAMP}.log" CURRENT_PID=$$ -if [[ "$only_list_tests" != "" ]]; then - echo "" - echo "The following tests will be run:" -fi -let number_of_individual_tests=0 +# do the first level of test filtering +filtered_integtest_list=() let test_index=0 -for FULL_TEST_NAME in "${integtest_list[@]}"; do +for FULL_TEST_NAME in "${initial_integtest_list[@]}"; do test_name=`basename ${FULL_TEST_NAME}` requested_test=`echo ${test_name} | egrep -i ${requested_test_names:-${test_name}}` excluded_test=`echo ${test_name} | egrep -i ${excluded_test_names:-nullnullnull}` if [[ "${requested_test}" != "" ]] && [[ "${excluded_test}" == "" ]]; then - let number_of_individual_tests=${number_of_individual_tests}+1 - if [[ "$only_list_tests" != "" ]]; then - echo " ${FULL_TEST_NAME}" - fi + filtered_integtest_list+=("${FULL_TEST_NAME}") fi let test_index=${test_index}+1 done -let total_number_of_tests=${number_of_individual_tests}*${individual_test_requested_iterations}*${full_set_requested_interations} + +# reduce the list of tests to a random subset, if requested +if [[ $random_subset_count -gt 0 ]]; then + filtered_integtest_list=($(shuf -n "$random_subset_count" -e "${filtered_integtest_list[@]}")) +fi + if [[ "$only_list_tests" != "" ]]; then + echo "" + echo "The following tests will be run:" + for FULL_TEST_NAME in "${filtered_integtest_list[@]}"; do + echo " ${FULL_TEST_NAME}" + done exit 0 fi +let number_of_individual_tests=${#filtered_integtest_list[@]} +let total_number_of_tests=${number_of_individual_tests}*${individual_test_requested_iterations}*${full_set_requested_interations} # run the tests let overall_test_index=0 # this is only used for user feedback let full_set_loop_count=0 while [[ ${full_set_loop_count} -lt ${full_set_requested_interations} ]]; do let test_index=0 - for FULL_TEST_NAME in "${integtest_list[@]}"; do + for FULL_TEST_NAME in "${filtered_integtest_list[@]}"; do test_repo=`dirname ${FULL_TEST_NAME}` test_name=`basename ${FULL_TEST_NAME}` CURRENT_TIMESTAMP=`date '+%Y%m%d%H%M%S'` From 36aed2282c65742996ff94f55f8ca1dd826182cf Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Tue, 28 Apr 2026 15:13:13 -0500 Subject: [PATCH 2/3] Added support to the bundle script for multiple instances of the -r, -k, and -x options --- scripts/dunedaq_integtest_bundle.sh | 266 +++++++++++++-------------- scripts/list_available_integtests.sh | 44 +++-- 2 files changed, 161 insertions(+), 149 deletions(-) diff --git a/scripts/dunedaq_integtest_bundle.sh b/scripts/dunedaq_integtest_bundle.sh index b6aa027..bca09d4 100755 --- a/scripts/dunedaq_integtest_bundle.sh +++ b/scripts/dunedaq_integtest_bundle.sh @@ -65,8 +65,9 @@ eval set -- "$GETOPT_TEMP" let individual_test_requested_iterations=1 let full_set_requested_interations=1 let stop_on_failure=0 -requested_test_names= -excluded_test_names= +repo_list=() +requested_test_names="" +excluded_test_names="" let random_subset_count=0 only_list_tests="" PYTEST_COMMAND="pytest -s --tb=short" # our core pytest command, with DAQ printout included and short pytest traceback @@ -79,44 +80,24 @@ while true; do exit 0 ;; -r) - if [[ "$2" == "all" ]]; then - echo "" - echo "Building the list of _all_ integtests..." - initial_integtest_list=(`list_available_integtests.sh 2>/dev/null`) - if [[ ${#initial_integtest_list[@]} -eq 0 ]]; then - echo "" - echo "*** No integtests were found!" - echo "" - exit 3 - fi - elif [[ "$2" == "local" ]]; then - echo "" - echo "Building the list of _local_ integtests..." - initial_integtest_list=(`list_available_integtests.sh local 2>/dev/null`) - if [[ ${#initial_integtest_list[@]} -eq 0 ]]; then - echo "" - echo "*** No integtests were found in local repositories!" - echo "" - exit 3 - fi - else - repo_list_string=`echo $2 | sed 's/|/ /g'` - initial_integtest_list=(`list_available_integtests.sh ${repo_list_string} 2>/dev/null`) - if [[ ${#initial_integtest_list[@]} -eq 0 ]]; then - echo "" - echo "*** No integtests were found in the \"${repo_list_string}\" repo(s)." - echo "" - exit 3 - fi - fi + repo_list_string=`echo $2 | sed 's/|/ /g'` + repo_list+=("${repo_list_string}") shift 2 ;; -k|--include) - requested_test_names=$2 + if [[ "${requested_test_names}" == "" ]]; then + requested_test_names="$2" + else + requested_test_names="${requested_test_names}|$2" + fi shift 2 ;; -x|--exclude) - excluded_test_names=$2 + if [[ "${excluded_test_names}" == "" ]]; then + excluded_test_names=$2 + else + excluded_test_names="${excluded_test_names}|$2" + fi shift 2 ;; -n) @@ -177,12 +158,33 @@ if [[ "${PYTEST_OPTIONS}" != "" ]]; then fi # run the integtests from the daqsystemtest repo if no repo was specified -if [[ "${initial_integtest_list}" == "" ]]; then - initial_integtest_list=(`list_available_integtests.sh daqsystemtest 2>/dev/null`) +if [[ "${#repo_list}" -eq 0 ]]; then echo "" echo "Integtests from the _daqsystemtest_ repo will be run..." fi +for requested_repo in "${repo_list[@]}"; do + if [[ "${requested_repo}" == "all" ]]; then + echo "" + echo "Building the list of _all_ integtests..." + break + fi +done +for requested_repo in "${repo_list[@]}"; do + if [[ "${requested_repo}" == "local" ]]; then + echo "" + echo "Building the list of _local_ integtests..." + break + fi +done +initial_integtest_list=(`list_available_integtests.sh ${repo_list[@]} 2>/dev/null`) +if [[ ${#initial_integtest_list[@]} -eq 0 ]]; then + echo "" + echo "*** No integtests were found in the \"${repo_list[@]}\" repo(s)." + echo "" + exit 3 +fi + # check if the numad daemon is running numad_grep_output=`ps -ef | grep numad | grep -v grep` if [[ "${numad_grep_output}" != "" ]]; then @@ -233,10 +235,12 @@ if [[ $random_subset_count -gt 0 ]]; then fi if [[ "$only_list_tests" != "" ]]; then + let idx=0 echo "" echo "The following tests will be run:" for FULL_TEST_NAME in "${filtered_integtest_list[@]}"; do - echo " ${FULL_TEST_NAME}" + let idx+=1 + echo " ${idx} ${FULL_TEST_NAME}" done exit 0 fi @@ -257,118 +261,114 @@ while [[ ${full_set_loop_count} -lt ${full_set_requested_interations} ]]; do # the test results are located. That file, in turn, allows this script to find the directory # for the current test, and make a copy of it if the test fails. export DUNEDAQ_INTEGTEST_BUNDLE_INFO="${INITIAL_TIMESTAMP};${CURRENT_PID};${CURRENT_TIMESTAMP}" - requested_test=`echo ${test_name} | egrep -i ${requested_test_names:-${test_name}}` - excluded_test=`echo ${test_name} | egrep -i ${excluded_test_names:-nullnullnull}` - if [[ "${requested_test}" != "" ]] && [[ "${excluded_test}" == "" ]]; then - let individual_loop_count=0 - while [[ ${individual_loop_count} -lt ${individual_test_requested_iterations} ]]; do - let overall_test_index=${overall_test_index}+1 - echo "" - echo -e "\U0001F535 \033[0;34mStarting test ${overall_test_index} of ${total_number_of_tests}...\033[0m \U0001F535" | CaptureOutput ${ITGRUNNER_LOG_FILE} + let individual_loop_count=0 + while [[ ${individual_loop_count} -lt ${individual_test_requested_iterations} ]]; do + let overall_test_index=${overall_test_index}+1 + echo "" + echo -e "\U0001F535 \033[0;34mStarting test ${overall_test_index} of ${total_number_of_tests}...\033[0m \U0001F535" | CaptureOutput ${ITGRUNNER_LOG_FILE} - echo -e "\u2B95 \033[0;1mRunning ${FULL_TEST_NAME}\033[0m \u2B05" | CaptureOutput ${ITGRUNNER_LOG_FILE} - if [[ -e "./${test_name}" ]]; then - ${PYTEST_COMMAND} ./${test_name} | CaptureOutputNoANSI ${ITGRUNNER_LOG_FILE} - elif [[ -e "${DBT_AREA_ROOT}/sourcecode/${test_repo}/integtest/${test_name}" ]]; then - if [[ -w "${DBT_AREA_ROOT}" ]]; then - ${PYTEST_COMMAND} ${DBT_AREA_ROOT}/sourcecode/${test_repo}/integtest/${test_name} | CaptureOutputNoANSI ${ITGRUNNER_LOG_FILE} - else - ${PYTEST_COMMAND} -p no:cacheprovider --no-summary ${DBT_AREA_ROOT}/sourcecode/${test_repo}/integtest/${test_name} | CaptureOutputNoANSI ${ITGRUNNER_LOG_FILE} - fi + echo -e "\u2B95 \033[0;1mRunning ${FULL_TEST_NAME}\033[0m \u2B05" | CaptureOutput ${ITGRUNNER_LOG_FILE} + if [[ -e "./${test_name}" ]]; then + ${PYTEST_COMMAND} ./${test_name} | CaptureOutputNoANSI ${ITGRUNNER_LOG_FILE} + elif [[ -e "${DBT_AREA_ROOT}/sourcecode/${test_repo}/integtest/${test_name}" ]]; then + if [[ -w "${DBT_AREA_ROOT}" ]]; then + ${PYTEST_COMMAND} ${DBT_AREA_ROOT}/sourcecode/${test_repo}/integtest/${test_name} | CaptureOutputNoANSI ${ITGRUNNER_LOG_FILE} else - share_envvar_name="${test_repo^^}_SHARE" # double caret converts env var to uppercase - ${PYTEST_COMMAND} -p no:cacheprovider --no-summary ${!share_envvar_name}/integtest/${test_name} | CaptureOutputNoANSI ${ITGRUNNER_LOG_FILE} + ${PYTEST_COMMAND} -p no:cacheprovider --no-summary ${DBT_AREA_ROOT}/sourcecode/${test_repo}/integtest/${test_name} | CaptureOutputNoANSI ${ITGRUNNER_LOG_FILE} fi - let pytest_return_code=${PIPESTATUS[0]} + else + share_envvar_name="${test_repo^^}_SHARE" # double caret converts env var to uppercase + ${PYTEST_COMMAND} -p no:cacheprovider --no-summary ${!share_envvar_name}/integtest/${test_name} | CaptureOutputNoANSI ${ITGRUNNER_LOG_FILE} + fi + let pytest_return_code=${PIPESTATUS[0]} - let individual_loop_count=${individual_loop_count}+1 + let individual_loop_count=${individual_loop_count}+1 - # check if the test failed - if [[ ${pytest_return_code} -ne 0 ]]; then - # 15-Dec-2025, KAB: if the test failed for a reason other than it - # couldn't be found, make a copy of the pytest directory. This allows - # testers to take a look at the results within a reasonable time frame. - # (If we can't find the "jq" JSON utility, we simply note that fact - # and continue.) - # This code makes use of a bread-crumb file that is created by the - # integrationtest infrastructure. - if [[ ${pytest_return_code} -ne 4 ]]; then - if [[ "`which jq 2>/dev/null`" != "" ]]; then - current_pytest_rundir="" - mapfile -t bundle_info_files < <(find "${pytest_user_dir}" -type f -name "bundle_script_info.json" -printf '%T@ %p\n' | grep -v 'failed-' | sort -nr | awk '{print $2}') - for info_file in "${bundle_info_files[@]}"; do - script_start_time=`jq -r .bundle_script_start_time ${info_file}` - script_pid=`jq -r .bundle_script_process_id ${info_file}` - individual_test_start_time=`jq -r .individual_test_start_time ${info_file}` - if [[ ${script_start_time} -eq ${INITIAL_TIMESTAMP} ]] && \ - [[ ${script_pid} -eq ${CURRENT_PID} ]] && \ - [[ ${individual_test_start_time} -eq ${CURRENT_TIMESTAMP} ]]; then - current_pytest_rundir=$info_file - break - fi - done + # check if the test failed + if [[ ${pytest_return_code} -ne 0 ]]; then + # 15-Dec-2025, KAB: if the test failed for a reason other than it + # couldn't be found, make a copy of the pytest directory. This allows + # testers to take a look at the results within a reasonable time frame. + # (If we can't find the "jq" JSON utility, we simply note that fact + # and continue.) + # This code makes use of a bread-crumb file that is created by the + # integrationtest infrastructure. + if [[ ${pytest_return_code} -ne 4 ]]; then + if [[ "`which jq 2>/dev/null`" != "" ]]; then + current_pytest_rundir="" + mapfile -t bundle_info_files < <(find "${pytest_user_dir}" -type f -name "bundle_script_info.json" -printf '%T@ %p\n' | grep -v 'failed-' | sort -nr | awk '{print $2}') + for info_file in "${bundle_info_files[@]}"; do + script_start_time=`jq -r .bundle_script_start_time ${info_file}` + script_pid=`jq -r .bundle_script_process_id ${info_file}` + individual_test_start_time=`jq -r .individual_test_start_time ${info_file}` + if [[ ${script_start_time} -eq ${INITIAL_TIMESTAMP} ]] && \ + [[ ${script_pid} -eq ${CURRENT_PID} ]] && \ + [[ ${individual_test_start_time} -eq ${CURRENT_TIMESTAMP} ]]; then + current_pytest_rundir=$info_file + break + fi + done - was_successfully_copied="" - if [[ "${current_pytest_rundir}" != "" ]]; then - pytest_tmpdir=`echo ${current_pytest_rundir} | xargs -r dirname | xargs -r dirname` - if [[ "${pytest_tmpdir}" != "" ]]; then - pytest_rootdir=`echo ${pytest_tmpdir} | xargs -r dirname` - pytest_basedir=`echo ${pytest_tmpdir} | xargs -r basename` - if [[ "${pytest_rootdir}" != "" ]] && [[ "${pytest_basedir}" != "" ]]; then - new_dir="${pytest_rootdir}/failed-${pytest_basedir}" - echo "" - echo -e "\U1F535 Copying the files from failed test ${pytest_tmpdir} to ${new_dir}. \U1F535" - echo -e "\U1F535 Please note that copied directories from failed tests typically get cleaned up after 26 hours, \U1F535" - echo -e "\U1F535 or when 10 newer failures happen, whichever comes first. \U1F535" - cp -pR "${pytest_tmpdir}" "${new_dir}" - if [[ $? == 0 ]]; then - was_successfully_copied="yes" - # 18-Dec-2025, KAB: added the removal of the "current" symbolic links - # from inside the copied directory (since they get broken in the copying) - rm -f "${new_dir}/configcurrent" - rm -f "${new_dir}/runcurrent" - fi + was_successfully_copied="" + if [[ "${current_pytest_rundir}" != "" ]]; then + pytest_tmpdir=`echo ${current_pytest_rundir} | xargs -r dirname | xargs -r dirname` + if [[ "${pytest_tmpdir}" != "" ]]; then + pytest_rootdir=`echo ${pytest_tmpdir} | xargs -r dirname` + pytest_basedir=`echo ${pytest_tmpdir} | xargs -r basename` + if [[ "${pytest_rootdir}" != "" ]] && [[ "${pytest_basedir}" != "" ]]; then + new_dir="${pytest_rootdir}/failed-${pytest_basedir}" + echo "" + echo -e "\U1F535 Copying the files from failed test ${pytest_tmpdir} to ${new_dir}. \U1F535" + echo -e "\U1F535 Please note that copied directories from failed tests typically get cleaned up after 26 hours, \U1F535" + echo -e "\U1F535 or when 10 newer failures happen, whichever comes first. \U1F535" + cp -pR "${pytest_tmpdir}" "${new_dir}" + if [[ $? == 0 ]]; then + was_successfully_copied="yes" + # 18-Dec-2025, KAB: added the removal of the "current" symbolic links + # from inside the copied directory (since they get broken in the copying) + rm -f "${new_dir}/configcurrent" + rm -f "${new_dir}/runcurrent" fi fi fi - if [[ "${was_successfully_copied}" == "" ]]; then - echo "" - echo -e "\U1f7e1 WARNING: Unable to copy the pytest directory for this failed test (${current_pytest_rundir}). \U1f7e1" - fi - else - echo "" - echo -e "\U1f7e1 WARNING: Unable to find the 'jq' utility which is needed to help identify which pytest directory to copy for this failed test. \U1f7e1" fi - - # remove stale and surplus directories from failed tests - test_dirs_to_remove=() - mapfile -t all_failed_test_dirs < <(find ${pytest_user_dir} -maxdepth 1 -type d -printf '%T@ %p\n' | sort -nr | awk '{print $2}' | grep 'failed-') - surplus_dirs=("${all_failed_test_dirs[@]:10}") - for test_dir in "${surplus_dirs[@]}"; do - test_dirs_to_remove+=(${test_dir}) - done - stale_failed_test_dirs=(`find ${pytest_user_dir} -maxdepth 1 -type d -name 'failed-*' -cmin +1560 -print`) - for test_dir in "${stale_failed_test_dirs[@]}"; do - test_dirs_to_remove+=(${test_dir}) - done - if [[ ${#test_dirs_to_remove[@]} -gt 0 ]];then + if [[ "${was_successfully_copied}" == "" ]]; then echo "" - echo -e "\U1F535 Removing ${#test_dirs_to_remove[@]} old failed test directory(ies). \U1F535" - for test_dir in "${test_dirs_to_remove[@]}"; do - if [[ -e "${test_dir}" ]]; then - rm -rf "${test_dir}" - fi - done + echo -e "\U1f7e1 WARNING: Unable to copy the pytest directory for this failed test (${current_pytest_rundir}). \U1f7e1" fi + else + echo "" + echo -e "\U1f7e1 WARNING: Unable to find the 'jq' utility which is needed to help identify which pytest directory to copy for this failed test. \U1f7e1" fi - # exit out of this script if the user has requested that we stop on a failure - if [[ ${stop_on_failure} -gt 0 ]]; then - break 3 + # remove stale and surplus directories from failed tests + test_dirs_to_remove=() + mapfile -t all_failed_test_dirs < <(find ${pytest_user_dir} -maxdepth 1 -type d -printf '%T@ %p\n' | sort -nr | awk '{print $2}' | grep 'failed-') + surplus_dirs=("${all_failed_test_dirs[@]:10}") + for test_dir in "${surplus_dirs[@]}"; do + test_dirs_to_remove+=(${test_dir}) + done + stale_failed_test_dirs=(`find ${pytest_user_dir} -maxdepth 1 -type d -name 'failed-*' -cmin +1560 -print`) + for test_dir in "${stale_failed_test_dirs[@]}"; do + test_dirs_to_remove+=(${test_dir}) + done + if [[ ${#test_dirs_to_remove[@]} -gt 0 ]];then + echo "" + echo -e "\U1F535 Removing ${#test_dirs_to_remove[@]} old failed test directory(ies). \U1F535" + for test_dir in "${test_dirs_to_remove[@]}"; do + if [[ -e "${test_dir}" ]]; then + rm -rf "${test_dir}" + fi + done fi fi - done - fi + + # exit out of this script if the user has requested that we stop on a failure + if [[ ${stop_on_failure} -gt 0 ]]; then + break 3 + fi + fi + done let test_index=${test_index}+1 done diff --git a/scripts/list_available_integtests.sh b/scripts/list_available_integtests.sh index 9e9dc26..770fb7a 100755 --- a/scripts/list_available_integtests.sh +++ b/scripts/list_available_integtests.sh @@ -3,11 +3,11 @@ if [[ "$1" == "--help" ]] || [[ "$1" == "-h" ]] || [[ "$1" == "-?" ]]; then echo - echo "Usage: `basename $0` [optional list of repo names|local|all]" + echo "Usage: `basename $0` [optional list of repo names]" echo " e.g. `basename $0` daqsystemtest" echo " If no repo name is specified, integtests for all repos are listed." - echo " If a special repo name of \"local\" is specified, only integtests for repos" - echo " in the local software area are listed." + echo " If a special repo name of \"local\" is specified, integtests for repos in the" + echo " local software area are listed." echo " If a special repo name of \"all\" is specified, integtests for all repos are listed." echo exit @@ -16,25 +16,37 @@ fi echo "" >&2 repo_list=() if [[ $# -ge 1 ]]; then - if [[ "$1" == "local" ]]; then - echo "Looking for integtests in _local_ repos..." >&2 - echo "" >&2 - repo_list=(`list_repos_with_integtests.sh local 2>/dev/null`) - elif [[ "$1" == "all" ]]; then - echo "Looking for integtests in _all_ repos..." >&2 - echo "" >&2 - repo_list=(`list_repos_with_integtests.sh 2>/dev/null`) - else - echo "Looking for integtests in the _$@_ repo(s)..." >&2 - echo "" >&2 - repo_list=("$@") - fi + for arg in "$@" + do + # create a string that we'll use to check if a repo is already in the list + repo_list_string=$(IFS=\| ; echo "${repo_list[*]}") + repo_list_string="|${repo_list_string}|" + + if [[ "$arg" == "local" ]] || [[ "$arg" == "all" ]]; then + echo "Looking for integtests in _${arg}_ repos..." >&2 + echo "" >&2 + temp_list=(`list_repos_with_integtests.sh ${arg} 2>/dev/null`) + for candidate_repo in "${temp_list[@]}"; do + if [[ "`echo \"${repo_list_string}\" | grep \"|${candidate_repo}|\"`" == "" ]]; then + repo_list+=("${candidate_repo}") + fi + done + else + candidate_repo=$arg + if [[ "`echo \"${repo_list_string}\" | grep \"|${candidate_repo}|\"`" == "" ]]; then + repo_list+=("${candidate_repo}") + fi + fi + done else echo "Looking for integtests in _all_ repos..." >&2 echo "" >&2 repo_list=(`list_repos_with_integtests.sh 2>/dev/null`) fi +echo "Looking for integtests in the _${repo_list[@]}_ repo(s)..." >&2 +echo "" >&2 + for repo_name in "${repo_list[@]}"; do share_envvar_name="${repo_name^^}_SHARE" # double caret converts env var to uppercase # ${!var} returns what var points to From a1488c53633a53d0400045323cec9dd264c0b848 Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Wed, 29 Apr 2026 07:50:04 -0500 Subject: [PATCH 3/3] Added support for excluded repositories in the arguments to the global bundle script. --- scripts/dunedaq_integtest_bundle.sh | 37 +++++++++++---- scripts/list_available_integtests.sh | 67 +++++++++++++++++++++++----- 2 files changed, 85 insertions(+), 19 deletions(-) diff --git a/scripts/dunedaq_integtest_bundle.sh b/scripts/dunedaq_integtest_bundle.sh index bca09d4..c220d40 100755 --- a/scripts/dunedaq_integtest_bundle.sh +++ b/scripts/dunedaq_integtest_bundle.sh @@ -16,6 +16,9 @@ Options: - it can be a pipe-delimited string with a list of repos, e.g. 'dfmodules|trigger' - it can have the special value of \"all\" - integtests in all repos will be run - it can have the special value of \"local\" - integtests in locally-cloned repos will be run + -R + - this can be the name of a single repo + - it can be a pipe-delimited string with a list of repos, e.g. 'dfmodules|trigger' -k, --include -x, --exclude --random-subset : randomly picks the specified number of tests from the results of -r/-k/-x @@ -55,7 +58,7 @@ CaptureOutput() { tee -a $1 } -GETOPT_TEMP=`getopt -o hr:k:x:n:N: --long help,stop-on-failure,concise-output,include:,exclude:,tmpdir:,verbosity:,trigger-full-rc-output:,random-subset:,list-only,pytest-options: -- "$@"` +GETOPT_TEMP=`getopt -o hr:R:k:x:n:N: --long help,stop-on-failure,concise-output,include:,exclude:,tmpdir:,verbosity:,trigger-full-rc-output:,random-subset:,list-only,pytest-options: -- "$@"` if [ $? -ne 0 ]; then usage exit 1 @@ -65,7 +68,8 @@ eval set -- "$GETOPT_TEMP" let individual_test_requested_iterations=1 let full_set_requested_interations=1 let stop_on_failure=0 -repo_list=() +requested_repo_list=() +excluded_repo_names="" requested_test_names="" excluded_test_names="" let random_subset_count=0 @@ -81,7 +85,15 @@ while true; do ;; -r) repo_list_string=`echo $2 | sed 's/|/ /g'` - repo_list+=("${repo_list_string}") + requested_repo_list+=("${repo_list_string}") + shift 2 + ;; + -R) + if [[ "${excluded_repo_names}" == "" ]]; then + excluded_repo_names="$2" + else + excluded_repo_names="${excluded_repo_names}|$2" + fi shift 2 ;; -k|--include) @@ -158,29 +170,34 @@ if [[ "${PYTEST_OPTIONS}" != "" ]]; then fi # run the integtests from the daqsystemtest repo if no repo was specified -if [[ "${#repo_list}" -eq 0 ]]; then +if [[ "${#requested_repo_list}" -eq 0 ]]; then + requested_repo_list+=("daqsystemtest") echo "" echo "Integtests from the _daqsystemtest_ repo will be run..." fi -for requested_repo in "${repo_list[@]}"; do +for requested_repo in "${requested_repo_list[@]}"; do if [[ "${requested_repo}" == "all" ]]; then echo "" echo "Building the list of _all_ integtests..." break fi done -for requested_repo in "${repo_list[@]}"; do +for requested_repo in "${requested_repo_list[@]}"; do if [[ "${requested_repo}" == "local" ]]; then echo "" echo "Building the list of _local_ integtests..." break fi done -initial_integtest_list=(`list_available_integtests.sh ${repo_list[@]} 2>/dev/null`) +if [[ "${excluded_repo_names}" == "" ]]; then + initial_integtest_list=(`list_available_integtests.sh ${requested_repo_list[@]} 2>/dev/null`) +else + initial_integtest_list=(`list_available_integtests.sh ${requested_repo_list[@]} -x ${excluded_repo_names} 2>/dev/null`) +fi if [[ ${#initial_integtest_list[@]} -eq 0 ]]; then echo "" - echo "*** No integtests were found in the \"${repo_list[@]}\" repo(s)." + echo "*** No integtests were found in the \"${requested_repo_list[@]}\" repo(s) [with \"${excluded_repo_names}\" repos excluded]." echo "" exit 3 fi @@ -274,10 +291,14 @@ while [[ ${full_set_loop_count} -lt ${full_set_requested_interations} ]]; do if [[ -w "${DBT_AREA_ROOT}" ]]; then ${PYTEST_COMMAND} ${DBT_AREA_ROOT}/sourcecode/${test_repo}/integtest/${test_name} | CaptureOutputNoANSI ${ITGRUNNER_LOG_FILE} else + # remove the trailing "--" in PYTEST_COMMAND since we are adding more pytest options here + PYTEST_COMMAND=`echo ${PYTEST_COMMAND} | sed 's/--$//'` ${PYTEST_COMMAND} -p no:cacheprovider --no-summary ${DBT_AREA_ROOT}/sourcecode/${test_repo}/integtest/${test_name} | CaptureOutputNoANSI ${ITGRUNNER_LOG_FILE} fi else share_envvar_name="${test_repo^^}_SHARE" # double caret converts env var to uppercase + # remove the trailing "--" in PYTEST_COMMAND since we are adding more pytest options here + PYTEST_COMMAND=`echo ${PYTEST_COMMAND} | sed 's/--$//'` ${PYTEST_COMMAND} -p no:cacheprovider --no-summary ${!share_envvar_name}/integtest/${test_name} | CaptureOutputNoANSI ${ITGRUNNER_LOG_FILE} fi let pytest_return_code=${PIPESTATUS[0]} diff --git a/scripts/list_available_integtests.sh b/scripts/list_available_integtests.sh index 770fb7a..3f0f328 100755 --- a/scripts/list_available_integtests.sh +++ b/scripts/list_available_integtests.sh @@ -1,17 +1,52 @@ #!/bin/bash # 19-Dec-2025, KAB -if [[ "$1" == "--help" ]] || [[ "$1" == "-h" ]] || [[ "$1" == "-?" ]]; then - echo - echo "Usage: `basename $0` [optional list of repo names]" - echo " e.g. `basename $0` daqsystemtest" - echo " If no repo name is specified, integtests for all repos are listed." - echo " If a special repo name of \"local\" is specified, integtests for repos in the" - echo " local software area are listed." - echo " If a special repo name of \"all\" is specified, integtests for all repos are listed." - echo - exit +usage() { + declare -r script_name=$(basename "$0") + echo """ +Usage: +"${script_name}" [option(s)] [optional list of repo names] + + Example: `basename $0` daqsystemtest + If no repo name is specified, integtests for all repos are listed. + If a special repo name of \"local\" is specified, integtests for repos in the + local software area are listed. + If a special repo name of \"all\" is specified, integtests for all repos are listed. + +Options: + -h, --help : prints out usage information + -x, --exclude +""" +} + +GETOPT_TEMP=`getopt -o hx: --long help,exclude: -- "$@"` +if [ $? -ne 0 ]; then + usage + exit 1 fi +eval set -- "$GETOPT_TEMP" + +excluded_repo_names="" +while true; do + case "$1" in + -h|--help) + usage + exit 0 + ;; + -x|--exclude) + if [[ "${excluded_repo_names}" == "" ]]; then + excluded_repo_names=$2 + else + excluded_repo_names="${excluded_repo_names}|$2" + fi + shift 2 + ;; + --) + shift + break + ;; + esac +done echo "" >&2 repo_list=() @@ -30,7 +65,7 @@ if [[ $# -ge 1 ]]; then if [[ "`echo \"${repo_list_string}\" | grep \"|${candidate_repo}|\"`" == "" ]]; then repo_list+=("${candidate_repo}") fi - done + done else candidate_repo=$arg if [[ "`echo \"${repo_list_string}\" | grep \"|${candidate_repo}|\"`" == "" ]]; then @@ -44,6 +79,16 @@ else repo_list=(`list_repos_with_integtests.sh 2>/dev/null`) fi +# filter out excluded repos +filtered_repo_list=() +for REPO_NAME in "${repo_list[@]}"; do + excluded_repo=`echo ${REPO_NAME} | egrep -i ${excluded_repo_names:-nullnullnull}` + if [[ "${excluded_repo}" == "" ]]; then + filtered_repo_list+=("${REPO_NAME}") + fi +done +repo_list=("${filtered_repo_list[@]}") + echo "Looking for integtests in the _${repo_list[@]}_ repo(s)..." >&2 echo "" >&2