From f5ffb3d115aa86cae6b876c63ac0cfcbb46aeaa9 Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Mon, 11 May 2026 13:02:54 +0100 Subject: [PATCH 1/8] Initial changes to script --- run_hyperion.sh | 123 ++++++++++++++++++++++++++---------------------- 1 file changed, 67 insertions(+), 56 deletions(-) diff --git a/run_hyperion.sh b/run_hyperion.sh index f0299e9bac..847e9feba2 100755 --- a/run_hyperion.sh +++ b/run_hyperion.sh @@ -12,10 +12,12 @@ BLUEAPI_CONFIG=$CONFIG_DIR/blueapi_config.yaml SUPERVISOR_CONFIG=$CONFIG_DIR/supervisor/supervisor_config.yaml CLIENT_CONFIG=$CONFIG_DIR/supervisor/client_config.yaml STOMP_CONFIG=$CONFIG_DIR/blueapi_config.yaml -DO_CALLBACKS=1 HEALTHCHECK_PORT=5005 +SUPERVISOR_HEALTHCHECK_PORT=5006 CALLBACK_WATCHDOG_PORT=5005 CALLBACK_MODE=0mq +START_HYPERION_SUPERVISOR=0 +START_HYPERION_BLUEAPI=0 for option in "$@"; do case $option in @@ -40,12 +42,13 @@ for option in "$@"; do ;; --blueapi) MODE=blueapi + START_HYPERION_BLUEAPI=1 CALLBACK_WATCHDOG_PORT=5006 ;; --supervisor) - MODE=supervisor - DO_CALLBACKS=0 - HEALTHCHECK_PORT=5006 + MODE=blueapi + START_HYPERION_SUPERVISOR=1 + SUPERVISOR_HEALTHCHECK_PORT=5006 ;; --stomp) CALLBACK_MODE=stomp @@ -63,14 +66,15 @@ Options: options. --no-start Used to specify that the script should be run without starting the server. --dev Enable dev mode to run from a local workspace on a development machine. - --udc Start hyperion in UDC mode taking instructions from agamemnon in a monolithic process - --blueapi Start hyperion in blueapi mode taking instructions from the supervisor - --supervisor Start hyperion in supervisor mode, taking commands from Agamemnon and feeding them to - an instance running in blueapi mode. + --udc (Re)start hyperion in UDC mode taking instructions from agamemnon in a monolithic process + --blueapi (Re)start hyperio-blueapi taking instructions from the supervisor + --supervisor (Re)start hyperion-supervisor, taking commands from Agamemnon and feeding them to + hyperion-blueapi. --stomp Start external callbacks in stomp mode instead of 0mq (the default) --help This help By default this script will start an Hyperion server unless the --no-start flag is specified. +Note: --udc is exclusive with --supervisor and --blueapi. END exit 0 ;; @@ -82,15 +86,18 @@ END done kill_active_apps () { - if [ $MODE = "supervisor" ]; then + if [ $MODE = "blueapi" ]; then + echo "Killing vanilla hyperion instances" + pkill -e -f "mx-bluesky/.venv/bin/python .*--mode (gda|udc)" + fi + if [[ $START_HYPERION_SUPERVISOR == 1 || $MODE = "udc" ]]; then # supervisor mode kills only supervisor echo "Killing active instances of hyperion supervisor..." pkill -e -f "mx-bluesky/.venv/bin/python .*--mode supervisor" - else + fi + if [[ $START_HYPERION_BLUEAPI == 1 || $MODE = "udc" ]]; then echo "Killing active instances of hyperion-blueapi" pkill -e -f "python .*mx-bluesky/.venv/bin/blueapi .*serve" - echo "Killing vanilla hyperion instances" - pkill -e -f "mx-bluesky/.venv/bin/python .*--mode (gda|udc)" echo "Killing hyperion-callbacks" pkill -e -f "mx-bluesky/.venv/bin/python .*hyperion-callbacks" fi @@ -104,6 +111,30 @@ check_user () { fi } +wait_for_healthcheck () { + local APP=$1 + local HEALTHCHECK_PORT=$2 + local HEALTHCHECK_ENDPOINT=$3 + echo "$(date) Waiting for $APP to start" + for i in {1..30} + do + echo "$(date)" + curl --head -X GET http://localhost:$HEALTHCHECK_PORT/$HEALTHCHECK_ENDPOINT >/dev/null + ret_value=$? + if [ $ret_value -ne 0 ]; then + sleep 1 + else + break + fi + done + if [ $ret_value -ne 0 ]; then + echo "$(date) $APP Failed to start!!!!" + exit 1 + else + echo "$(date) Hyperion started" + fi +} + if [ -z "${BEAMLINE}" ]; then echo "BEAMLINE environment variable is not set and the --beamline parameter is not specified." echo "Please set the option -b, --beamline=BEAMLINE to set it manually" @@ -159,68 +190,48 @@ if [[ $START == 1 ]]; then echo "Debug log file set to $DEBUG_LOG_DIR" export DEBUG_LOG_DIR mkdir -p "$DEBUG_LOG_DIR" - if [ $MODE = "supervisor" ]; then + source .venv/bin/activate + + if [[ $START_HYPERION_SUPERVISOR == 1 ]]; then start_log_path=$LOG_DIR/supervisor_start_log.log else start_log_path=$LOG_DIR/start_log.log fi callback_start_log_path=$LOG_DIR/callback_start_log.log - source .venv/bin/activate - - declare -A h_and_cb_args=( ["IN_DEV"]="$IN_DEV" ) - declare -A h_and_cb_arg_strings=( ["IN_DEV"]="--dev" ) - - h_commands="--mode $MODE " + h_commands="" cb_commands="--watchdog-port $CALLBACK_WATCHDOG_PORT " - if [ $MODE = "supervisor" ]; then - h_commands+="--client-config ${CLIENT_CONFIG} --supervisor-config ${SUPERVISOR_CONFIG} " + + if [[ "$IN_DEV" == true ]]; then + h_commands+="--dev " + cb_commands+="--dev " fi + if [ "${CALLBACK_MODE}" = "stomp" ]; then - cb_commands+="--stomp-config $STOMP_CONFIG" + cb_commands+="--stomp-config $STOMP_CONFIG " fi - for i in "${!h_and_cb_args[@]}" - do - if [ "${h_and_cb_args[$i]}" != false ]; then - h_commands+="${h_and_cb_arg_strings[$i]} "; - cb_commands+="${h_and_cb_arg_strings[$i]} "; - fi; - done unset PYEPICS_LIBCA - if [ $MODE = "blueapi" ]; then - echo "Starting hyperion in blueapi mode, start log is $start_log_path" + if [[ $START_HYPERION_BLUEAPI == 1 ]]; then + echo "Starting hyperion-blueapi, start log is $start_log_path" blueapi --config $BLUEAPI_CONFIG serve > $start_log_path 2>&1 & - HEALTHCHECK_ENDPOINT="healthz" - else - echo "Starting hyperion in mode $MODE with hyperion $h_commands, start_log is $start_log_path" + wait_for_healthcheck hyperion-blueapi $HEALTHCHECK_PORT healthz + fi + if [[ $START_HYPERION_SUPERVISOR == 1 ]]; then + h_commands+="--mode supervisor --client-config ${CLIENT_CONFIG} --supervisor-config ${SUPERVISOR_CONFIG} " + echo "Starting hyperion-supervisor with hyperion $h_commands, start_log is $start_log_path" hyperion `echo $h_commands;`>$start_log_path 2>&1 & - HEALTHCHECK_ENDPOINT="status" + wait_for_healthcheck hyperion-supervisor $SUPERVISOR_HEALTHCHECK_PORT status + elif [[ $MODE = "udc" ]]; then + h_commands+="--mode udc " + echo "Starting hyperion udc with hyperion $h_commands, start_log is $start_log_path" + hyperion `echo $h_commands;`>$start_log_path 2>&1 & + wait_for_healthcheck hyperion $HEALTHCHECK_PORT status fi - if [[ $DO_CALLBACKS == 1 ]]; then + if [[ $START_HYPERION_BLUEAPI == 1 || $MODE = "udc" ]]; then echo "Starting hyperion-callbacks with hyperion-callbacks $cb_commands, start_log is $callback_start_log_path" hyperion-callbacks `echo $cb_commands;`>$callback_start_log_path 2>&1 & fi - echo "$(date) Waiting for Hyperion to start" - - for i in {1..30} - do - echo "$(date)" - curl --head -X GET http://localhost:$HEALTHCHECK_PORT/$HEALTHCHECK_ENDPOINT >/dev/null - ret_value=$? - if [ $ret_value -ne 0 ]; then - sleep 1 - else - break - fi - done - - if [ $ret_value -ne 0 ]; then - echo "$(date) Hyperion Failed to start!!!!" - exit 1 - else - echo "$(date) Hyperion started" - fi fi sleep 1 From 8d9923fc8c38b96600b479137872937b6c34e90f Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Mon, 11 May 2026 13:26:14 +0100 Subject: [PATCH 2/8] Make hyperion termination regexp find python3 processes Widen scope to find processes regardless of parent directory name --- run_hyperion.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/run_hyperion.sh b/run_hyperion.sh index 847e9feba2..11f8f880dd 100755 --- a/run_hyperion.sh +++ b/run_hyperion.sh @@ -86,20 +86,20 @@ END done kill_active_apps () { - if [ $MODE = "blueapi" ]; then + if [[ $MODE = "blueapi" || $STOP == 1 ]]; then echo "Killing vanilla hyperion instances" - pkill -e -f "mx-bluesky/.venv/bin/python .*--mode (gda|udc)" + pkill -e -f "\.venv/bin/python3? .*--mode (gda|udc)" fi - if [[ $START_HYPERION_SUPERVISOR == 1 || $MODE = "udc" ]]; then + if [[ $START_HYPERION_SUPERVISOR == 1 || $MODE = "udc" || $STOP == 1 ]]; then # supervisor mode kills only supervisor echo "Killing active instances of hyperion supervisor..." - pkill -e -f "mx-bluesky/.venv/bin/python .*--mode supervisor" + pkill -e -f "\.venv/bin/python3? .*--mode supervisor" fi - if [[ $START_HYPERION_BLUEAPI == 1 || $MODE = "udc" ]]; then + if [[ $START_HYPERION_BLUEAPI == 1 || $MODE = "udc" || $STOP == 1 ]]; then echo "Killing active instances of hyperion-blueapi" - pkill -e -f "python .*mx-bluesky/.venv/bin/blueapi .*serve" + pkill -e -f "python3? .*/\.venv/bin/blueapi .*serve" echo "Killing hyperion-callbacks" - pkill -e -f "mx-bluesky/.venv/bin/python .*hyperion-callbacks" + pkill -e -f "\.venv/bin/python3? .*hyperion-callbacks" fi } From f830e13e5d7f54d90dea14d2584498b837f1720b Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Mon, 11 May 2026 14:36:13 +0100 Subject: [PATCH 3/8] Add gda.mx.hyperion.blueapi.enabled option to docs Fix vanilla UDC not always stopped --- docs/user/hyperion/configuration.rst | 1 + run_hyperion.sh | 6 ++---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/user/hyperion/configuration.rst b/docs/user/hyperion/configuration.rst index d0ed16fd33..2e50e63c15 100644 --- a/docs/user/hyperion/configuration.rst +++ b/docs/user/hyperion/configuration.rst @@ -30,6 +30,7 @@ Also note that some of these configuration properties will be removed in a futur "gda.mx.hyperion.xrc.use_roi_mode", "boolean", "If true then ROI mode is used." "gda.mx.udc.hyperion.enable", "boolean", "Enables Hyperion UDC mode in GDA - with this turned off, legacy UDC is performed by GDA." "gda.mx.hyperion.enable_beamstop_diode_check", "boolean", "If true, enables an extended beamstop position check + "gda.mx.hyperion.blueapi.enabled", "boolean", "If true then enables blueapi + supervisor, otherwise monolithic UDC is used." during UDC default state script measuring the diode current out-of and in beam. Otherwise the beamstop position is moved to the data collection position." diff --git a/run_hyperion.sh b/run_hyperion.sh index 11f8f880dd..5c5b2c978c 100755 --- a/run_hyperion.sh +++ b/run_hyperion.sh @@ -86,10 +86,8 @@ END done kill_active_apps () { - if [[ $MODE = "blueapi" || $STOP == 1 ]]; then - echo "Killing vanilla hyperion instances" - pkill -e -f "\.venv/bin/python3? .*--mode (gda|udc)" - fi + echo "Killing vanilla hyperion instances" + pkill -e -f "\.venv/bin/python3? .*--mode (gda|udc)" if [[ $START_HYPERION_SUPERVISOR == 1 || $MODE = "udc" || $STOP == 1 ]]; then # supervisor mode kills only supervisor echo "Killing active instances of hyperion supervisor..." From 8eb1ea9cbb3048c490613f3fb4138a36a9002218 Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Tue, 12 May 2026 10:09:16 +0100 Subject: [PATCH 4/8] Fix docs table syntax --- docs/user/hyperion/configuration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/hyperion/configuration.rst b/docs/user/hyperion/configuration.rst index 2e50e63c15..ce47181719 100644 --- a/docs/user/hyperion/configuration.rst +++ b/docs/user/hyperion/configuration.rst @@ -30,9 +30,9 @@ Also note that some of these configuration properties will be removed in a futur "gda.mx.hyperion.xrc.use_roi_mode", "boolean", "If true then ROI mode is used." "gda.mx.udc.hyperion.enable", "boolean", "Enables Hyperion UDC mode in GDA - with this turned off, legacy UDC is performed by GDA." "gda.mx.hyperion.enable_beamstop_diode_check", "boolean", "If true, enables an extended beamstop position check - "gda.mx.hyperion.blueapi.enabled", "boolean", "If true then enables blueapi + supervisor, otherwise monolithic UDC is used." during UDC default state script measuring the diode current out-of and in beam. Otherwise the beamstop position is moved to the data collection position." + "gda.mx.hyperion.blueapi.enabled", "boolean", "If true then enables blueapi + supervisor, otherwise monolithic UDC is used." Beamline configuration/calibration files ======================================== From 6806c3310991bd097a9ab9b00f7b797633760323 Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Tue, 12 May 2026 10:09:43 +0100 Subject: [PATCH 5/8] Update uv.lock --- uv.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uv.lock b/uv.lock index a3cbc9a23d..d0622ee54f 100644 --- a/uv.lock +++ b/uv.lock @@ -807,8 +807,8 @@ wheels = [ [[package]] name = "dls-dodal" -version = "2.2.4.dev10+g0649293ed" -source = { git = "https://github.com/DiamondLightSource/dodal.git?rev=main#0649293ed97538713ef77ab6e6f55fe984abd616" } +version = "2.3.1.dev2+gf637d5d25" +source = { git = "https://github.com/DiamondLightSource/dodal.git?rev=main#f637d5d258172bb1a6daff15dc96e005daedcbbf" } dependencies = [ { name = "aiofiles", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "aiohttp", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, From e20e6e18dc0b5dbb4a405fca88b70be2c0338c8b Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Tue, 12 May 2026 13:39:00 +0100 Subject: [PATCH 6/8] Start blueapi in a separate process group --- run_hyperion.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/run_hyperion.sh b/run_hyperion.sh index 5c5b2c978c..6db2a3d16f 100755 --- a/run_hyperion.sh +++ b/run_hyperion.sh @@ -212,7 +212,9 @@ if [[ $START == 1 ]]; then unset PYEPICS_LIBCA if [[ $START_HYPERION_BLUEAPI == 1 ]]; then echo "Starting hyperion-blueapi, start log is $start_log_path" - blueapi --config $BLUEAPI_CONFIG serve > $start_log_path 2>&1 & + # start in a separate process group to avoid GDA sending it a SIGINT on + # GDA server shutdown + ( set -m; nohup blueapi --config $BLUEAPI_CONFIG serve > $start_log_path 2>&1 & ) wait_for_healthcheck hyperion-blueapi $HEALTHCHECK_PORT healthz fi if [[ $START_HYPERION_SUPERVISOR == 1 ]]; then From 006aa74207114299d2e52d0541d97916f0fe87e6 Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Tue, 12 May 2026 13:49:22 +0100 Subject: [PATCH 7/8] Update uv.lock --- uv.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uv.lock b/uv.lock index d0622ee54f..b92491885e 100644 --- a/uv.lock +++ b/uv.lock @@ -807,8 +807,8 @@ wheels = [ [[package]] name = "dls-dodal" -version = "2.3.1.dev2+gf637d5d25" -source = { git = "https://github.com/DiamondLightSource/dodal.git?rev=main#f637d5d258172bb1a6daff15dc96e005daedcbbf" } +version = "2.3.1.dev3+g604855599" +source = { git = "https://github.com/DiamondLightSource/dodal.git?rev=main#6048555990595e43c65490d23ffe99c9ffa80a7d" } dependencies = [ { name = "aiofiles", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "aiohttp", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, From a6b8dff7e5588df32bf443815e276c8dc26d433b Mon Sep 17 00:00:00 2001 From: Robert Tuck Date: Tue, 12 May 2026 14:26:08 +0100 Subject: [PATCH 8/8] Start all processes in separate process group --- run_hyperion.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/run_hyperion.sh b/run_hyperion.sh index 6db2a3d16f..3ccd0bc434 100755 --- a/run_hyperion.sh +++ b/run_hyperion.sh @@ -220,17 +220,17 @@ if [[ $START == 1 ]]; then if [[ $START_HYPERION_SUPERVISOR == 1 ]]; then h_commands+="--mode supervisor --client-config ${CLIENT_CONFIG} --supervisor-config ${SUPERVISOR_CONFIG} " echo "Starting hyperion-supervisor with hyperion $h_commands, start_log is $start_log_path" - hyperion `echo $h_commands;`>$start_log_path 2>&1 & + ( set -m; hyperion `echo $h_commands;`>$start_log_path 2>&1 & ) wait_for_healthcheck hyperion-supervisor $SUPERVISOR_HEALTHCHECK_PORT status elif [[ $MODE = "udc" ]]; then h_commands+="--mode udc " echo "Starting hyperion udc with hyperion $h_commands, start_log is $start_log_path" - hyperion `echo $h_commands;`>$start_log_path 2>&1 & + ( set -m; hyperion `echo $h_commands;`>$start_log_path 2>&1 & ) wait_for_healthcheck hyperion $HEALTHCHECK_PORT status fi if [[ $START_HYPERION_BLUEAPI == 1 || $MODE = "udc" ]]; then echo "Starting hyperion-callbacks with hyperion-callbacks $cb_commands, start_log is $callback_start_log_path" - hyperion-callbacks `echo $cb_commands;`>$callback_start_log_path 2>&1 & + ( set -m; hyperion-callbacks `echo $cb_commands;`>$callback_start_log_path 2>&1 & ) fi fi