Prevent notebook execution from running twice

i14/templates/align-template.yaml

@@ -69,13 +69,15 @@ spec:
         value: /tmp
       command: [bash]
       source: |
-        python -m papermill /tmp/notebook.ipynb /tmp/notebook-parametrized.ipynb \
+        python -m papermill /tmp/notebook.ipynb "{{`{{ inputs.parameters.outpath }}`}}/align-notebook.ipynb" \
           -p inpath "{{`{{ inputs.parameters.inpath }}`}}" \
           -p maskpath "{{`{{ inputs.parameters.maskpath }}`}}" \
           -p complexpath "{{`{{ inputs.parameters.complexpath }}`}}" \
-          -p outpath "{{`{{ inputs.parameters.outpath }}`}}"
-        python -m jupyter nbconvert /tmp/notebook-parametrized.ipynb --to notebook --execute --allow-errors --output align-notebook --output-dir "{{`{{ inputs.parameters.outpath }}`}}"
-        python -m jupyter nbconvert "{{`{{ inputs.parameters.outpath }}`}}/align-notebook.ipynb" --to html --output align-output --output-dir "{{`{{ inputs.parameters.outpath }}`}}"
+          -p outpath "{{`{{ inputs.parameters.outpath }}`}}" \
+          --prepare-only
+        python -m jupyter nbconvert "{{`{{ inputs.parameters.outpath }}`}}/align-notebook.ipynb" \
+          --to html --execute --output align-output \
+          --output-dir "{{`{{ inputs.parameters.outpath }}`}}"
       volumeMounts:
       - name: session
         mountPath: "{{`{{ workflow.parameters.visitdir }}`}}"

i14/templates/dpc-batch-template.yaml

@@ -72,13 +72,17 @@ spec:
         value: /tmp
       command: [bash]
       source: |
-        python -m papermill /tmp/notebook.ipynb /tmp/notebook-parametrized.ipynb \
+        python -m papermill /tmp/notebook.ipynb \
+          "{{`{{ inputs.parameters.outputFolder }}`}}/dpc-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" \
           -p inpath "{{`{{ workflow.parameters.visitdir }}`}}/scan/i14-{{`{{ inputs.parameters.scanNumber }}`}}.nxs" \
           -p process_configfile /tmp/config.yaml \
           -p outpath "{{`{{ inputs.parameters.outputFolder }}`}}/i14-{{`{{ inputs.parameters.scanNumber }}`}}_dpc-phase.nxs" \
-          -p auto_processing False
-        python -m jupyter nbconvert /tmp/notebook-parametrized.ipynb --to notebook --execute --allow-errors --output dpc-notebook-{{`{{ inputs.parameters.scanNumber }}`}} --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}"
-        python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/dpc-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" --to html --no-input --output dpc-output-{{`{{ inputs.parameters.scanNumber }}`}} --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}"
+          -p auto_processing False \
+          --prepare-only
+        python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/dpc-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" \
+          --to html  --execute --no-input \
+          --output dpc-output-{{`{{ inputs.parameters.scanNumber }}`}} \
+          --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}"
       volumeMounts:
       - name: session
         mountPath: "{{`{{ workflow.parameters.visitdir }}`}}"

i14/templates/mask-template.yaml

@@ -76,15 +76,17 @@ spec:
         value: /tmp
       command: [bash]
       source: |
-        python -m papermill /tmp/notebook.ipynb /tmp/notebook-parametrized.ipynb \
+        python -m papermill /tmp/notebook.ipynb "{{`{{ inputs.parameters.outpath }}`}}/mask-notebook.ipynb" \
           -p inpath "{{`{{ inputs.parameters.inpath }}`}}" \
           -p outpath "{{`{{ inputs.parameters.outpath }}`}}" \
           -p padding "{{`{{ inputs.parameters.padding }}`}}" \
           -p min_size "{{`{{ inputs.parameters.minSize }}`}}" \
           -p erosion_radius "{{`{{ inputs.parameters.erosionRadius }}`}}" \
-          -p dilation_radius "{{`{{ inputs.parameters.dilationRadius }}`}}"
-        python -m jupyter nbconvert /tmp/notebook-parametrized.ipynb --to notebook --execute --allow-errors --output mask-notebook --output-dir "{{`{{ inputs.parameters.outpath }}`}}"
-        python -m jupyter nbconvert "{{`{{ inputs.parameters.outpath }}`}}/mask-notebook.ipynb" --to html --output mask-output --output-dir "{{`{{ inputs.parameters.outpath }}`}}"
+          -p dilation_radius "{{`{{ inputs.parameters.dilationRadius }}`}}" \
+          --prepare-only
+        python -m jupyter nbconvert "{{`{{ inputs.parameters.outpath }}`}}/mask-notebook.ipynb" \
+          --to html  --execute --output mask-output \
+          --output-dir "{{`{{ inputs.parameters.outpath }}`}}"
       volumeMounts:
       - name: session
         mountPath: "{{`{{ workflow.parameters.visitdir }}`}}"

i14/templates/phase-template.yaml

@@ -68,13 +68,16 @@ spec:
         value: /tmp
       command: [bash]
       source: |
-        python -m papermill /tmp/notebook.ipynb /tmp/notebook-parametrized.ipynb \
+        python -m papermill /tmp/notebook.ipynb "{{`{{ inputs.parameters.outpath }}`}}/phase-notebook.ipynb" \
           -p inpath "{{`{{ inputs.parameters.inpath }}`}}" \
           -p maskpath "{{`{{ inputs.parameters.maskpath }}`}}" \
           -p normalise "{{`{{ inputs.parameters.normalise }}`}}" \
-          -p outpath "{{`{{ inputs.parameters.outpath }}`}}"
-        python -m jupyter nbconvert /tmp/notebook-parametrized.ipynb --to notebook --execute --allow-errors --output phase-notebook --output-dir "{{`{{ inputs.parameters.outpath }}`}}"
-        python -m jupyter nbconvert "{{`{{ inputs.parameters.outpath }}`}}/phase-notebook.ipynb" --to html --output phase-output --output-dir "{{`{{ inputs.parameters.outpath }}`}}"
+          -p outpath "{{`{{ inputs.parameters.outpath }}`}}" \
+          --prepare-only
+        python -m jupyter nbconvert "{{`{{ inputs.parameters.outpath }}`}}/phase-notebook.ipynb" \
+          --to html --execute \
+          --output phase-output \
+          --output-dir "{{`{{ inputs.parameters.outpath }}`}}"
       volumeMounts:
       - name: session
         mountPath: "{{`{{ workflow.parameters.visitdir }}`}}"

i14/templates/xanes-auto-template.yaml

@@ -128,7 +128,8 @@ spec:
         value: /opt/venv/lib64/python3.12/site-packages/hdf5plugin/plugins/
       command: [bash]
       source: |
-        python -m papermill /tmp/notebook.ipynb /tmp/notebook-parametrized.ipynb \
+        python -m papermill /tmp/notebook.ipynb \
+          "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-{{`{{ inputs.parameters.first }}`}}-{{`{{ inputs.parameters.last }}`}}/xanes-notebook.ipynb" \
           -p first "{{`{{ inputs.parameters.first }}`}}" \
           -p last "{{`{{ inputs.parameters.last }}`}}" \
           -p exclude "{{`{{ inputs.parameters.exclude }}`}}" \
@@ -142,9 +143,11 @@ spec:
           -p sztol 0.9 \
           -p ref_index None \
           -p max_fractional_shift 0.2 \
-          -p auto_processing False
-        python -m jupyter nbconvert /tmp/notebook-parametrized.ipynb --to notebook --execute --allow-errors --output xanes-notebook --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-{{`{{ inputs.parameters.first }}`}}-{{`{{ inputs.parameters.last }}`}}"
-        python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-{{`{{ inputs.parameters.first }}`}}-{{`{{ inputs.parameters.last }}`}}/xanes-notebook.ipynb" --to html --output xanes-output --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-{{`{{ inputs.parameters.first }}`}}-{{`{{ inputs.parameters.last }}`}}"
+          -p auto_processing False \
+          --prepare-only
+        python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-{{`{{ inputs.parameters.first }}`}}-{{`{{ inputs.parameters.last }}`}}/xanes-notebook.ipynb" \
+          --to html --execute --output xanes-output \
+          --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-{{`{{ inputs.parameters.first }}`}}-{{`{{ inputs.parameters.last }}`}}"
       volumeMounts:
       - name: session
         mountPath: "{{`{{ workflow.parameters.visitdir }}`}}"

i14/templates/xanes-point-template.yaml

@@ -72,14 +72,16 @@ spec:
         value: /tmp
       command: [bash]
       source: |
-        python -m papermill /tmp/notebook.ipynb /tmp/notebook-parametrized.ipynb \
+        python -m papermill /tmp/notebook.ipynb "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-point-notebook.ipynb" \
           -p inpath "{{`{{ workflow.parameters.visitdir }}`}}/scan/i14-{{`{{ inputs.parameters.scanNumber }}`}}.nxs" \
           -p outpath "{{`{{ inputs.parameters.outputFolder }}`}}/i14-{{`{{ inputs.parameters.scanNumber }}`}}xanes_point.txt" \
           -p edge_element "{{`{{ inputs.parameters.edgeElement }}`}}" \
           -p normalise "{{`{{ inputs.parameters.normalise }}`}}" \
-          -p auto_processing False
-        python -m jupyter nbconvert /tmp/notebook-parametrized.ipynb --to notebook --execute --allow-errors --output xanes-point-notebook --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}"
-        python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-point-notebook.ipynb" --to html --output xanes-point-output --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}"
+          -p auto_processing False \
+          --prepare-only
+        python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-point-notebook.ipynb" \
+          --to html --execute --output xanes-point-output \
+          --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}"
       volumeMounts:
       - name: session
         mountPath: "{{`{{ workflow.parameters.visitdir }}`}}"

i14/templates/xanes-sparse-template.yaml

@@ -95,7 +95,8 @@ spec:
         value: /opt/venv/lib64/python3.12/site-packages/hdf5plugin/plugins/
       command: [bash]
       source: |
-        python -m papermill /tmp/notebook.ipynb /tmp/notebook-parametrized.ipynb \
+        python -m papermill /tmp/notebook.ipynb \
+          "{{`{{ inputs.parameters.outputFolder }}`}}/xanes_sparse_{{`{{ inputs.parameters.first }}`}}_{{`{{ inputs.parameters.last }}`}}/xanes-sparse-notebook.ipynb" \
           -p first "{{`{{ inputs.parameters.first }}`}}" \
           -p last "{{`{{ inputs.parameters.last }}`}}" \
           -p exclude "{{`{{ inputs.parameters.exclude }}`}}" \
@@ -108,9 +109,11 @@ spec:
           -p num_short_iteration 75 \
           -p num_final_iteration 2000 \
           -p normalised "{{`{{ inputs.parameters.normalise }}`}}" \
-          -p auto_processing False
-        python -m jupyter nbconvert /tmp/notebook-parametrized.ipynb --to notebook --execute --allow-errors --output xanes-sparse-notebook --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}/xanes_sparse_{{`{{ inputs.parameters.first }}`}}_{{`{{ inputs.parameters.last }}`}}"
-        python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xanes_sparse_{{`{{ inputs.parameters.first }}`}}_{{`{{ inputs.parameters.last }}`}}/xanes-sparse-notebook.ipynb" --to html --output xanes-sparse-output --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}/xanes_sparse_{{`{{ inputs.parameters.first }}`}}_{{`{{ inputs.parameters.last }}`}}"
+          -p auto_processing False \
+          --prepare-only
+        python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xanes_sparse_{{`{{ inputs.parameters.first }}`}}_{{`{{ inputs.parameters.last }}`}}/xanes-sparse-notebook.ipynb" \
+          --to html --execute --output xanes-sparse-output \
+          --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}/xanes_sparse_{{`{{ inputs.parameters.first }}`}}_{{`{{ inputs.parameters.last }}`}}"
       volumeMounts:
       - name: session
         mountPath: "{{`{{ workflow.parameters.visitdir }}`}}"

i14/templates/xrd1d-batch-template.yaml

@@ -100,17 +100,20 @@ spec:
         value: /tmp
       command: [bash]
       source: |
-        python -m papermill /tmp/notebook.ipynb /tmp/notebook-parametrized.ipynb \
+        python -m papermill /tmp/notebook.ipynb \
+          "{{`{{ inputs.parameters.outputFolder }}`}}/xrd1d-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" \
           -p inpath "{{`{{ workflow.parameters.visitdir }}`}}/scan/i14-{{`{{ inputs.parameters.scanNumber }}`}}.nxs" \
           -p process_configfile "{{`{{ inputs.parameters.xrdProcessConfigUpload }}`}}" \
           -p outpath "{{`{{ inputs.parameters.outputFolder }}`}}/i14-{{`{{ inputs.parameters.scanNumber }}`}}-xrd1d.nxs" \
           -p calibration_path "{{`{{ inputs.parameters.calibrationPath }}`}}" \
           -p mask_path "{{`{{ inputs.parameters.maskPath }}`}}" \
           -p flat_path "{{`{{ inputs.parameters.flatPath }}`}}" \
           -p background_type "{{`{{ inputs.parameters.backgroundType }}`}}" \
-          -p polynomial_order "{{`{{ inputs.parameters.polynomialOrder }}`}}"
-        python -m jupyter nbconvert /tmp/notebook-parametrized.ipynb --to notebook --execute --allow-errors --output xrd1d-notebook-{{`{{ inputs.parameters.scanNumber }}`}} --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}"
-        python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xrd1d-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" --to html --no-input --output xrd1d-output-{{`{{ inputs.parameters.scanNumber }}`}} --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}"
+          -p polynomial_order "{{`{{ inputs.parameters.polynomialOrder }}`}}" \
+          --prepare-only
+        python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xrd1d-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" \
+          --to html --execute --no-input --output xrd1d-output-{{`{{ inputs.parameters.scanNumber }}`}} \
+          --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}"
       volumeMounts:
       - name: session
         mountPath: "{{`{{ workflow.parameters.visitdir }}`}}"

i14/templates/xrd2d-batch-template.yaml

@@ -89,15 +89,19 @@ spec:
         value: /tmp
       command: [bash]
       source: |
-        python -m papermill /tmp/notebook.ipynb /tmp/notebook-parametrized.ipynb \
+        python -m papermill /tmp/notebook.ipynb \
+          "{{`{{ inputs.parameters.outputFolder }}`}}/xrd2d-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" \
           -p inpath "{{`{{ workflow.parameters.visitdir }}`}}/scan/i14-{{`{{ inputs.parameters.scanNumber }}`}}.nxs" \
           -p process_configfile "{{`{{ inputs.parameters.xrdProcessConfigUpload }}`}}" \
           -p outpath "{{`{{ inputs.parameters.outputFolder }}`}}/i14-{{`{{ inputs.parameters.scanNumber }}`}}-xrd2d.nxs" \
           -p calibration_path "{{`{{ inputs.parameters.calibrationPath }}`}}" \
           -p mask_path "{{`{{ inputs.parameters.maskPath }}`}}" \
-          -p flat_path "{{`{{ inputs.parameters.flatPath }}`}}"
-        python -m jupyter nbconvert /tmp/notebook-parametrized.ipynb --to notebook --execute --allow-errors --output xrd2d-notebook-{{`{{ inputs.parameters.scanNumber }}`}} --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}"
-        python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xrd2d-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" --to html --no-input --output xrd2d-output-{{`{{ inputs.parameters.scanNumber }}`}} --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}"
+          -p flat_path "{{`{{ inputs.parameters.flatPath }}`}}" \
+          --prepare-only
+        python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xrd2d-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" \
+          --to html --execute --no-input \
+          --output xrd2d-output-{{`{{ inputs.parameters.scanNumber }}`}} \
+          --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}"
       volumeMounts:
       - name: session
         mountPath: "{{`{{ workflow.parameters.visitdir }}`}}"