ci: drop pytest in favor of Sprocket test

.github/workflows/build-and-test.yaml

@@ -8,5 +8,5 @@ jobs:
         uses: ./.github/workflows/docker-build.yaml
         secrets: inherit
     call-test:
-        uses: ./.github/workflows/pytest.yaml
+        uses: ./.github/workflows/sprocket-test.yaml
         needs: call-build

.github/workflows/sprocket-lint.yaml

@@ -3,12 +3,17 @@ name: Workflows Sprocket Lint
 on: [push]
 
 jobs:
-    sprocket_lint:
-        runs-on: ubuntu-latest
-        steps:
-        - uses: actions/checkout@v2
-        - name: Run sprocket
-          uses: stjude-rust-labs/sprocket-action@main
-          with:
-            lint: true
-            except: KnownRules
+  sprocket_lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Run 'sprocket lint'
+      uses: stjude-rust-labs/sprocket-action@main
+      with:
+        lint: true
+        except: KnownRules
+        ignore-patterns: methylation
+    - name: Run 'sprocket format'
+      uses: stjude-rust-labs/sprocket-action@main
+      with:
+        action: format

.github/workflows/sprocket-test.yaml

@@ -0,0 +1,41 @@
+name: Sprocket Test
+
+on:
+  workflow_call:
+
+jobs:
+  list-files:
+    runs-on: ubuntu-latest
+    outputs:
+        files: ${{ steps.set-files.outputs.files }}
+    steps:
+        - name: checkout
+          uses: actions/checkout@v4
+        - name: set files
+          id: set-files
+          run: echo "files=$(find . -name '*.wdl' -print -o -path "./template" -prune | jq -ncR '[inputs]')" >> $GITHUB_OUTPUT
+  sprocket_test:
+    needs: list-files
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        file: ${{ fromJson(needs.list-files.outputs.files) }}
+        runner: [sprocket]
+      fail-fast: false
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        lfs: true
+    - name: Update Rust
+      run: rustup update stable && rustup default stable
+    - name: Build Sprocket
+      run: |
+        cargo install sprocket --locked
+    - name: Update containers
+      run: |
+        ./developer_scripts/update_container_tags.sh ${GITHUB_REF##*/}
+    - name: Run sprocket test
+      env: 
+        RUNNER: ${{ matrix.runner }}
+      run: |
+        sprocket dev test --filter-tag reference --filter-tag slow --filter-tag high_mem ${{ matrix.file }}

.github/workflows/validate-inputs.yaml

@@ -12,4 +12,4 @@ jobs:
           with:
             action: validate
             wdl_files: "workflows/reference/bwa-db-build.wdl,workflows/reference/gatk-reference.wdl,workflows/reference/qc-reference.wdl,workflows/reference/star-db-build.wdl,workflows/reference/star-db-build.wdl"
-            inputs_files: "workflows/reference/inputs/bwa-db-build-inputs.json,workflows/reference/inputs/gatk-reference-inputs.json,workflows/reference/inputs/qc-reference-inputs.json,workflows/reference/inputs/star-db-build-inputs.json,workflows/reference/inputs/star-db-build-mouse-contaminant-inputs.json"
+            inputs_files: "@workflows/reference/inputs/bwa-db-build-inputs.json,@workflows/reference/inputs/gatk-reference-inputs.json,@workflows/reference/inputs/qc-reference-inputs.json,@workflows/reference/inputs/star-db-build-inputs.json,@workflows/reference/inputs/star-db-build-mouse-contaminant-inputs.json"

.gitignore

@@ -1,3 +1,4 @@
+pytest/
 # Ignore common bioinformatics formats used in these workflows.
 # But only if they are in the root of this repo
 /*.fastq.gz

data_structures/flag_filter.wdl

@@ -106,7 +106,7 @@ task validate_string_is_12bit_int {
     >>>
 
     runtime {
-        container: "ghcr.io/stjudecloud/util:3.0.3"
+        container: "ghcr.io/stjudecloud/util:3.0.4"
         maxRetries: 1
     }
 }

data_structures/read_group.wdl

@@ -82,7 +82,9 @@ workflow read_group_to_string {
 
     input {
         ReadGroup read_group
-        Array[String] required_fields = []
+        Array[String] required_fields = [
+            "SM",
+        ]
         Boolean format_as_sam_record = false
         Boolean restrictive = true
     }
@@ -136,7 +138,7 @@ task get_read_groups {
 
     runtime {
         disks: "~{disk_size_gb} GB"
-        container: "ghcr.io/stjudecloud/util:3.0.3"
+        container: "ghcr.io/stjudecloud/util:3.0.4"
         maxRetries: 1
     }
 }
@@ -147,22 +149,20 @@ task validate_read_group {
     }
 
     parameter_meta {
-        read_group: "`ReadGroup` struct to validate"
         required_fields: "Array of read group fields that must be defined. The ID field is always required and does not need to be specified."
+        read_group: "`ReadGroup` struct to validate"
         restrictive: "If true, run a stricter validation of field values. Otherwise, check against SAM spec-defined values."
     }
 
     input {
+        Array[String] required_fields
         ReadGroup read_group
-        Array[String] required_fields = []
-        Boolean restrictive = true
+        Boolean restrictive
     }
 
     # The SAM spec allows any printable ASCII character in header fields.
     String sam_spec_pattern = "[\\ -~]+"
     # We have the opinion that is too permissive for ID and SM.
-    String id_pattern = "id"
-    String sample_pattern = "sample.?"
     String restrictive_pattern = "\\ "  # Disallow spaces
     Array[String] platforms = [
         "CAPILLARY",
@@ -182,11 +182,9 @@ task validate_read_group {
     command <<<
         exit_code=0
         if ~{restrictive}; then
-            if [[ ~{read_group.ID} =~ ^~{id_pattern}$ ]] \
-                || [[ ~{read_group.ID} =~ ~{restrictive_pattern} ]]
+            if [[ "~{read_group.ID}" =~ ~{restrictive_pattern} ]]
             then
-                >&2 echo "ID (~{read_group.ID}) must not match patterns:"
-                >&2 echo "'~{id_pattern}' or '~{restrictive_pattern}'"
+                >&2 echo "ID must not contain spaces"
                 exit_code=1
             fi
         fi
@@ -202,11 +200,9 @@ task validate_read_group {
         fi
         if ~{defined(read_group.SM)}; then
             if ~{restrictive}; then
-                if [[ "~{read_group.SM}" =~ ^~{sample_pattern}$ ]] \
-                    || [[ "~{read_group.SM}" =~ ~{restrictive_pattern} ]]
+                if [[ "~{read_group.SM}" =~ ~{restrictive_pattern} ]]
                 then
-                    >&2 echo "SM must not match patterns:"
-                    >&2 echo "'~{sample_pattern}' or '~{restrictive_pattern}'"
+                    >&2 echo "SM must not contain spaces"
                     exit_code=1
                 fi
             fi
@@ -367,7 +363,7 @@ task validate_read_group {
     >>>
 
     runtime {
-        container: "ghcr.io/stjudecloud/util:3.0.3"
+        container: "ghcr.io/stjudecloud/util:3.0.4"
         maxRetries: 1
     }
 }
@@ -391,7 +387,7 @@ task inner_read_group_to_string {
 
     input {
         ReadGroup read_group
-        Boolean format_as_sam_record = false
+        Boolean format_as_sam_record
     }
 
     String delimiter = if format_as_sam_record
@@ -425,7 +421,7 @@ task inner_read_group_to_string {
     }
 
     runtime {
-        container: "ghcr.io/stjudecloud/util:3.0.3"
+        container: "ghcr.io/stjudecloud/util:3.0.4"
         maxRetries: 1
     }
 }

data_structures/test/flag_filter.yaml

@@ -0,0 +1,54 @@
+validate_string_is_12bit_int:
+  - name: valid_numbers
+    inputs:
+      number:
+        - "5"
+        - "0x900"
+        - "01"
+        - "4095"
+        - "0"
+        - "072"
+    assertions:
+      stderr:
+        - Input number \(.*\) is valid
+  - name: invalid_numbers
+    inputs:
+      number:
+        - "0x1000"
+        - ""
+        - string
+        - this is not a number
+        - "000000000011"
+        - "-1"
+        - "08"
+    assertions:
+      exit_code: 42
+      stderr:
+        - Input number \(.*\) is invalid
+  - name: too_big_decimal_fails
+    inputs:
+      number:
+        - "4096"
+        - "9999"
+    assertions:
+      exit_code: 42
+      stderr:
+        - Input number \(.*\) interpreted as decimal
+        - But number must be less than 4096!
+validate_flag_filter:
+  - name: valid_FlagFilter_passes
+    inputs:
+      flags:
+        - include_if_all: "3"
+          exclude_if_any: "0xF04"
+          include_if_any: "03"
+          exclude_if_all: "4095"
+  - name: invalid_FlagFilter_fails
+    inputs:
+      flags:
+        - include_if_all: "3"
+          exclude_if_any: "0xF04"
+          include_if_any: "03"
+          exclude_if_all: "" # empty string should trigger a fail
+    assertions:
+      should_fail: true