Looper/pipestat/issue update

.github/workflows/tests.yml

@@ -0,0 +1,81 @@
+name: Tests
+
+on:
+  push:
+    branches: [master, dev, looper-update]
+  pull_request:
+    branches: [master, dev]
+  workflow_dispatch:
+    inputs:
+      run_integration:
+        description: "Run integration tests (requires self-hosted runner)"
+        required: false
+        default: "false"
+
+jobs:
+  # --------------------------------------------------------------------------
+  # Tier 1: Unit tests — no genome data or bioinformatics tools required.
+  # Runs on every push and pull request.
+  # --------------------------------------------------------------------------
+  unit-tests:
+    name: Unit tests (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.11", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Python dependencies
+        run: pip install -r requirements.txt pytest
+
+      - name: Run unit tests
+        run: pytest tests/test_unit.py -v --tb=short
+
+  # --------------------------------------------------------------------------
+  # Tier 2: Integration tests — full pipeline runs.
+  # Requires a self-hosted runner with genome indices and tools installed.
+  # Triggered manually via workflow_dispatch or by setting
+  # RUN_INTEGRATION_TESTS=true in the environment.
+  # --------------------------------------------------------------------------
+  integration-tests:
+    name: Integration tests (${{ matrix.scenario }})
+    if: >
+      github.event_name == 'workflow_dispatch' &&
+      github.event.inputs.run_integration == 'true'
+    runs-on: self-hosted
+    strategy:
+      fail-fast: false
+      matrix:
+        scenario:
+          - se_basic
+          - pe_basic
+          - se_groseq
+          - se_umi
+          - pe_umi
+          - se_fastp
+          - se_fastx
+          - se_fqdedup
+          - se_scale
+          - se_no_complexity
+          - se_nofifo
+          - se_coverage
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Python dependencies
+        run: pip install -r requirements.txt pytest
+
+      - name: Run integration test for ${{ matrix.scenario }}
+        env:
+          RUN_INTEGRATION_TESTS: "true"
+        run: >
+          pytest tests/test_integration.py -v --tb=short
+          -k "${{ matrix.scenario }}"

Makefile

@@ -1,5 +1,52 @@
 test:
-	python pipelines/peppro.py  -P 3 -M 100 -O peppro_test -R -S test -G hg38  -Q single  -C peppro.yaml  --genome-size hs --prealignments rCRSd human_repeats -I examples/data/test_R1.fq.gz
+	python pipelines/peppro.py -P 3 -M 100 -O peppro_test -R -S test -G hg38 \
+		-Q single -C peppro.yaml \
+		--protocol PRO \
+		--prealignment-names rCRSd human_repeats \
+		--genome-index $$(refgenie seek hg38/bowtie2_index --seek-key dir) \
+		--chrom-sizes $$(refgenie seek hg38/fasta --seek-key chrom_sizes) \
+		--pipestat-schema peppro_output_schema.yaml \
+		-I examples/data/test_r1.fq.gz
+
+# -----------------------------------------------------------------------
+# Test suite targets
+# -----------------------------------------------------------------------
+
+# Run only unit tests (no genome data or external tools required)
+test-unit:
+	pytest tests/test_unit.py -v --tb=short
+
+# Run a single integration scenario (e.g. make test-se SCENARIO=se_basic)
+SCENARIO ?= se_basic
+test-scenario:
+	RUN_INTEGRATION_TESTS=true pytest tests/test_integration.py -v --tb=short -k "$(SCENARIO)"
+
+# Run all SE integration scenarios
+test-se:
+	RUN_INTEGRATION_TESTS=true pytest tests/test_integration.py -v --tb=short \
+		-k "se_basic or se_groseq or se_umi or se_fastp or se_fastx or se_fqdedup or se_scale or se_no_complexity or se_nofifo or se_coverage"
+
+# Run all PE integration scenarios
+test-pe:
+	RUN_INTEGRATION_TESTS=true pytest tests/test_integration.py -v --tb=short \
+		-k "pe_basic or pe_umi"
+
+# Run recovery regression tests
+test-recovery:
+	RUN_INTEGRATION_TESTS=true pytest tests/test_integration.py -v --tb=short \
+		-k "recovery"
+
+# Run all integration tests (SE + PE + recovery)
+test-integration:
+	RUN_INTEGRATION_TESTS=true pytest tests/test_integration.py -v --tb=short
+
+# Run both unit and integration tests
+test-all:
+	RUN_INTEGRATION_TESTS=true pytest tests/ -v --tb=short
+
+# Regenerate test FASTQ data files from the source R1 read file
+test-data:
+	bash tests/scripts/generate_test_data.sh
 
 docker:
 	docker build -t databio/peppro -f containers/peppro.Dockerfile .

PEPPROr/DESCRIPTION

@@ -1,10 +1,10 @@
 Package: PEPPROr
 Title: Functions and libraries to analyze pro-seq (or gro-seq) data
-Version: 0.0.2.0000
+Version: 0.0.3.0000
 Authors@R: person("Jason", "Smith", email = "jasonsmith@virginia.edu", role = c("aut", "cre"))
 Maintainer: Jason Smith <jasonsmith@virginia.edu>
 Description: Installs required libraries to calculate the fraction of reads in features, to plot library complexity curves, TSS enrichments, and fragment length distributions. 
-Depends: R (>= 3.5.1), data.table, pepr, ggplot2, optigrab, GenomicDistributions
+Depends: R (>= 3.5.1), data.table, pepr, ggplot2, GenomicDistributions
 License: BSD 2-Clause "Simplified" License
 Encoding: UTF-8
 LazyData: true

PEPPROr/R/PEPPROr.R

@@ -2361,7 +2361,7 @@ calcCountsTable = function(project, results_subdir) {
 #' @export
 createAssetsSummary <- function(project, output_dir, results_subdir) {
     # Convenience
-    project_name <- config(project)$name
+    project_name <- pepr::config(project)$name
 
     # Create assets_summary file
     project_samples <- pepr::sampleTable(project)$sample_name

README.md

@@ -9,6 +9,30 @@
 
 PEPPRO is a pipeline designed to process PRO-seq (and GRO-seq) data. For more information see: http://peppro.databio.org/
 
+## Install
+
+```bash
+pip install piper pipestat looper
+```
+
+**Note:** The pypiper PyPI package is `piper` (not `pypiper`, which is an unrelated package).
+
+## Testing
+
+Unit tests need no special setup:
+
+```bash
+pytest tests/test_unit.py -v
+```
+
+Integration tests require bioinformatics tools via [bulker](https://bulker.io). Use the wrapper script:
+
+```bash
+bash tests/scripts/test-integration.sh
+```
+
+This runs `bulker exec databio/peppro:1.1.0` to provide samtools, bowtie2, bedtools, etc. Do NOT run integration tests without bulker — they will fail with missing tools.
+
 ## Docs
 
 Develop docs with: