Nf test local subworkflows

.github/workflows/ci.yml

@@ -12,6 +12,7 @@ on:
 env:
   NXF_ANSI_LOG: false
   NFT_MAX_SHARDS: 5
+  SOURCE_BRANCH: ${{ github.head_ref || github.ref_name }}
 
 concurrency:
   group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}"
@@ -51,14 +52,15 @@ jobs:
         with:
           version: "${{ matrix.NXF_VER }}"
 
-      - name: Disk space cleanup
-        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
+      # - name: Disk space cleanup
+      #   uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
 
       - name: Install nf-test
         run: |
           conda install -c bioconda nf-test
 
-      - name: "Run ${{ matrix.filter }} tests | ${{ matrix.shard }}/${{ env.NFT_MAX_SHARDS }}"
+      - name: "Run ${{ matrix.filter }} tests (changed) | ${{ matrix.shard }}/${{ env.NFT_MAX_SHARDS }}"
+        if: ${{ env.SOURCE_BRANCH != 'main' }}
         run: |
           $CONDA/bin/nf-test test \
             --ci \
@@ -67,6 +69,15 @@ jobs:
             --filter ${{ matrix.filter }} \
             --junitxml=default.xml
 
+      - name: "Run ${{ matrix.filter }} tests (all) | ${{ matrix.shard }}/${{ env.NFT_MAX_SHARDS }}"
+        if: ${{ env.SOURCE_BRANCH == 'main' }}
+        run: |
+          $CONDA/bin/nf-test test \
+            --ci \
+            --shard ${{ matrix.shard }}/${{ env.NFT_MAX_SHARDS }} \
+            --filter ${{ matrix.filter }} \
+            --junitxml=default.xml
+
       - name: Publish Test Report
         uses: mikepenz/action-junit-report@v3
         if: always() # always run even if the previous step fails

assets/schema_input.json

@@ -9,11 +9,15 @@
         "properties": {
             "sample": {
                 "type": "string",
-                "meta": ["id", "sample"]
+                "pattern": "^[^\\. ]+$",
+                "meta": ["id", "sample"],
+                "errorMessage": "Sample name should be a string that does not contain dots or spaces"
             },
             "family": {
                 "type": "string",
-                "meta": ["family"]
+                "pattern": "^[^\\. ]+$",
+                "meta": ["family"],
+                "errorMessage": "Family name should be a string that does not contain dots or spaces"
             },
             "cram": {
                 "oneOf": [

conf/test.config

@@ -56,7 +56,6 @@ params {
     vcfanno           = true
     vcfanno_config    = "${projectDir}/assets/vcfanno.toml"
     vcfanno_resources = "https://github.com/brentp/vcfanno/raw/master/example/exac.vcf.gz;https://github.com/brentp/vcfanno/raw/master/example/exac.vcf.gz.tbi"
-
 }
 
 process {

nf-test.config

@@ -1,6 +1,6 @@
 config {
 
-    testsDir "tests"
+    testsDir "."
     workDir ".nf-test"
     configFile "tests/nextflow.config"
     profile "nf_test,docker"
@@ -10,4 +10,6 @@ config {
         load "[email protected]"
     }
 
+    triggers "conf/modules.config", "nextflow.config"
+
 }

tests/subworkflows/local/bam_call_elprep/main.nf.test → subworkflows/local/bam_call_elprep/tests/main.nf.test

@@ -1,7 +1,7 @@
 nextflow_workflow {
 
     name "Test Workflow BAM_CALL_ELPREP"
-    script "subworkflows/local/bam_call_elprep/main.nf"
+    script "../main.nf"
     workflow "BAM_CALL_ELPREP"
 
     tag "subworkflows"

subworkflows/local/bam_call_vardictjava/main.nf

@@ -16,7 +16,6 @@ workflow BAM_CALL_VARDICTJAVA {
         ch_fai               // channel: [mandatory] [ val(meta), path(fai) ] => fasta reference index
         ch_dbsnp             // channel: [optional]  [ path(vcf) ] => the dbnsp vcf file
         ch_dbsnp_tbi         // channel: [optional]  [ path(tbi) ] => the dbsnp vcf index file
-        filter               // boolean: filter the VCFs
 
     main:
     def ch_versions = Channel.empty()

tests/subworkflows/local/bam_call_vardictjava/main.nf.test → subworkflows/local/bam_call_vardictjava/tests/main.nf.test

@@ -1,7 +1,7 @@
 nextflow_workflow {
 
     name "Test Workflow BAM_CALL_VARDICTJAVA"
-    script "subworkflows/local/bam_call_vardictjava/main.nf"
+    script "../main.nf"
     workflow "BAM_CALL_VARDICTJAVA"
 
     tag "subworkflows"
@@ -43,7 +43,6 @@ nextflow_workflow {
                 ])
                 input[3] = [[],[]]
                 input[4] = [[],[]]
-                input[5] = false
                 """
             }
         }

tests/subworkflows/local/cram_call_gatk4/main.nf.test → subworkflows/local/cram_call_gatk4/tests/main.nf.test

@@ -1,7 +1,7 @@
 nextflow_workflow {
 
     name "Test Workflow CRAM_CALL_GATK4"
-    script "subworkflows/local/cram_call_gatk4/main.nf"
+    script "../main.nf"
     workflow "CRAM_CALL_GATK4"
 
     tag "subworkflows"

tests/subworkflows/local/cram_prepare_samtools_bedtools/main.nf.test → subworkflows/local/cram_prepare_samtools_bedtools/tests/main.nf.test

@@ -1,7 +1,7 @@
 nextflow_workflow {
 
     name "Test Workflow CRAM_PREPARE_SAMTOOLS_BEDTOOLS"
-    script "subworkflows/local/cram_prepare_samtools_bedtools/main.nf"
+    script "../main.nf"
     workflow "CRAM_PREPARE_SAMTOOLS_BEDTOOLS"
 
     tag "subworkflows"
@@ -49,7 +49,7 @@ nextflow_workflow {
                     },
                     workflow.out.ready_beds,
                     workflow.out.reports
-                ).match("default - WGS") }
+                ).match() }
             )
         }
 
@@ -96,7 +96,7 @@ nextflow_workflow {
                     },
                     workflow.out.ready_beds,
                     workflow.out.reports
-                ).match("default - WES common ROI") }
+                ).match() }
             )
         }
 
@@ -143,7 +143,7 @@ nextflow_workflow {
                     },
                     workflow.out.ready_beds,
                     workflow.out.reports
-                ).match("default - WES") }
+                ).match() }
             )
         }
 
@@ -197,7 +197,7 @@ nextflow_workflow {
                     },
                     workflow.out.ready_beds,
                     workflow.out.reports
-                ).match("default - merge") }
+                ).match() }
             )
         }
 

tests/subworkflows/local/cram_prepare_samtools_bedtools/main.nf.test.snap → subworkflows/local/cram_prepare_samtools_bedtools/tests/main.nf.test.snap

@@ -1,5 +1,5 @@
 {
-    "default - merge": {
+    "cram_prepare_samtools_bedtools - default - WES common ROI": {
         "content": [
             [
                 [
@@ -8,9 +8,9 @@
                         "sample": "NA24143",
                         "family": "Ashkenazim",
                         "family_samples": "NA24143",
-                        "duplicate_count": 2
+                        "duplicate_count": 1
                     },
-                    "NA24143.cram,readsMD5:be28f434d6f7bcfa398488a6611d89c1",
+                    "/nf-cmgg/test-datasets/raw/germline/data/genomics/homo_sapiens/illumina/crams/NA24143.cram",
                     "NA24143.cram.crai"
                 ]
             ],
@@ -21,9 +21,9 @@
                         "sample": "NA24143",
                         "family": "Ashkenazim",
                         "family_samples": "NA24143",
-                        "duplicate_count": 2
+                        "duplicate_count": 1
                     },
-                    "NA24143.bam,readsMD5:be28f434d6f7bcfa398488a6611d89c1",
+                    "NA24143.bam,readsMD5:77afffb023e537869c5c6ebf31187ded",
                     "NA24143.bam.bai"
                 ]
             ],
@@ -34,22 +34,22 @@
                         "sample": "NA24143",
                         "family": "Ashkenazim",
                         "family_samples": "NA24143",
-                        "duplicate_count": 2
+                        "duplicate_count": 1
                     },
-                    "NA24143_intersect.bed:md5,b87069698afefb15282d069e56110046"
+                    "NA24143.intersect.bed:md5,b87069698afefb15282d069e56110046"
                 ]
             ],
             [
 
             ]
         ],
         "meta": {
-            "nf-test": "0.9.0",
-            "nextflow": "24.04.4"
+            "nf-test": "0.9.1",
+            "nextflow": "24.10.0"
         },
-        "timestamp": "2024-10-23T10:19:20.53195135"
+        "timestamp": "2024-11-14T16:07:03.620603158"
     },
-    "default - WGS": {
+    "cram_prepare_samtools_bedtools - default - WES": {
         "content": [
             [
                 [
@@ -76,20 +76,20 @@
                         "family_samples": "NA24143",
                         "duplicate_count": 1
                     },
-                    "NA24143.filter.bed:md5,85a5568a6976ed455caa712991b30ac2"
+                    "NA24143.intersect.bed:md5,b87069698afefb15282d069e56110046"
                 ]
             ],
             [
 
             ]
         ],
         "meta": {
-            "nf-test": "0.9.0",
-            "nextflow": "24.04.4"
+            "nf-test": "0.9.1",
+            "nextflow": "24.10.0"
         },
-        "timestamp": "2024-10-23T10:17:48.433466279"
+        "timestamp": "2024-11-14T16:07:24.688580575"
     },
-    "default - WES": {
+    "cram_prepare_samtools_bedtools - default - WGS": {
         "content": [
             [
                 [
@@ -116,20 +116,20 @@
                         "family_samples": "NA24143",
                         "duplicate_count": 1
                     },
-                    "NA24143_intersect.bed:md5,b87069698afefb15282d069e56110046"
+                    "NA24143.filter.bed:md5,85a5568a6976ed455caa712991b30ac2"
                 ]
             ],
             [
 
             ]
         ],
         "meta": {
-            "nf-test": "0.9.0",
-            "nextflow": "24.04.4"
+            "nf-test": "0.9.1",
+            "nextflow": "24.10.0"
         },
-        "timestamp": "2024-10-23T10:18:36.993186258"
+        "timestamp": "2024-11-14T16:06:42.311563291"
     },
-    "default - WES common ROI": {
+    "cram_prepare_samtools_bedtools - default - merge": {
         "content": [
             [
                 [
@@ -138,9 +138,9 @@
                         "sample": "NA24143",
                         "family": "Ashkenazim",
                         "family_samples": "NA24143",
-                        "duplicate_count": 1
+                        "duplicate_count": 2
                     },
-                    "/nf-cmgg/test-datasets/raw/germline/data/genomics/homo_sapiens/illumina/crams/NA24143.cram",
+                    "NA24143.cram,readsMD5:be28f434d6f7bcfa398488a6611d89c1",
                     "NA24143.cram.crai"
                 ]
             ],
@@ -151,9 +151,9 @@
                         "sample": "NA24143",
                         "family": "Ashkenazim",
                         "family_samples": "NA24143",
-                        "duplicate_count": 1
+                        "duplicate_count": 2
                     },
-                    "NA24143.bam,readsMD5:77afffb023e537869c5c6ebf31187ded",
+                    "NA24143.bam,readsMD5:be28f434d6f7bcfa398488a6611d89c1",
                     "NA24143.bam.bai"
                 ]
             ],
@@ -164,19 +164,19 @@
                         "sample": "NA24143",
                         "family": "Ashkenazim",
                         "family_samples": "NA24143",
-                        "duplicate_count": 1
+                        "duplicate_count": 2
                     },
-                    "NA24143_intersect.bed:md5,b87069698afefb15282d069e56110046"
+                    "NA24143.intersect.bed:md5,b87069698afefb15282d069e56110046"
                 ]
             ],
             [
 
             ]
         ],
         "meta": {
-            "nf-test": "0.9.0",
-            "nextflow": "24.04.4"
+            "nf-test": "0.9.1",
+            "nextflow": "24.10.0"
         },
-        "timestamp": "2024-10-23T10:18:13.865281894"
+        "timestamp": "2024-11-14T16:08:20.712943795"
     }
 }

tests/subworkflows/local/gvcf_joint_genotype_gatk4/main.nf.test → subworkflows/local/gvcf_joint_genotype_gatk4/tests/main.nf.test

@@ -1,7 +1,7 @@
 nextflow_workflow {
 
     name "Test Workflow GVCF_JOINT_GENOTYPE_GATK4"
-    script "subworkflows/local/gvcf_joint_genotype_gatk4/main.nf"
+    script "../main.nf"
     workflow "GVCF_JOINT_GENOTYPE_GATK4"
 
     tag "subworkflows"

tests/subworkflows/local/input_split_bedtools/main.nf.test → subworkflows/local/input_split_bedtools/tests/main.nf.test

@@ -1,7 +1,7 @@
 nextflow_workflow {
 
     name "Test Workflow INPUT_SPLIT_BEDTOOLS"
-    script "subworkflows/local/input_split_bedtools/main.nf"
+    script "../main.nf"
     workflow "INPUT_SPLIT_BEDTOOLS"
 
     tag "subworkflows"

tests/subworkflows/local/vcf_annotation/main.nf.test → subworkflows/local/vcf_annotation/tests/main.nf.test

@@ -1,7 +1,7 @@
 nextflow_workflow {
 
     name "Test Workflow VCF_ANNOTATION"
-    script "subworkflows/local/vcf_annotation/main.nf"
+    script "../main.nf"
     workflow "VCF_ANNOTATION"
 
     tag "subworkflows"
@@ -44,9 +44,9 @@ nextflow_workflow {
             assertAll(
                 { assert workflow.success },
                 { assert snapshot(
-                    workflow.out.annotated_vcfs.collect { it.collect { it instanceof Map ? it.groupTarget : file(it).name } },
+                    workflow.out.annotated_vcfs.collect { [ it[0].groupTarget, it[1][-7..-1] ]},
                     workflow.out.reports
-                ).match("default") }
+                ).match() }
             )
         }
 
@@ -87,9 +87,9 @@ nextflow_workflow {
             assertAll(
                 { assert workflow.success },
                 { assert snapshot(
-                    workflow.out.annotated_vcfs.collect { it.collect { it instanceof Map ? it.groupTarget : file(it).name } },
+                    workflow.out.annotated_vcfs.collect { [ it[0].groupTarget, it[1][-7..-1] ] },
                     workflow.out.reports
-                ).match("vcfanno") }
+                ).match() }
             )
         }