PIP-1671-add-to-dockstore (#69)

.circleci/config.yml

@@ -57,6 +57,15 @@ commands:
       - run: curl -Ls https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz | tar xJ && sudo mv shellcheck-stable/shellcheck
           /usr/local/bin
 
+  add_docker_to_inputs:
+    description: Add docker to input JSONs
+    steps:
+      - run:
+          command: |
+            for i in tests/unit/json/*.json tests/integration/json/*.json tests/functional/json/*.json; do
+              python tests/add_docker_to_input.py $i $WGBS_DOCKER_IMAGE_TAG
+            done
+
 jobs:
   lint:
     <<: *py37
@@ -95,6 +104,7 @@ jobs:
     steps:
       - checkout
       - set_environment_variables
+      - add_docker_to_inputs
       - run:
           name: Run pytest-workflow WDL tests using tox
           command: |

.dockstore.yml

@@ -0,0 +1,12 @@
+---
+version: 1.2
+workflows:
+  - name: wgbs-pipeline
+    publish: true
+    subclass: WDL
+    primaryDescriptorPath: /wgbs-pipeline.wdl
+    testParameterFiles:
+      - /dockstore_test_input.json
+    authors:
+      - name: Paul Sud
+        orcid: https://orcid.org/0000-0003-1667-4694

dockstore_test_input.json

@@ -0,0 +1,17 @@
+{
+  "wgbs.benchmark_mode": true,
+  "wgbs.extra_reference": "https://storage.googleapis.com/dockstore_test_input/wgbs-pipeline/conversion_control.fa.gz",
+  "wgbs.fastqs": [
+    [
+      [
+        "https://storage.googleapis.com/dockstore_test_input/wgbs-pipeline/sample5_data_1_200000.fastq.gz",
+        "https://storage.googleapis.com/dockstore_test_input/wgbs-pipeline/sample5_data_2_200000.fastq.gz"
+      ]
+    ]
+  ],
+  "wgbs.reference": "https://storage.googleapis.com/dockstore_test_input/wgbs-pipeline/sacCer3.fa.gz",
+  "wgbs.sample_names": [
+    "sample5"
+  ],
+  "wgbs.underconversion_sequence_name": "NC_001416.1"
+}

docs/CONTRIBUTING.md

@@ -23,4 +23,4 @@ $ tox
 ## Useful Tips
 
 * To build Docker images manually, run `docker build . -f Dockerfile -t MY_REPO:MY_TAG`
-* You can run individual WDL tests manually with `tests/wdl/test.sh tests/wdl/test_task/test_TASK_NAME.wdl tests/wdl/test_task/test_TASK_NAME_input.json DOCKER_IMAGE`. This will invoke `caper` for you.
+* You can run individual WDL tests manually with `tests/wdl/test.sh tests/wdl/test_task/test_TASK_NAME.wdl tests/wdl/test_task/test_TASK_NAME_input.json. This will invoke `caper` for you. Note you will need to first update the input JSON with `docker` input.

tests/add_docker_to_input.py

@@ -0,0 +1,24 @@
+import argparse
+import json
+
+
+def main():
+    parser = _get_parser()
+    args = parser.parse_args()
+    with open(args.input_json_path) as f:
+        input_json = json.load(f)
+    workflow_name = next(iter(input_json.keys())).split(".")[0]
+    input_json["{}.docker".format(workflow_name)] = args.docker_image
+    with open(args.input_json_path, "w") as f:
+        json.dump(input_json, f, indent=2)
+
+
+def _get_parser():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("input_json_path")
+    parser.add_argument("docker_image")
+    return parser
+
+
+if __name__ == "__main__":
+    main()

tests/caper_run.sh

@@ -12,11 +12,6 @@ if [ $# -lt 2 ]; then
     exit 1
 fi
 
-if [ -z "${WGBS_DOCKER_IMAGE_TAG}" ]; then
-    echo "Must specify WGBS_DOCKER_IMAGE_TAG via environment variable."
-    exit 1
-fi
-
 WDL=$1
 INPUT=$2
 WORKFLOW_OPTIONS="tests/pytest_workflow_options.json"
@@ -30,6 +25,6 @@ if [ $# -gt 2 ]; then
     WORKFLOW_OPTIONS="tests/pytest_workflow_no_relative_output_paths.json"
 fi
 
-echo "Running caper with WDL ${WDL}, input ${INPUT}, workflow options ${WORKFLOW_OPTIONS}, and image ${WGBS_DOCKER_IMAGE_TAG}"
+echo "Running caper with WDL ${WDL}, input ${INPUT}, workflow options ${WORKFLOW_OPTIONS}"
 
-caper run "${WDL}" -i "${INPUT}" --docker "${WGBS_DOCKER_IMAGE_TAG}" -o "./${WORKFLOW_OPTIONS}"
+caper run "${WDL}" -i "${INPUT}" -o "./${WORKFLOW_OPTIONS}"

tests/integration/wdl/test_bscaller.wdl

@@ -8,6 +8,7 @@ workflow test_bscaller {
     File index
     String sample_name
     String sample_barcode
+    String docker
 
     call wgbs.bscaller { input:
         reference = reference,
@@ -16,6 +17,7 @@ workflow test_bscaller {
         csi = csi,
         index = index,
         sample_barcode = sample_barcode,
-        sample_name = sample_name
+        sample_name = sample_name,
+        docker = docker,
     }
 }

tests/integration/wdl/test_calculate_average_coverage.wdl

@@ -4,10 +4,12 @@ workflow test_calculate_average_coverage {
     File bam
     File chrom_sizes
     Int num_cpus
+    String docker
 
     call wgbs.calculate_average_coverage { input:
         bam = bam,
         chrom_sizes = chrom_sizes,
         num_cpus = num_cpus,
+        docker = docker,
     }
 }

tests/integration/wdl/test_calculate_bed_pearson_correlation.wdl

@@ -3,9 +3,11 @@ import "../../../wgbs-pipeline.wdl" as wgbs
 workflow test_calculate_bed_pearson_correlation {
     File bed1
     File bed2
+    String docker
 
     call wgbs.calculate_bed_pearson_correlation { input:
         bed1 = bed1,
         bed2 = bed2,
+        docker = docker,
     }
 }

tests/integration/wdl/test_extract.wdl

@@ -7,6 +7,7 @@ workflow test_extract {
     File bcf
     File bcf_csi
     String sample_barcode
+    String docker
 
     call wgbs.extract { input:
         reference = reference,
@@ -15,5 +16,6 @@ workflow test_extract {
         bcf = bcf,
         bcf_csi = bcf_csi,
         sample_barcode = sample_barcode,
+        docker = docker,
     }
 }

tests/integration/wdl/test_index.wdl

@@ -4,10 +4,12 @@ workflow test_index {
     File configuration_file
     File reference
     File extra_reference
+    String docker
 
     call wgbs.index { input:
         configuration_file = configuration_file,
         reference = reference,
-        extra_reference = extra_reference
+        extra_reference = extra_reference,
+        docker = docker,
     }
 }

tests/integration/wdl/test_make_conf.wdl

@@ -8,6 +8,7 @@ workflow test_make_conf {
     Boolean benchmark_mode = false
     String? include_file
     String? underconversion_sequence_name
+    String docker
 
     call wgbs.make_conf { input:
         num_threads = num_threads,
@@ -16,6 +17,7 @@ workflow test_make_conf {
         extra_reference = extra_reference,
         include_file = include_file,
         underconversion_sequence_name = underconversion_sequence_name,
-        benchmark_mode = benchmark_mode
+        benchmark_mode = benchmark_mode,
+        docker = docker,
     }
 }

tests/integration/wdl/test_make_coverage_bigwig.wdl

@@ -3,9 +3,11 @@ import "../../../wgbs-pipeline.wdl" as wgbs
 workflow test_make_coverage_bigwig {
     File encode_bed
     File chrom_sizes
+    String docker
 
     call wgbs.make_coverage_bigwig { input:
         encode_bed = encode_bed,
         chrom_sizes = chrom_sizes,
+        docker = docker,
     }
 }

tests/integration/wdl/test_make_metadata_csv.wdl

@@ -4,10 +4,12 @@ workflow test_make_metadata_csv {
     Array[String] sample_names
     Array[Array[Array[File]]] fastqs
     String barcode_prefix
+    String docker
 
     call wgbs.make_metadata_csv { input:
         sample_names = sample_names,
         fastqs = write_json(fastqs),
         barcode_prefix = barcode_prefix,
+        docker = docker,
     }
 }

tests/integration/wdl/test_map.wdl

@@ -7,13 +7,15 @@ workflow test_map {
     Array[File] fastqs
     String sample_name
     String sample_barcode = "sample_sample5"
+    String docker
 
     call wgbs.map { input:
         reference = reference,
         index = index,
         gemBS_json = gemBS_json,
         fastqs = fastqs,
         sample_barcode = sample_barcode,
-        sample_name = sample_name
+        sample_name = sample_name,
+        docker = docker,
     }
 }

tests/integration/wdl/test_prepare.wdl

@@ -6,12 +6,14 @@ workflow test_prepare {
     File index
     String reference
     String extra_reference
+    String docker
 
     call wgbs.prepare { input:
         configuration_file = configuration_file,
         metadata_file = metadata_file,
         index = index,
         reference = reference,
-        extra_reference = extra_reference
+        extra_reference = extra_reference,
+        docker = docker,
     }
 }

tests/integration/wdl/test_qc_report.wdl

@@ -6,12 +6,14 @@ workflow test_qc_report {
     File gemBS_json
     File contig_sizes
     String sample_barcode
+    String docker
 
     call wgbs.qc_report { input:
         reference = reference,
         gemBS_json = gemBS_json,
         contig_sizes = contig_sizes,
         map_qc_json = map_qc_json,
-        sample_barcode = sample_barcode
+        sample_barcode = sample_barcode,
+        docker = docker,
     }
 }

tests/unit/wdl/test_make_coverage_bigwig.wdl

@@ -3,9 +3,11 @@ import "../../../wgbs-pipeline.wdl" as wgbs
 workflow test_make_coverage_bigwig {
     File encode_bed
     File chrom_sizes
+    String docker
 
     call wgbs.make_coverage_bigwig { input:
         encode_bed = encode_bed,
         chrom_sizes = chrom_sizes,
+        docker = docker,
     }
 }

tox.ini

@@ -21,7 +21,6 @@ deps =
     caper==0.8.2.1
     pytest
     pytest-workflow>=1.3.0
-passenv = WGBS_DOCKER_IMAGE_TAG
 
 [testenv:lint]
 basepython = python3.7