diff --git a/.circleci/config.yml b/.circleci/config.yml index 60ae787..3f4cf53 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -57,6 +57,15 @@ commands: - run: curl -Ls https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz | tar xJ && sudo mv shellcheck-stable/shellcheck /usr/local/bin + add_docker_to_inputs: + description: Add docker to input JSONs + steps: + - run: + command: | + for i in tests/unit/json/*.json tests/integration/json/*.json tests/functional/json/*.json; do + python tests/add_docker_to_input.py $i $WGBS_DOCKER_IMAGE_TAG + done + jobs: lint: <<: *py37 @@ -95,6 +104,7 @@ jobs: steps: - checkout - set_environment_variables + - add_docker_to_inputs - run: name: Run pytest-workflow WDL tests using tox command: | diff --git a/.dockstore.yml b/.dockstore.yml new file mode 100644 index 0000000..f75ff1f --- /dev/null +++ b/.dockstore.yml @@ -0,0 +1,12 @@ +--- +version: 1.2 +workflows: + - name: wgbs-pipeline + publish: true + subclass: WDL + primaryDescriptorPath: /wgbs-pipeline.wdl + testParameterFiles: + - /dockstore_test_input.json + authors: + - name: Paul Sud + orcid: https://orcid.org/0000-0003-1667-4694 diff --git a/dockstore_test_input.json b/dockstore_test_input.json new file mode 100644 index 0000000..78f9c3c --- /dev/null +++ b/dockstore_test_input.json @@ -0,0 +1,17 @@ +{ + "wgbs.benchmark_mode": true, + "wgbs.extra_reference": "https://storage.googleapis.com/dockstore_test_input/wgbs-pipeline/conversion_control.fa.gz", + "wgbs.fastqs": [ + [ + [ + "https://storage.googleapis.com/dockstore_test_input/wgbs-pipeline/sample5_data_1_200000.fastq.gz", + "https://storage.googleapis.com/dockstore_test_input/wgbs-pipeline/sample5_data_2_200000.fastq.gz" + ] + ] + ], + "wgbs.reference": "https://storage.googleapis.com/dockstore_test_input/wgbs-pipeline/sacCer3.fa.gz", + "wgbs.sample_names": [ + "sample5" + ], + "wgbs.underconversion_sequence_name": "NC_001416.1" +} diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 5792413..b8124d8 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -23,4 +23,4 @@ $ tox ## Useful Tips * To build Docker images manually, run `docker build . -f Dockerfile -t MY_REPO:MY_TAG` -* You can run individual WDL tests manually with `tests/wdl/test.sh tests/wdl/test_task/test_TASK_NAME.wdl tests/wdl/test_task/test_TASK_NAME_input.json DOCKER_IMAGE`. This will invoke `caper` for you. +* You can run individual WDL tests manually with `tests/wdl/test.sh tests/wdl/test_task/test_TASK_NAME.wdl tests/wdl/test_task/test_TASK_NAME_input.json. This will invoke `caper` for you. Note you will need to first update the input JSON with `docker` input. diff --git a/tests/add_docker_to_input.py b/tests/add_docker_to_input.py new file mode 100644 index 0000000..d79b5d1 --- /dev/null +++ b/tests/add_docker_to_input.py @@ -0,0 +1,24 @@ +import argparse +import json + + +def main(): + parser = _get_parser() + args = parser.parse_args() + with open(args.input_json_path) as f: + input_json = json.load(f) + workflow_name = next(iter(input_json.keys())).split(".")[0] + input_json["{}.docker".format(workflow_name)] = args.docker_image + with open(args.input_json_path, "w") as f: + json.dump(input_json, f, indent=2) + + +def _get_parser(): + parser = argparse.ArgumentParser() + parser.add_argument("input_json_path") + parser.add_argument("docker_image") + return parser + + +if __name__ == "__main__": + main() diff --git a/tests/caper_run.sh b/tests/caper_run.sh index e35b6b9..05b7b13 100755 --- a/tests/caper_run.sh +++ b/tests/caper_run.sh @@ -12,11 +12,6 @@ if [ $# -lt 2 ]; then exit 1 fi -if [ -z "${WGBS_DOCKER_IMAGE_TAG}" ]; then - echo "Must specify WGBS_DOCKER_IMAGE_TAG via environment variable." - exit 1 -fi - WDL=$1 INPUT=$2 WORKFLOW_OPTIONS="tests/pytest_workflow_options.json" @@ -30,6 +25,6 @@ if [ $# -gt 2 ]; then WORKFLOW_OPTIONS="tests/pytest_workflow_no_relative_output_paths.json" fi -echo "Running caper with WDL ${WDL}, input ${INPUT}, workflow options ${WORKFLOW_OPTIONS}, and image ${WGBS_DOCKER_IMAGE_TAG}" +echo "Running caper with WDL ${WDL}, input ${INPUT}, workflow options ${WORKFLOW_OPTIONS}" -caper run "${WDL}" -i "${INPUT}" --docker "${WGBS_DOCKER_IMAGE_TAG}" -o "./${WORKFLOW_OPTIONS}" +caper run "${WDL}" -i "${INPUT}" -o "./${WORKFLOW_OPTIONS}" diff --git a/tests/integration/wdl/test_bscaller.wdl b/tests/integration/wdl/test_bscaller.wdl index ea4a206..75f485c 100644 --- a/tests/integration/wdl/test_bscaller.wdl +++ b/tests/integration/wdl/test_bscaller.wdl @@ -8,6 +8,7 @@ workflow test_bscaller { File index String sample_name String sample_barcode + String docker call wgbs.bscaller { input: reference = reference, @@ -16,6 +17,7 @@ workflow test_bscaller { csi = csi, index = index, sample_barcode = sample_barcode, - sample_name = sample_name + sample_name = sample_name, + docker = docker, } } diff --git a/tests/integration/wdl/test_calculate_average_coverage.wdl b/tests/integration/wdl/test_calculate_average_coverage.wdl index ba2212d..d4d5a8f 100644 --- a/tests/integration/wdl/test_calculate_average_coverage.wdl +++ b/tests/integration/wdl/test_calculate_average_coverage.wdl @@ -4,10 +4,12 @@ workflow test_calculate_average_coverage { File bam File chrom_sizes Int num_cpus + String docker call wgbs.calculate_average_coverage { input: bam = bam, chrom_sizes = chrom_sizes, num_cpus = num_cpus, + docker = docker, } } diff --git a/tests/integration/wdl/test_calculate_bed_pearson_correlation.wdl b/tests/integration/wdl/test_calculate_bed_pearson_correlation.wdl index d644158..c93a265 100644 --- a/tests/integration/wdl/test_calculate_bed_pearson_correlation.wdl +++ b/tests/integration/wdl/test_calculate_bed_pearson_correlation.wdl @@ -3,9 +3,11 @@ import "../../../wgbs-pipeline.wdl" as wgbs workflow test_calculate_bed_pearson_correlation { File bed1 File bed2 + String docker call wgbs.calculate_bed_pearson_correlation { input: bed1 = bed1, bed2 = bed2, + docker = docker, } } diff --git a/tests/integration/wdl/test_extract.wdl b/tests/integration/wdl/test_extract.wdl index f438601..5e04638 100644 --- a/tests/integration/wdl/test_extract.wdl +++ b/tests/integration/wdl/test_extract.wdl @@ -7,6 +7,7 @@ workflow test_extract { File bcf File bcf_csi String sample_barcode + String docker call wgbs.extract { input: reference = reference, @@ -15,5 +16,6 @@ workflow test_extract { bcf = bcf, bcf_csi = bcf_csi, sample_barcode = sample_barcode, + docker = docker, } } diff --git a/tests/integration/wdl/test_index.wdl b/tests/integration/wdl/test_index.wdl index 673dd7a..603862a 100644 --- a/tests/integration/wdl/test_index.wdl +++ b/tests/integration/wdl/test_index.wdl @@ -4,10 +4,12 @@ workflow test_index { File configuration_file File reference File extra_reference + String docker call wgbs.index { input: configuration_file = configuration_file, reference = reference, - extra_reference = extra_reference + extra_reference = extra_reference, + docker = docker, } } diff --git a/tests/integration/wdl/test_make_conf.wdl b/tests/integration/wdl/test_make_conf.wdl index 370396e..2914eef 100644 --- a/tests/integration/wdl/test_make_conf.wdl +++ b/tests/integration/wdl/test_make_conf.wdl @@ -8,6 +8,7 @@ workflow test_make_conf { Boolean benchmark_mode = false String? include_file String? underconversion_sequence_name + String docker call wgbs.make_conf { input: num_threads = num_threads, @@ -16,6 +17,7 @@ workflow test_make_conf { extra_reference = extra_reference, include_file = include_file, underconversion_sequence_name = underconversion_sequence_name, - benchmark_mode = benchmark_mode + benchmark_mode = benchmark_mode, + docker = docker, } } diff --git a/tests/integration/wdl/test_make_coverage_bigwig.wdl b/tests/integration/wdl/test_make_coverage_bigwig.wdl index 0a73352..bcfc44a 100644 --- a/tests/integration/wdl/test_make_coverage_bigwig.wdl +++ b/tests/integration/wdl/test_make_coverage_bigwig.wdl @@ -3,9 +3,11 @@ import "../../../wgbs-pipeline.wdl" as wgbs workflow test_make_coverage_bigwig { File encode_bed File chrom_sizes + String docker call wgbs.make_coverage_bigwig { input: encode_bed = encode_bed, chrom_sizes = chrom_sizes, + docker = docker, } } diff --git a/tests/integration/wdl/test_make_metadata_csv.wdl b/tests/integration/wdl/test_make_metadata_csv.wdl index 8d5ec2d..5edd092 100644 --- a/tests/integration/wdl/test_make_metadata_csv.wdl +++ b/tests/integration/wdl/test_make_metadata_csv.wdl @@ -4,10 +4,12 @@ workflow test_make_metadata_csv { Array[String] sample_names Array[Array[Array[File]]] fastqs String barcode_prefix + String docker call wgbs.make_metadata_csv { input: sample_names = sample_names, fastqs = write_json(fastqs), barcode_prefix = barcode_prefix, + docker = docker, } } diff --git a/tests/integration/wdl/test_map.wdl b/tests/integration/wdl/test_map.wdl index dccd027..fd79504 100644 --- a/tests/integration/wdl/test_map.wdl +++ b/tests/integration/wdl/test_map.wdl @@ -7,6 +7,7 @@ workflow test_map { Array[File] fastqs String sample_name String sample_barcode = "sample_sample5" + String docker call wgbs.map { input: reference = reference, @@ -14,6 +15,7 @@ workflow test_map { gemBS_json = gemBS_json, fastqs = fastqs, sample_barcode = sample_barcode, - sample_name = sample_name + sample_name = sample_name, + docker = docker, } } diff --git a/tests/integration/wdl/test_prepare.wdl b/tests/integration/wdl/test_prepare.wdl index b06cd89..7ab9b5e 100644 --- a/tests/integration/wdl/test_prepare.wdl +++ b/tests/integration/wdl/test_prepare.wdl @@ -6,12 +6,14 @@ workflow test_prepare { File index String reference String extra_reference + String docker call wgbs.prepare { input: configuration_file = configuration_file, metadata_file = metadata_file, index = index, reference = reference, - extra_reference = extra_reference + extra_reference = extra_reference, + docker = docker, } } diff --git a/tests/integration/wdl/test_qc_report.wdl b/tests/integration/wdl/test_qc_report.wdl index b546bc6..9eb85be 100644 --- a/tests/integration/wdl/test_qc_report.wdl +++ b/tests/integration/wdl/test_qc_report.wdl @@ -6,12 +6,14 @@ workflow test_qc_report { File gemBS_json File contig_sizes String sample_barcode + String docker call wgbs.qc_report { input: reference = reference, gemBS_json = gemBS_json, contig_sizes = contig_sizes, map_qc_json = map_qc_json, - sample_barcode = sample_barcode + sample_barcode = sample_barcode, + docker = docker, } } diff --git a/tests/unit/wdl/test_make_coverage_bigwig.wdl b/tests/unit/wdl/test_make_coverage_bigwig.wdl index 0a73352..bcfc44a 100644 --- a/tests/unit/wdl/test_make_coverage_bigwig.wdl +++ b/tests/unit/wdl/test_make_coverage_bigwig.wdl @@ -3,9 +3,11 @@ import "../../../wgbs-pipeline.wdl" as wgbs workflow test_make_coverage_bigwig { File encode_bed File chrom_sizes + String docker call wgbs.make_coverage_bigwig { input: encode_bed = encode_bed, chrom_sizes = chrom_sizes, + docker = docker, } } diff --git a/tox.ini b/tox.ini index 6e926ad..7663ef9 100644 --- a/tox.ini +++ b/tox.ini @@ -21,7 +21,6 @@ deps = caper==0.8.2.1 pytest pytest-workflow>=1.3.0 -passenv = WGBS_DOCKER_IMAGE_TAG [testenv:lint] basepython = python3.7 diff --git a/wgbs-pipeline.wdl b/wgbs-pipeline.wdl index 41b560a..9549ee2 100755 --- a/wgbs-pipeline.wdl +++ b/wgbs-pipeline.wdl @@ -4,6 +4,7 @@ workflow wgbs { caper_docker: "encodedcc/wgbs-pipeline:1.1.7" caper_singularity: "docker://encodedcc/wgbs-pipeline:1.1.7" croo_out_def: "https://raw.githubusercontent.com/ENCODE-DCC/wgbs-pipeline/dev/croo_out_def.json" + description: "ENCODE WGBS pipeline, see https://github.com/ENCODE-DCC/wgbs-pipeline for details." } File reference @@ -67,6 +68,9 @@ workflow wgbs { Int? qc_report_num_cpus Int? qc_report_ram_gb + String docker = "encodedcc/wgbs-pipeline:1.1.7" + String singularity = "docker://encodedcc/wgbs-pipeline:1.1.7" + # Don't need metadata csv to create indexes if (!index_only) { call make_metadata_csv { input: @@ -76,6 +80,8 @@ workflow wgbs { num_cpus = make_metadata_csv_num_cpus, ram_gb = make_metadata_csv_ram_gb, disk_size_gb = make_metadata_csv_disk_size_gb, + docker = docker, + singularity = singularity, } } @@ -90,6 +96,8 @@ workflow wgbs { num_cpus = make_conf_num_cpus, ram_gb = make_conf_ram_gb, disk_size_gb = make_conf_disk_size_gb, + docker = docker, + singularity = singularity, } if (!defined(indexed_reference)) { @@ -100,6 +108,8 @@ workflow wgbs { num_cpus = index_num_cpus, ram_gb = index_ram_gb, disk_size_gb = index_disk_size_gb, + docker = docker, + singularity = singularity, } } @@ -116,6 +126,8 @@ workflow wgbs { num_cpus = prepare_num_cpus, ram_gb = prepare_ram_gb, disk_size_gb = prepare_disk_size_gb, + docker = docker, + singularity = singularity, } } @@ -138,6 +150,8 @@ workflow wgbs { disk_size_gb = map_disk_size_gb, sort_threads = map_sort_threads, sort_memory = map_sort_memory, + docker = docker, + singularity = singularity, } call bscaller { input: @@ -151,6 +165,8 @@ workflow wgbs { num_cpus = bscaller_num_cpus, ram_gb = bscaller_ram_gb, disk_size_gb = bscaller_disk_size_gb, + docker = docker, + singularity = singularity, } call calculate_average_coverage { input: @@ -159,6 +175,8 @@ workflow wgbs { num_cpus = calculate_average_coverage_num_cpus, ram_gb = calculate_average_coverage_ram_gb, disk_size_gb = calculate_average_coverage_disk_size_gb, + docker = docker, + singularity = singularity, } call extract { input: @@ -173,6 +191,8 @@ workflow wgbs { num_cpus = extract_num_cpus, ram_gb = extract_ram_gb, disk_size_gb = extract_disk_size_gb, + docker = docker, + singularity = singularity, } call make_coverage_bigwig { input: @@ -181,6 +201,8 @@ workflow wgbs { num_cpus = make_coverage_bigwig_num_cpus, ram_gb = make_coverage_bigwig_ram_gb, disk_size_gb = make_coverage_bigwig_disk_size_gb, + docker = docker, + singularity = singularity, } call qc_report { input: @@ -192,6 +214,8 @@ workflow wgbs { num_cpus = qc_report_num_cpus, ram_gb = qc_report_ram_gb, disk_size_gb = qc_report_disk_size_gb, + docker = docker, + singularity = singularity, } } @@ -203,6 +227,8 @@ workflow wgbs { num_cpus = calculate_bed_pearson_correlation_num_cpus, ram_gb = calculate_bed_pearson_correlation_ram_gb, disk_size_gb = calculate_bed_pearson_correlation_disk_size_gb, + docker = docker, + singularity = singularity, } } } @@ -215,6 +241,8 @@ task make_metadata_csv { Int? num_cpus = 1 Int? ram_gb = 2 Int? disk_size_gb = 10 + String docker + String singularity = "" command { set -euo pipefail @@ -233,6 +261,8 @@ task make_metadata_csv { cpu: "${num_cpus}" memory: "${ram_gb} GB" disks: "local-disk ${disk_size_gb} SSD" + docker: docker + singularity: singularity } } @@ -247,6 +277,8 @@ task make_conf { Int? num_cpus = 1 Int? ram_gb = 2 Int? disk_size_gb = 10 + String docker + String singularity = "" command { set -euo pipefail @@ -269,6 +301,8 @@ task make_conf { cpu: "${num_cpus}" memory: "${ram_gb} GB" disks: "local-disk ${disk_size_gb} SSD" + docker: docker + singularity: singularity } } @@ -281,6 +315,8 @@ task prepare { Int? num_cpus = 8 Int? ram_gb = 32 Int? disk_size_gb = 500 + String docker + String singularity = "" command { set -euo pipefail @@ -302,6 +338,8 @@ task prepare { cpu: "${num_cpus}" memory: "${ram_gb} GB" disks: "local-disk ${disk_size_gb} HDD" + docker: docker + singularity: singularity } } @@ -312,6 +350,8 @@ task index { Int? num_cpus = 8 Int? ram_gb = 64 Int? disk_size_gb = 500 + String docker + String singularity = "" command { set -euo pipefail @@ -345,6 +385,8 @@ task index { cpu: "${num_cpus}" memory: "${ram_gb} GB" disks: "local-disk ${disk_size_gb} HDD" + docker: docker + singularity: singularity } } @@ -360,6 +402,8 @@ task map { Int? disk_size_gb = 500 Int? sort_threads String? sort_memory + String docker + String singularity = "" command { set -euo pipefail @@ -388,6 +432,8 @@ task map { cpu: "${num_cpus}" memory: "${ram_gb} GB" disks: "local-disk ${disk_size_gb} HDD" + docker: docker + singularity: singularity } } @@ -397,6 +443,8 @@ task calculate_average_coverage { Int? num_cpus = 1 Int? ram_gb = 8 Int? disk_size_gb = 200 + String docker + String singularity = "" command { python3 "$(which calculate_average_coverage.py)" \ @@ -414,6 +462,8 @@ task calculate_average_coverage { cpu: "${num_cpus}" memory: "${ram_gb} GB" disks: "local-disk ${disk_size_gb} SSD" + docker: docker + singularity: singularity } } @@ -428,6 +478,8 @@ task bscaller { Int? num_cpus = 8 Int? ram_gb = 32 Int? disk_size_gb = 500 + String docker + String singularity = "" command { set -euo pipefail @@ -449,6 +501,8 @@ task bscaller { cpu: "${num_cpus}" memory: "${ram_gb} GB" disks: "local-disk ${disk_size_gb} HDD" + docker: docker + singularity: singularity } } @@ -464,6 +518,8 @@ task extract { Int? num_cpus = 8 Int? ram_gb = 96 Int? disk_size_gb = 500 + String docker + String singularity = "" command { set -euo pipefail @@ -518,6 +574,8 @@ task extract { cpu: "${num_cpus}" memory: "${ram_gb} GB" disks: "local-disk ${disk_size_gb} SSD" + docker: docker + singularity: singularity } } @@ -527,6 +585,8 @@ task make_coverage_bigwig { Int? num_cpus = 4 Int? ram_gb = 8 Int? disk_size_gb = 50 + String docker + String singularity = "" command { set -euo pipefail @@ -547,6 +607,8 @@ task make_coverage_bigwig { cpu: "${num_cpus}" memory: "${ram_gb} GB" disks: "local-disk ${disk_size_gb} SSD" + docker: docker + singularity: singularity } } @@ -556,6 +618,8 @@ task calculate_bed_pearson_correlation { Int? num_cpus = 1 Int? ram_gb = 16 Int? disk_size_gb = 50 + String docker + String singularity = "" command { python3 "$(which calculate_bed_pearson_correlation.py)" --bedmethyls ${bed1} ${bed2} --outfile bed_pearson_correlation_qc.json @@ -569,6 +633,8 @@ task calculate_bed_pearson_correlation { cpu: "${num_cpus}" memory: "${ram_gb} GB" disks: "local-disk ${disk_size_gb} SSD" + docker: docker + singularity: singularity } } @@ -581,6 +647,8 @@ task qc_report { Int? num_cpus = 1 Int? ram_gb = 4 Int? disk_size_gb = 50 + String docker + String singularity = "" command { set -euo pipefail @@ -605,5 +673,7 @@ task qc_report { cpu: "${num_cpus}" memory: "${ram_gb} GB" disks: "local-disk ${disk_size_gb} HDD" + docker: docker + singularity: singularity } }