diff --git a/.circleci/config.yml b/.circleci/config.yml index 6139a51..b75430d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -20,32 +20,65 @@ machine_defaults: &machine_defaults image: ubuntu-1604:201903-01 working_directory: ~/long-read-rna-pipeline -make_tag: &make_tag - name: make docker image tag - command: | - echo "export TAG=quay.io/encode-dcc/${CIRCLE_PROJECT_REPONAME}:${CIRCLE_BRANCH}_${CIRCLE_WORKFLOW_ID}" > ${BASH_ENV} +commands: + set_env_variables: + description: "Set environment ready for docker running and pushing" + steps: + - run: + command: | + echo "export TAG=quay.io/encode-dcc/${CIRCLE_PROJECT_REPONAME}:${CIRCLE_BRANCH}_${CIRCLE_WORKFLOW_ID}" >> ${BASH_ENV} + echo "export CROMWELL=./cromwell-49.jar" >> ${BASH_ENV} + echo "export WOMTOOL=./womtool-49.jar" >> ${BASH_ENV} + source ${BASH_ENV} -only_dev_and_master: &only_dev_and_master - filters: - branches: - only: - - /^dev.*/ - - master + install_python_requirements: + description: "Install pytest workflow requirements" + steps: + - run: pyenv global 3.7.0 + - run: pip install --upgrade pip + - run: pip install -r requirements.txt + + download_file: + description: "Download a file using wget" + parameters: + file_url: + type: string + steps: + - run: wget <> + + prepare_pytest_environment: + description: "Get machine ready to run caper and pytest-workflow" + steps: + - set_env_variables + - install_python_requirements + - download_file: + file_url: "https://github.com/broadinstitute/cromwell/releases/download/49/cromwell-49.jar" + - download_file: + file_url: "https://github.com/broadinstitute/cromwell/releases/download/49/womtool-49.jar" + + run_test_tag: + description: "Run tagged test" + parameters: + tag: + type: string + wt: + type: string + steps: + - run: + command: pytest -v --tag <> --wt <> --symlink --kwd + no_output_timeout: 60m # Jobs -orbs: - singularity: singularity/singularity@1.0.3 jobs: build: <<: *defaults steps: - checkout - setup_remote_docker - - run: *make_tag + - set_env_variables - run: name: build image command: | - source ${BASH_ENV} echo "pulling template!" docker pull quay.io/encode-dcc/long-read-rna-pipeline:template docker login -u=${QUAY_ROBOT_USER} -p=${QUAY_ROBOT_USER_TOKEN} quay.io @@ -59,10 +92,9 @@ jobs: steps: - checkout - setup_remote_docker - - run: *make_tag + - set_env_variables - run: command: | - source ${BASH_ENV} docker pull $TAG docker login -u=${QUAY_ROBOT_USER} -p=${QUAY_ROBOT_USER_TOKEN} quay.io docker tag $TAG quay.io/encode-dcc/long-read-rna-pipeline:template @@ -74,35 +106,12 @@ jobs: <<: *machine_defaults steps: - checkout - - run: *make_tag + - prepare_pytest_environment - run: command: | - pyenv global 3.5.2 - source ${BASH_ENV} - test/test.sh get-splice-junctions.wdl test/test_task/test_get_splice_junctions_input.json $TAG docker + test/caper_run.sh get-splice-junctions.wdl test/test_task/test_get_splice_junctions_input.json python3 src/compare_md5.py --keys_to_inspect get_splice_junctions.get_splice_junctions_.splice_junctions \ - --metadata_json test_get_splice_junctions_input.metadata.json \ - --reference_json test/test_task/test_get_splice_junctions_reference_md5.json \ - --outfile test_get_splice_junctions_input.result.json - cat test_get_splice_junctions_input.result.json - python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < test_get_splice_junctions_input.result.json - no_output_timeout: 30m - - test_get_splice_junctions_singularity: - <<: *machine_defaults - steps: - - checkout - - run: *make_tag - - run: sudo apt-get update - - singularity/install-go - - singularity/debian-install-3 - - run: - command: | - pyenv global 3.5.2 - source ${BASH_ENV} - test/test.sh get-splice-junctions.wdl test/test_task/test_get_splice_junctions_input.json $TAG singularity - python3 src/compare_md5.py --keys_to_inspect get_splice_junctions.get_splice_junctions_.splice_junctions \ - --metadata_json test_get_splice_junctions_input.metadata.json \ + --metadata_json metadata.json \ --reference_json test/test_task/test_get_splice_junctions_reference_md5.json \ --outfile test_get_splice_junctions_input.result.json cat test_get_splice_junctions_input.result.json @@ -113,35 +122,12 @@ jobs: <<: *machine_defaults steps: - checkout - - run: *make_tag - - run: - command: | - pyenv global 3.5.2 - source ${BASH_ENV} - test/test.sh test/test_task/test_minimap2.wdl test/test_task/test_minimap2_input.json $TAG docker - python3 src/compare_md5.py --keys_to_inspect test_minimap2.skipNfirstlines.output_file test_minimap2.minimap2.mapping_qc \ - --metadata_json test_minimap2_input.metadata.json \ - --reference_json test/test_task/test_minimap2_reference_md5.json \ - --outfile test_minimap2_input.result.json - cat test_minimap2_input.result.json - python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < test_minimap2_input.result.json - no_output_timeout: 30m - - test_minimap2_singularity: - <<: *machine_defaults - steps: - - checkout - - run: *make_tag - - run: sudo apt-get update - - singularity/install-go - - singularity/debian-install-3 + - prepare_pytest_environment - run: command: | - pyenv global 3.5.2 - source ${BASH_ENV} - test/test.sh test/test_task/test_minimap2.wdl test/test_task/test_minimap2_input.json $TAG singularity + test/caper_run.sh test/test_task/test_minimap2.wdl test/test_task/test_minimap2_input.json python3 src/compare_md5.py --keys_to_inspect test_minimap2.skipNfirstlines.output_file test_minimap2.minimap2.mapping_qc \ - --metadata_json test_minimap2_input.metadata.json \ + --metadata_json metadata.json \ --reference_json test/test_task/test_minimap2_reference_md5.json \ --outfile test_minimap2_input.result.json cat test_minimap2_input.result.json @@ -152,43 +138,16 @@ jobs: <<: *machine_defaults steps: - checkout - - run: *make_tag + - prepare_pytest_environment - run: command: | - pyenv global 3.5.2 - source ${BASH_ENV} - test/test.sh test/test_task/test_transcriptclean.wdl test/test_task/test_transcriptclean_input.json $TAG docker + test/caper_run.sh test/test_task/test_transcriptclean.wdl test/test_task/test_transcriptclean_input.json python3 src/compare_md5.py \ --keys_to_inspect test_transcriptclean.transcriptclean.transcript_log \ test_transcriptclean.transcriptclean.transcript_error_log \ test_transcriptclean.transcriptclean.corrected_fasta \ test_transcriptclean.skipNfirstlines.output_file \ - --metadata_json test_transcriptclean_input.metadata.json \ - --reference_json test/test_task/test_transcriptclean_reference_md5.json \ - --outfile test_transcriptclean_input.result.json - cat test_transcriptclean_input.result.json - python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < test_transcriptclean_input.result.json - no_output_timeout: 30m - - test_transcriptclean_singularity: - <<: *machine_defaults - steps: - - checkout - - run: *make_tag - - run: sudo apt-get update - - singularity/install-go - - singularity/debian-install-3 - - run: - command: | - pyenv global 3.5.2 - source ${BASH_ENV} - test/test.sh test/test_task/test_transcriptclean.wdl test/test_task/test_transcriptclean_input.json $TAG singularity - python3 src/compare_md5.py \ - --keys_to_inspect test_transcriptclean.transcriptclean.transcript_log \ - test_transcriptclean.transcriptclean.transcript_error_log \ - test_transcriptclean.transcriptclean.corrected_fasta \ - test_transcriptclean.skipNfirstlines.output_file \ - --metadata_json test_transcriptclean_input.metadata.json \ + --metadata_json metadata.json \ --reference_json test/test_task/test_transcriptclean_reference_md5.json \ --outfile test_transcriptclean_input.result.json cat test_transcriptclean_input.result.json @@ -199,37 +158,13 @@ jobs: <<: *machine_defaults steps: - checkout - - run: *make_tag + - prepare_pytest_environment - run: command: | - pyenv global 3.5.2 - source ${BASH_ENV} - test/test.sh test/test_task/test_filter_transcriptclean.wdl test/test_task/test_filter_transcriptclean_input.json $TAG docker + test/caper_run.sh test/test_task/test_filter_transcriptclean.wdl test/test_task/test_filter_transcriptclean_input.json python3 src/compare_md5.py \ --keys_to_inspect test_filter_transcriptclean.skipNfirstlines.output_file \ - --metadata_json test_filter_transcriptclean_input.metadata.json \ - --reference_json test/test_task/test_filter_transcriptclean_reference_md5.json \ - --outfile test_filter_transcriptclean_input.result.json - cat test_filter_transcriptclean_input.result.json - python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < test_filter_transcriptclean_input.result.json - no_output_timeout: 30m - - test_filter_transcriptclean_singularity: - <<: *machine_defaults - steps: - - checkout - - run: *make_tag - - run: sudo apt-get update - - singularity/install-go - - singularity/debian-install-3 - - run: - command: | - pyenv global 3.5.2 - source ${BASH_ENV} - test/test.sh test/test_task/test_filter_transcriptclean.wdl test/test_task/test_filter_transcriptclean_input.json $TAG singularity - python3 src/compare_md5.py \ - --keys_to_inspect test_filter_transcriptclean.skipNfirstlines.output_file \ - --metadata_json test_filter_transcriptclean_input.metadata.json \ + --metadata_json metadata.json \ --reference_json test/test_task/test_filter_transcriptclean_reference_md5.json \ --outfile test_filter_transcriptclean_input.result.json cat test_filter_transcriptclean_input.result.json @@ -240,37 +175,13 @@ jobs: <<: *machine_defaults steps: - checkout - - run: *make_tag + - prepare_pytest_environment - run: command: | - pyenv global 3.5.2 - source ${BASH_ENV} - test/test.sh test/test_task/test_init_talon_db.wdl test/test_task/test_init_talon_db_input.json $TAG docker - python3 src/compare_md5.py \ - --keys_to_inspect test_init_talon_db.init_talon_db.database test_init_talon_db.init_talon_db.talon_inputs \ - --metadata_json test_init_talon_db_input.metadata.json \ - --reference_json test/test_task/test_init_talon_db_reference_md5.json \ - --outfile test_init_talon_db_input.result.json - cat test_init_talon_db_input.result.json - python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < test_init_talon_db_input.result.json - no_output_timeout: 30m - - test_init_talon_database_singularity: - <<: *machine_defaults - steps: - - checkout - - run: *make_tag - - run: sudo apt-get update - - singularity/install-go - - singularity/debian-install-3 - - run: - command: | - pyenv global 3.5.2 - source ${BASH_ENV} - test/test.sh test/test_task/test_init_talon_db.wdl test/test_task/test_init_talon_db_input.json $TAG singularity + test/caper_run.sh test/test_task/test_init_talon_db.wdl test/test_task/test_init_talon_db_input.json python3 src/compare_md5.py \ --keys_to_inspect test_init_talon_db.init_talon_db.database test_init_talon_db.init_talon_db.talon_inputs \ - --metadata_json test_init_talon_db_input.metadata.json \ + --metadata_json metadata.json \ --reference_json test/test_task/test_init_talon_db_reference_md5.json \ --outfile test_init_talon_db_input.result.json cat test_init_talon_db_input.result.json @@ -281,37 +192,13 @@ jobs: <<: *machine_defaults steps: - checkout - - run: *make_tag + - prepare_pytest_environment - run: command: | - pyenv global 3.5.2 - source ${BASH_ENV} - test/test.sh test/test_task/test_init_talon_db.wdl test/test_task/test_init_talon_db_idxprefix_input.json $TAG docker - python3 src/compare_md5.py \ - --keys_to_inspect test_init_talon_db.init_talon_db.database test_init_talon_db.init_talon_db.talon_inputs \ - --metadata_json test_init_talon_db_idxprefix_input.metadata.json \ - --reference_json test/test_task/test_init_talon_db_idxprefix_reference_md5.json \ - --outfile test_init_talon_db_idxprefix_input.result.json - cat test_init_talon_db_idxprefix_input.result.json - python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < test_init_talon_db_idxprefix_input.result.json - no_output_timeout: 30m - - test_init_talon_database_idxprefix_singularity: - <<: *machine_defaults - steps: - - checkout - - run: *make_tag - - run: sudo apt-get update - - singularity/install-go - - singularity/debian-install-3 - - run: - command: | - pyenv global 3.5.2 - source ${BASH_ENV} - test/test.sh test/test_task/test_init_talon_db.wdl test/test_task/test_init_talon_db_idxprefix_input.json $TAG singularity + test/caper_run.sh test/test_task/test_init_talon_db.wdl test/test_task/test_init_talon_db_idxprefix_input.json python3 src/compare_md5.py \ --keys_to_inspect test_init_talon_db.init_talon_db.database test_init_talon_db.init_talon_db.talon_inputs \ - --metadata_json test_init_talon_db_idxprefix_input.metadata.json \ + --metadata_json metadata.json \ --reference_json test/test_task/test_init_talon_db_idxprefix_reference_md5.json \ --outfile test_init_talon_db_idxprefix_input.result.json cat test_init_talon_db_idxprefix_input.result.json @@ -322,37 +209,13 @@ jobs: <<: *machine_defaults steps: - checkout - - run: *make_tag - - run: - command: | - pyenv global 3.5.2 - source ${BASH_ENV} - test/test.sh test/test_task/test_talon.wdl test/test_task/test_talon_input.json $TAG docker - python3 src/compare_md5.py \ - --keys_to_inspect test_talon.talon.talon_log \ - --metadata_json test_talon_input.metadata.json \ - --reference_json test/test_task/test_talon_reference_md5.json \ - --outfile test_talon_input.result.json - cat test_talon_input.result.json - python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < test_talon_input.result.json - no_output_timeout: 30m - - test_talon_singularity: - <<: *machine_defaults - steps: - - checkout - - run: *make_tag - - run: sudo apt-get update - - singularity/install-go - - singularity/debian-install-3 + - prepare_pytest_environment - run: command: | - pyenv global 3.5.2 - source ${BASH_ENV} - test/test.sh test/test_task/test_talon.wdl test/test_task/test_talon_input.json $TAG singularity + test/caper_run.sh test/test_task/test_talon.wdl test/test_task/test_talon_input.json python3 src/compare_md5.py \ --keys_to_inspect test_talon.talon.talon_log \ - --metadata_json test_talon_input.metadata.json \ + --metadata_json metadata.json \ --reference_json test/test_task/test_talon_reference_md5.json \ --outfile test_talon_input.result.json cat test_talon_input.result.json @@ -363,39 +226,14 @@ jobs: <<: *machine_defaults steps: - checkout - - run: *make_tag + - prepare_pytest_environment - run: command: | - pyenv global 3.5.2 - source ${BASH_ENV} - test/test.sh test/test_task/test_create_abundance_from_talon_db.wdl test/test_task/test_create_abundance_from_talon_db_input.json $TAG docker + test/caper_run.sh test/test_task/test_create_abundance_from_talon_db.wdl test/test_task/test_create_abundance_from_talon_db_input.json python3 src/compare_md5.py \ --keys_to_inspect test_create_abundance_from_talon_db.create_abundance_from_talon_db.talon_abundance \ test_create_abundance_from_talon_db.create_abundance_from_talon_db.number_of_genes_detected \ - --metadata_json test_create_abundance_from_talon_db_input.metadata.json \ - --reference_json test/test_task/test_create_abundance_from_talon_db_reference_md5.json \ - --outfile test_create_abundance_from_talon_db_input.result.json - cat test_create_abundance_from_talon_db_input.result.json - python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < test_create_abundance_from_talon_db_input.result.json - no_output_timeout: 30m - - test_create_abundance_from_talon_db_singularity: - <<: *machine_defaults - steps: - - checkout - - run: *make_tag - - run: sudo apt-get update - - singularity/install-go - - singularity/debian-install-3 - - run: - command: | - pyenv global 3.5.2 - source ${BASH_ENV} - test/test.sh test/test_task/test_create_abundance_from_talon_db.wdl test/test_task/test_create_abundance_from_talon_db_input.json $TAG singularity - python3 src/compare_md5.py \ - --keys_to_inspect test_create_abundance_from_talon_db.create_abundance_from_talon_db.talon_abundance \ - test_create_abundance_from_talon_db.create_abundance_from_talon_db.number_of_genes_detected \ - --metadata_json test_create_abundance_from_talon_db_input.metadata.json \ + --metadata_json metadata.json \ --reference_json test/test_task/test_create_abundance_from_talon_db_reference_md5.json \ --outfile test_create_abundance_from_talon_db_input.result.json cat test_create_abundance_from_talon_db_input.result.json @@ -406,37 +244,13 @@ jobs: <<: *machine_defaults steps: - checkout - - run: *make_tag - - run: - command: | - pyenv global 3.5.2 - source ${BASH_ENV} - test/test.sh test/test_task/test_create_gtf_from_talon_db.wdl test/test_task/test_create_gtf_from_talon_db_input.json $TAG docker - python3 src/compare_md5.py \ - --keys_to_inspect test_create_gtf_from_talon_db.create_gtf_from_talon_db.gtf \ - --metadata_json test_create_gtf_from_talon_db_input.metadata.json \ - --reference_json test/test_task/test_create_gtf_from_talon_db_reference_md5.json \ - --outfile test_create_gtf_from_talon_db_input.result.json - cat test_create_gtf_from_talon_db_input.result.json - python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < test_create_gtf_from_talon_db_input.result.json - no_output_timeout: 30m - - test_create_gtf_from_talon_db_singularity: - <<: *machine_defaults - steps: - - checkout - - run: *make_tag - - run: sudo apt-get update - - singularity/install-go - - singularity/debian-install-3 + - prepare_pytest_environment - run: command: | - pyenv global 3.5.2 - source ${BASH_ENV} - test/test.sh test/test_task/test_create_gtf_from_talon_db.wdl test/test_task/test_create_gtf_from_talon_db_input.json $TAG singularity + test/caper_run.sh test/test_task/test_create_gtf_from_talon_db.wdl test/test_task/test_create_gtf_from_talon_db_input.json python3 src/compare_md5.py \ --keys_to_inspect test_create_gtf_from_talon_db.create_gtf_from_talon_db.gtf \ - --metadata_json test_create_gtf_from_talon_db_input.metadata.json \ + --metadata_json metadata.json \ --reference_json test/test_task/test_create_gtf_from_talon_db_reference_md5.json \ --outfile test_create_gtf_from_talon_db_input.result.json cat test_create_gtf_from_talon_db_input.result.json @@ -447,37 +261,13 @@ jobs: <<: *machine_defaults steps: - checkout - - run: *make_tag + - prepare_pytest_environment - run: command: | - pyenv global 3.5.2 - source ${BASH_ENV} - test/test.sh test/test_task/test_calculate_spearman.wdl test/test_task/test_calculate_spearman_input.json $TAG docker + test/caper_run.sh test/test_task/test_calculate_spearman.wdl test/test_task/test_calculate_spearman_input.json python3 src/compare_md5.py \ --keys_to_inspect test_calculate_spearman.calculate_spearman.spearman \ - --metadata_json test_calculate_spearman_input.metadata.json \ - --reference_json test/test_task/test_calculate_spearman_reference_md5.json \ - --outfile test_calculate_spearman_input.result.json - cat test_calculate_spearman_input.result.json - python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < test_calculate_spearman_input.result.json - no_output_timeout: 30m - - test_calculate_spearman_singularity: - <<: *machine_defaults - steps: - - checkout - - run: *make_tag - - run: sudo apt-get update - - singularity/install-go - - singularity/debian-install-3 - - run: - command: | - pyenv global 3.5.2 - source ${BASH_ENV} - test/test.sh test/test_task/test_calculate_spearman.wdl test/test_task/test_calculate_spearman_input.json $TAG singularity - python3 src/compare_md5.py \ - --keys_to_inspect test_calculate_spearman.calculate_spearman.spearman \ - --metadata_json test_calculate_spearman_input.metadata.json \ + --metadata_json metadata.json \ --reference_json test/test_task/test_calculate_spearman_reference_md5.json \ --outfile test_calculate_spearman_input.result.json cat test_calculate_spearman_input.result.json @@ -488,12 +278,10 @@ jobs: <<: *machine_defaults steps: - checkout - - run: *make_tag + - prepare_pytest_environment - run: command: | - pyenv global 3.5.2 - source ${BASH_ENV} - test/test.sh long-read-rna-pipeline.wdl test/test_workflow/test_workflow_input.json $TAG docker + test/caper_run.sh long-read-rna-pipeline.wdl test/test_workflow/test_workflow_input.json python3 src/compare_md5.py \ --keys_to_inspect long_read_rna_pipeline.minimap2.mapping_qc \ long_read_rna_pipeline.transcriptclean.transcript_log \ @@ -503,36 +291,7 @@ jobs: long_read_rna_pipeline.create_abundance_from_talon_db.number_of_genes_detected \ long_read_rna_pipeline.create_abundance_from_talon_db.talon_abundance \ long_read_rna_pipeline.create_gtf_from_talon_db.gtf \ - --metadata_json test_workflow_input.metadata.json \ - --reference_json test/test_workflow/test_workflow_reference_md5.json \ - --outfile test_workflow_input.result.json - cat test_workflow_input.result.json - python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < test_workflow_input.result.json - no_output_timeout: 30m - - test_workflow_onerep_singularity: - <<: *machine_defaults - steps: - - checkout - - run: *make_tag - - run: sudo apt-get update - - singularity/install-go - - singularity/debian-install-3 - - run: - command: | - pyenv global 3.5.2 - source ${BASH_ENV} - test/test.sh long-read-rna-pipeline.wdl test/test_workflow/test_workflow_input.json $TAG singularity - python3 src/compare_md5.py \ - --keys_to_inspect long_read_rna_pipeline.minimap2.mapping_qc \ - long_read_rna_pipeline.transcriptclean.transcript_log \ - long_read_rna_pipeline.transcriptclean.transcript_error_log \ - long_read_rna_pipeline.transcriptclean.corrected_fasta \ - long_read_rna_pipeline.talon.talon_log \ - long_read_rna_pipeline.create_abundance_from_talon_db.number_of_genes_detected \ - long_read_rna_pipeline.create_abundance_from_talon_db.talon_abundance \ - long_read_rna_pipeline.create_gtf_from_talon_db.gtf \ - --metadata_json test_workflow_input.metadata.json \ + --metadata_json metadata.json \ --reference_json test/test_workflow/test_workflow_reference_md5.json \ --outfile test_workflow_input.result.json cat test_workflow_input.result.json @@ -543,12 +302,10 @@ jobs: <<: *machine_defaults steps: - checkout - - run: *make_tag + - prepare_pytest_environment - run: command: | - pyenv global 3.5.2 - source ${BASH_ENV} - test/test.sh long-read-rna-pipeline.wdl test/test_workflow/test_workflow_2reps_input.json $TAG docker + test/caper_run.sh long-read-rna-pipeline.wdl test/test_workflow/test_workflow_2reps_input.json python3 src/compare_md5.py \ --keys_to_inspect long_read_rna_pipeline.minimap2.mapping_qc \ long_read_rna_pipeline.transcriptclean.transcript_log \ @@ -559,37 +316,7 @@ jobs: long_read_rna_pipeline.create_abundance_from_talon_db.talon_abundance \ long_read_rna_pipeline.create_gtf_from_talon_db.gtf \ long_read_rna_pipeline.calculate_spearman.spearman \ - --metadata_json test_workflow_2reps_input.metadata.json \ - --reference_json test/test_workflow/test_workflow_2reps_reference_md5.json \ - --outfile test_workflow_2reps_input.result.json - cat test_workflow_2reps_input.result.json - python3 -c "import sys; import json; data=json.loads(sys.stdin.read()); sys.exit(int(not data['match_overall']))" < test_workflow_2reps_input.result.json - no_output_timeout: 30m - - test_workflow_tworep_singularity: - <<: *machine_defaults - steps: - - checkout - - run: *make_tag - - run: sudo apt-get update - - singularity/install-go - - singularity/debian-install-3 - - run: - command: | - pyenv global 3.5.2 - source ${BASH_ENV} - test/test.sh long-read-rna-pipeline.wdl test/test_workflow/test_workflow_2reps_input.json $TAG singularity - python3 src/compare_md5.py \ - --keys_to_inspect long_read_rna_pipeline.minimap2.mapping_qc \ - long_read_rna_pipeline.transcriptclean.transcript_log \ - long_read_rna_pipeline.transcriptclean.transcript_error_log \ - long_read_rna_pipeline.transcriptclean.corrected_fasta \ - long_read_rna_pipeline.talon.talon_log \ - long_read_rna_pipeline.create_abundance_from_talon_db.number_of_genes_detected \ - long_read_rna_pipeline.create_abundance_from_talon_db.talon_abundance \ - long_read_rna_pipeline.create_gtf_from_talon_db.gtf \ - long_read_rna_pipeline.calculate_spearman.spearman \ - --metadata_json test_workflow_2reps_input.metadata.json \ + --metadata_json metadata.json \ --reference_json test/test_workflow/test_workflow_2reps_reference_md5.json \ --outfile test_workflow_2reps_input.result.json cat test_workflow_2reps_input.result.json @@ -604,92 +331,45 @@ workflows: - test_get_splice_junctions_docker: requires: - build - - test_get_splice_junctions_singularity: - requires: - - build - test_minimap2_docker: requires: - build - - test_minimap2_singularity: - requires: - - build - test_transcriptclean_docker: requires: - build - - test_transcriptclean_singularity: - requires: - - build - test_init_talon_database_docker: requires: - build - - test_init_talon_database_singularity: - requires: - - build - test_init_talon_database_idxprefix_docker: requires: - build - - test_init_talon_database_idxprefix_singularity: - requires: - - build - test_talon_docker: requires: - build - - test_talon_singularity: - requires: - - build - test_create_abundance_from_talon_db_docker: requires: - build - - test_create_abundance_from_talon_db_singularity: - requires: - - build - test_create_gtf_from_talon_db_docker: requires: - build - - test_create_gtf_from_talon_db_singularity: - requires: - - build - test_calculate_spearman_docker: requires: - build - - test_calculate_spearman_singularity: - requires: - - build - test_workflow_onerep_docker: -# <<: *only_dev_and_master - requires: - - build - - test_workflow_onerep_singularity: - <<: *only_dev_and_master requires: - build - test_workflow_tworep_docker: -# <<: *only_dev_and_master - requires: - - build - - test_workflow_tworep_singularity: - <<: *only_dev_and_master requires: - build - push_template: requires: - test_get_splice_junctions_docker - - test_get_splice_junctions_singularity - test_minimap2_docker - - test_minimap2_singularity - test_transcriptclean_docker - - test_transcriptclean_singularity - test_init_talon_database_docker - - test_init_talon_database_singularity - test_talon_docker - - test_talon_singularity - test_create_abundance_from_talon_db_docker - - test_create_abundance_from_talon_db_singularity - test_create_gtf_from_talon_db_docker - - test_create_gtf_from_talon_db_singularity - test_calculate_spearman_docker - - test_calculate_spearman_singularity - test_workflow_onerep_docker - - test_workflow_onerep_singularity - test_workflow_tworep_docker - - test_workflow_tworep_singularity \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9fb59ef --- /dev/null +++ b/requirements.txt @@ -0,0 +1,64 @@ +aadict==0.2.3 +argcomplete==1.11.1 +asset==0.6.13 +attrs==19.3.0 +autouri==0.1.4 +awscli==1.18.72 +boto3==1.13.22 +botocore==1.16.22 +cachetools==4.1.0 +caper==0.8.2.1 +certifi==2020.4.5.1 +cffi==1.14.0 +chardet==3.0.4 +colorama==0.4.3 +coloredlogs==11.3 +cryptography==2.9.2 +dateparser==0.7.4 +docker==4.2.1 +docutils==0.15.2 +filelock==3.0.12 +globre==0.1.5 +google-api-core==1.17.0 +google-auth==1.16.0 +google-cloud-core==1.3.0 +google-cloud-storage==1.28.1 +google-resumable-media==0.5.1 +googleapis-common-protos==1.52.0 +humanfriendly==8.2 +idna==2.9 +importlib-metadata==1.6.0 +jmespath==0.10.0 +jsonschema==3.2.0 +lark-parser==0.8.5 +miniwdl==0.7.4 +more-itertools==8.3.0 +packaging==20.4 +pluggy==0.13.1 +protobuf==3.12.2 +py==1.8.1 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +pycparser==2.20 +pygtail==0.11.1 +pyhocon==0.3.55 +pyOpenSSL==19.1.0 +pyparsing==2.4.7 +pyrsistent==0.16.0 +pytest==5.4.3 +pytest-workflow @ git+https://github.com/LUMC/pytest-workflow.git@4cbc3bb6b9840dafdfc584d62c1e7f9503f6291b +python-dateutil==2.8.1 +python-json-logger==0.1.11 +pytz==2020.1 +PyYAML==5.3.1 +regex==2020.5.14 +requests==2.23.0 +rsa==3.4.2 +s3transfer==0.3.3 +six==1.15.0 +tzlocal==2.1 +urllib3==1.25.9 +wcwidth==0.2.3 +websocket-client==0.57.0 +xdg==4.0.1 +zipp==3.1.0 diff --git a/src/test/__init__.py b/src/test/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/test/test_dataframe_utils.py b/src/test/test_dataframe_utils.py deleted file mode 100644 index 1028e33..0000000 --- a/src/test/test_dataframe_utils.py +++ /dev/null @@ -1,34 +0,0 @@ -from ..dataframe_utils import ( - remove_genomic_transcripts, - filter_startswith_prefix, - calculate_abundances_aggregated_by_gene, -) -from pandas import DataFrame - -DF = DataFrame( - { - "transcript_novelty": ["Genomic", "foo", "bar", "Genomic"], - "annot_gene_id": ["FOOid1", "BARid2", "FOOid1", "BAZid4"], - "numeric_values": [1, 2, 3, 4], - } -) - - -def test_remove_genomic_transcripts(): - assert len(remove_genomic_transcripts(DF).index) == 2 - - -def test_filter_startswith_prefix_nomatch(): - assert filter_startswith_prefix(DF, "not_matching_prefix").equals(DF) - - -def test_filter_startswith_prefix_match(): - filtered = filter_startswith_prefix(DF, "FOO") - assert len(filtered.index) == 2 - - -def test_calculate_abundances_aggregated_by_gene(): - aggregated = calculate_abundances_aggregated_by_gene(DF, "numeric_values") - assert aggregated.FOOid1 == 4 - assert aggregated.BARid2 == 2 - assert aggregated.BAZid4 == 4 diff --git a/test/caper_run.sh b/test/caper_run.sh new file mode 100755 index 0000000..d9aae33 --- /dev/null +++ b/test/caper_run.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e + +echo "Running cromwell $CROMWELL, womtool $WOMTOOL and image $TAG" + +if [[ $# -eq 1 ]]; then + caper run $1 --docker $TAG --cromwell $CROMWELL --womtool $WOMTOOL --options ./test/cromwell_options.json -m metadata.json +fi + +if [[ $# -eq 2 ]]; then + caper run $1 --docker $TAG --cromwell $CROMWELL --womtool $WOMTOOL -i $2 --options ./test/cromwell_options.json -m metadata.json +fi diff --git a/test/cromwell_options.json b/test/cromwell_options.json new file mode 100644 index 0000000..d9f9fc6 --- /dev/null +++ b/test/cromwell_options.json @@ -0,0 +1,5 @@ +{ +"default_runtime_attributes": { + "docker_user": "$EUID" + } +} diff --git a/test/test.sh b/test/test.sh deleted file mode 100755 index 5edcfbd..0000000 --- a/test/test.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -set -e # exit on error - -if [ $# -lt 3 ]; then - echo "Usage: ./test.sh [WDL] [INPUT_JSON] [DOCKER_IMAGE]" - echo "Make sure to have cromwell-40.jar in your \$PATH as an executable (chmod +x)." - exit 1 -fi - -WDL=$1 -INPUT=$2 -DOCKER_IMAGE=$3 - -if [ -f "cromwell-40.jar" ]; then - echo "cromwell-40.jar already available, skipping download." -else - wget -N -c https://github.com/broadinstitute/cromwell/releases/download/40/cromwell-40.jar -fi - -CROMWELL_JAR=cromwell-40.jar -BACKEND_CONF=backends/backend.conf -RESULT_PREFIX=$(basename ${INPUT} .json) -METADATA=${RESULT_PREFIX}.metadata.json # metadata -RESULT=${RESULT_PREFIX}.result.json # output - -if [ $4 = "docker" ]; then - # Write workflow option JSON file for docker - BACKEND=Local - TMP_WF_OPT=$RESULT_PREFIX.test_longrna_wf_opt.json - cat > $TMP_WF_OPT << EOM - { - "default_runtime_attributes" : { - "docker" : "$DOCKER_IMAGE" - } - } -EOM -fi - -if [ $4 = "singularity" ]; then - #build singularity container - SINGULARITY_PULLFOLDER=~/.singularity singularity pull docker://${DOCKER_IMAGE} - # Write workflow option JSON file for singularity - BACKEND=singularity - TMP_WF_OPT=$RESULT_PREFIX.test_longrna_wf_opt.json - SINGULARITY_IMAGE=$(echo ${DOCKER_IMAGE} | sed 's/quay\.io\/encode-dcc\///g' | sed 's/:/_/' | sed 's/$/\.sif/') - cat > $TMP_WF_OPT << EOM - { - "default_runtime_attributes" : { - "singularity_container" : "~/long-read-rna-pipeline/$SINGULARITY_IMAGE" - } - } -EOM -fi - -java -Dconfig.file=${BACKEND_CONF} -Dbackend.default=${BACKEND} -jar ${CROMWELL_JAR} run ${WDL} -i ${INPUT} -o ${TMP_WF_OPT} -m ${METADATA} - -rm -f ${TMP_WF_OPT} \ No newline at end of file