diff --git a/constants/constants.py b/constants/constants.py index c84844e7..e517b0b2 100644 --- a/constants/constants.py +++ b/constants/constants.py @@ -113,7 +113,11 @@ class Constants(object): DATA_SET_GENERIC = "Generic" ## default name for a dataset ### NextClade link - NEXTCLADE_LINK = "https://clades.nextstrain.org/?input-fasta=" + NEXTCLADE_LINK_sars_cov_2 = "https://clades.nextstrain.org/?dataset-name=sars-cov-2&input-fasta=" + NEXTCLADE_LINK_A_H3N2 = "https://clades.nextstrain.org/?dataset-name=flu_h3n2_ha&input-fasta=" + NEXTCLADE_LINK_A_H1N1 = "https://clades.nextstrain.org/?dataset-name=flu_h1n1pdm_ha&input-fasta=" + NEXTCLADE_LINK_B_Yamagata = "https://clades.nextstrain.org/?dataset-name=flu_yam_ha&input-fasta=" + NEXTCLADE_LINK_B_Victoria = "https://clades.nextstrain.org/?dataset-name=flu_vic_ha&input-fasta=" AUSPICE_LINK = "https://auspice.us/" ## NUMBER OF SETs to paginate diff --git a/constants/software_names.py b/constants/software_names.py index bd6a1191..11792d5d 100644 --- a/constants/software_names.py +++ b/constants/software_names.py @@ -96,10 +96,10 @@ class SoftwareNames(object): "Number of reads R1", "Number of reads R2", "Average read length R1", "Average read length R2", "STDEV read length R1", "STDEV read length R2", - "Total bases",] + "Total reads",] SOFTWARE_ILLUMINA_stat_collect_show_percentage = [ "Number of reads R1", "Number of reads R2", - "Total bases",] + "Total reads",] SOFTWARE_RabbitQC = os.path.join(settings.DIR_SOFTWARE, "RabbitQC/rabbit_qc") SOFTWARE_RabbitQC_name = "RabbitQC" diff --git a/deprecated_tests/tests_software_single.py b/deprecated_tests/tests_software_single.py index 1f770cc9..cde82521 100644 --- a/deprecated_tests/tests_software_single.py +++ b/deprecated_tests/tests_software_single.py @@ -63,229 +63,19 @@ def setUp(self): def tearDown(self): pass - @override_settings(MEDIA_ROOT=getattr(settings, "MEDIA_ROOT_TEST", None)) - def test_process_second_stage_analysis_single_file(self): + def test_run_clean_minion_2(self): """ - test global method + Test run fastq and trimmomatic all together """ - self.assertEquals(getattr(settings, "MEDIA_ROOT_TEST", None), getattr(settings, "MEDIA_ROOT", None)) - self.utils.remove_dir(getattr(settings, "MEDIA_ROOT_TEST", None)) - self.utils.make_path(getattr(settings, "MEDIA_ROOT_TEST", None)) - - gb_file = os.path.join(self.baseDirectory, ConstantsTestsCase.MANAGING_DIR, ConstantsTestsCase.MANAGING_FILES_GBK) - fasta_file = os.path.join(self.baseDirectory, ConstantsTestsCase.MANAGING_DIR, ConstantsTestsCase.MANAGING_FILES_FASTA) - file_1 = os.path.join(self.baseDirectory, ConstantsTestsCase.DIR_FASTQ, ConstantsTestsCase.FASTQ1_1) - file_2 = os.path.join(self.baseDirectory, ConstantsTestsCase.DIR_FASTQ, ConstantsTestsCase.FASTQ1_2) - - try: - user = User.objects.get(username=ConstantsTestsCase.TEST_USER_NAME) - except User.DoesNotExist: - user = User() - user.username = ConstantsTestsCase.TEST_USER_NAME - user.is_active = False - user.password = ConstantsTestsCase.TEST_USER_NAME - user.save() - - ref_name = "second_stagis_single_file2" - try: - reference = Reference.objects.get(name=ref_name) - except Reference.DoesNotExist: - reference = Reference() - reference.name = ref_name - reference.reference_fasta.name = fasta_file - reference.reference_fasta_name = os.path.basename(fasta_file) - reference.reference_genbank.name = gb_file - reference.reference_genbank_name = os.path.basename(gb_file) - reference.owner = user - reference.save() - - temp_dir = self.utils.get_temp_dir() - self.utils.copy_file(file_1, os.path.join(temp_dir, ConstantsTestsCase.FASTQ1_1)) - self.utils.copy_file(file_2, os.path.join(temp_dir, ConstantsTestsCase.FASTQ1_2)) - - sample_name = "run_snippyis_single_file1" - try: - sample = Sample.objects.get(name=sample_name) - except Sample.DoesNotExist: - sample = Sample() - sample.name = sample_name - sample.is_valid_1 = True - sample.file_name_1 = ConstantsTestsCase.FASTQ1_1 - sample.path_name_1.name = os.path.join(temp_dir, ConstantsTestsCase.FASTQ1_1) - sample.is_valid_2 = False - sample.file_name_2 = ConstantsTestsCase.FASTQ1_2 - sample.path_name_2.name = os.path.join(temp_dir, ConstantsTestsCase.FASTQ1_2) - sample.set_type_of_fastq_sequencing(Constants.FORMAT_FASTQ_illumina) - sample.owner = user - sample.save() - - project_name = "file_naais_single_filee_3" - try: - project = Project.objects.get(name=project_name) - except Project.DoesNotExist: - project = Project() - project.name = project_name - project.reference = reference - project.owner = user - project.save() - - ## create project_sample - project_sample = ProjectSample() - project_sample.sample = sample - project_sample.project = project - project_sample.save() - - ### copy to trimmomatic files - self.utils.copy_file(file_1, project_sample.sample.get_trimmomatic_file(TypePath.MEDIA_ROOT, True)) - self.utils.copy_file(file_2, project_sample.sample.get_trimmomatic_file(TypePath.MEDIA_ROOT, False)) - - manageDatabase = ManageDatabase() - metaKeyAndValue = MetaKeyAndValue() - meta_key_project_sample = metaKeyAndValue.get_meta_key_queue_by_project_sample_id(project_sample.id) - manageDatabase.set_project_sample_metakey(project_sample, user, meta_key_project_sample, MetaKeyAndValue.META_VALUE_Queue, "meta_sample.description") - self.assertTrue(self.software.process_second_stage_snippy_coverage_freebayes(project_sample, user)) - try: - project_sample = ProjectSample.objects.get(pk=project_sample.id) - except ProjectSample.DoesNotExist: - self.fail("Must exist") - self.assertTrue(project_sample.is_finished) - self.assertTrue(project_sample.is_mask_consensus_sequences) - - ### test the files - self.assertTrue(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_CONSENSUS_FASTA, SoftwareNames.SOFTWARE_SNIPPY_name))) - self.assertFalse(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_CONSENSUS_FA, SoftwareNames.SOFTWARE_SNIPPY_name))) - self.assertTrue(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_CONSENSUS_FASTA, SoftwareNames.SOFTWARE_SNIPPY_name).index(SoftwareNames.SOFTWARE_SNIPPY_name.lower()) != -1) - - self.assertTrue(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_BAM, SoftwareNames.SOFTWARE_SNIPPY_name))) - self.assertTrue(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_BAM_BAI, SoftwareNames.SOFTWARE_SNIPPY_name))) - self.assertFalse(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_DEPTH, SoftwareNames.SOFTWARE_SNIPPY_name))) - self.assertTrue(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_DEPTH_GZ, SoftwareNames.SOFTWARE_SNIPPY_name))) - self.assertTrue(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_DEPTH_GZ_TBI, SoftwareNames.SOFTWARE_SNIPPY_name))) - self.assertTrue(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_VCF, SoftwareNames.SOFTWARE_SNIPPY_name))) - self.assertTrue(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_VCF_GZ, SoftwareNames.SOFTWARE_SNIPPY_name))) - self.assertTrue(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_CSV, SoftwareNames.SOFTWARE_SNIPPY_name))) - self.assertTrue(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_TAB, SoftwareNames.SOFTWARE_SNIPPY_name))) - self.assertTrue(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_REF_FASTA, SoftwareNames.SOFTWARE_SNIPPY_name))) - self.assertTrue(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_REF_FASTA, SoftwareNames.SOFTWARE_SNIPPY_name))) - self.assertTrue(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_REF_FASTA_FAI, SoftwareNames.SOFTWARE_SNIPPY_name))) - - expected_file = os.path.join(self.baseDirectory, ConstantsTestsCase.MANAGING_DIR, "run_snippyis_single_file1.tab") - self.assertTrue(filecmp.cmp(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_TAB, SoftwareNames.SOFTWARE_SNIPPY_name), - expected_file)) - self.assertTrue(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_TAB, SoftwareNames.SOFTWARE_SNIPPY_name))) - - ## freebayes - self.assertTrue(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_VCF, SoftwareNames.SOFTWARE_FREEBAYES_name).index(SoftwareNames.SOFTWARE_FREEBAYES_name.lower()) != -1) - self.assertTrue(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_VCF, SoftwareNames.SOFTWARE_FREEBAYES_name))) - self.assertTrue(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_TAB, SoftwareNames.SOFTWARE_FREEBAYES_name))) - expected_file = os.path.join(self.baseDirectory, ConstantsTestsCase.MANAGING_DIR, "run_freebayes_single_file1.tab") - self.assertTrue(filecmp.cmp(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_TAB, SoftwareNames.SOFTWARE_FREEBAYES_name), - expected_file)) - self.assertTrue(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_VCF_GZ, SoftwareNames.SOFTWARE_FREEBAYES_name))) - - ## test freebayes clean - self.assertTrue(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_TAB, SoftwareNames.SOFTWARE_FREEBAYES_name))) - self.assertTrue(os.path.getsize(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_TAB, SoftwareNames.SOFTWARE_FREEBAYES_name)) > 10) - - ## test consensus file - self.assertTrue(os.path.exists(project_sample.get_consensus_file(TypePath.MEDIA_ROOT))) - self.assertTrue(os.path.getsize(project_sample.get_consensus_file(TypePath.MEDIA_ROOT)) > 10) - self.assertTrue(filecmp.cmp(project_sample.get_consensus_file(TypePath.MEDIA_ROOT), - project_sample.get_backup_consensus_file())) - - ### human file name, snippy tab - self.assertTrue(os.path.exists(project_sample.get_file_output_human(TypePath.MEDIA_ROOT, FileType.FILE_TAB, self.software_names.get_snippy_name()))) - self.assertTrue(os.path.getsize(project_sample.get_file_output_human(TypePath.MEDIA_ROOT, FileType.FILE_TAB, self.software_names.get_snippy_name())) > 10) - - ### set flag that is finished - list_meta = manageDatabase.get_project_sample_metakey(project_sample, MetaKeyAndValue.META_KEY_Count_Hits, None) - self.assertTrue(len(list_meta) == 1) - self.assertEquals(MetaKeyAndValue.META_VALUE_Success, list_meta[0].value) - self.assertEquals(MetaKeyAndValue.META_KEY_Count_Hits, list_meta[0].meta_tag.name) - - ### get the hits value - decode_coverage = DecodeObjects() - count_hits = decode_coverage.decode_result(list_meta[0].description) - self.assertEquals(0, count_hits.get_hits_50_90()) - self.assertEquals(3, count_hits.get_hits_less_50()) - self.assertEquals(3, count_hits.get_total_50_50_90()) - self.assertEquals(125, count_hits.get_total()) - - self.assertEquals(3, project_sample.count_variations.var_less_50) - self.assertEquals(0, project_sample.count_variations.var_bigger_50_90) - self.assertEquals(122, project_sample.count_variations.var_bigger_90) - self.assertEquals(0, project_sample.alert_first_level) - self.assertEquals(0, project_sample.alert_second_level) - - ### test mixed infections - self.assertEquals('0.815100969586423', '{}'.format(project_sample.mixed_infections.average_value)) - self.assertEquals('No', project_sample.mixed_infections.tag.name) - self.assertFalse(project_sample.mixed_infections.has_master_vector) - - list_meta = manageDatabase.get_project_sample_metakey(project_sample, MetaKeyAndValue.META_KEY_Snippy_Freebayes, None) - self.assertEquals(1, len(list_meta)) - self.assertEquals(MetaKeyAndValue.META_VALUE_Success, list_meta[0].value) - self.assertEquals(MetaKeyAndValue.META_KEY_Snippy_Freebayes, list_meta[0].meta_tag.name) - - decode_result = DecodeObjects() - result = decode_result.decode_result(list_meta[0].description) - self.assertTrue(result is not None) - self.assertEquals("Snippy-3.2-dev; (--mapqual 20 --mincov 10 --minfrac 0.51)", result.get_software(self.software_names.get_snippy_name())) - self.assertEquals("Freebayes-v1.1.0-54-g49413aa; (--min-mapping-quality 20 --min-base-quality 20 --min-coverage 100 --min-alternate-count 10 --min-alternate-fraction 0.01 --ploidy 2 -V)",\ - result.get_software(self.software_names.get_freebayes_name())) - self.assertTrue(result.is_software_present(self.software_names.get_freebayes_name())) - self.assertFalse(result.is_software_present(self.software_names.get_trimmomatic_name())) - - meta_value = manageDatabase.get_project_sample_metakey(project_sample, MetaKeyAndValue.META_KEY_Coverage, MetaKeyAndValue.META_VALUE_Success) - decode_coverage = DecodeObjects() - coverage = decode_coverage.decode_result(meta_value.description) - self.assertEqual(coverage.get_coverage('PA', Coverage.COVERAGE_ALL), "527.4") - self.assertEqual(coverage.get_coverage('MP', Coverage.COVERAGE_ALL), "2198.8") - self.assertEqual(coverage.get_coverage('HA', Coverage.COVERAGE_ALL), "1449.3") - self.assertEqual(coverage.get_coverage('PA', Coverage.COVERAGE_MORE_9), "100.0") - self.assertEqual(coverage.get_coverage('MP', Coverage.COVERAGE_MORE_9), "100.0") - self.assertEqual(coverage.get_coverage('HA', Coverage.COVERAGE_MORE_9), "100.0") - self.assertEqual(coverage.get_coverage('PA', Coverage.COVERAGE_MORE_0), "100.0") - self.assertEqual(coverage.get_coverage('MP', Coverage.COVERAGE_MORE_0), "100.0") - self.assertEqual(coverage.get_coverage('HA', Coverage.COVERAGE_MORE_0), "100.0") - - lst_meta_sample = manageDatabase.get_project_sample_metakey(project_sample, meta_key_project_sample, None) - self.assertEquals(2, len(lst_meta_sample)) - self.assertEquals(MetaKeyAndValue.META_VALUE_Queue, lst_meta_sample[1].value) - self.assertEquals(MetaKeyAndValue.META_VALUE_Success, lst_meta_sample[0].value) - - ### check if the images exist - for gene in self.utils.get_elements_from_db(reference, user): - output_image = project_sample.get_global_file_by_element(TypePath.MEDIA_ROOT, ProjectSample.PREFIX_FILE_COVERAGE, gene, FileExtensions.FILE_PNG) - self.assertTrue(os.path.exists(output_image)) - - meta_value = manageDatabase.get_project_sample_metakey_last(project_sample, - MetaKeyAndValue.META_KEY_bam_stats, - MetaKeyAndValue.META_VALUE_Success) - self.assertTrue(not meta_value is None) - result = decode_coverage.decode_result(meta_value.description) - self.assertEqual('87818', result.get_value_by_key(MetaKeyAndValue.SAMTOOLS_flagstat_mapped_reads)) - self.assertEqual('87818', result.get_value_by_key(MetaKeyAndValue.SAMTOOLS_flagstat_total_reads)) - self.assertEqual(None, result.get_value_by_key('xpto')) - - self.utils.remove_dir(temp_dir) - self.utils.remove_dir(getattr(settings, "MEDIA_ROOT", None)) - - - @override_settings(MEDIA_ROOT=getattr(settings, "MEDIA_ROOT_TEST", None)) - def test_process_second_stage_analysis_single_file_no_freebayes(self): - """ - test global method - """ - self.assertEquals(getattr(settings, "MEDIA_ROOT_TEST", None), getattr(settings, "MEDIA_ROOT", None)) - self.utils.remove_dir(getattr(settings, "MEDIA_ROOT_TEST", None)) - self.utils.make_path(getattr(settings, "MEDIA_ROOT_TEST", None)) - - gb_file = os.path.join(self.baseDirectory, ConstantsTestsCase.MANAGING_DIR, ConstantsTestsCase.MANAGING_FILES_GBK) - fasta_file = os.path.join(self.baseDirectory, ConstantsTestsCase.MANAGING_DIR, ConstantsTestsCase.MANAGING_FILES_FASTA) - file_1 = os.path.join(self.baseDirectory, ConstantsTestsCase.DIR_FASTQ, ConstantsTestsCase.FASTQ1_1) - file_2 = os.path.join(self.baseDirectory, ConstantsTestsCase.DIR_FASTQ, ConstantsTestsCase.FASTQ1_2) - + uploadFiles = UploadFiles() + to_test = True + (version, file) = uploadFiles.get_file_to_upload(to_test) + self.assertEqual("2", version) + self.assertEqual(os.path.join(self.baseDirectory, "db/type_identification/test_db_influenza_typing_v2.fasta"), file) + uploadFiles.upload_file(version, file) ## upload file + + file_name = os.path.join(self.baseDirectory, ConstantsTestsCase.DIR_FASTQ, "covid/test_minion_seq.fastq.gz") + self.assertTrue(file_name) try: user = User.objects.get(username=ConstantsTestsCase.TEST_USER_NAME) except User.DoesNotExist: @@ -295,24 +85,10 @@ def test_process_second_stage_analysis_single_file_no_freebayes(self): user.password = ConstantsTestsCase.TEST_USER_NAME user.save() - ref_name = "second_stagis_single_file2" - try: - reference = Reference.objects.get(name=ref_name) - except Reference.DoesNotExist: - reference = Reference() - reference.name = ref_name - reference.reference_fasta.name = fasta_file - reference.reference_fasta_name = os.path.basename(fasta_file) - reference.reference_genbank.name = gb_file - reference.reference_genbank_name = os.path.basename(gb_file) - reference.owner = user - reference.save() - temp_dir = self.utils.get_temp_dir() - self.utils.copy_file(file_1, os.path.join(temp_dir, ConstantsTestsCase.FASTQ1_1)) - self.utils.copy_file(file_2, os.path.join(temp_dir, ConstantsTestsCase.FASTQ1_2)) + self.utils.copy_file(file_name, os.path.join(temp_dir, ConstantsTestsCase.FASTQ1_1)) - sample_name = "run_snippyis_single_file1" + sample_name = "run_nanofilt" try: sample = Sample.objects.get(name=sample_name) except Sample.DoesNotExist: @@ -322,199 +98,163 @@ def test_process_second_stage_analysis_single_file_no_freebayes(self): sample.file_name_1 = ConstantsTestsCase.FASTQ1_1 sample.path_name_1.name = os.path.join(temp_dir, ConstantsTestsCase.FASTQ1_1) sample.is_valid_2 = False - sample.file_name_2 = ConstantsTestsCase.FASTQ1_2 - sample.path_name_2.name = os.path.join(temp_dir, ConstantsTestsCase.FASTQ1_2) - sample.set_type_of_fastq_sequencing(Constants.FORMAT_FASTQ_illumina) + sample.type_of_fastq = Sample.TYPE_OF_FASTQ_minion sample.owner = user sample.save() - - project_name = "file_naais_single_filee_3" - try: - project = Project.objects.get(name=project_name) - except Project.DoesNotExist: - project = Project() - project.name = project_name - project.reference = reference - project.owner = user - project.save() - - ## create project_sample - project_sample = ProjectSample() - project_sample.sample = sample - project_sample.project = project - project_sample.save() - ### copy to trimmomatic files, don't copy trimmommatic files to project - ##self.utils.copy_file(file_1, project_sample.sample.get_trimmomatic_file(TypePath.MEDIA_ROOT, True)) - ##self.utils.copy_file(file_2, project_sample.sample.get_trimmomatic_file(TypePath.MEDIA_ROOT, False)) - - ### turn off freebayes + ### turn off abricate default_software = DefaultSoftware() default_software.test_all_defaults(user) - self.assertTrue(default_software.is_software_to_run(SoftwareNames.SOFTWARE_FREEBAYES_name, - user, ConstantsSettings.TECHNOLOGY_illumina)) - ### turn off freebayes + self.assertTrue(default_software.is_software_to_run(SoftwareNames.SOFTWARE_ABRICATE_name, + user, ConstantsSettings.TECHNOLOGY_minion)) is_to_run = False - default_software.set_software_to_run(SoftwareNames.SOFTWARE_FREEBAYES_name, user, - ConstantsSettings.TECHNOLOGY_illumina, is_to_run) - self.assertFalse(default_software.is_software_to_run(SoftwareNames.SOFTWARE_FREEBAYES_name, - user, ConstantsSettings.TECHNOLOGY_illumina)) - ### set mincov 30 for snippy + default_software.set_software_to_run(SoftwareNames.SOFTWARE_ABRICATE_name, user, + ConstantsSettings.TECHNOLOGY_minion, is_to_run) + self.assertFalse(default_software.is_software_to_run(SoftwareNames.SOFTWARE_ABRICATE_name, + user, ConstantsSettings.TECHNOLOGY_minion)) + + b_make_identify_species = False + self.assertTrue(self.software_minion.run_clean_minion(sample, user, b_make_identify_species)) + default_project_software = DefaultProjectSoftware() + self.assertFalse(default_project_software.is_to_run_abricate(user, sample, + ConstantsSettings.TECHNOLOGY_minion)) + try: - software = SoftwareSettings.objects.get(name=SoftwareNames.SOFTWARE_SNIPPY_name, owner=user, - type_of_use = SoftwareSettings.TYPE_OF_USE_global, - technology__name = ConstantsSettings.TECHNOLOGY_illumina) - self.assertTrue(software.is_used_in_global()) + software = SoftwareSettings.objects.get(name=SoftwareNames.SOFTWARE_NanoFilt_name, owner=user,\ + type_of_use = SoftwareSettings.TYPE_OF_USE_sample, + technology__name = ConstantsSettings.TECHNOLOGY_minion, + is_obsolete = False) except SoftwareSettings.DoesNotExist: - self.fail("Must exist this software name") + self.fail("Must must exist") parameters = Parameter.objects.filter(software=software) - self.assertTrue(3, len(parameters)) + self.assertTrue(5, len(parameters)) + for parameter in parameters: + if (parameter.name == "-q"): + parameter.parameter = "5" + parameter.save() + break + + tag_name_mixed= "No" + sample = Sample.objects.get(name=sample_name) + if (b_make_identify_species): + self.fail("Must have b_make_identify_species FALSE") + self.assertEqual(tag_name_mixed, sample.mixed_infections_tag.name) + self.assertEqual("A-H5N5", sample.type_subtype) + + sample.mixed_infections_tag = None + sample.save() + + sample = Sample.objects.get(name=sample_name) + self.assertEqual(None, sample.mixed_infections_tag) + + try: + mixed_infections_tag = MixedInfectionsTag.objects.get(name=tag_name_mixed) + except MixedInfectionsTag.DoesNotExist as e: + self.fail("Must have tag mixed") + + if (not sample.mixed_infections_tag is None): sample.mixed_infections_tag.delete() + else: + self.assertEqual(Constants.EMPTY_VALUE_NA, sample.mixed_infections_tag.name) + self.assertEqual(Constants.EMPTY_VALUE_NA, sample.type_subtype) + + self.assertEqual(0, sample.number_alerts) + + ### run again + default_project_software.set_abricate_to_run(user, sample, + ConstantsSettings.TECHNOLOGY_minion, True) + ## global + self.assertFalse(default_software.is_software_to_run(SoftwareNames.SOFTWARE_ABRICATE_name, + user, ConstantsSettings.TECHNOLOGY_minion)) + ### project sample + self.assertTrue(default_project_software.is_to_run_abricate(user, + sample, ConstantsSettings.TECHNOLOGY_minion)) + self.assertTrue(self.software_minion.run_clean_minion(sample, user, + default_project_software.is_to_run_abricate(user, + sample, ConstantsSettings.TECHNOLOGY_minion) )) - ### test set default - self.assertEqual("10", parameters[1].parameter) - parameter = parameters[1] - parameter.parameter = 5 - parameter.save() - self.assertEqual("--mapqual 20 --mincov 5 --minfrac 0.51", default_software.get_snippy_parameters(user)) + parameters = Parameter.objects.filter(software=software) + self.assertTrue(5, len(parameters)) + for parameter in parameters: + if (parameter.name == "-q"): + self.assertEqual("5", parameter.parameter) + break; + + self.assertTrue(os.path.exists(os.path.join(temp_dir, os.path.basename(sample.get_fastq(TypePath.MEDIA_ROOT, True))))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, Constants.DIR_PROCESSED_PROCESSED, os.path.basename(sample.get_nanofilt_file(TypePath.MEDIA_ROOT))))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, os.path.basename(sample.get_rabbitQC_output(TypePath.MEDIA_ROOT))))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, Constants.DIR_PROCESSED_PROCESSED, os.path.basename(sample.get_rabbitQC_nanofilt(TypePath.MEDIA_ROOT))))) - default_software_project = DefaultProjectSoftware() - default_software_project.test_all_defaults(user, project, None, None) ## the user can have defaults yet - default_software_project.test_default_db(SoftwareNames.SOFTWARE_FREEBAYES_name, user, - SoftwareSettings.TYPE_OF_USE_project_sample, - None, project_sample, None, ConstantsSettings.TECHNOLOGY_illumina) - - ## test if it is necessary to run freebayes - self.assertFalse(default_software_project.is_to_run_freebayes(user, project_sample)) - manageDatabase = ManageDatabase() - metaKeyAndValue = MetaKeyAndValue() - meta_key_project_sample = metaKeyAndValue.get_meta_key_queue_by_project_sample_id(project_sample.id) - manageDatabase.set_project_sample_metakey(project_sample, user, meta_key_project_sample, MetaKeyAndValue.META_VALUE_Queue, "meta_sample.description") - self.assertTrue(self.software.process_second_stage_snippy_coverage_freebayes(project_sample, user)) - ## test if freebays run - default_project_software = DefaultProjectSoftware() - self.assertFalse(default_project_software.is_to_run_freebayes(user, project_sample)) - - try: - project_sample = ProjectSample.objects.get(pk=project_sample.id) - except ProjectSample.DoesNotExist: - self.fail("Must exist") - self.assertTrue(project_sample.is_finished) - self.assertTrue(project_sample.is_mask_consensus_sequences) - - ### test the files - self.assertTrue(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_CONSENSUS_FASTA, SoftwareNames.SOFTWARE_SNIPPY_name))) - self.assertFalse(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_CONSENSUS_FA, SoftwareNames.SOFTWARE_SNIPPY_name))) - self.assertTrue(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_CONSENSUS_FASTA, SoftwareNames.SOFTWARE_SNIPPY_name).index(SoftwareNames.SOFTWARE_SNIPPY_name.lower()) != -1) - - self.assertTrue(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_BAM, SoftwareNames.SOFTWARE_SNIPPY_name))) - self.assertTrue(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_BAM_BAI, SoftwareNames.SOFTWARE_SNIPPY_name))) - self.assertFalse(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_DEPTH, SoftwareNames.SOFTWARE_SNIPPY_name))) - self.assertTrue(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_DEPTH_GZ, SoftwareNames.SOFTWARE_SNIPPY_name))) - self.assertTrue(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_DEPTH_GZ_TBI, SoftwareNames.SOFTWARE_SNIPPY_name))) - self.assertTrue(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_VCF, SoftwareNames.SOFTWARE_SNIPPY_name))) - self.assertTrue(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_VCF_GZ, SoftwareNames.SOFTWARE_SNIPPY_name))) - self.assertTrue(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_CSV, SoftwareNames.SOFTWARE_SNIPPY_name))) - self.assertTrue(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_TAB, SoftwareNames.SOFTWARE_SNIPPY_name))) - self.assertTrue(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_REF_FASTA, SoftwareNames.SOFTWARE_SNIPPY_name))) - self.assertTrue(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_REF_FASTA, SoftwareNames.SOFTWARE_SNIPPY_name))) - self.assertTrue(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_REF_FASTA_FAI, SoftwareNames.SOFTWARE_SNIPPY_name))) - - expected_file = os.path.join(self.baseDirectory, ConstantsTestsCase.MANAGING_DIR, "run_snippyis_single_file1.tab") - self.assertTrue(filecmp.cmp(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_TAB, SoftwareNames.SOFTWARE_SNIPPY_name), - expected_file)) - self.assertTrue(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_TAB, SoftwareNames.SOFTWARE_SNIPPY_name))) - - ## freebayes - self.assertTrue(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_VCF, SoftwareNames.SOFTWARE_FREEBAYES_name).index(SoftwareNames.SOFTWARE_FREEBAYES_name.lower()) != -1) - self.assertFalse(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_VCF, SoftwareNames.SOFTWARE_FREEBAYES_name))) - self.assertFalse(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_TAB, SoftwareNames.SOFTWARE_FREEBAYES_name))) - expected_file = os.path.join(self.baseDirectory, ConstantsTestsCase.MANAGING_DIR, "run_freebayes_single_file1.tab") - self.assertFalse(os.path.exists(project_sample.get_file_output(TypePath.MEDIA_ROOT, FileType.FILE_VCF_GZ, SoftwareNames.SOFTWARE_FREEBAYES_name))) - - ## test consensus file - self.assertTrue(os.path.exists(project_sample.get_consensus_file(TypePath.MEDIA_ROOT))) - self.assertTrue(os.path.getsize(project_sample.get_consensus_file(TypePath.MEDIA_ROOT)) > 10) - self.assertTrue(filecmp.cmp(project_sample.get_consensus_file(TypePath.MEDIA_ROOT), - project_sample.get_backup_consensus_file())) + list_meta = manageDatabase.get_sample_metakey(sample, MetaKeyAndValue.META_KEY_Number_And_Average_Reads, None) + self.assertEqual(2, len(list_meta)) + self.assertEquals(MetaKeyAndValue.META_VALUE_Success, list_meta[0].value) + self.assertEquals(MetaKeyAndValue.META_KEY_Number_And_Average_Reads, list_meta[0].meta_tag.name) + + decodeResultAverageAndNumberReads = DecodeObjects() + result_average = decodeResultAverageAndNumberReads.decode_result(list_meta[0].description) + self.assertEqual('3000', result_average.number_file_1) + self.assertEqual('1158.4', result_average.average_file_1) + self.assertEqual(None, result_average.number_file_2) + self.assertEqual(None, result_average.average_file_2) - ### human file name, snippy tab - self.assertTrue(os.path.exists(project_sample.get_file_output_human(TypePath.MEDIA_ROOT, FileType.FILE_TAB, self.software_names.get_snippy_name()))) - self.assertTrue(os.path.getsize(project_sample.get_file_output_human(TypePath.MEDIA_ROOT, FileType.FILE_TAB, self.software_names.get_snippy_name())) > 10) - - ### set flag that is finished - list_meta = manageDatabase.get_project_sample_metakey(project_sample, MetaKeyAndValue.META_KEY_Count_Hits, None) - self.assertTrue(len(list_meta) == 1) + list_meta = manageDatabase.get_sample_metakey(sample, MetaKeyAndValue.META_KEY_Fastq_Trimmomatic, None) + self.assertTrue(len(list_meta) == 0) + list_meta = manageDatabase.get_sample_metakey(sample, MetaKeyAndValue.META_KEY_NanoStat_NanoFilt, None) + self.assertEqual(2, len(list_meta)) self.assertEquals(MetaKeyAndValue.META_VALUE_Success, list_meta[0].value) - self.assertEquals(MetaKeyAndValue.META_KEY_Count_Hits, list_meta[0].meta_tag.name) + self.assertEquals(MetaKeyAndValue.META_KEY_NanoStat_NanoFilt, list_meta[0].meta_tag.name) + self.assertEquals("Success, NanoStat(1.4.0), NanoFilt(2.6.0)", list_meta[0].description) + self.assertEquals("Success, NanoStat(1.4.0), NanoFilt(2.6.0)", list_meta[1].description) - ### get the hits value - decode_coverage = DecodeObjects() - count_hits = decode_coverage.decode_result(list_meta[0].description) - self.assertEquals(0, count_hits.get_hits_50_90()) - self.assertEquals(0, count_hits.get_hits_less_50()) - self.assertEquals(0, count_hits.get_total_50_50_90()) - self.assertEquals(0, count_hits.get_total()) - - self.assertEquals(0, project_sample.count_variations.var_less_50) - self.assertEquals(0, project_sample.count_variations.var_bigger_50_90) - self.assertEquals(0, project_sample.count_variations.var_bigger_90) - self.assertEquals(0, project_sample.alert_first_level) - self.assertEquals(0, project_sample.alert_second_level) - - ### test mixed infections - self.assertEquals(Constants.EMPTY_VALUE_NA, project_sample.mixed_infections.tag.name) + list_meta = manageDatabase.get_sample_metakey(sample, MetaKeyAndValue.META_KEY_NanoStat_NanoFilt_Software, None) + self.assertEqual(2, len(list_meta)) + self.assertEquals(MetaKeyAndValue.META_VALUE_Success, list_meta[0].value) + self.assertEquals(MetaKeyAndValue.META_KEY_NanoStat_NanoFilt_Software, list_meta[0].meta_tag.name) + result_data = decodeResultAverageAndNumberReads.decode_result(list_meta[0].description) + vect_soft = result_data.get_list_software_instance(self.software_names.get_NanoStat_name()) + self.assertEqual(2, len(vect_soft)) + + self.assertEqual("Mean read length", vect_soft[0].get_vect_key_values()[0].key) + self.assertEqual("1,298.4", vect_soft[0].get_vect_key_values()[0].value) + self.assertEqual("Total bases", vect_soft[0].get_vect_key_values()[-1].key) + self.assertEqual("3,895,144.0", vect_soft[0].get_vect_key_values()[-1].value) + self.assertEqual("Mean read length", vect_soft[1].get_vect_key_values()[0].key) + self.assertEqual("1,158.4", vect_soft[1].get_vect_key_values()[0].value) + self.assertEqual("Total bases", vect_soft[1].get_vect_key_values()[-1].key) + self.assertEqual("3,475,144.0", vect_soft[1].get_vect_key_values()[-1].value) + + stats_illumina, total_reads = self.software.get_stats_from_sample_reads(sample) + self.assertEqual('Mean read length', stats_illumina[0][0]) + self.assertEqual('1,298.4', stats_illumina[0][1]) + self.assertEqual(3000, total_reads) + + sample = Sample.objects.get(name=sample_name) + self.assertEqual("A-H5N5", sample.type_subtype) + # self.assertEqual("H5N5", sample.type_subtype) + self.assertEqual("No", sample.mixed_infections_tag.name) + self.assertEqual(0, sample.number_alerts) + + list_meta = manageDatabase.get_sample_metakey(sample, MetaKeyAndValue.META_KEY_NanoStat_NanoFilt, None) + self.assertEquals(2, len(list_meta)) + self.assertEquals(MetaKeyAndValue.META_VALUE_Success, list_meta[0].value) + self.assertEquals("Success, NanoStat(1.4.0), NanoFilt(2.6.0)", list_meta[0].description) - list_meta = manageDatabase.get_project_sample_metakey(project_sample, MetaKeyAndValue.META_KEY_Snippy_Freebayes, None) - self.assertEquals(1, len(list_meta)) + ### software + list_meta = manageDatabase.get_sample_metakey(sample, MetaKeyAndValue.META_KEY_NanoStat_NanoFilt_Software, None) + self.assertEquals(2, len(list_meta)) self.assertEquals(MetaKeyAndValue.META_VALUE_Success, list_meta[0].value) - self.assertEquals(MetaKeyAndValue.META_KEY_Snippy_Freebayes, list_meta[0].meta_tag.name) + self.assertEquals(MetaKeyAndValue.META_KEY_NanoStat_NanoFilt_Software, list_meta[0].meta_tag.name) decode_result = DecodeObjects() result = decode_result.decode_result(list_meta[0].description) - self.assertTrue(result is not None) - self.assertEquals("Snippy-3.2-dev; (--mapqual 20 --mincov 5 --minfrac 0.51)", result.get_software(self.software_names.get_snippy_name())) - self.assertEquals("MSA Masker-1.0; (--c 4; for coverages less than 5 in 30% of the regions.)", result.get_software(self.software_names.get_msa_masker_name())) - self.assertEquals("", result.get_software(self.software_names.get_freebayes_name())) - self.assertTrue(result.is_software_present(self.software_names.get_snippy_name())) - self.assertFalse(result.is_software_present(self.software_names.get_freebayes_name())) - self.assertFalse(result.is_software_present(self.software_names.get_trimmomatic_name())) - - meta_value = manageDatabase.get_project_sample_metakey(project_sample, MetaKeyAndValue.META_KEY_Coverage, MetaKeyAndValue.META_VALUE_Success) - decode_coverage = DecodeObjects() - coverage = decode_coverage.decode_result(meta_value.description) - self.assertEqual(coverage.get_coverage('PA', Coverage.COVERAGE_ALL), "527.4") - self.assertEqual(coverage.get_coverage('MP', Coverage.COVERAGE_ALL), "2198.8") - self.assertEqual(coverage.get_coverage('HA', Coverage.COVERAGE_ALL), "1449.3") - self.assertEqual(coverage.get_coverage('PA', Coverage.COVERAGE_MORE_9), "100.0") - self.assertEqual(coverage.get_coverage('MP', Coverage.COVERAGE_MORE_9), "100.0") - self.assertEqual(coverage.get_coverage('HA', Coverage.COVERAGE_MORE_9), "100.0") - self.assertEqual(coverage.get_coverage('PA', Coverage.COVERAGE_MORE_0), "100.0") - self.assertEqual(coverage.get_coverage('MP', Coverage.COVERAGE_MORE_0), "100.0") - self.assertEqual(coverage.get_coverage('HA', Coverage.COVERAGE_MORE_0), "100.0") - - lst_meta_sample = manageDatabase.get_project_sample_metakey(project_sample, meta_key_project_sample, None) - self.assertEquals(2, len(lst_meta_sample)) - self.assertEquals(MetaKeyAndValue.META_VALUE_Queue, lst_meta_sample[1].value) - self.assertEquals(MetaKeyAndValue.META_VALUE_Success, lst_meta_sample[0].value) - - ### check if the images exist - for gene in self.utils.get_elements_from_db(reference, user): - output_image = project_sample.get_global_file_by_element(TypePath.MEDIA_ROOT, ProjectSample.PREFIX_FILE_COVERAGE, gene, FileExtensions.FILE_PNG) - self.assertTrue(os.path.exists(output_image)) - - meta_value = manageDatabase.get_project_sample_metakey_last(project_sample, - MetaKeyAndValue.META_KEY_bam_stats, - MetaKeyAndValue.META_VALUE_Success) - self.assertTrue(not meta_value is None) - decode_masking_vcf = DecodeObjects() - result = decode_masking_vcf.decode_result(meta_value.description) - self.assertEqual('87818', result.get_value_by_key(MetaKeyAndValue.SAMTOOLS_flagstat_mapped_reads)) - self.assertEqual('87818', result.get_value_by_key(MetaKeyAndValue.SAMTOOLS_flagstat_total_reads)) - self.assertEqual(None, result.get_value_by_key('xpto')) - - self.utils.remove_dir(temp_dir) - self.utils.remove_dir(getattr(settings, "MEDIA_ROOT", None)) - - + self.assertTrue(not result is None) + self.assertEquals("NanoStat-1.4.0", + result.get_software(self.software_names.get_NanoStat_name())) + self.assertEquals("NanoFilt-2.6.0; (-q 5 -l 50 --headcrop 70 --tailcrop 70)",\ + result.get_software(self.software_names.get_NanoFilt_name())) + self.assertEquals("RabbitQC-0.0.1; (-w 3 -D)",\ + result.get_software(self.software_names.get_rabbitQC_name())) + + ## remove all files + cmd = "rm -r %s*" % (temp_dir); os.system(cmd) \ No newline at end of file diff --git a/fluwebvirus/settings.py b/fluwebvirus/settings.py index f3afa2fc..e33ea9ec 100644 --- a/fluwebvirus/settings.py +++ b/fluwebvirus/settings.py @@ -250,6 +250,8 @@ DATE_FORMAT_FOR_TABLE = '%d-%m-%Y' DATETIME_FORMAT_FOR_TABLE = '%d-%m-%Y %H:%M' +DATE_FORMAT_FOR_SHOW = '%Y-%m-%d' +DATETIME_FORMAT_FOR_TABLE_SHOW = '%Y-%m-%d %H:%M' DATETIME_FORMAT = '%d-%m-%Y %H:%M' DATETIME_INPUT_FORMATS = ['%d-%m-%Y', '%d/%m/%Y'] ## it's necessary to look which kind of date is returned from forms to correct the format diff --git a/fluwebvirus/settings_test.py b/fluwebvirus/settings_test.py index c3e11d46..054dc930 100644 --- a/fluwebvirus/settings_test.py +++ b/fluwebvirus/settings_test.py @@ -230,6 +230,8 @@ DATE_FORMAT_FOR_TABLE = '%d-%m-%Y' DATETIME_FORMAT_FOR_TABLE = '%d-%m-%Y %H:%M' +DATE_FORMAT_FOR_SHOW = '%Y-%m-%d' +DATETIME_FORMAT_FOR_TABLE_SHOW = '%Y-%m-%d %H:%M' DATETIME_FORMAT = '%d-%m-%Y %H:%M' DATETIME_INPUT_FORMATS = ['%d-%m-%Y', '%d/%m/%Y'] ## it's necessary to look which kind of date is returned from forms to correct the format diff --git a/managing_files/management/commands/collect_global_files.py b/managing_files/management/commands/collect_global_files.py index a13b55f9..e8c30e42 100644 --- a/managing_files/management/commands/collect_global_files.py +++ b/managing_files/management/commands/collect_global_files.py @@ -7,7 +7,7 @@ from managing_files.models import Project from utils.collect_extra_data import CollectExtraData from django.contrib.auth.models import User -import logging +import logging, datetime class Command(BaseCommand): ''' @@ -38,12 +38,15 @@ def handle(self, *args, **options): self.stdout.write("Starting for project_id: " + str(project_id)) self.logger_production.info("Starting for project_id: " + str(project_id)) self.logger_debug.info("Starting for project_id: " + str(project_id)) - - print(list(Project.objects.all())) try: project = Project.objects.get(pk=project_id) if (user_id == None): user = project.owner else: user = User.objects.get(pk=user_id) + + ### change date + project.last_change_date = datetime.datetime.now() + project.save() + collect_extra_data.collect_extra_data_for_project(project, user) self.stdout.write("End") except Project.DoesNotExist as e: diff --git a/managing_files/models.py b/managing_files/models.py index a2e69964..418df48f 100644 --- a/managing_files/models.py +++ b/managing_files/models.py @@ -788,6 +788,8 @@ class Project(models.Model): PERCENTAGE_validated_minor_variants = 51 ## only pass <= 50 PROJECT_FILE_NAME_SAMPLE_RESULT_TSV = "Sample_list.tsv" ### first column ID instead of 'sample name' to be compatible with Phandango e Microreact PROJECT_FILE_NAME_SAMPLE_RESULT_CSV = "Sample_list.csv" ### first column ID instead of 'sample name' to be compatible with Phandango e Microreact + PROJECT_FILE_NAME_SAMPLE_RESULT_SETTINGS_TSV = "Sample_list_settings.tsv" ### first column ID instead of 'sample name' to be compatible with Phandango e Microreact + PROJECT_FILE_NAME_SAMPLE_RESULT_SETTINGS_CSV = "Sample_list_settings.csv" ### first column ID instead of 'sample name' to be compatible with Phandango e Microreact PROJECT_FILE_NAME_SAMPLE_RESULT_CSV_simple = "Sample_list_simple.csv" ### first column must be ID because of manging_files.ajax_views.show_phylo_canvas ### this file is only used for to show the manging_files.views.ShowSampleProjectsView PROJECT_FILE_NAME_SAMPLE_RESULT_json = "Sample_list_simple.json" ### first column ID instead of 'sample name' to be compatible with Phandango e Microreact, to download to @@ -796,6 +798,8 @@ class Project(models.Model): PROJECT_FILE_NAME_Pangolin_lineage = "PangolinLineage.csv" ### has the result of pangolin lineage + PROJECT_FILE_NAME_all_files_zipped = "AllFiles.zip" ### Several files zipped + ## put the type file here to clean if there isn't enough sequences to create the trees and alignments vect_clean_file = [PROJECT_FILE_NAME_MAFFT, PROJECT_FILE_NAME_FASTTREE,\ PROJECT_FILE_NAME_FASTTREE_tree,\ @@ -877,6 +881,9 @@ def _clean_name(self, name_to_clean, dict_to_clean = { ' ' : '_', '(' : '', ')' name_to_clean = name_to_clean.replace(key, dict_to_clean[key]) return name_to_clean + def get_clean_project_name(self): + return self._clean_name(self.name) + def get_global_file_by_project(self, type_path, file_name): """ type_path: constants.TypePath -> MEDIA_ROOT, MEDIA_URL diff --git a/managing_files/views.py b/managing_files/views.py index 6a8e0fda..52d75ae8 100644 --- a/managing_files/views.py +++ b/managing_files/views.py @@ -1,6 +1,7 @@ # Create your views here. import ntpath, os, logging, sys +from datetime import datetime from django.views import generic from braces.views import LoginRequiredMixin, FormValidMessageMixin from django.urls import reverse_lazy @@ -41,6 +42,7 @@ from django.template.defaultfilters import filesizeformat from settings.constants_settings import ConstantsSettings from settings.models import Software as SoftwareSettings +from utils.support_django_template import get_link_for_dropdown_item # http://www.craigderington.me/generic-list-view-with-django-tables/ @@ -254,16 +256,12 @@ def get_context_data(self, **kwargs): ## list of all samples in CSV and TSV csv_file = self.utils.get_sample_list_by_user(self.request.user.id, "MEDIA_ROOT", FileExtensions.FILE_CSV) if os.path.exists(csv_file): - context['list_samples_file_csv'] = mark_safe(' Download - ' +\ - os.path.basename(csv_file) + '') + context['list_samples_file_csv'] = get_link_for_dropdown_item( + self.utils.get_sample_list_by_user(self.request.user.id, "MEDIA_URL", FileExtensions.FILE_CSV)) tsv_file = self.utils.get_sample_list_by_user(self.request.user.id, "MEDIA_ROOT", FileExtensions.FILE_TSV) if os.path.exists(tsv_file): - context['list_samples_file_tsv'] = mark_safe(' Download - ' +\ - os.path.basename(tsv_file) + '') + context['list_samples_file_tsv'] = get_link_for_dropdown_item( + self.utils.get_sample_list_by_user(self.request.user.id, "MEDIA_URL", FileExtensions.FILE_TSV)) context['table'] = table context['nav_sample'] = True @@ -1020,7 +1018,7 @@ def get_context_data(self, **kwargs): context['href_trimmonatic_quality_2'] = "Not available" #### data from illumina stat - stat_data = self.software.get_stats_from_sample_reads(sample) + stat_data, total_reads = self.software.get_stats_from_sample_reads(sample) if not stat_data is None: context['data_illuminastat'] = stat_data else: ### other like Minion @@ -1036,7 +1034,7 @@ def get_context_data(self, **kwargs): else: context['href_trimmonatic_quality_1'] = "Not available" #### data from nanoStat - stat_data = self.software.get_stats_from_sample_reads(sample) + stat_data, total_reads = self.software.get_stats_from_sample_reads(sample) if not stat_data is None: context['data_nanostat'] = stat_data ### software @@ -1598,9 +1596,43 @@ def get_context_data(self, **kwargs): ## Files context['coverage_file'] = project.get_global_file_by_project_web(Project.PROJECT_FILE_NAME_COVERAGE) context['main_variations_snippy_file'] = project.get_global_file_by_project_web(Project.PROJECT_FILE_NAME_TAB_VARIATIONS_SNIPPY) - context['sample_file_result_csv'] = project.get_global_file_by_project_web(Project.PROJECT_FILE_NAME_SAMPLE_RESULT_CSV) - context['sample_file_result_tsv'] = project.get_global_file_by_project_web(Project.PROJECT_FILE_NAME_SAMPLE_RESULT_TSV) - context['sample_file_all_consensus'] = project.get_global_file_by_project_web(Project.PROJECT_FILE_NAME_SAMPLE_RESULT_all_consensus) + + ## coverage + if os.path.exists(project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_COVERAGE)): + context['samples_file_coverage'] = get_link_for_dropdown_item( + project.get_global_file_by_project(TypePath.MEDIA_URL, Project.PROJECT_FILE_NAME_COVERAGE)) + ## variants + if os.path.exists(project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_TAB_VARIATIONS_SNIPPY)): + context['samples_file_variants'] = get_link_for_dropdown_item( + project.get_global_file_by_project(TypePath.MEDIA_URL, Project.PROJECT_FILE_NAME_TAB_VARIATIONS_SNIPPY)) + ## minor intra host + if os.path.exists(project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_TAB_VARIATIONS_FREEBAYES)): + context['samples_file_minor_intra_host'] = get_link_for_dropdown_item( + project.get_global_file_by_project(TypePath.MEDIA_URL, Project.PROJECT_FILE_NAME_TAB_VARIATIONS_FREEBAYES)) + ## all files zipped + if os.path.exists(project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_all_files_zipped)): + context['download_all_files'] = get_link_for_dropdown_item( + project.get_global_file_by_project(TypePath.MEDIA_URL, Project.PROJECT_FILE_NAME_all_files_zipped), + "{}_{}_{}".format(os.path.splitext(Project.PROJECT_FILE_NAME_all_files_zipped)[0], + project.get_clean_project_name(), datetime.now().strftime(settings.DATE_FORMAT_FOR_SHOW))) + + + if os.path.exists(project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_SAMPLE_RESULT_CSV)): + context['sample_file_result_csv'] = get_link_for_dropdown_item( + project.get_global_file_by_project(TypePath.MEDIA_URL, Project.PROJECT_FILE_NAME_SAMPLE_RESULT_CSV)) + if os.path.exists(project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_SAMPLE_RESULT_TSV)): + context['sample_file_result_tsv'] = get_link_for_dropdown_item( + project.get_global_file_by_project(TypePath.MEDIA_URL, Project.PROJECT_FILE_NAME_SAMPLE_RESULT_TSV)) + if os.path.exists(project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_SAMPLE_RESULT_SETTINGS_CSV)): + context['samples_file_settings_statistics_csv'] = get_link_for_dropdown_item( + project.get_global_file_by_project(TypePath.MEDIA_URL, Project.PROJECT_FILE_NAME_SAMPLE_RESULT_SETTINGS_CSV)) + if os.path.exists(project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_SAMPLE_RESULT_SETTINGS_TSV)): + context['samples_file_settings_statistics_tsv'] = get_link_for_dropdown_item( + project.get_global_file_by_project(TypePath.MEDIA_URL, Project.PROJECT_FILE_NAME_SAMPLE_RESULT_SETTINGS_TSV)) + + if os.path.exists(project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_SAMPLE_RESULT_all_consensus)): + context['sample_file_all_consensus'] = get_link_for_dropdown_item( + project.get_global_file_by_project(TypePath.MEDIA_URL, Project.PROJECT_FILE_NAME_SAMPLE_RESULT_all_consensus)) ### need to test because in the past this file was not created context['freebays_variations_50_file'] = project.get_global_file_by_project_web(Project.PROJECT_FILE_NAME_TAB_VARIATIONS_FREEBAYES) @@ -1633,21 +1665,17 @@ def get_context_data(self, **kwargs): (os.path.exists(file_pangolin_result) and software_pangolin.pangolin_results_out_date(project)) ): context['update_pangolin'] = True context['update_pangolin_message'] = mark_safe(software_pangolin.get_update_message(project)) - + + ## pangolin file if (project.number_passed_sequences > 0 and os.path.exists(file_pangolin_result)): - context['pangolin_lineage'] = project.get_global_file_by_project_web(Project.PROJECT_FILE_NAME_Pangolin_lineage) + context['pangolin_lineage'] = get_link_for_dropdown_item(project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_Pangolin_lineage)) #### nextclade link -# if (is_sars_cov_2 and \ -# os.path.exists(project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_SAMPLE_RESULT_all_consensus)) and \ -# settings.SHOW_NEXTCLADE_LINK): ## docker versions doesn't show NextClade link if (os.path.exists(project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_SAMPLE_RESULT_all_consensus)) and \ settings.SHOW_NEXTCLADE_LINK): ## docker versions doesn't show NextClade link - context['nextclade_link'] = "{}{}://{}{}".format( - Constants.NEXTCLADE_LINK, - settings.WEB_SITE_HTTP_NAME, - get_current_site(self.request), - project.get_global_file_by_project(TypePath.MEDIA_URL, Project.PROJECT_FILE_NAME_SAMPLE_RESULT_all_consensus)) + context = _get_constext_nextclade( + project.get_global_file_by_project(TypePath.MEDIA_URL, Project.PROJECT_FILE_NAME_SAMPLE_RESULT_all_consensus), + context, get_current_site(self.request), is_sars_cov_2) return context @@ -1959,22 +1987,48 @@ def get_context_data(self, **kwargs): context['software_used'] = software_used #### nextclade link -# software_pangolin = SoftwarePangolin() -# if (software_pangolin.is_ref_sars_cov_2(project_sample.project.reference.get_reference_fasta(TypePath.MEDIA_ROOT)) and \ -# os.path.exists(project_sample.get_consensus_file(TypePath.MEDIA_ROOT)) and \ -# settings.SHOW_NEXTCLADE_LINK): ## docker versions doesn't show NextClade link + software_pangolin = SoftwarePangolin() + is_sars_cov_2 = software_pangolin.is_ref_sars_cov_2(project_sample.project.reference.get_reference_fasta(TypePath.MEDIA_ROOT)) if (os.path.exists(project_sample.get_consensus_file(TypePath.MEDIA_ROOT)) and \ settings.SHOW_NEXTCLADE_LINK): ## docker versions doesn't show NextClade link - context['nextclade_link'] = "{}{}://{}{}".format( - Constants.NEXTCLADE_LINK, - settings.WEB_SITE_HTTP_NAME, - get_current_site(self.request), - project_sample.get_consensus_file(TypePath.MEDIA_URL)) + context = _get_constext_nextclade(project_sample.get_consensus_file(TypePath.MEDIA_URL), + context, get_current_site(self.request), is_sars_cov_2) except ProjectSample.DoesNotExist: context['error_cant_see'] = 1 return context +def _get_constext_nextclade(media_url_path, context, current_site, is_sars_cov_2): + + ## sarscov 2 + if (is_sars_cov_2): + context['nextclade_link_covid'] = "{}{}://{}{}".format( + Constants.NEXTCLADE_LINK_sars_cov_2, + settings.WEB_SITE_HTTP_NAME, + current_site, + media_url_path) + else: + context['nextclade_link_a_h3n2'] = "{}{}://{}{}".format( + Constants.NEXTCLADE_LINK_A_H3N2, + settings.WEB_SITE_HTTP_NAME, + current_site, + media_url_path) + context['nextclade_link_a_h1n1'] = "{}{}://{}{}".format( + Constants.NEXTCLADE_LINK_A_H1N1, + settings.WEB_SITE_HTTP_NAME, + current_site, + media_url_path) + context['nextclade_link_b_yamagata'] = "{}{}://{}{}".format( + Constants.NEXTCLADE_LINK_B_Yamagata, + settings.WEB_SITE_HTTP_NAME, + current_site, + media_url_path) + context['nextclade_link_b_victoria'] = "{}{}://{}{}".format( + Constants.NEXTCLADE_LINK_B_Victoria, + settings.WEB_SITE_HTTP_NAME, + current_site, + media_url_path) + return context def is_all_check_box_in_session(vect_check_to_test, request): """ diff --git a/static/tests/global_project/AllProjects_1.csv b/static/tests/global_project/AllProjects_1.csv index 81509f8e..a39ce2b4 100644 --- a/static/tests/global_project/AllProjects_1.csv +++ b/static/tests/global_project/AllProjects_1.csv @@ -1,3 +1,3 @@ Name,#samples,Reference,Last access,Created -several_names_test_create_tree_6000,4,second_stage_2_ test_create_tree,14-12-2002 20:30 +several_names_test_create_tree_6000,4,second_stage_2_ test_create_tree,2002-12-14 20:30 Total Projects,1 diff --git a/static/tests/global_project/AllProjects_1.tsv b/static/tests/global_project/AllProjects_1.tsv index 9e91c4dd..5f054bf8 100644 --- a/static/tests/global_project/AllProjects_1.tsv +++ b/static/tests/global_project/AllProjects_1.tsv @@ -1,3 +1,3 @@ "Name" "#samples" "Reference" "Last access" "Created" -"several_names_test_create_tree_6000" "4" "second_stage_2_ test_create_tree" "14-12-2002 20:30" +"several_names_test_create_tree_6000" "4" "second_stage_2_ test_create_tree" "2002-12-14 20:30" "Total Projects" "1" diff --git a/static/tests/global_project/AllSamples_1.csv b/static/tests/global_project/AllSamples_1.csv index aad0add7..c3011906 100644 --- a/static/tests/global_project/AllSamples_1.csv +++ b/static/tests/global_project/AllSamples_1.csv @@ -1,7 +1,7 @@ -,,,,,,,,,,,,,Trimmo. stats.,Trimmo. stats.,Trimmo. stats.,Trimmo. stats.,Trimmo. stats.,,,,, -sample,fastq1,fastq2,data set,vaccine status,week,onset date,collection date,lab reception date,latitude,longitude,type-subtype,xpto_1,Technology,Sample Down sized,Input Read Pairs,Both Surviving,Forward Only Surviving,Reverse Only Surviving,Dropped,FastQC,Trimmomatic,Uploaded Date,#Projects,Project Names +,,,,,,,,,,,,,,,Trimmo. stats.,Trimmo. stats.,Trimmo. stats.,Trimmo. stats.,Trimmo. stats.,,,,, +sample,fastq1,fastq2,data set,vaccine status,week,onset date,collection date,lab reception date,latitude,longitude,classification,xpto_1,Technology,Sample Down sized,Input Read Pairs,Both Surviving,Forward Only Surviving,Reverse Only Surviving,Dropped,FastQC,Trimmomatic,Uploaded Date,#Projects,Project Names EVA011_S54,EVA011_S54_L001_R1_001.fastq.gz,EVA011_S54_L001_R2_001.fastq.gz,,,,,,,,,"xpto, zpto",xpto_1 xpto_1,Illumina,False,,,,,,FastQC-0.11.9; (let's go again),Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic),Date,1,several_names_test_create_tree_6000 -EVA003_S91,EVA003_S91_L001_R1_001.fastq.gz,EVA003_S91_L001_R2_001.fastq.gz,,,,,,,,,"xpto, zpto",xpto_1 xpto_1,Illumina,False,xpto,xpto1,xpto2,,xpto4,FastQC-0.11.9; (let's go again),Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic),Date,1,several_names_test_create_tree_6000 -EVA002_S52,EVA002_S52_L001_R1_001.fastq.gz,EVA002_S52_L001_R2_001.fastq.gz,,,,,,,,,"xpto, zpto",xpto_1 xpto_1,Illumina,False,xpto,xpto1,xpto2,,xpto4,FastQC-0.11.9; (let's go again),Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic),Date,1,several_names_test_create_tree_6000 -EVA001_S66,EVA001_S66_L001_R1_001.fastq.gz,EVA001_S66_L001_R2_001.fastq.gz,,,,,,,,,"xpto, zpto",xpto_1 xpto_1,Illumina,False,xpto,xpto1,xpto2,xpto3,xpto4,FastQC-0.11.9; (let's go again),Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic),Date,1,several_names_test_create_tree_6000 +EVA003_S91,EVA003_S91_L001_R1_001.fastq.gz,EVA003_S91_L001_R2_001.fastq.gz,,,,,,,,,"xpto, zpto",xpto_1 xpto_1,Illumina,False,10,5,2,,44,FastQC-0.11.9; (let's go again),Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic),Date,1,several_names_test_create_tree_6000 +EVA002_S52,EVA002_S52_L001_R1_001.fastq.gz,EVA002_S52_L001_R2_001.fastq.gz,,,,,,,,,"xpto, zpto",xpto_1 xpto_1,Illumina,False,10,5,2,,44,FastQC-0.11.9; (let's go again),Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic),Date,1,several_names_test_create_tree_6000 +EVA001_S66,EVA001_S66_L001_R1_001.fastq.gz,EVA001_S66_L001_R2_001.fastq.gz,,,,,,,,,"xpto, zpto",xpto_1 xpto_1,Illumina,False,10,5,2,33,44,FastQC-0.11.9; (let's go again),Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic),Date,1,several_names_test_create_tree_6000 Total Samples,4 diff --git a/static/tests/global_project/AllSamples_1.tsv b/static/tests/global_project/AllSamples_1.tsv index c998dc9b..ef19f7b9 100644 --- a/static/tests/global_project/AllSamples_1.tsv +++ b/static/tests/global_project/AllSamples_1.tsv @@ -1,7 +1,7 @@ -"" "" "" "" "" "" "" "" "" "" "" "" "" "Trimmo. stats." "Trimmo. stats." "Trimmo. stats." "Trimmo. stats." "Trimmo. stats." "" "" "" "" "" -"sample" "fastq1" "fastq2" "data set" "vaccine status" "week" "onset date" "collection date" "lab reception date" "latitude" "longitude" "type-subtype" "xpto_1" "Technology" "Sample Down sized" "Input Read Pairs" "Both Surviving" "Forward Only Surviving" "Reverse Only Surviving" "Dropped" "FastQC" "Trimmomatic" "Uploaded Date" "#Projects" "Project Names" +"" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "Trimmo. stats." "Trimmo. stats." "Trimmo. stats." "Trimmo. stats." "Trimmo. stats." "" "" "" "" "" +"sample" "fastq1" "fastq2" "data set" "vaccine status" "week" "onset date" "collection date" "lab reception date" "latitude" "longitude" "classification" "xpto_1" "Technology" "Sample Down sized" "Input Read Pairs" "Both Surviving" "Forward Only Surviving" "Reverse Only Surviving" "Dropped" "FastQC" "Trimmomatic" "Uploaded Date" "#Projects" "Project Names" "EVA011_S54" "EVA011_S54_L001_R1_001.fastq.gz" "EVA011_S54_L001_R2_001.fastq.gz" "" "" "" "" "" "" "" "" "xpto, zpto" "xpto_1 xpto_1" "Illumina" "False" "" "" "" "" "" "FastQC-0.11.9; (let's go again)" "Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic)" "Date" "1" "several_names_test_create_tree_6000" -"EVA003_S91" "EVA003_S91_L001_R1_001.fastq.gz" "EVA003_S91_L001_R2_001.fastq.gz" "" "" "" "" "" "" "" "" "xpto, zpto" "xpto_1 xpto_1" "Illumina" "False" "xpto" "xpto1" "xpto2" "" "xpto4" "FastQC-0.11.9; (let's go again)" "Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic)" "Date" "1" "several_names_test_create_tree_6000" -"EVA002_S52" "EVA002_S52_L001_R1_001.fastq.gz" "EVA002_S52_L001_R2_001.fastq.gz" "" "" "" "" "" "" "" "" "xpto, zpto" "xpto_1 xpto_1" "Illumina" "False" "xpto" "xpto1" "xpto2" "" "xpto4" "FastQC-0.11.9; (let's go again)" "Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic)" "Date" "1" "several_names_test_create_tree_6000" -"EVA001_S66" "EVA001_S66_L001_R1_001.fastq.gz" "EVA001_S66_L001_R2_001.fastq.gz" "" "" "" "" "" "" "" "" "xpto, zpto" "xpto_1 xpto_1" "Illumina" "False" "xpto" "xpto1" "xpto2" "xpto3" "xpto4" "FastQC-0.11.9; (let's go again)" "Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic)" "Date" "1" "several_names_test_create_tree_6000" +"EVA003_S91" "EVA003_S91_L001_R1_001.fastq.gz" "EVA003_S91_L001_R2_001.fastq.gz" "" "" "" "" "" "" "" "" "xpto, zpto" "xpto_1 xpto_1" "Illumina" "False" "10" "5" "2" "" "44" "FastQC-0.11.9; (let's go again)" "Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic)" "Date" "1" "several_names_test_create_tree_6000" +"EVA002_S52" "EVA002_S52_L001_R1_001.fastq.gz" "EVA002_S52_L001_R2_001.fastq.gz" "" "" "" "" "" "" "" "" "xpto, zpto" "xpto_1 xpto_1" "Illumina" "False" "10" "5" "2" "" "44" "FastQC-0.11.9; (let's go again)" "Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic)" "Date" "1" "several_names_test_create_tree_6000" +"EVA001_S66" "EVA001_S66_L001_R1_001.fastq.gz" "EVA001_S66_L001_R2_001.fastq.gz" "" "" "" "" "" "" "" "" "xpto, zpto" "xpto_1 xpto_1" "Illumina" "False" "10" "5" "2" "33" "44" "FastQC-0.11.9; (let's go again)" "Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic)" "Date" "1" "several_names_test_create_tree_6000" "Total Samples" "4" diff --git a/static/tests/global_project/insa_flu_sample_output.csv b/static/tests/global_project/insa_flu_sample_output.csv index 9cef8db3..72318837 100644 --- a/static/tests/global_project/insa_flu_sample_output.csv +++ b/static/tests/global_project/insa_flu_sample_output.csv @@ -1,6 +1,6 @@ -Reference name,second_stage_2_ test_create_tree,,,,,,,,,,,,,,,,,,,,, -id,fastq1,fastq2,data set,vaccine status,week,onset date,collection date,lab reception date,latitude,longitude,type-subtype,putative mixed-infection,xpto,Technology,Sample Down sized,Available Reads/Mapped Reads/Mapped Percentage,FastQC,Trimmomatic,Mask consensus by sites,SPAdes,Snippy,Threshold to mask consensus sequences -EVA011_S54,EVA011_S54_L001_R1_001.fastq.gz,EVA011_S54_L001_R2_001.fastq.gz,,,,,,,,,"xpto, zpto",,xpto xpto,Illumina,False,87818/87818/100.0,FastQC-0.11.9; (let's go again),Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic),"Element:element_name -> (Mask sites:2,5 Mask regions:1-30,32-50,320-500 Mask from beginning:2 Mask from end:2)",,,Not applied -EVA003_S91,EVA003_S91_L001_R1_001.fastq.gz,EVA003_S91_L001_R2_001.fastq.gz,,,,,,,,,"xpto, zpto",,xpto xpto,Illumina,False,87818/87818/100.0,FastQC-0.11.9; (let's go again),Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic),"Element:element_name -> (Mask sites:2,5 Mask regions:1-30,32-50,320-500 Mask from beginning:2 Mask from end:2)",SPAdes-3.11.1; (let's go),Snippy-3.2-dev; (let's go_2222),Not applied -EVA002_S52,EVA002_S52_L001_R1_001.fastq.gz,EVA002_S52_L001_R2_001.fastq.gz,,,,,,,,,"xpto, zpto",,xpto xpto,Illumina,False,87818/87818/100.0,FastQC-0.11.9; (let's go again),Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic),"Element:element_name -> (Mask sites:2,5 Mask regions:1-30,32-50,320-500 Mask from beginning:2 Mask from end:2)",SPAdes-3.11.1; (let's go),Snippy-3.2-dev; (let's go_2222),Not applied -EVA001_S66,EVA001_S66_L001_R1_001.fastq.gz,EVA001_S66_L001_R2_001.fastq.gz,,,,,,,,,"xpto, zpto",,xpto xpto,Illumina,False,87818/87818/100.0,FastQC-0.11.9; (let's go again),Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic),"Element:element_name -> (Mask sites:2,5 Mask regions:1-30,32-50,320-500 Mask from beginning:2 Mask from end:2)",SPAdes-3.11.1; (let's go),Snippy-3.2-dev; (let's go_2222),Not applied +Reference name,second_stage_2_ test_create_tree,,,,,,,,,,,, +id,fastq1,fastq2,data set,vaccine status,week,onset date,collection date,lab reception date,latitude,longitude,classification,putative mixed-infection,xpto +EVA011_S54,EVA011_S54_L001_R1_001.fastq.gz,EVA011_S54_L001_R2_001.fastq.gz,,,,,,,,,"xpto, zpto",,xpto xpto +EVA003_S91,EVA003_S91_L001_R1_001.fastq.gz,EVA003_S91_L001_R2_001.fastq.gz,,,,,,,,,"xpto, zpto",,xpto xpto +EVA002_S52,EVA002_S52_L001_R1_001.fastq.gz,EVA002_S52_L001_R2_001.fastq.gz,,,,,,,,,"xpto, zpto",,xpto xpto +EVA001_S66,EVA001_S66_L001_R1_001.fastq.gz,EVA001_S66_L001_R2_001.fastq.gz,,,,,,,,,"xpto, zpto",,xpto xpto diff --git a/static/tests/global_project/insa_flu_sample_output.json b/static/tests/global_project/insa_flu_sample_output.json index 35b3f4bc..9104fa9b 100644 --- a/static/tests/global_project/insa_flu_sample_output.json +++ b/static/tests/global_project/insa_flu_sample_output.json @@ -1 +1 @@ -{"second_stage_2_ test_create_tree": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "type-subtype": "", "putative mixed-infection": "", "xpto_1": ""}, "EVA011_S54": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "type-subtype": "xpto, zpto", "putative mixed-infection": "", "xpto_1": "xpto_1 xpto_1"}, "EVA003_S91": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "type-subtype": "xpto, zpto", "putative mixed-infection": "", "xpto_1": "xpto_1 xpto_1"}, "EVA002_S52": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "type-subtype": "xpto, zpto", "putative mixed-infection": "", "xpto_1": "xpto_1 xpto_1"}, "EVA001_S66": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "type-subtype": "xpto, zpto", "putative mixed-infection": "", "xpto_1": "xpto_1 xpto_1"}} \ No newline at end of file +{"second_stage_2_ test_create_tree": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "classification": "", "putative mixed-infection": "", "xpto_1": ""}, "EVA011_S54": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "classification": "xpto, zpto", "putative mixed-infection": "", "xpto_1": "xpto_1 xpto_1"}, "EVA003_S91": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "classification": "xpto, zpto", "putative mixed-infection": "", "xpto_1": "xpto_1 xpto_1"}, "EVA002_S52": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "classification": "xpto, zpto", "putative mixed-infection": "", "xpto_1": "xpto_1 xpto_1"}, "EVA001_S66": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "classification": "xpto, zpto", "putative mixed-infection": "", "xpto_1": "xpto_1 xpto_1"}} \ No newline at end of file diff --git a/static/tests/global_project/insa_flu_sample_output.tsv b/static/tests/global_project/insa_flu_sample_output.tsv index 3bc6f51a..9940e5c7 100644 --- a/static/tests/global_project/insa_flu_sample_output.tsv +++ b/static/tests/global_project/insa_flu_sample_output.tsv @@ -1,6 +1,6 @@ -"Reference name" "second_stage_2_ test_create_tree" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" -"id" "fastq1" "fastq2" "data set" "vaccine status" "week" "onset date" "collection date" "lab reception date" "latitude" "longitude" "type-subtype" "putative mixed-infection" "xpto" "Technology" "Sample Down sized" "Available Reads/Mapped Reads/Mapped Percentage" "FastQC" "Trimmomatic" "Mask consensus by sites" "SPAdes" "Snippy" "Threshold to mask consensus sequences" -"EVA011_S54" "EVA011_S54_L001_R1_001.fastq.gz" "EVA011_S54_L001_R2_001.fastq.gz" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto xpto" "Illumina" "False" "87818/87818/100.0" "FastQC-0.11.9; (let's go again)" "Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic)" "Element:element_name -> (Mask sites:2,5 Mask regions:1-30,32-50,320-500 Mask from beginning:2 Mask from end:2)" "" "" "Not applied" -"EVA003_S91" "EVA003_S91_L001_R1_001.fastq.gz" "EVA003_S91_L001_R2_001.fastq.gz" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto xpto" "Illumina" "False" "87818/87818/100.0" "FastQC-0.11.9; (let's go again)" "Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic)" "Element:element_name -> (Mask sites:2,5 Mask regions:1-30,32-50,320-500 Mask from beginning:2 Mask from end:2)" "SPAdes-3.11.1; (let's go)" "Snippy-3.2-dev; (let's go_2222)" "Not applied" -"EVA002_S52" "EVA002_S52_L001_R1_001.fastq.gz" "EVA002_S52_L001_R2_001.fastq.gz" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto xpto" "Illumina" "False" "87818/87818/100.0" "FastQC-0.11.9; (let's go again)" "Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic)" "Element:element_name -> (Mask sites:2,5 Mask regions:1-30,32-50,320-500 Mask from beginning:2 Mask from end:2)" "SPAdes-3.11.1; (let's go)" "Snippy-3.2-dev; (let's go_2222)" "Not applied" -"EVA001_S66" "EVA001_S66_L001_R1_001.fastq.gz" "EVA001_S66_L001_R2_001.fastq.gz" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto xpto" "Illumina" "False" "87818/87818/100.0" "FastQC-0.11.9; (let's go again)" "Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic)" "Element:element_name -> (Mask sites:2,5 Mask regions:1-30,32-50,320-500 Mask from beginning:2 Mask from end:2)" "SPAdes-3.11.1; (let's go)" "Snippy-3.2-dev; (let's go_2222)" "Not applied" +"Reference name" "second_stage_2_ test_create_tree" "" "" "" "" "" "" "" "" "" "" "" "" +"id" "fastq1" "fastq2" "data set" "vaccine status" "week" "onset date" "collection date" "lab reception date" "latitude" "longitude" "classification" "putative mixed-infection" "xpto" +"EVA011_S54" "EVA011_S54_L001_R1_001.fastq.gz" "EVA011_S54_L001_R2_001.fastq.gz" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto xpto" +"EVA003_S91" "EVA003_S91_L001_R1_001.fastq.gz" "EVA003_S91_L001_R2_001.fastq.gz" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto xpto" +"EVA002_S52" "EVA002_S52_L001_R1_001.fastq.gz" "EVA002_S52_L001_R2_001.fastq.gz" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto xpto" +"EVA001_S66" "EVA001_S66_L001_R1_001.fastq.gz" "EVA001_S66_L001_R2_001.fastq.gz" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto xpto" diff --git a/static/tests/global_project/insa_flu_sample_output_1.json b/static/tests/global_project/insa_flu_sample_output_1.json index 92e81c2e..34c93a39 100755 --- a/static/tests/global_project/insa_flu_sample_output_1.json +++ b/static/tests/global_project/insa_flu_sample_output_1.json @@ -1 +1 @@ -{"second_stage_2_ test_create_tree": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "type-subtype": "", "putative mixed-infection": "", "xpto": ""}, "EVA011_S54": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "type-subtype": "xpto, zpto", "putative mixed-infection": "", "xpto": "xpto xpto"}, "EVA003_S91": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "type-subtype": "xpto, zpto", "putative mixed-infection": "", "xpto": "xpto xpto"}, "EVA002_S52": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "type-subtype": "xpto, zpto", "putative mixed-infection": "", "xpto": "xpto xpto"}, "EVA001_S66": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "type-subtype": "xpto, zpto", "putative mixed-infection": "", "xpto": "xpto xpto"}} \ No newline at end of file +{"second_stage_2_ test_create_tree": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "classification": "", "putative mixed-infection": "", "xpto": ""}, "EVA011_S54": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "classification": "xpto, zpto", "putative mixed-infection": "", "xpto": "xpto xpto"}, "EVA003_S91": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "classification": "xpto, zpto", "putative mixed-infection": "", "xpto": "xpto xpto"}, "EVA002_S52": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "classification": "xpto, zpto", "putative mixed-infection": "", "xpto": "xpto xpto"}, "EVA001_S66": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "classification": "xpto, zpto", "putative mixed-infection": "", "xpto": "xpto xpto"}} \ No newline at end of file diff --git a/static/tests/global_project/insa_flu_sample_output_2.csv b/static/tests/global_project/insa_flu_sample_output_2.csv index b8d0bd2e..a89dbb6f 100755 --- a/static/tests/global_project/insa_flu_sample_output_2.csv +++ b/static/tests/global_project/insa_flu_sample_output_2.csv @@ -1,6 +1,6 @@ -Reference name:,second_stage_2_ test_create_tree,,,,,,,,,,,,,,,,Trimmo. stats.,Trimmo. stats.,Trimmo. stats.,Trimmo. stats.,Trimmo. stats.,,,,, -id,fastq1,fastq2,data set,vaccine status,week,onset date,collection date,lab reception date,latitude,longitude,type-subtype,putative mixed-infection,xpto_1,Technology,Sample Down sized,Available Reads/Mapped Reads/Mapped Percentage,Input Read Pairs,Both Surviving,Forward Only Surviving,Reverse Only Surviving,Dropped,FastQC,Trimmomatic,SPAdes,Snippy,Threshold to mask consensus sequences -EVA011_S54,EVA011_S54_L001_R1_001.fastq.gz,EVA011_S54_L001_R2_001.fastq.gz,,,,,,,,,"xpto, zpto",,xpto_1 xpto_1,Illumina,False,87818/87818/100.0,,,,,,FastQC-0.11.9; (let's go again),Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic),,,70 -EVA003_S91,EVA003_S91_L001_R1_001.fastq.gz,EVA003_S91_L001_R2_001.fastq.gz,,,,,,,,,"xpto, zpto",,xpto_1 xpto_1,Illumina,False,87818/87818/100.0,xpto,xpto1,xpto2,,xpto4,FastQC-0.11.9; (let's go again),Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic),SPAdes-3.11.1; (let's go),Snippy-3.2-dev; (let's go_2222),70 -EVA002_S52,EVA002_S52_L001_R1_001.fastq.gz,EVA002_S52_L001_R2_001.fastq.gz,,,,,,,,,"xpto, zpto",,xpto_1 xpto_1,Illumina,False,87818/87818/100.0,xpto,xpto1,xpto2,,xpto4,FastQC-0.11.9; (let's go again),Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic),SPAdes-3.11.1; (let's go),Snippy-3.2-dev; (let's go_2222),70 -EVA001_S66,EVA001_S66_L001_R1_001.fastq.gz,EVA001_S66_L001_R2_001.fastq.gz,,,,,,,,,"xpto, zpto",,xpto_1 xpto_1,Illumina,False,87818/87818/100.0,xpto,xpto1,xpto2,xpto3,xpto4,FastQC-0.11.9; (let's go again),Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic),SPAdes-3.11.1; (let's go),Snippy-3.2-dev; (let's go_2222),Not applied +Reference name,second_stage_2_ test_create_tree,,,,,,,,,,,, +id,fastq1,fastq2,data set,vaccine status,week,onset date,collection date,lab reception date,latitude,longitude,classification,putative mixed-infection,xpto_1 +EVA011_S54,EVA011_S54_L001_R1_001.fastq.gz,EVA011_S54_L001_R2_001.fastq.gz,,,,,,,,,"xpto, zpto",,xpto_1 xpto_1 +EVA003_S91,EVA003_S91_L001_R1_001.fastq.gz,EVA003_S91_L001_R2_001.fastq.gz,,,,,,,,,"xpto, zpto",,xpto_1 xpto_1 +EVA002_S52,EVA002_S52_L001_R1_001.fastq.gz,EVA002_S52_L001_R2_001.fastq.gz,,,,,,,,,"xpto, zpto",,xpto_1 xpto_1 +EVA001_S66,EVA001_S66_L001_R1_001.fastq.gz,EVA001_S66_L001_R2_001.fastq.gz,,,,,,,,,"xpto, zpto",,xpto_1 xpto_1 diff --git a/static/tests/global_project/insa_flu_sample_output_2.tsv b/static/tests/global_project/insa_flu_sample_output_2.tsv index 56a146e4..88830b6e 100755 --- a/static/tests/global_project/insa_flu_sample_output_2.tsv +++ b/static/tests/global_project/insa_flu_sample_output_2.tsv @@ -1,6 +1,6 @@ -"Reference name:" "second_stage_2_ test_create_tree" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "Trimmo. stats." "Trimmo. stats." "Trimmo. stats." "Trimmo. stats." "Trimmo. stats." "" "" "" "" "" -"id" "fastq1" "fastq2" "data set" "vaccine status" "week" "onset date" "collection date" "lab reception date" "latitude" "longitude" "type-subtype" "putative mixed-infection" "xpto_1" "Technology" "Sample Down sized" "Available Reads/Mapped Reads/Mapped Percentage" "Input Read Pairs" "Both Surviving" "Forward Only Surviving" "Reverse Only Surviving" "Dropped" "FastQC" "Trimmomatic" "SPAdes" "Snippy" "Threshold to mask consensus sequences" -"EVA011_S54" "EVA011_S54_L001_R1_001.fastq.gz" "EVA011_S54_L001_R2_001.fastq.gz" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto_1 xpto_1" "Illumina" "False" "87818/87818/100.0" "" "" "" "" "" "FastQC-0.11.9; (let's go again)" "Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic)" "" "" "70" -"EVA003_S91" "EVA003_S91_L001_R1_001.fastq.gz" "EVA003_S91_L001_R2_001.fastq.gz" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto_1 xpto_1" "Illumina" "False" "87818/87818/100.0" "xpto" "xpto1" "xpto2" "" "xpto4" "FastQC-0.11.9; (let's go again)" "Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic)" "SPAdes-3.11.1; (let's go)" "Snippy-3.2-dev; (let's go_2222)" "70" -"EVA002_S52" "EVA002_S52_L001_R1_001.fastq.gz" "EVA002_S52_L001_R2_001.fastq.gz" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto_1 xpto_1" "Illumina" "False" "87818/87818/100.0" "xpto" "xpto1" "xpto2" "" "xpto4" "FastQC-0.11.9; (let's go again)" "Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic)" "SPAdes-3.11.1; (let's go)" "Snippy-3.2-dev; (let's go_2222)" "70" -"EVA001_S66" "EVA001_S66_L001_R1_001.fastq.gz" "EVA001_S66_L001_R2_001.fastq.gz" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto_1 xpto_1" "Illumina" "False" "87818/87818/100.0" "xpto" "xpto1" "xpto2" "xpto3" "xpto4" "FastQC-0.11.9; (let's go again)" "Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic)" "SPAdes-3.11.1; (let's go)" "Snippy-3.2-dev; (let's go_2222)" "Not applied" +"Reference name" "second_stage_2_ test_create_tree" "" "" "" "" "" "" "" "" "" "" "" "" +"id" "fastq1" "fastq2" "data set" "vaccine status" "week" "onset date" "collection date" "lab reception date" "latitude" "longitude" "classification" "putative mixed-infection" "xpto_1" +"EVA011_S54" "EVA011_S54_L001_R1_001.fastq.gz" "EVA011_S54_L001_R2_001.fastq.gz" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto_1 xpto_1" +"EVA003_S91" "EVA003_S91_L001_R1_001.fastq.gz" "EVA003_S91_L001_R2_001.fastq.gz" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto_1 xpto_1" +"EVA002_S52" "EVA002_S52_L001_R1_001.fastq.gz" "EVA002_S52_L001_R2_001.fastq.gz" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto_1 xpto_1" +"EVA001_S66" "EVA001_S66_L001_R1_001.fastq.gz" "EVA001_S66_L001_R2_001.fastq.gz" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto_1 xpto_1" diff --git a/static/tests/global_project/insa_flu_sample_output_2_ont.csv b/static/tests/global_project/insa_flu_sample_output_2_ont.csv index 6da11c7c..e3a7e3b2 100755 --- a/static/tests/global_project/insa_flu_sample_output_2_ont.csv +++ b/static/tests/global_project/insa_flu_sample_output_2_ont.csv @@ -1,10 +1,10 @@ -Reference name:,second_stage_2_ test_create_tree,,,,,,,,,,,,,,,,Trimmo. stats.,Trimmo. stats.,Trimmo. stats.,Trimmo. stats.,Trimmo. stats.,NanoStat stats.,NanoStat stats.,NanoStat stats.,NanoStat stats.,NanoStat stats.,NanoStat stats.,NanoStat stats.,NanoStat stats.,,,,,,,,,,,, -id,fastq1,fastq2,data set,vaccine status,week,onset date,collection date,lab reception date,latitude,longitude,type-subtype,putative mixed-infection,xpto_1,Technology,Sample Down sized,Available Reads/Mapped Reads/Mapped Percentage,Input Read Pairs,Both Surviving,Forward Only Surviving,Reverse Only Surviving,Dropped,Mean read length - ONT,Mean read quality - ONT,Median read length - ONT,Median read quality - ONT,Number of reads - ONT,Read length N50 - ONT,STDEV read length - ONT,Total bases - ONT,NanoStat,RabbitQC,NanoFilt,FastQC,Trimmomatic,Medaka,Samtools,Minimum depth of coverage per site to validate the sequence,MSA Masker,SPAdes,Snippy,Threshold to mask consensus sequences -EVA011_S54_ONT,EVA011_S54_L001_R1_001.fastq.gz,,,,,,,,,,"xpto, zpto",,xpto_1 xpto_1,ONT,False,0/0/0,,,,,,0,1,2,3,4,5,6,7,NanoStat-1.4.0,RabbitQC-0.0.1; (-w 3 -D),NanoFilt-2.6.0; (let's go again),,,,,,,,,70 -EVA003_S91_ONT,EVA003_S91_L001_R1_001.fastq.gz,,,,,,,,,,"xpto, zpto",,xpto_1 xpto_1,ONT,False,12505/12505/100.0,,,,,,0,1,2,3,4,5,6,7,NanoStat-1.4.0,RabbitQC-0.0.1; (-w 3 -D),NanoFilt-2.6.0; (let's go again),,,Medaka-1.2.1; (consensus )/Medaka-1.2.1; (variant --verbose),Samtools-1.3; (depth -aa -q 10),Minimum depth of coverage per site to validate the sequence; (Threshold:6007),MSA Masker-1.0; (MSA parameters; for coverages less than 6007 in 40% of the regions.),,,70 -EVA002_S52_ONT,EVA002_S52_L001_R1_001.fastq.gz,,,,,,,,,,"xpto, zpto",,xpto_1 xpto_1,ONT,False,0/0/0,,,,,,0,1,2,3,4,5,6,7,NanoStat-1.4.0,RabbitQC-0.0.1; (-w 3 -D),NanoFilt-2.6.0; (let's go again),,,Medaka-1.2.1; (consensus )/Medaka-1.2.1; (variant --verbose),Samtools-1.3; (depth -aa -q 10),Minimum depth of coverage per site to validate the sequence; (Threshold:6006),MSA Masker-1.0; (MSA parameters; for coverages less than 6006 in 40% of the regions.),,,70 -EVA001_S66_ONT,EVA001_S66_L001_R1_001.fastq.gz,,,,,,,,,,"xpto, zpto",,xpto_1 xpto_1,ONT,False,0/0/0,,,,,,0,1,2,3,4,5,6,7,NanoStat-1.4.0,RabbitQC-0.0.1; (-w 3 -D),NanoFilt-2.6.0; (let's go again),,,Medaka-1.2.1; (consensus )/Medaka-1.2.1; (variant --verbose),Samtools-1.3; (depth -aa -q 10),Minimum depth of coverage per site to validate the sequence; (Threshold:6005),MSA Masker-1.0; (MSA parameters; for coverages less than 6005 in 40% of the regions.),,,70 -EVA011_S54,EVA011_S54_L001_R1_001.fastq.gz,EVA011_S54_L001_R2_001.fastq.gz,,,,,,,,,"xpto, zpto",,xpto_1 xpto_1,Illumina,False,87818/87818/100.0,,,,,,,,,,,,,,,,,FastQC-0.11.9; (let's go again),Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic),,,,,,,70 -EVA003_S91,EVA003_S91_L001_R1_001.fastq.gz,EVA003_S91_L001_R2_001.fastq.gz,,,,,,,,,"xpto, zpto",,xpto_1 xpto_1,Illumina,False,87818/87818/100.0,xpto,xpto1,xpto2,,xpto4,,,,,,,,,,,,FastQC-0.11.9; (let's go again),Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic),,,,,SPAdes-3.11.1; (let's go),Snippy-3.2-dev; (let's go_2222),70 -EVA002_S52,EVA002_S52_L001_R1_001.fastq.gz,EVA002_S52_L001_R2_001.fastq.gz,,,,,,,,,"xpto, zpto",,xpto_1 xpto_1,Not defined,False,0/0/0,xpto,xpto1,xpto2,,xpto4,,,,,,,,,,,,FastQC-0.11.9; (let's go again),Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic),,,,,SPAdes-3.11.1; (let's go),Snippy-3.2-dev; (let's go_2222),70 -EVA001_S66,EVA001_S66_L001_R1_001.fastq.gz,EVA001_S66_L001_R2_001.fastq.gz,,,,,,,,,"xpto, zpto",,xpto_1 xpto_1,Not defined,False,0/0/0,xpto,xpto1,xpto2,xpto3,xpto4,,,,,,,,,,,,FastQC-0.11.9; (let's go again),Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic),,,,,SPAdes-3.11.1; (let's go),Snippy-3.2-dev; (let's go_2222),Not applied +Reference name,second_stage_2_ test_create_tree,,,,,,,,,,,, +id,fastq1,fastq2,data set,vaccine status,week,onset date,collection date,lab reception date,latitude,longitude,classification,putative mixed-infection,xpto_1 +EVA011_S54_ONT,EVA011_S54_L001_R1_001.fastq.gz,,,,,,,,,,"xpto, zpto",,xpto_1 xpto_1 +EVA003_S91_ONT,EVA003_S91_L001_R1_001.fastq.gz,,,,,,,,,,"xpto, zpto",,xpto_1 xpto_1 +EVA002_S52_ONT,EVA002_S52_L001_R1_001.fastq.gz,,,,,,,,,,"xpto, zpto",,xpto_1 xpto_1 +EVA001_S66_ONT,EVA001_S66_L001_R1_001.fastq.gz,,,,,,,,,,"xpto, zpto",,xpto_1 xpto_1 +EVA011_S54,EVA011_S54_L001_R1_001.fastq.gz,EVA011_S54_L001_R2_001.fastq.gz,,,,,,,,,"xpto, zpto",,xpto_1 xpto_1 +EVA003_S91,EVA003_S91_L001_R1_001.fastq.gz,EVA003_S91_L001_R2_001.fastq.gz,,,,,,,,,"xpto, zpto",,xpto_1 xpto_1 +EVA002_S52,EVA002_S52_L001_R1_001.fastq.gz,EVA002_S52_L001_R2_001.fastq.gz,,,,,,,,,"xpto, zpto",,xpto_1 xpto_1 +EVA001_S66,EVA001_S66_L001_R1_001.fastq.gz,EVA001_S66_L001_R2_001.fastq.gz,,,,,,,,,"xpto, zpto",,xpto_1 xpto_1 diff --git a/static/tests/global_project/insa_flu_sample_output_2_ont.tsv b/static/tests/global_project/insa_flu_sample_output_2_ont.tsv index 7da6b173..806ba9f0 100755 --- a/static/tests/global_project/insa_flu_sample_output_2_ont.tsv +++ b/static/tests/global_project/insa_flu_sample_output_2_ont.tsv @@ -1,10 +1,10 @@ -"Reference name:" "second_stage_2_ test_create_tree" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "Trimmo. stats." "Trimmo. stats." "Trimmo. stats." "Trimmo. stats." "Trimmo. stats." "NanoStat stats." "NanoStat stats." "NanoStat stats." "NanoStat stats." "NanoStat stats." "NanoStat stats." "NanoStat stats." "NanoStat stats." "" "" "" "" "" "" "" "" "" "" "" "" -"id" "fastq1" "fastq2" "data set" "vaccine status" "week" "onset date" "collection date" "lab reception date" "latitude" "longitude" "type-subtype" "putative mixed-infection" "xpto_1" "Technology" "Sample Down sized" "Available Reads/Mapped Reads/Mapped Percentage" "Input Read Pairs" "Both Surviving" "Forward Only Surviving" "Reverse Only Surviving" "Dropped" "Mean read length - ONT" "Mean read quality - ONT" "Median read length - ONT" "Median read quality - ONT" "Number of reads - ONT" "Read length N50 - ONT" "STDEV read length - ONT" "Total bases - ONT" "NanoStat" "RabbitQC" "NanoFilt" "FastQC" "Trimmomatic" "Medaka" "Samtools" "Minimum depth of coverage per site to validate the sequence" "MSA Masker" "SPAdes" "Snippy" "Threshold to mask consensus sequences" -"EVA011_S54_ONT" "EVA011_S54_L001_R1_001.fastq.gz" "" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto_1 xpto_1" "ONT" "False" "0/0/0" "" "" "" "" "" "0" "1" "2" "3" "4" "5" "6" "7" "NanoStat-1.4.0" "RabbitQC-0.0.1; (-w 3 -D)" "NanoFilt-2.6.0; (let's go again)" "" "" "" "" "" "" "" "" "70" -"EVA003_S91_ONT" "EVA003_S91_L001_R1_001.fastq.gz" "" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto_1 xpto_1" "ONT" "False" "12505/12505/100.0" "" "" "" "" "" "0" "1" "2" "3" "4" "5" "6" "7" "NanoStat-1.4.0" "RabbitQC-0.0.1; (-w 3 -D)" "NanoFilt-2.6.0; (let's go again)" "" "" "Medaka-1.2.1; (consensus )/Medaka-1.2.1; (variant --verbose)" "Samtools-1.3; (depth -aa -q 10)" "Minimum depth of coverage per site to validate the sequence; (Threshold:6007)" "MSA Masker-1.0; (MSA parameters; for coverages less than 6007 in 40% of the regions.)" "" "" "70" -"EVA002_S52_ONT" "EVA002_S52_L001_R1_001.fastq.gz" "" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto_1 xpto_1" "ONT" "False" "0/0/0" "" "" "" "" "" "0" "1" "2" "3" "4" "5" "6" "7" "NanoStat-1.4.0" "RabbitQC-0.0.1; (-w 3 -D)" "NanoFilt-2.6.0; (let's go again)" "" "" "Medaka-1.2.1; (consensus )/Medaka-1.2.1; (variant --verbose)" "Samtools-1.3; (depth -aa -q 10)" "Minimum depth of coverage per site to validate the sequence; (Threshold:6006)" "MSA Masker-1.0; (MSA parameters; for coverages less than 6006 in 40% of the regions.)" "" "" "70" -"EVA001_S66_ONT" "EVA001_S66_L001_R1_001.fastq.gz" "" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto_1 xpto_1" "ONT" "False" "0/0/0" "" "" "" "" "" "0" "1" "2" "3" "4" "5" "6" "7" "NanoStat-1.4.0" "RabbitQC-0.0.1; (-w 3 -D)" "NanoFilt-2.6.0; (let's go again)" "" "" "Medaka-1.2.1; (consensus )/Medaka-1.2.1; (variant --verbose)" "Samtools-1.3; (depth -aa -q 10)" "Minimum depth of coverage per site to validate the sequence; (Threshold:6005)" "MSA Masker-1.0; (MSA parameters; for coverages less than 6005 in 40% of the regions.)" "" "" "70" -"EVA011_S54" "EVA011_S54_L001_R1_001.fastq.gz" "EVA011_S54_L001_R2_001.fastq.gz" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto_1 xpto_1" "Illumina" "False" "87818/87818/100.0" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "FastQC-0.11.9; (let's go again)" "Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic)" "" "" "" "" "" "" "70" -"EVA003_S91" "EVA003_S91_L001_R1_001.fastq.gz" "EVA003_S91_L001_R2_001.fastq.gz" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto_1 xpto_1" "Illumina" "False" "87818/87818/100.0" "xpto" "xpto1" "xpto2" "" "xpto4" "" "" "" "" "" "" "" "" "" "" "" "FastQC-0.11.9; (let's go again)" "Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic)" "" "" "" "" "SPAdes-3.11.1; (let's go)" "Snippy-3.2-dev; (let's go_2222)" "70" -"EVA002_S52" "EVA002_S52_L001_R1_001.fastq.gz" "EVA002_S52_L001_R2_001.fastq.gz" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto_1 xpto_1" "Not defined" "False" "0/0/0" "xpto" "xpto1" "xpto2" "" "xpto4" "" "" "" "" "" "" "" "" "" "" "" "FastQC-0.11.9; (let's go again)" "Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic)" "" "" "" "" "SPAdes-3.11.1; (let's go)" "Snippy-3.2-dev; (let's go_2222)" "70" -"EVA001_S66" "EVA001_S66_L001_R1_001.fastq.gz" "EVA001_S66_L001_R2_001.fastq.gz" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto_1 xpto_1" "Not defined" "False" "0/0/0" "xpto" "xpto1" "xpto2" "xpto3" "xpto4" "" "" "" "" "" "" "" "" "" "" "" "FastQC-0.11.9; (let's go again)" "Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic)" "" "" "" "" "SPAdes-3.11.1; (let's go)" "Snippy-3.2-dev; (let's go_2222)" "Not applied" +"Reference name" "second_stage_2_ test_create_tree" "" "" "" "" "" "" "" "" "" "" "" "" +"id" "fastq1" "fastq2" "data set" "vaccine status" "week" "onset date" "collection date" "lab reception date" "latitude" "longitude" "classification" "putative mixed-infection" "xpto_1" +"EVA011_S54_ONT" "EVA011_S54_L001_R1_001.fastq.gz" "" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto_1 xpto_1" +"EVA003_S91_ONT" "EVA003_S91_L001_R1_001.fastq.gz" "" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto_1 xpto_1" +"EVA002_S52_ONT" "EVA002_S52_L001_R1_001.fastq.gz" "" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto_1 xpto_1" +"EVA001_S66_ONT" "EVA001_S66_L001_R1_001.fastq.gz" "" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto_1 xpto_1" +"EVA011_S54" "EVA011_S54_L001_R1_001.fastq.gz" "EVA011_S54_L001_R2_001.fastq.gz" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto_1 xpto_1" +"EVA003_S91" "EVA003_S91_L001_R1_001.fastq.gz" "EVA003_S91_L001_R2_001.fastq.gz" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto_1 xpto_1" +"EVA002_S52" "EVA002_S52_L001_R1_001.fastq.gz" "EVA002_S52_L001_R2_001.fastq.gz" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto_1 xpto_1" +"EVA001_S66" "EVA001_S66_L001_R1_001.fastq.gz" "EVA001_S66_L001_R2_001.fastq.gz" "" "" "" "" "" "" "" "" "xpto, zpto" "" "xpto_1 xpto_1" diff --git a/static/tests/global_project/insa_flu_sample_output_2_ont_settings.csv b/static/tests/global_project/insa_flu_sample_output_2_ont_settings.csv new file mode 100755 index 00000000..0aed375e --- /dev/null +++ b/static/tests/global_project/insa_flu_sample_output_2_ont_settings.csv @@ -0,0 +1,10 @@ +Reference name:,second_stage_2_ test_create_tree,,,,,,,,Trimmo. stats.,Trimmo. stats.,Trimmo. stats.,Trimmo. stats.,Trimmo. stats.,NanoStat stats.,NanoStat stats.,NanoStat stats.,NanoStat stats.,NanoStat stats.,NanoStat stats.,NanoStat stats.,NanoStat stats.,,,,,,,,,,,, +id,fastq1,fastq2,sample date created,Technology,Sample Down sized,Available Reads for Mapping,Mapped Reads,Mapped Percentage,Input Read Pairs,Both Surviving,Forward Only Surviving,Reverse Only Surviving,Dropped,Mean read length - ONT,Mean read quality - ONT,Median read length - ONT,Median read quality - ONT,Number of reads - ONT,Read length N50 - ONT,STDEV read length - ONT,Total bases - ONT,NanoStat,RabbitQC,NanoFilt,FastQC,Trimmomatic,Medaka,Samtools,Minimum depth of coverage per site to validate the sequence,MSA Masker,SPAdes,Snippy,Threshold to mask consensus sequences +EVA011_S54_ONT,EVA011_S54_L001_R1_001.fastq.gz,,2010-10-23,ONT,False,-,-,-,,,,,,0,1,2,3,4,5,6,7,NanoStat-1.4.0,RabbitQC-0.0.1; (-w 3 -D),NanoFilt-2.6.0; (let's go again),,,,,,,,,70 +EVA003_S91_ONT,EVA003_S91_L001_R1_001.fastq.gz,,2010-10-23,ONT,False,4,12505,312625.0,,,,,,0,1,2,3,4,5,6,7,NanoStat-1.4.0,RabbitQC-0.0.1; (-w 3 -D),NanoFilt-2.6.0; (let's go again),,,Medaka-1.2.1; (consensus )/Medaka-1.2.1; (variant --verbose),Samtools-1.3; (depth -aa -q 10),Minimum depth of coverage per site to validate the sequence; (Threshold:6007),MSA Masker-1.0; (MSA parameters; for coverages less than 6007 in 40% of the regions.),,,70 +EVA002_S52_ONT,EVA002_S52_L001_R1_001.fastq.gz,,2010-10-23,ONT,False,-,-,-,,,,,,0,1,2,3,4,5,6,7,NanoStat-1.4.0,RabbitQC-0.0.1; (-w 3 -D),NanoFilt-2.6.0; (let's go again),,,Medaka-1.2.1; (consensus )/Medaka-1.2.1; (variant --verbose),Samtools-1.3; (depth -aa -q 10),Minimum depth of coverage per site to validate the sequence; (Threshold:6006),MSA Masker-1.0; (MSA parameters; for coverages less than 6006 in 40% of the regions.),,,70 +EVA001_S66_ONT,EVA001_S66_L001_R1_001.fastq.gz,,2010-10-23,ONT,False,-,-,-,,,,,,0,1,2,3,4,5,6,7,NanoStat-1.4.0,RabbitQC-0.0.1; (-w 3 -D),NanoFilt-2.6.0; (let's go again),,,Medaka-1.2.1; (consensus )/Medaka-1.2.1; (variant --verbose),Samtools-1.3; (depth -aa -q 10),Minimum depth of coverage per site to validate the sequence; (Threshold:6005),MSA Masker-1.0; (MSA parameters; for coverages less than 6005 in 40% of the regions.),,,70 +EVA011_S54,EVA011_S54_L001_R1_001.fastq.gz,EVA011_S54_L001_R2_001.fastq.gz,2010-10-23,Illumina,False,-,-,-,,,,,,,,,,,,,,,,,FastQC-0.11.9; (let's go again),Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic),,,,,,,70 +EVA003_S91,EVA003_S91_L001_R1_001.fastq.gz,EVA003_S91_L001_R2_001.fastq.gz,2010-10-23,Illumina,False,-,-,-,xpto,xpto1,xpto2,,xpto4,,,,,,,,,,,,FastQC-0.11.9; (let's go again),Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic),,,,,SPAdes-3.11.1; (let's go),Snippy-3.2-dev; (let's go_2222),70 +EVA002_S52,EVA002_S52_L001_R1_001.fastq.gz,EVA002_S52_L001_R2_001.fastq.gz,2010-10-23,Not defined,False,-,-,-,xpto,xpto1,xpto2,,xpto4,,,,,,,,,,,,FastQC-0.11.9; (let's go again),Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic),,,,,SPAdes-3.11.1; (let's go),Snippy-3.2-dev; (let's go_2222),70 +EVA001_S66,EVA001_S66_L001_R1_001.fastq.gz,EVA001_S66_L001_R2_001.fastq.gz,2010-10-23,Not defined,False,-,-,-,xpto,xpto1,xpto2,xpto3,xpto4,,,,,,,,,,,,FastQC-0.11.9; (let's go again),Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic),,,,,SPAdes-3.11.1; (let's go),Snippy-3.2-dev; (let's go_2222),Not applied diff --git a/static/tests/global_project/insa_flu_sample_output_2_ont_simple.csv b/static/tests/global_project/insa_flu_sample_output_2_ont_simple.csv index 5caa4cb8..f20e9aa3 100755 --- a/static/tests/global_project/insa_flu_sample_output_2_ont_simple.csv +++ b/static/tests/global_project/insa_flu_sample_output_2_ont_simple.csv @@ -1,4 +1,4 @@ -id,data set,vaccine status,week,onset date,collection date,lab reception date,latitude,longitude,type-subtype,putative mixed-infection,xpto_1 +id,data set,vaccine status,week,onset date,collection date,lab reception date,latitude,longitude,classification,putative mixed-infection,xpto_1 second_stage_2_ test_create_tree,,,,,,,,,,, EVA011_S54_ONT,,,,,,,,,"xpto, zpto",,xpto_1 xpto_1 EVA003_S91_ONT,,,,,,,,,"xpto, zpto",,xpto_1 xpto_1 diff --git a/static/tests/global_project/insa_flu_sample_output_2_settings.csv b/static/tests/global_project/insa_flu_sample_output_2_settings.csv new file mode 100755 index 00000000..7453af20 --- /dev/null +++ b/static/tests/global_project/insa_flu_sample_output_2_settings.csv @@ -0,0 +1,6 @@ +Reference name:,second_stage_2_ test_create_tree,,,,,,,,Trimmo. stats.,Trimmo. stats.,Trimmo. stats.,Trimmo. stats.,Trimmo. stats.,,,,,, +id,fastq1,fastq2,sample date created,Technology,Sample Down sized,Available Reads for Mapping,Mapped Reads,Mapped Percentage,Input Read Pairs,Both Surviving,Forward Only Surviving,Reverse Only Surviving,Dropped,FastQC,illumina_stat,Trimmomatic,SPAdes,Snippy,Threshold to mask consensus sequences +EVA011_S54,EVA011_S54_L001_R1_001.fastq.gz,EVA011_S54_L001_R2_001.fastq.gz,2010-10-23,Illumina,False,20001,87818,439.1,,,,,,FastQC-0.11.9; (let's go again),illumina_stat,Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic),,,70 +EVA003_S91,EVA003_S91_L001_R1_001.fastq.gz,EVA003_S91_L001_R2_001.fastq.gz,2010-10-23,Illumina,False,20001,87818,439.1,10,5,2,,44,FastQC-0.11.9; (let's go again),illumina_stat,Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic),SPAdes-3.11.1; (let's go),Snippy-3.2-dev; (let's go_2222),70 +EVA002_S52,EVA002_S52_L001_R1_001.fastq.gz,EVA002_S52_L001_R2_001.fastq.gz,2010-10-23,Illumina,False,20001,87818,439.1,10,5,2,,44,FastQC-0.11.9; (let's go again),illumina_stat,Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic),SPAdes-3.11.1; (let's go),Snippy-3.2-dev; (let's go_2222),70 +EVA001_S66,EVA001_S66_L001_R1_001.fastq.gz,EVA001_S66_L001_R2_001.fastq.gz,2010-10-23,Illumina,False,20001,87818,439.1,10,5,2,33,44,FastQC-0.11.9; (let's go again),illumina_stat,Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic),SPAdes-3.11.1; (let's go),Snippy-3.2-dev; (let's go_2222),Not applied diff --git a/static/tests/global_project/insa_flu_sample_output_2_simple.csv b/static/tests/global_project/insa_flu_sample_output_2_simple.csv new file mode 100755 index 00000000..b6062b70 --- /dev/null +++ b/static/tests/global_project/insa_flu_sample_output_2_simple.csv @@ -0,0 +1,6 @@ +id,data set,vaccine status,week,onset date,collection date,lab reception date,latitude,longitude,classification,putative mixed-infection,xpto_1 +second_stage_2_ test_create_tree,,,,,,,,,,, +EVA011_S54,,,,,,,,,"xpto, zpto",,xpto_1 xpto_1 +EVA003_S91,,,,,,,,,"xpto, zpto",,xpto_1 xpto_1 +EVA002_S52,,,,,,,,,"xpto, zpto",,xpto_1 xpto_1 +EVA001_S66,,,,,,,,,"xpto, zpto",,xpto_1 xpto_1 diff --git a/static/tests/global_project/insa_flu_sample_output_ont.json b/static/tests/global_project/insa_flu_sample_output_ont.json index 4ba0dc07..e15b5501 100755 --- a/static/tests/global_project/insa_flu_sample_output_ont.json +++ b/static/tests/global_project/insa_flu_sample_output_ont.json @@ -1 +1 @@ -{"second_stage_2_ test_create_tree": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "type-subtype": "", "putative mixed-infection": "", "xpto_1": ""}, "EVA011_S54_ONT": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "type-subtype": "xpto, zpto", "putative mixed-infection": "", "xpto_1": "xpto_1 xpto_1"}, "EVA003_S91_ONT": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "type-subtype": "xpto, zpto", "putative mixed-infection": "", "xpto_1": "xpto_1 xpto_1"}, "EVA002_S52_ONT": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "type-subtype": "xpto, zpto", "putative mixed-infection": "", "xpto_1": "xpto_1 xpto_1"}, "EVA001_S66_ONT": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "type-subtype": "xpto, zpto", "putative mixed-infection": "", "xpto_1": "xpto_1 xpto_1"}, "EVA011_S54": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "type-subtype": "xpto, zpto", "putative mixed-infection": "", "xpto_1": "xpto_1 xpto_1"}, "EVA003_S91": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "type-subtype": "xpto, zpto", "putative mixed-infection": "", "xpto_1": "xpto_1 xpto_1"}, "EVA002_S52": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "type-subtype": "xpto, zpto", "putative mixed-infection": "", "xpto_1": "xpto_1 xpto_1"}, "EVA001_S66": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "type-subtype": "xpto, zpto", "putative mixed-infection": "", "xpto_1": "xpto_1 xpto_1"}} \ No newline at end of file +{"second_stage_2_ test_create_tree": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "classification": "", "putative mixed-infection": "", "xpto_1": ""}, "EVA011_S54_ONT": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "classification": "xpto, zpto", "putative mixed-infection": "", "xpto_1": "xpto_1 xpto_1"}, "EVA003_S91_ONT": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "classification": "xpto, zpto", "putative mixed-infection": "", "xpto_1": "xpto_1 xpto_1"}, "EVA002_S52_ONT": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "classification": "xpto, zpto", "putative mixed-infection": "", "xpto_1": "xpto_1 xpto_1"}, "EVA001_S66_ONT": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "classification": "xpto, zpto", "putative mixed-infection": "", "xpto_1": "xpto_1 xpto_1"}, "EVA011_S54": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "classification": "xpto, zpto", "putative mixed-infection": "", "xpto_1": "xpto_1 xpto_1"}, "EVA003_S91": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "classification": "xpto, zpto", "putative mixed-infection": "", "xpto_1": "xpto_1 xpto_1"}, "EVA002_S52": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "classification": "xpto, zpto", "putative mixed-infection": "", "xpto_1": "xpto_1 xpto_1"}, "EVA001_S66": {"data set": "", "vaccine status": "", "week": "", "onset date": "", "collection date": "", "lab reception date": "", "latitude": "", "longitude": "", "classification": "xpto, zpto", "putative mixed-infection": "", "xpto_1": "xpto_1 xpto_1"}} \ No newline at end of file diff --git a/static/tests/global_project/insa_flu_sample_output_settings.csv b/static/tests/global_project/insa_flu_sample_output_settings.csv new file mode 100755 index 00000000..b4a6235d --- /dev/null +++ b/static/tests/global_project/insa_flu_sample_output_settings.csv @@ -0,0 +1,6 @@ +Reference name,second_stage_2_ test_create_tree,,,,,,,,,,,,, +id,fastq1,fastq2,sample date created,Technology,Sample Down sized,Available Reads for Mapping,Mapped Reads,Mapped Percentage,FastQC,Trimmomatic,Mask consensus by sites,SPAdes,Snippy,Threshold to mask consensus sequences +EVA011_S54,EVA011_S54_L001_R1_001.fastq.gz,EVA011_S54_L001_R2_001.fastq.gz,2010-10-23,Illumina,False,-,-,-,FastQC-0.11.9; (let's go again),Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic),"Element:element_name -> (Mask sites:2,5 Mask regions:1-30,32-50,320-500 Mask from beginning:2 Mask from end:2)",,,Not applied +EVA003_S91,EVA003_S91_L001_R1_001.fastq.gz,EVA003_S91_L001_R2_001.fastq.gz,2010-10-23,Illumina,False,-,-,-,FastQC-0.11.9; (let's go again),Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic),"Element:element_name -> (Mask sites:2,5 Mask regions:1-30,32-50,320-500 Mask from beginning:2 Mask from end:2)",SPAdes-3.11.1; (let's go),Snippy-3.2-dev; (let's go_2222),Not applied +EVA002_S52,EVA002_S52_L001_R1_001.fastq.gz,EVA002_S52_L001_R2_001.fastq.gz,2010-10-23,Illumina,False,-,-,-,FastQC-0.11.9; (let's go again),Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic),"Element:element_name -> (Mask sites:2,5 Mask regions:1-30,32-50,320-500 Mask from beginning:2 Mask from end:2)",SPAdes-3.11.1; (let's go),Snippy-3.2-dev; (let's go_2222),Not applied +EVA001_S66,EVA001_S66_L001_R1_001.fastq.gz,EVA001_S66_L001_R2_001.fastq.gz,2010-10-23,Illumina,False,-,-,-,FastQC-0.11.9; (let's go again),Trimmomatic-0.39; (let's go again)/Trimmomatic-0.39; (let's go againsecond version trimmomatic),"Element:element_name -> (Mask sites:2,5 Mask regions:1-30,32-50,320-500 Mask from beginning:2 Mask from end:2)",SPAdes-3.11.1; (let's go),Snippy-3.2-dev; (let's go_2222),Not applied diff --git a/static/tests/global_project/insa_flu_sample_output_simple.csv b/static/tests/global_project/insa_flu_sample_output_simple.csv index 6e3af43f..f71a6c72 100755 --- a/static/tests/global_project/insa_flu_sample_output_simple.csv +++ b/static/tests/global_project/insa_flu_sample_output_simple.csv @@ -1,4 +1,4 @@ -id,data set,vaccine status,week,onset date,collection date,lab reception date,latitude,longitude,type-subtype,putative mixed-infection,xpto +id,data set,vaccine status,week,onset date,collection date,lab reception date,latitude,longitude,classification,putative mixed-infection,xpto second_stage_2_ test_create_tree,,,,,,,,,,, EVA011_S54,,,,,,,,,"xpto, zpto",,xpto xpto EVA003_S91,,,,,,,,,"xpto, zpto",,xpto xpto diff --git a/templates/project_sample/project_sample_show.html b/templates/project_sample/project_sample_show.html index ef8f98a6..f81848ac 100644 --- a/templates/project_sample/project_sample_show.html +++ b/templates/project_sample/project_sample_show.html @@ -292,24 +292,58 @@
- Result List: {{ sample_file_result_csv }} - {{ sample_file_result_tsv }} - - {% if pangolin_lineage %} - Pangolin Lineage: {{ pangolin_lineage }} - {% endif %} +
+ +
+ {% if sample_file_result_csv %} {{ sample_file_result_csv }} {% endif %} + {% if sample_file_result_tsv %} {{ sample_file_result_tsv }} {% endif %} + {% if samples_file_settings_statistics_csv %} +
+ {{ samples_file_settings_statistics_csv }} + {% endif %} + {% if samples_file_settings_statistics_tsv %} {{ samples_file_settings_statistics_tsv }} {% endif %} + + {% if sample_file_all_consensus %} +
+ {{ sample_file_all_consensus }} + {% endif %} + {% if samples_file_coverage or samples_file_variants or samples_file_minor_intra_host %} +
+ {% if samples_file_coverage %} {{ samples_file_coverage }} {% endif %} + {% if samples_file_variants %} {{ samples_file_variants }} {% endif %} + {% if samples_file_minor_intra_host %} {{ samples_file_minor_intra_host }} {% endif %} + {% endif %} + + {% if pangolin_lineage %} +
+ {{ pangolin_lineage }} + {% endif %} + {% if download_all_files %} +
+ {{ download_all_files }} + {% endif %} +
+
- All consensus: {{ sample_file_all_consensus }} - {% if nextclade_link %} - - - - - - - - + {% if nextclade_link_covid or nextclade_link_a_h3n2 %} +
+ +
+ {% if nextclade_link_covid %} + (SARS-Cov 2) + {% else %} + (A-H3N2) + (A-H1N1) + (B-Yamagata) + (B-Vitoria) + {% endif %} +
+
{% endif %}
diff --git a/templates/project_sample/project_sample_single_detail.html b/templates/project_sample/project_sample_single_detail.html index 088014ac..f02787db 100644 --- a/templates/project_sample/project_sample_single_detail.html +++ b/templates/project_sample/project_sample_single_detail.html @@ -124,15 +124,26 @@
Consensus: {{ consensus_file }} - {% if nextclade_link %} - - + + {% if nextclade_link_covid or nextclade_link_a_h3n2 %} +
+ +
+ {% if nextclade_link_covid %} + (SARS-Cov 2) + {% else %} + (A-H3N2) + (A-H1N1) + (B-Yamagata) + (B-Vitoria) + {% endif %} +
+
{% endif %}
{% if medaka_variants_file %} diff --git a/templates/samples/samples.html b/templates/samples/samples.html index db9a4154..bea834b6 100644 --- a/templates/samples/samples.html +++ b/templates/samples/samples.html @@ -39,9 +39,9 @@ Add Samples
- Add Sample metadata + Add One Sample
- Add Samples from csv/tsv file + Add Multiple Samples Add Fastq Files Update metadata
diff --git a/tests_insaflu/tests_collect_extra_data.py b/tests_insaflu/tests_collect_extra_data.py index e170371d..6133c993 100644 --- a/tests_insaflu/tests_collect_extra_data.py +++ b/tests_insaflu/tests_collect_extra_data.py @@ -339,17 +339,25 @@ def test_create_tree_and_alignments(self): ### samples test expected_file_samples = os.path.join(self.baseDirectory, ConstantsTestsCase.DIR_GLOBAL_PROJECT, "insa_flu_sample_output.csv") - out_file = collect_extra_data.collect_sample_table(project, Constants.SEPARATOR_COMMA) - + out_file = collect_extra_data.collect_sample_table(project, Constants.SEPARATOR_COMMA, CollectExtraData.SAMPLE_LIST_list) + self.assertTrue(os.path.exists(out_file)) + self.assertTrue(filecmp.cmp(out_file, expected_file_samples)) + self.utils.copy_file(out_file, + project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_SAMPLE_RESULT_CSV)) + if (os.path.exists(out_file)): os.unlink(out_file) + + ### samples test + expected_file_samples = os.path.join(self.baseDirectory, ConstantsTestsCase.DIR_GLOBAL_PROJECT, "insa_flu_sample_output_settings.csv") + out_file = collect_extra_data.collect_sample_table(project, Constants.SEPARATOR_COMMA, CollectExtraData.SAMPLE_LIST_list_settings) self.assertTrue(os.path.exists(out_file)) self.assertTrue(filecmp.cmp(out_file, expected_file_samples)) + self.utils.copy_file(out_file, + project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_SAMPLE_RESULT_SETTINGS_CSV)) if (os.path.exists(out_file)): os.unlink(out_file) ### samples test CSV simple expected_file_samples = os.path.join(self.baseDirectory, ConstantsTestsCase.DIR_GLOBAL_PROJECT, "insa_flu_sample_output_simple.csv") - b_simple = True - out_file = collect_extra_data.collect_sample_table(project, Constants.SEPARATOR_COMMA, b_simple) - + out_file = collect_extra_data.collect_sample_table(project, Constants.SEPARATOR_COMMA, CollectExtraData.SAMPLE_LIST_simple) self.assertTrue(os.path.exists(out_file)) self.assertTrue(filecmp.cmp(out_file, expected_file_samples)) self.utils.copy_file(out_file, @@ -358,8 +366,7 @@ def test_create_tree_and_alignments(self): ### samples test TSV expected_file_samples = os.path.join(self.baseDirectory, ConstantsTestsCase.DIR_GLOBAL_PROJECT, "insa_flu_sample_output.tsv") - out_file = collect_extra_data.collect_sample_table(project, Constants.SEPARATOR_TAB) - + out_file = collect_extra_data.collect_sample_table(project, Constants.SEPARATOR_TAB, CollectExtraData.SAMPLE_LIST_list) self.assertTrue(os.path.exists(out_file)) self.assertTrue(filecmp.cmp(out_file, expected_file_samples)) if (os.path.exists(out_file)): os.unlink(out_file) @@ -367,7 +374,6 @@ def test_create_tree_and_alignments(self): ### collect variations from snippy out_file = collect_extra_data.collect_variations_snippy(project) expected_file_samples = os.path.join(self.baseDirectory, ConstantsTestsCase.DIR_GLOBAL_PROJECT, "insa_flu_variations_snippy.tsv") - self.assertTrue(filecmp.cmp(out_file, expected_file_samples)) if (os.path.exists(out_file)): os.unlink(out_file) @@ -375,6 +381,8 @@ def test_create_tree_and_alignments(self): vect_type_remove = ['ins', 'del'] out_file = collect_extra_data.collect_variations_freebayes(project, vect_type_remove) expected_file_samples = os.path.join(self.baseDirectory, ConstantsTestsCase.DIR_GLOBAL_PROJECT, "insa_flu_variations_freebayes.tsv") + self.utils.copy_file(out_file, + project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_TAB_VARIATIONS_FREEBAYES)) self.assertTrue(filecmp.cmp(out_file, expected_file_samples)) if (os.path.exists(out_file)): os.unlink(out_file) @@ -392,6 +400,12 @@ def test_create_tree_and_alignments(self): self.assertTrue(filecmp.cmp(out_file, expected_file_samples)) if (os.path.exists(out_file)): os.unlink(out_file) + ### zip file, zip files change every time that are produced + collect_extra_data.zip_several_files(project) + out_file = project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_all_files_zipped) + self.assertTrue(os.path.exists(out_file)) + self.assertTrue(os.path.getsize(out_file) > 1616) + ### masking data manageDatabase = ManageDatabase() geneticElement = GeneticElement() @@ -431,7 +445,7 @@ def test_create_tree_and_alignments(self): self.assertTrue(filecmp.cmp(project_sample.get_consensus_file(TypePath.MEDIA_ROOT), expected_file_samples)) self.assertTrue(os.path.exists(project_sample.get_backup_consensus_file())) self.assertFalse(filecmp.cmp(project_sample.get_consensus_file(TypePath.MEDIA_ROOT), project_sample.get_backup_consensus_file())) - + ### get sample result file self.utils.remove_dir(temp_dir) self.utils.remove_dir(getattr(settings, "MEDIA_ROOT", None)) @@ -544,14 +558,21 @@ def test_create_tree_and_alignments_2(self): manage_database.set_sample_metakey(sample, user, MetaKeyAndValue.META_KEY_Identify_Sample_Software, MetaKeyAndValue.META_VALUE_Success, result_all_2.to_json()) + ### set statistics of number of reads + key_values = KeyValues() + for _, key in enumerate(SoftwareNames.SOFTWARE_ILLUMINA_stat_collect): + key_values.add_key_value(KeyValue(key, str(_ + 10000))) + result_all_2.add_software(SoftwareDesc(SoftwareNames.SOFTWARE_ILLUMINA_stat, "", "", + key_values)) + key_values = None if (n_id < 6004): key_values = KeyValues() - key_values.add_key_value(KeyValue("Input Read Pairs:", "xpto")) - key_values.add_key_value(KeyValue("Both Surviving:", "xpto1")) - key_values.add_key_value(KeyValue("Forward Only Surviving:", "xpto2")) - if (n_id == 6001): key_values.add_key_value(KeyValue("Reverse Only Surviving:", "xpto3")) - key_values.add_key_value(KeyValue("Dropped:", "xpto4")) + key_values.add_key_value(KeyValue("Input Read Pairs:", "10")) + key_values.add_key_value(KeyValue("Both Surviving:", "5")) + key_values.add_key_value(KeyValue("Forward Only Surviving:", "2")) + if (n_id == 6001): key_values.add_key_value(KeyValue("Reverse Only Surviving:", "33")) + key_values.add_key_value(KeyValue("Dropped:", "44")) result_all_2.add_software(SoftwareDesc(software_names.get_trimmomatic_name(), software_names.get_trimmomatic_version(), parameters, key_values)) result_all_2.add_software(SoftwareDesc(software_names.get_trimmomatic_name(), @@ -619,21 +640,27 @@ def test_create_tree_and_alignments_2(self): ### samples test expected_file_samples = os.path.join(self.baseDirectory, ConstantsTestsCase.DIR_GLOBAL_PROJECT, "insa_flu_sample_output_2.csv") - out_file = collect_extra_data.collect_sample_table(project, Constants.SEPARATOR_COMMA) - + out_file = collect_extra_data.collect_sample_table(project, Constants.SEPARATOR_COMMA, CollectExtraData.SAMPLE_LIST_list) + self.assertTrue(os.path.exists(out_file)) + self.assertTrue(filecmp.cmp(out_file, expected_file_samples)) + if (os.path.exists(out_file)): os.unlink(out_file) + + expected_file_samples = os.path.join(self.baseDirectory, ConstantsTestsCase.DIR_GLOBAL_PROJECT, "insa_flu_sample_output_2_settings.csv") + out_file = collect_extra_data.collect_sample_table(project, Constants.SEPARATOR_COMMA, CollectExtraData.SAMPLE_LIST_list_settings) self.assertTrue(os.path.exists(out_file)) self.assertTrue(filecmp.cmp(out_file, expected_file_samples)) if (os.path.exists(out_file)): os.unlink(out_file) - b_simple = True - out_file = collect_extra_data.collect_sample_table(project, Constants.SEPARATOR_COMMA, b_simple) + expected_file_samples = os.path.join(self.baseDirectory, ConstantsTestsCase.DIR_GLOBAL_PROJECT, "insa_flu_sample_output_2_simple.csv") + out_file = collect_extra_data.collect_sample_table(project, Constants.SEPARATOR_COMMA, CollectExtraData.SAMPLE_LIST_simple) self.assertTrue(os.path.exists(out_file)) self.utils.copy_file(out_file, project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_SAMPLE_RESULT_CSV_simple)) + self.assertTrue(filecmp.cmp(out_file, expected_file_samples)) + if (os.path.exists(out_file)): os.unlink(out_file) expected_file_samples = os.path.join(self.baseDirectory, ConstantsTestsCase.DIR_GLOBAL_PROJECT, "insa_flu_sample_output_2.tsv") - out_file = collect_extra_data.collect_sample_table(project, Constants.SEPARATOR_TAB) - + out_file = collect_extra_data.collect_sample_table(project, Constants.SEPARATOR_TAB, CollectExtraData.SAMPLE_LIST_list) self.assertTrue(os.path.exists(out_file)) self.assertTrue(filecmp.cmp(out_file, expected_file_samples)) if (os.path.exists(out_file)): os.unlink(out_file) @@ -914,8 +941,8 @@ def test_create_tree_and_alignments_3(self): if (count_before < 6007): key_values = KeyValues() for _, key in enumerate(SoftwareNames.SOFTWARE_NANOSTAT_vect_info_to_collect): - key_values.add_key_value(KeyValue(key, _)) - if (n_id == 6005): key_values.add_key_value(KeyValue(key, _ * 2)) + key_values.add_key_value(KeyValue(key, str(_))) + if (n_id == 6005): key_values.add_key_value(KeyValue(key, str(_ * 2))) result_all_2.add_software(SoftwareDesc(software_names.get_NanoStat_name(), software_names.get_NanoStat_version(), software_names.get_NanoStat_parameters(), key_values)) result_all_2.add_software(SoftwareDesc(software_names.get_rabbitQC_name(), software_names.get_rabbitQC_version(), @@ -1004,23 +1031,28 @@ def test_create_tree_and_alignments_3(self): ### samples test expected_file_samples = os.path.join(self.baseDirectory, ConstantsTestsCase.DIR_GLOBAL_PROJECT, "insa_flu_sample_output_2_ont.csv") - out_file = collect_extra_data.collect_sample_table(project, Constants.SEPARATOR_COMMA) + out_file = collect_extra_data.collect_sample_table(project, Constants.SEPARATOR_COMMA, CollectExtraData.SAMPLE_LIST_list) + self.assertTrue(os.path.exists(out_file)) + self.assertTrue(filecmp.cmp(out_file, expected_file_samples)) + if (os.path.exists(out_file)): os.unlink(out_file) + + expected_file_samples = os.path.join(self.baseDirectory, ConstantsTestsCase.DIR_GLOBAL_PROJECT, "insa_flu_sample_output_2_ont_settings.csv") + out_file = collect_extra_data.collect_sample_table(project, Constants.SEPARATOR_COMMA, CollectExtraData.SAMPLE_LIST_list_settings) self.assertTrue(os.path.exists(out_file)) self.assertTrue(filecmp.cmp(out_file, expected_file_samples)) if (os.path.exists(out_file)): os.unlink(out_file) ### samples test expected_file_samples = os.path.join(self.baseDirectory, ConstantsTestsCase.DIR_GLOBAL_PROJECT, "insa_flu_sample_output_2_ont_simple.csv") - b_simple = True - out_file = collect_extra_data.collect_sample_table(project, Constants.SEPARATOR_COMMA, b_simple) + out_file = collect_extra_data.collect_sample_table(project, Constants.SEPARATOR_COMMA, CollectExtraData.SAMPLE_LIST_simple) self.assertTrue(os.path.exists(out_file)) self.assertTrue(filecmp.cmp(out_file, expected_file_samples)) - self.utils.copy_file(out_file, + self.utils.copy_file(out_file, project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_SAMPLE_RESULT_CSV_simple)) if (os.path.exists(out_file)): os.unlink(out_file) expected_file_samples = os.path.join(self.baseDirectory, ConstantsTestsCase.DIR_GLOBAL_PROJECT, "insa_flu_sample_output_2_ont.tsv") - out_file = collect_extra_data.collect_sample_table(project, Constants.SEPARATOR_TAB) + out_file = collect_extra_data.collect_sample_table(project, Constants.SEPARATOR_TAB, CollectExtraData.SAMPLE_LIST_list) self.assertTrue(os.path.exists(out_file)) self.assertTrue(filecmp.cmp(out_file, expected_file_samples)) if (os.path.exists(out_file)): os.unlink(out_file) diff --git a/tests_insaflu/tests_software.py b/tests_insaflu/tests_software.py index 3a9650af..3aa05e4c 100644 --- a/tests_insaflu/tests_software.py +++ b/tests_insaflu/tests_software.py @@ -407,9 +407,10 @@ def test_run_fastq_and_trimmomatic(self): self.assertEqual('139.1', result_average.average_file_2) ### get key values of data - stats_illumina = self.software.get_stats_from_sample_reads(sample) + stats_illumina, total_reads = self.software.get_stats_from_sample_reads(sample) self.assertEqual('Number of reads R1', stats_illumina[0][0]) self.assertEqual('44,425', stats_illumina[0][1]) + self.assertEqual(82508, total_reads) self.assertEqual('41,254', stats_illumina[0][2]) self.assertEqual('-3,171.0', stats_illumina[0][3]) self.assertEqual('92.9', stats_illumina[0][4]) @@ -418,7 +419,7 @@ def test_run_fastq_and_trimmomatic(self): self.assertEqual('142.0', stats_illumina[1][2]) self.assertEqual('-1.6', stats_illumina[1][3]) self.assertEqual('--', stats_illumina[1][4]) - self.assertEqual('Total bases', stats_illumina[-1][0]) + self.assertEqual('Total reads', stats_illumina[-1][0]) self.assertEqual('88,850', stats_illumina[-1][1]) self.assertEqual('82,508', stats_illumina[-1][2]) self.assertEqual('-6,342.0', stats_illumina[-1][3]) @@ -517,9 +518,10 @@ def test_run_fastq_and_trimmomatic_2(self): self.assertEqual('143.6', result_average.average_file_2) ### get key values of data - stats_illumina = self.software.get_stats_from_sample_reads(sample) + stats_illumina, total_reads = self.software.get_stats_from_sample_reads(sample) self.assertEqual('Number of reads R1', stats_illumina[0][0]) self.assertEqual('44,425', stats_illumina[0][1]) + self.assertEqual(88850, total_reads) self.assertEqual('--', stats_illumina[0][2]) self.assertEqual('0.0', stats_illumina[0][3]) self.assertEqual('100', stats_illumina[0][4]) @@ -528,7 +530,7 @@ def test_run_fastq_and_trimmomatic_2(self): self.assertEqual('--', stats_illumina[1][2]) self.assertEqual('0.0', stats_illumina[1][3]) self.assertEqual('--', stats_illumina[1][4]) - self.assertEqual('Total bases', stats_illumina[-1][0]) + self.assertEqual('Total reads', stats_illumina[-1][0]) self.assertEqual('88,850', stats_illumina[-1][1]) self.assertEqual('--', stats_illumina[-1][2]) self.assertEqual('0.0', stats_illumina[-1][3]) diff --git a/tests_insaflu/tests_software_minion.py b/tests_insaflu/tests_software_minion.py index 9ef846a7..6e3dde00 100644 --- a/tests_insaflu/tests_software_minion.py +++ b/tests_insaflu/tests_software_minion.py @@ -14,6 +14,7 @@ from utils.result import DecodeObjects, Coverage, MaskingConsensus from constants.meta_key_and_values import MetaKeyAndValue from utils.software_minion import SoftwareMinion +from utils.software import Software from utils.utils import Utils from constants.software_names import SoftwareNames from django.test.utils import override_settings @@ -27,6 +28,7 @@ class Test(TestCase): software_minion = SoftwareMinion() + software = Software() utils = Utils() constants = Constants() software_names = SoftwareNames() @@ -298,6 +300,11 @@ def test_run_clean_minion_2(self): self.assertEqual("Total bases", vect_soft[1].get_vect_key_values()[-1].key) self.assertEqual("3,475,144.0", vect_soft[1].get_vect_key_values()[-1].value) + stats_illumina, total_reads = self.software.get_stats_from_sample_reads(sample) + self.assertEqual('Mean read length', stats_illumina[0][0]) + self.assertEqual('1,298.4', stats_illumina[0][1]) + self.assertEqual(3000, total_reads) + sample = Sample.objects.get(name=sample_name) self.assertEqual("A-H5N5", sample.type_subtype) # self.assertEqual("H5N5", sample.type_subtype) diff --git a/utils/collect_extra_data.py b/utils/collect_extra_data.py index e169be4c..ecc78eba 100644 --- a/utils/collect_extra_data.py +++ b/utils/collect_extra_data.py @@ -30,10 +30,17 @@ class CollectExtraData(object): classdocs ''' - HEADER_SAMPLE_OUT_CSV = "id,fastq1,fastq2,data set,vaccine status,week,onset date,collection date,lab reception date,latitude,longitude,type-subtype,putative mixed-infection" - HEADER_SAMPLE_OUT_CSV_for_sample = "sample,fastq1,fastq2,data set,vaccine status,week,onset date,collection date,lab reception date,latitude,longitude,type-subtype" - HEADER_SAMPLE_OUT_CSV_simple = "id,data set,vaccine status,week,onset date,collection date,lab reception date,latitude,longitude,type-subtype,putative mixed-infection" + HEADER_SAMPLE_OUT_CSV = "id,fastq1,fastq2,data set,vaccine status,week,onset date,collection date,lab reception date,latitude,longitude,classification,putative mixed-infection" + HEADER_SAMPLE_OUT_CSV_statistics = "id,fastq1,fastq2,sample date created" + HEADER_SAMPLE_OUT_CSV_simple = "id,data set,vaccine status,week,onset date,collection date,lab reception date,latitude,longitude,classification,putative mixed-infection" + ## this is used for global sample list. It has all samples per user + HEADER_SAMPLE_OUT_CSV_for_sample = "sample,fastq1,fastq2,data set,vaccine status,week,onset date,collection date,lab reception date,latitude,longitude,classification" + ## Type of sample list + SAMPLE_LIST_simple = 0 ## used in the tree + SAMPLE_LIST_list = 1 ## list of samples with metadata + SAMPLE_LIST_list_settings = 2 ## list of samples with settings + utils = Utils() software = Software() logger_debug = logging.getLogger("fluWebVirus.debug") @@ -153,6 +160,7 @@ def __collect_update_pangolin_lineage(self, project, user, b_mark_sge_success = def __collect_update_extra_metadata_for_project(self, project, user): """ Only for update metadata + It is not necessary """ ### get the taskID and seal it process_controler = ProcessControler() @@ -167,6 +175,9 @@ def __collect_update_extra_metadata_for_project(self, project, user): self.calculate_global_files(Project.PROJECT_FILE_NAME_SAMPLE_RESULT_TSV, project, user) ## IMPORTANT -> this need to be after of Project.PROJECT_FILE_NAME_SAMPLE_RESULT_CSV self.calculate_global_files(Project.PROJECT_FILE_NAME_SAMPLE_RESULT_json, project, user) + + ### zip several files to download + self.zip_several_files(project) except: ## finished with error process_SGE.set_process_controler(user, process_controler.get_name_project(project), ProcessControler.FLAG_ERROR) @@ -237,8 +248,10 @@ def __collect_extra_data_for_project(self, project, user): self.calculate_global_files(Project.PROJECT_FILE_NAME_TAB_VARIATIONS_FREEBAYES_with_snps_indels, project, user) ## collect sample table with plus type and subtype, mixed infection, equal to upload table self.calculate_global_files(Project.PROJECT_FILE_NAME_SAMPLE_RESULT_CSV, project, user) + self.calculate_global_files(Project.PROJECT_FILE_NAME_SAMPLE_RESULT_SETTINGS_CSV, project, user) self.calculate_global_files(Project.PROJECT_FILE_NAME_SAMPLE_RESULT_CSV_simple, project, user) self.calculate_global_files(Project.PROJECT_FILE_NAME_SAMPLE_RESULT_TSV, project, user) + self.calculate_global_files(Project.PROJECT_FILE_NAME_SAMPLE_RESULT_SETTINGS_TSV, project, user) ## IMPORTANT -> this need to be after of Project.PROJECT_FILE_NAME_SAMPLE_RESULT_CSV self.calculate_global_files(Project.PROJECT_FILE_NAME_SAMPLE_RESULT_json, project, user) @@ -261,6 +274,9 @@ def __collect_extra_data_for_project(self, project, user): ### Refresh sample list, something can would change self.collect_sample_list(user) + + ### zip several files to download + self.zip_several_files(project) except: ## finished with error process_SGE.set_process_controler(user, process_controler.get_name_project(project), ProcessControler.FLAG_ERROR) @@ -382,18 +398,27 @@ def calculate_global_files(self, type_file, project, user): out_file_file_system = project.get_global_file_by_project(TypePath.MEDIA_ROOT, type_file) elif (type_file == Project.PROJECT_FILE_NAME_SAMPLE_RESULT_CSV): ## samples csv - out_file = self.collect_sample_table(project, Constants.SEPARATOR_COMMA) + out_file = self.collect_sample_table(project, Constants.SEPARATOR_COMMA, CollectExtraData.SAMPLE_LIST_list) + out_file_file_system = project.get_global_file_by_project(TypePath.MEDIA_ROOT, type_file) + elif (type_file == Project.PROJECT_FILE_NAME_SAMPLE_RESULT_SETTINGS_CSV): + out_file = self.collect_sample_table(project, Constants.SEPARATOR_COMMA, CollectExtraData.SAMPLE_LIST_list_settings) out_file_file_system = project.get_global_file_by_project(TypePath.MEDIA_ROOT, type_file) elif (type_file == Project.PROJECT_FILE_NAME_SAMPLE_RESULT_CSV_simple): ## samples csv, simple version - b_simple = True ## doesn't have software information and fastq1, fastq2 - out_file = self.collect_sample_table(project, Constants.SEPARATOR_COMMA, b_simple) + out_file = self.collect_sample_table(project, Constants.SEPARATOR_COMMA, CollectExtraData.SAMPLE_LIST_simple) out_file_file_system = project.get_global_file_by_project(TypePath.MEDIA_ROOT, type_file) elif (type_file == Project.PROJECT_FILE_NAME_SAMPLE_RESULT_TSV): ## samples tsv - out_file = self.collect_sample_table(project, Constants.SEPARATOR_TAB) + out_file = self.collect_sample_table(project, Constants.SEPARATOR_TAB, CollectExtraData.SAMPLE_LIST_list) + out_file_file_system = project.get_global_file_by_project(TypePath.MEDIA_ROOT, type_file) + elif (type_file == Project.PROJECT_FILE_NAME_SAMPLE_RESULT_TSV): + ## samples tsv + out_file = self.collect_sample_table(project, Constants.SEPARATOR_TAB, CollectExtraData.SAMPLE_LIST_list) + out_file_file_system = project.get_global_file_by_project(TypePath.MEDIA_ROOT, type_file) + elif (type_file == Project.PROJECT_FILE_NAME_SAMPLE_RESULT_SETTINGS_TSV): + out_file = self.collect_sample_table(project, Constants.SEPARATOR_TAB, CollectExtraData.SAMPLE_LIST_list_settings) out_file_file_system = project.get_global_file_by_project(TypePath.MEDIA_ROOT, type_file) elif (type_file == Project.PROJECT_FILE_NAME_SAMPLE_RESULT_json): @@ -613,6 +638,50 @@ def merge_all_consensus_files(self, project): self.utils.merge_fasta_files(vect_to_process, out_file) return out_file + def zip_several_files(self, project): + + temp_dir = self.utils.get_temp_dir() + + ## coverage + if os.path.exists(project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_COVERAGE)): + self.utils.link_file(project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_COVERAGE), + os.path.join(temp_dir, Project.PROJECT_FILE_NAME_COVERAGE)) + ## variants + if os.path.exists(project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_TAB_VARIATIONS_SNIPPY)): + self.utils.link_file(project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_TAB_VARIATIONS_SNIPPY), + os.path.join(temp_dir, Project.PROJECT_FILE_NAME_TAB_VARIATIONS_SNIPPY)) + ## minor intra host + if os.path.exists(project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_TAB_VARIATIONS_FREEBAYES)): + self.utils.link_file(project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_TAB_VARIATIONS_FREEBAYES), + os.path.join(temp_dir, Project.PROJECT_FILE_NAME_TAB_VARIATIONS_FREEBAYES)) + ## sample file result + if os.path.exists(project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_SAMPLE_RESULT_CSV)): + self.utils.link_file(project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_SAMPLE_RESULT_CSV), + os.path.join(temp_dir, Project.PROJECT_FILE_NAME_SAMPLE_RESULT_CSV)) + if os.path.exists(project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_SAMPLE_RESULT_SETTINGS_CSV)): + self.utils.link_file(project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_SAMPLE_RESULT_SETTINGS_CSV), + os.path.join(temp_dir, Project.PROJECT_FILE_NAME_SAMPLE_RESULT_SETTINGS_CSV)) + + if os.path.exists(project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_SAMPLE_RESULT_all_consensus)): + self.utils.link_file(project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_SAMPLE_RESULT_all_consensus), + os.path.join(temp_dir, Project.PROJECT_FILE_NAME_SAMPLE_RESULT_all_consensus)) + + ### pangolin data + file_pangolin_result = project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_Pangolin_lineage) + ## pangolin file + if (project.number_passed_sequences > 0 and os.path.exists(file_pangolin_result)): + self.utils.link_file(project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_Pangolin_lineage), + os.path.join(temp_dir, Project.PROJECT_FILE_NAME_Pangolin_lineage)) + + ## all files zipped + zip_out = self.software.zip_files_in_path(temp_dir) + if os.path.exists(zip_out): + self.utils.move_file(zip_out, + project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_all_files_zipped)) + else: + self.utils.remove_file(project.get_global_file_by_project(TypePath.MEDIA_ROOT, Project.PROJECT_FILE_NAME_all_files_zipped)) + + def collect_variations_freebayes(self, project, vect_type_remove): """ collect freebayes variations @@ -656,12 +725,15 @@ def collect_variations_freebayes_only_one_file(self, file_in, file_out, vect_typ return file_out - def collect_sample_table(self, project, column_separator, b_simple = False): + def collect_sample_table(self, project, column_separator, type_list): """ collect sample table column_separator : COMMA or TAB id,fastq1,fastq2,data set,vaccine status,week,onset date,collection date,lab reception date,latitude,longitude - :param b_simple == True, doesn't have fastq1,fastq2 and all software information, created for the tree view + :param type_list + 0) simple, used to upload in the tree + 1) sample list, used to upload in the tree + 2) sample list with settings, used to upload in the tree """ manage_database = ManageDatabase() @@ -676,19 +748,24 @@ def collect_sample_table(self, project, column_separator, b_simple = False): with open(out_file, 'w', newline='') as handle_out: csv_writer = csv.writer(handle_out, delimiter=column_separator, quotechar='"', quoting=csv.QUOTE_MINIMAL if column_separator == Constants.SEPARATOR_COMMA else csv.QUOTE_ALL) - if (b_simple): vect_out_header = CollectExtraData.HEADER_SAMPLE_OUT_CSV_simple.split(',') - else: vect_out_header = CollectExtraData.HEADER_SAMPLE_OUT_CSV.split(',') + if (type_list == CollectExtraData.SAMPLE_LIST_list): vect_out_header = CollectExtraData.HEADER_SAMPLE_OUT_CSV.split(',') + elif (type_list == CollectExtraData.SAMPLE_LIST_simple): vect_out_header = CollectExtraData.HEADER_SAMPLE_OUT_CSV_simple.split(',') + ## settings + else: vect_out_header = CollectExtraData.HEADER_SAMPLE_OUT_CSV_statistics.split(',') ### extra tags - vect_tags = self.get_tags_for_samples_in_projects(project) - vect_out_header.extend(vect_tags) + if type_list in [CollectExtraData.SAMPLE_LIST_simple, CollectExtraData.SAMPLE_LIST_list]: + vect_tags = self.get_tags_for_samples_in_projects(project) + vect_out_header.extend(vect_tags) - ## some extra headers - if parse_pangolin.has_data(): vect_out_header.extend(["Lineage (Pangolin)", "Scorpio (Pangolin)"]) - if (not b_simple): vect_out_header.extend(["Technology", "Sample Down sized", "Available Reads/Mapped Reads/Mapped Percentage"]) + ## some extra headers + if parse_pangolin.has_data(): vect_out_header.extend(["Lineage Pangolin", "Scorpio Pangolin"]) + if type_list == CollectExtraData.SAMPLE_LIST_list_settings: + vect_out_header.extend(["Technology", "Sample Down sized", + "Available Reads for Mapping", "Mapped Reads", "Mapped Percentage"]) ### all information about the softwares - if (not b_simple): + if (type_list == CollectExtraData.SAMPLE_LIST_list_settings): ### set software names and versions ### array with software names (vect_tags_out_sample, vect_tags_out_project_sample, vect_not_add_software_name_output, @@ -723,11 +800,17 @@ def collect_sample_table(self, project, column_separator, b_simple = False): len(SoftwareNames.VECT_INSAFLU_PARAMETER))) csv_writer.writerow(vect_out) + ## add reference name to the file + elif (type_list in [CollectExtraData.SAMPLE_LIST_list]): + vect_out = ["Reference name", project.reference.name] + [''] * (len(vect_out_header) - 2) + csv_writer.writerow(vect_out) + ### write header csv_writer.writerow(vect_out_header) ### if file to need to have the reference under the header - if (b_simple): csv_writer.writerow([project.reference.name] + [''] * (len(vect_out_header) - 1)) + if (type_list == CollectExtraData.SAMPLE_LIST_simple): + csv_writer.writerow([project.reference.name] + [''] * (len(vect_out_header) - 1)) n_count = 0 for project_sample in project.project_samples.all(): @@ -735,50 +818,63 @@ def collect_sample_table(self, project, column_separator, b_simple = False): vect_out = [project_sample.sample.name] ### don't include file names, don't go to show in the tree - if (not b_simple): - ### fastq1 - vect_out.append(project_sample.sample.file_name_1) - ### fastq2 - vect_out.append(project_sample.sample.file_name_2 if project_sample.sample.file_name_2 != None and len(project_sample.sample.file_name_2) > 0 else '') - ### dataset - vect_out.append(project_sample.sample.data_set.name if project_sample.sample.data_set != None else '') - ### vaccine status - vect_out.append(project_sample.sample.vaccine_status.name if project_sample.sample.vaccine_status != None else '') - ### week - vect_out.append(str(project_sample.sample.week) if project_sample.sample.week != None else '') - ### onset date - vect_out.append(project_sample.sample.date_of_onset.strftime('%Y-%m-%d') if project_sample.sample.date_of_onset != None else '') - ### collection date - vect_out.append(project_sample.sample.date_of_collection.strftime('%Y-%m-%d') if project_sample.sample.date_of_collection != None else '') - ### lab reception date - vect_out.append(project_sample.sample.date_of_receipt_lab.strftime('%Y-%m-%d') if project_sample.sample.date_of_receipt_lab != None else '') - ### latitude - vect_out.append(str(project_sample.sample.geo_local.coords[0]) if project_sample.sample.geo_local != None else '') - ### longitude - vect_out.append(str(project_sample.sample.geo_local.coords[1]) if project_sample.sample.geo_local != None else '') - ### type_subtype - vect_out.append(project_sample.sample.type_subtype if project_sample.sample.type_subtype != None else '') - ### mixedinfection - vect_out.append(project_sample.mixed_infections.tag.name if project_sample.mixed_infections != None else '') - - ### print extra informations - query_set = TagNames.objects.filter(sample=project_sample.sample) - for tag_name_to_test in vect_tags: - b_print = False - for tag_names in query_set: - if (tag_names.tag_name.name == tag_name_to_test): - vect_out.append(tag_names.value) - b_print = True - break - if (not b_print): vect_out.append('') + if (type_list in [CollectExtraData.SAMPLE_LIST_simple, CollectExtraData.SAMPLE_LIST_list]): + + ## file names + if type_list == CollectExtraData.SAMPLE_LIST_list: + ### fastq1 + vect_out.append(project_sample.sample.file_name_1) + ### fastq2 + vect_out.append(project_sample.sample.file_name_2 if project_sample.sample.file_name_2 != None and len(project_sample.sample.file_name_2) > 0 else '') + + ### dataset + vect_out.append(project_sample.sample.data_set.name if project_sample.sample.data_set != None else '') + ### vaccine status + vect_out.append(project_sample.sample.vaccine_status.name if project_sample.sample.vaccine_status != None else '') + ### week + vect_out.append(str(project_sample.sample.week) if project_sample.sample.week != None else '') + ### onset date + vect_out.append(project_sample.sample.date_of_onset.strftime(settings.DATE_FORMAT_FOR_SHOW) if project_sample.sample.date_of_onset != None else '') + ### collection date + vect_out.append(project_sample.sample.date_of_collection.strftime(settings.DATE_FORMAT_FOR_SHOW) if project_sample.sample.date_of_collection != None else '') + ### lab reception date + vect_out.append(project_sample.sample.date_of_receipt_lab.strftime(settings.DATE_FORMAT_FOR_SHOW) if project_sample.sample.date_of_receipt_lab != None else '') + ### latitude + vect_out.append(str(project_sample.sample.geo_local.coords[0]) if project_sample.sample.geo_local != None else '') + ### longitude + vect_out.append(str(project_sample.sample.geo_local.coords[1]) if project_sample.sample.geo_local != None else '') + ### type_subtype + vect_out.append(project_sample.sample.type_subtype if project_sample.sample.type_subtype != None else '') + ### mixedinfection + vect_out.append(project_sample.mixed_infections.tag.name if project_sample.mixed_infections != None else '') + + ### print extra informations + query_set = TagNames.objects.filter(sample=project_sample.sample) + for tag_name_to_test in vect_tags: + b_print = False + for tag_names in query_set: + if (tag_names.tag_name.name == tag_name_to_test): + vect_out.append(tag_names.value) + b_print = True + break + if (not b_print): vect_out.append('') - ### information about the software - if (not b_simple): - ### pangolin if exists, not simple + ### pangolin if exists, simple if parse_pangolin.has_data(): vect_out.append(parse_pangolin.get_value(project_sample.seq_name_all_consensus, ParsePangolinResult.KEY_LINEAGE)) vect_out.append(parse_pangolin.get_value(project_sample.seq_name_all_consensus, ParsePangolinResult.KEY_SCORPIO)) + + ### information about the software, CollectExtraData.SAMPLE_LIST_list_settings + else: + ### fastq1 + vect_out.append(project_sample.sample.file_name_1) + ### fastq2 + vect_out.append(project_sample.sample.file_name_2 if project_sample.sample.file_name_2 != None and len(project_sample.sample.file_name_2) > 0 else '') + ### set date created + if settings.RUNNING_TEST: vect_out.append("2010-10-23") + else: vect_out.append(project_sample.sample.creation_date.strftime(settings.DATETIME_FORMAT_FOR_SHOW)) + ### print info about technology vect_out.append(project_sample.get_type_technology()) @@ -786,7 +882,7 @@ def collect_sample_table(self, project, column_separator, b_simple = False): vect_out.append("True" if manage_database.is_sample_downsized(project_sample.sample) else "False") ### print info about mapped stats - vect_out.append(self._get_mapped_stats_info(project_sample)) + vect_out.extend(self._get_mapped_stats_info(project_sample)) ### BEGIN info about software versions #### if project_sample.id in dict_all_results: @@ -829,11 +925,6 @@ def collect_sample_table(self, project, column_separator, b_simple = False): vect_out.append("Not applied") else: vect_out.append("") - else: ## simple, for the tree - ### pangolin if exists, simple - if parse_pangolin.has_data(): - vect_out.append(parse_pangolin.get_value(project_sample.seq_name_all_consensus, ParsePangolinResult.KEY_LINEAGE)) - vect_out.append(parse_pangolin.get_value(project_sample.seq_name_all_consensus, ParsePangolinResult.KEY_SCORPIO)) ### END save global parameters csv_writer.writerow(vect_out) @@ -865,15 +956,18 @@ def _get_mapped_stats_info(self, project_sample): result = self.software.get_statistics_bam(bam_file) manage_database.set_project_sample_metakey(project_sample, project_sample.sample.owner, MetaKeyAndValue.META_KEY_bam_stats, MetaKeyAndValue.META_VALUE_Success, result.to_json()) - else: + else: ## it's available yet decode_stats = DecodeObjects() result = decode_stats.decode_result(meta_value.description) + + ### get data from sample, original data + dt_data, total_reads = self.software.get_stats_from_sample_reads(project_sample.sample) mapped_reads = result.get_value_by_key(MetaKeyAndValue.SAMTOOLS_flagstat_mapped_reads) - total_reads = result.get_value_by_key(MetaKeyAndValue.SAMTOOLS_flagstat_total_reads) if not mapped_reads is None and not total_reads is None and self.utils.is_integer(mapped_reads) and self.utils.is_integer(total_reads): - if int(total_reads) > 0: return "{}/{}/{:.1f}".format(total_reads, mapped_reads, (int(mapped_reads)/float(total_reads))* 100) - return "{}/{}/nan".format(total_reads, mapped_reads) - return "0/0/0" + if int(total_reads) > 0: return [str(total_reads), + mapped_reads, "{:.1f}".format((int(mapped_reads)/float(total_reads)) * 100)] + return [str(total_reads), str(mapped_reads), "nan"] + return ['-'] * 3 def _get_info_for_software(self, tag_to_search, vect_out, dict_all_results, project_sample_id, vect_tags_info, vect_not_add_software_name = []): @@ -1215,7 +1309,7 @@ def _collect_sample_list(self, lst_samples, column_separator, b_test = False): if (not b_trimmomatic_stats and not b_nanostat_sats): vect_out = [''] * (len(vect_out_header) - 2) else: - vect_out = [''] * (star_stats_tag - 2) + vect_out = [''] * star_stats_tag if (b_trimmomatic_stats): vect_out.extend(["Trimmo. stats."] * len(SoftwareNames.SOFTWARE_TRIMMOMATIC_vect_info_to_collect)) if (b_nanostat_sats): vect_out.extend(["NanoStat stats."] * len(SoftwareNames.SOFTWARE_NANOSTAT_vect_info_to_collect)) vect_out.extend([''] * len(vect_tags_out_sample)) @@ -1250,11 +1344,11 @@ def _collect_sample_list(self, lst_samples, column_separator, b_test = False): ### week vect_out.append(str(sample.week) if sample.week != None else '') ### onset date - vect_out.append(sample.date_of_onset.strftime('%Y-%m-%d') if sample.date_of_onset != None else '') + vect_out.append(sample.date_of_onset.strftime(settings.DATE_FORMAT_FOR_SHOW) if sample.date_of_onset != None else '') ### collection date - vect_out.append(sample.date_of_collection.strftime('%Y-%m-%d') if sample.date_of_collection != None else '') + vect_out.append(sample.date_of_collection.strftime(settings.DATE_FORMAT_FOR_SHOW) if sample.date_of_collection != None else '') ### lab reception date - vect_out.append(sample.date_of_receipt_lab.strftime('%Y-%m-%d') if sample.date_of_receipt_lab != None else '') + vect_out.append(sample.date_of_receipt_lab.strftime(settings.DATE_FORMAT_FOR_SHOW) if sample.date_of_receipt_lab != None else '') ### latitude vect_out.append(str(sample.geo_local.coords[0]) if sample.geo_local != None else '') ### longitude @@ -1304,7 +1398,7 @@ def _collect_sample_list(self, lst_samples, column_separator, b_test = False): ## creation date if b_test: vect_out.append("Date") - else: vect_out.append(sample.creation_date.strftime(settings.DATETIME_FORMAT_FOR_TABLE)) + else: vect_out.append(sample.creation_date.strftime(settings.DATETIME_FORMAT_FOR_TABLE_SHOW)) ## info about projects vect_out.append("{}".format(ProjectSample.objects.filter(sample=sample, @@ -1378,10 +1472,10 @@ def _collect_project_list(self, lst_projects, column_separator, b_test = False): is_error=False).count())) vect_out.append(project.reference.name) vect_out.append("" if project.last_change_date is None else \ - project.last_change_date.strftime(settings.DATETIME_FORMAT_FOR_TABLE) ) + project.last_change_date.strftime(settings.DATETIME_FORMAT_FOR_TABLE_SHOW) ) if not b_test: vect_out.append("" if project.creation_date is None else \ - project.creation_date.strftime(settings.DATETIME_FORMAT_FOR_TABLE) ) + project.creation_date.strftime(settings.DATETIME_FORMAT_FOR_TABLE_SHOW) ) ### END save global parameters csv_writer.writerow(vect_out) diff --git a/utils/software.py b/utils/software.py index affdb790..3e79cb43 100644 --- a/utils/software.py +++ b/utils/software.py @@ -1375,23 +1375,27 @@ def get_key_values_stats_illumina(self, number_1, average_1, std1, number_2, ave "STDEV read length R1", "STDEV read length R2", """ result = Result() - total_bases = 0 + total_reads = 0 if not number_1 is None: result.add_key_value(KeyValue("Number of reads R1", "{:,.0f}".format(number_1))) - total_bases += number_1 + total_reads += number_1 if not average_1 is None: result.add_key_value(KeyValue("Average read length R1", str(average_1))) if not std1 is None: result.add_key_value(KeyValue("STDEV read length R1", str(std1))) if not number_2 is None: result.add_key_value(KeyValue("Number of reads R2", "{:,.0f}".format(number_2))) - total_bases += number_2 + total_reads += number_2 if not average_2 is None: result.add_key_value(KeyValue("Average read length R2", str(average_2))) if not std2 is None: result.add_key_value(KeyValue("STDEV read length R2", str(std2))) - result.add_key_value(KeyValue("Total bases", "{:,.0f}".format(total_bases))) + result.add_key_value(KeyValue("Total reads", "{:,.0f}".format(total_reads))) return result.key_values def get_stats_from_sample_reads(self, sample): """ Return stats to show on sample details + :out data_stat -> [[key name, value before filtering, value before filtering, percentage], + [key name, value before filtering, value before filtering, percentage], + [key name, value before filtering, value before filtering, percentage],... ] + :out total_reads_start -> number of reads before mapping """ if sample.is_type_fastq_gz_sequencing(): meta_key_software = MetaKeyAndValue.META_KEY_Fastq_Trimmomatic_Software @@ -1402,13 +1406,16 @@ def get_stats_from_sample_reads(self, sample): software_name = SoftwareNames.SOFTWARE_NanoStat_name show_percentage = SoftwareNames.SOFTWARE_NANOSTAT_vect_info_to_collect_show_percentage + ## return variables + data_stat, total_reads_start = [], -1 + decode_nanostat = DecodeObjects() manageDatabase = ManageDatabase() meta_data = manageDatabase.get_sample_metakey_last(sample, meta_key_software, None) - decode_nanostat = DecodeObjects() + if (meta_data is None): return data_stat, total_reads_start + result_data = decode_nanostat.decode_result(meta_data.description) vect_soft = result_data.get_list_software_instance(software_name) if (len(vect_soft) == 2): ## has data - data_nanostat = [] key_data_0 = vect_soft[0].get_vect_key_values() key_data_1 = vect_soft[1].get_vect_key_values() for _ in range(len(key_data_0)): @@ -1419,11 +1426,17 @@ def get_stats_from_sample_reads(self, sample): (len(value_0) > 0 and self.utils.is_float(value_0) and \ float(value_0) > 0 and len(value_1) > 0): percentage = "{:,.1f}".format(float(value_1) / float(value_0) * 100) - data_nanostat.append([key_data_0[_].key, key_data_0[_].value, key_data_1[_].value, + data_stat.append([key_data_0[_].key, key_data_0[_].value, key_data_1[_].value, "{:,.1f}".format(float(value_1) - float(value_0)), percentage]) - return data_nanostat + + ### get total reads + if (key_data_1[_].key in ("Number of reads R1", "Number of reads R2", + "Number of reads") and \ + self.utils.is_float(value_1)): + if total_reads_start == -1: total_reads_start = 0 + total_reads_start += int(float(value_1)) + elif (len(vect_soft) == 1): ## only has the fist analysis - data_nanostat = [] key_data_0 = vect_soft[0].get_vect_key_values() for _ in range(len(key_data_0)): percentage = "--" @@ -1432,9 +1445,17 @@ def get_stats_from_sample_reads(self, sample): (len(value_0) > 0 and self.utils.is_float(value_0) and \ float(value_0) > 0): percentage = "100" - data_nanostat.append([key_data_0[_].key, key_data_0[_].value, '--', '0.0', percentage]) - return data_nanostat - + data_stat.append([key_data_0[_].key, key_data_0[_].value, '--', '0.0', percentage]) + + ### get total reads + if (key_data_0[_].key in ("Number of reads", "Number of reads R1", + "Number of reads R2") and \ + self.utils.is_float(value_0)): + if total_reads_start == -1: total_reads_start = 0 + total_reads_start += int(float(value_0)) + return data_stat, None if total_reads_start == -1 else total_reads_start + + """ Global processing, fastQ, trimmomatic and GetSpecies """ @@ -1973,6 +1994,26 @@ def dos_2_unix(self, file_name): self.logger_production.error('Fail to run: ' + cmd) self.logger_debug.error('Fail to run: ' + cmd) raise Exception("Fail to create index") + + def zip_files_in_path(self, path_name): + """ + zip all files inside a directory and return a file + :out os.path.join(path_name, "zip_file_out.zip") + """ + if (not os.path.exists(path_name)): return + current_dir = os.getcwd() + os.chdir(path_name) ## change to path that will be compressed + + file_name_zip = "zip_file_out.zip" + cmd = "zip {} *".format(file_name_zip) + exist_status = os.system(cmd) + if (exist_status != 0): + os.chdir(current_dir) + self.logger_production.error('Fail to run: ' + cmd) + self.logger_debug.error('Fail to run: ' + cmd) + raise Exception("Fail to zip files in '{}'".format(path_name)) + os.chdir(current_dir) + return os.path.join(path_name, file_name_zip) def fasta_2_upper(self, file_name): """ diff --git a/utils/support_django_template.py b/utils/support_django_template.py new file mode 100644 index 00000000..307dfa25 --- /dev/null +++ b/utils/support_django_template.py @@ -0,0 +1,16 @@ +''' +Created on 23/01/2022 + +@author: mmp +Has several methods to support Django template language +''' +import os +from django.utils.safestring import mark_safe + +def get_link_for_dropdown_item(link_file, download_file_name = None): + """ returm HTML to dropdown item """ + return mark_safe(' Download - ' +\ + os.path.basename(link_file) + '')