[add] consensus add/remove to AllConsensus file

constants/software_names.py

@@ -310,6 +310,12 @@ class SoftwareNames(object):
 	SOFTWARE_MASK_CONSENSUS_BY_SITE_name_extended = "Mask consensus by sites"
 	SOFTWARE_MASK_CONSENSUS_BY_SITE_vesion = "1"
 	SOFTWARE_MASK_CONSENSUS_BY_SITE_parameters = ""	## don't have anything by default
+
+	### set consensus on AllConsensus file ON|OFF
+	SOFTWARE_GENERATE_CONSENSUS_name = "Generate consensus"
+	SOFTWARE_GENERATE_CONSENSUS_name_extended = "Generate consensus"
+	SOFTWARE_GENERATE_CONSENSUS_vesion = "1"
+	SOFTWARE_GENERATE_CONSENSUS_parameters = ""	## don't have anything by default
 
 	###################################
 	###################################
@@ -641,6 +647,15 @@ def get_software_mask_consensus_by_site_name(self): return self.SOFTWARE_MASK_CO
 	def get_software_mask_consensus_by_site_name_extended(self): return self.SOFTWARE_MASK_CONSENSUS_BY_SITE_name_extended
 	def get_software_mask_consensus_by_site_version(self): return self.SOFTWARE_MASK_CONSENSUS_BY_SITE_vesion
 	def get_software_mask_consensus_by_site_parameters(self): return self.SOFTWARE_MASK_CONSENSUS_BY_SITE_parameters
+
+	"""
+	return generate consensus
+	"""
+	def get_software_generate_consensus_(self): return self.SOFTWARE_GENERATE_CONSENSUS_name
+	def get_software_generate_consensus_name(self): return self.SOFTWARE_GENERATE_CONSENSUS_name
+	def get_software_generate_consensus_name_extended(self): return self.SOFTWARE_GENERATE_CONSENSUS_name_extended
+	def get_software_generate_consensus_version(self): return self.SOFTWARE_GENERATE_CONSENSUS_vesion
+	def get_software_generate_consensus_parameters(self): return self.SOFTWARE_GENERATE_CONSENSUS_parameters
 
 	"""
 	return 

fluwebvirus/settings.py

@@ -14,7 +14,7 @@
 from decouple import config
 
 ## define APP version
-APP_VERSION_NUMBER = "1.5.0"
+APP_VERSION_NUMBER = "1.5.1"
 
 ### running tests in command line
 RUN_TEST_IN_COMMAND_LINE = False

managing_files/management/commands/collect_global_files.py

@@ -47,6 +47,7 @@ def handle(self, *args, **options):
 			project.last_change_date = datetime.datetime.now()
 			project.save()
 
+			### collect extra data for projects
 			collect_extra_data.collect_extra_data_for_project(project, user)
 			self.stdout.write("End")
 		except Project.DoesNotExist as e:

managing_files/models.py

@@ -1066,12 +1066,13 @@ def get_backup_consensus_file(self):
 		return os.path.join(self.__get_path__(TypePath.MEDIA_ROOT, ProjectSample.PATH_MAIN_RESULT), "{}{}_backup{}".format(\
 					ProjectSample.FILE_CONSENSUS_FILE, self.sample.name, FileExtensions.FILE_FASTA))
 
-	def get_consensus_file_web(self):
+	def get_consensus_file_web(self, user_reject_file = False):
 		"""
 		get consensus file web
 		"""
 		out_file = self.get_consensus_file(TypePath.MEDIA_ROOT)
 		if (os.path.exists(out_file)):
+			if user_reject_file: return _('Rejected.')
 			return mark_safe('<a href="{}" download> {}</a>'.format(self.get_consensus_file(\
 						TypePath.MEDIA_URL), self.constants.short_name(os.path.basename(out_file), 15)))
 		return _('Not available.')

managing_files/tables.py

@@ -491,7 +491,8 @@ def render_consensus_file(self, record):
 		"""
 		return link to consensus file
 		"""
-		return record.get_consensus_file_web()
+		default_software = DefaultProjectSoftware()
+		return record.get_consensus_file_web(not default_software.include_consensus(record))
 
 	def render_results(self, record):
 		"""

managing_files/views.py

@@ -1906,7 +1906,8 @@ def get_context_data(self, **kwargs):
 			context['extra_data_sample_expand'] = (tag_names != None) ## (tag_names != None and tag_names.count()  > (Constants.START_EXPAND_SAMPLE_TAG_NAMES_ROWS))
 			if (tag_names != None): context['extra_data_sample'] = self.utils.grouped(tag_names, 4)
 
-			context['consensus_file'] = project_sample.get_consensus_file_web()
+			default_software = DefaultProjectSoftware()
+			context['consensus_file'] = project_sample.get_consensus_file_web(not default_software.include_consensus(project_sample))
 			software_used = []	### has a list with all software used... [name, parameters]
 			### only for illumina
 			decode_result = DecodeObjects()
@@ -1991,7 +1992,7 @@ def get_context_data(self, **kwargs):
 			software_pangolin = SoftwarePangolin()
 			is_sars_cov_2 = software_pangolin.is_ref_sars_cov_2(project_sample.project.reference.get_reference_fasta(TypePath.MEDIA_ROOT))
 			if (os.path.exists(project_sample.get_consensus_file(TypePath.MEDIA_ROOT)) and \
-					settings.SHOW_NEXTCLADE_LINK):		## docker versions doesn't show NextClade link
+					settings.SHOW_NEXTCLADE_LINK and default_software.include_consensus(project_sample)):		## docker versions doesn't show NextClade link
 				context = _get_constext_nextclade(project_sample.get_consensus_file(TypePath.MEDIA_URL),
 						context, get_current_site(self.request), is_sars_cov_2)
 

settings/ajax_views.py

@@ -92,7 +92,7 @@ def set_default_parameters(request):
 							manageDatabase = ManageDatabase()
 							process_SGE = ProcessSGE()
 
-							### change flag to nor finished
+							### change flag to not finished
 							project_sample.is_finished = False
 							project_sample.save()
 
@@ -327,6 +327,7 @@ def change_mask_consensus_in_project(project, masking_consensus_proposed):
 					MetaKeyAndValue.META_VALUE_Success, masking_consensus_proposed.to_json())
 		## need to mask all the project_sample if exist
 		for project_sample in project.project_samples.all():
+			if project_sample.is_deleted: continue
 			meta_value = manageDatabase.get_project_sample_metakey_last(project_sample,
 				MetaKeyAndValue.META_KEY_Masking_consensus, MetaKeyAndValue.META_VALUE_Success)
 			if not meta_value is None:
@@ -462,10 +463,48 @@ def turn_on_off_software(request):
 			try:
 				project, project_sample, sample = None, None, None
 				software = Software.objects.get(pk=software_id)
-				if not project_id is None: project = Project.objects.get(pk=project_id)
-				elif not project_sample_id is None:		## project sample 
+				if not project_id is None:				##	project 
+					project = Project.objects.get(pk=project_id)
+				elif not project_sample_id is None:		##	project sample 
 					project_sample = ProjectSample.objects.get(pk=project_sample_id)
+
+					## sample is not ready for run again, To prevent to many ON|OFF
+					if not project_sample.is_finished:
+						data['message'] = "You cannot do this operation. The project '{}' with sample '{}' is in pipeline to run.".format(
+							project_sample.project.name, project_sample.sample.name)
+						return JsonResponse(data)
 
+					### create a task to perform the analysis of snippy and freebayes
+					manageDatabase = ManageDatabase()
+					process_SGE = ProcessSGE()
+					metaKeyAndValue = MetaKeyAndValue()
+					try:
+						### the unique can be ON|OFF
+						if (software.name == SoftwareNames.SOFTWARE_FREEBAYES_name):
+							### change flag to not finished
+							project_sample.is_finished = False
+							project_sample.save()
+
+							(job_name_wait, job_name) = request.user.profile.get_name_sge_seq(Profile.SGE_PROCESS_dont_care, Profile.SGE_GLOBAL)
+							if (project_sample.is_sample_illumina()):
+								taskID = process_SGE.set_second_stage_snippy(project_sample, request.user, job_name, [job_name_wait])
+							else:
+								taskID = process_SGE.set_second_stage_medaka(project_sample, request.user, job_name, [job_name_wait])
+
+							### set project sample queue ID
+							manageDatabase.set_project_sample_metakey(project_sample, request.user,\
+											metaKeyAndValue.get_meta_key_queue_by_project_sample_id(project_sample.id),\
+											MetaKeyAndValue.META_VALUE_Queue, taskID)
+
+						## Generate Consensus
+						### need to collect global files again 
+						taskID = process_SGE.set_collect_global_files(project_sample.project, request.user)
+						manageDatabase.set_project_metakey(project, request.user, metaKeyAndValue.get_meta_key(\
+								MetaKeyAndValue.META_KEY_Queue_TaskID_Project, project.id),
+								MetaKeyAndValue.META_VALUE_Queue, taskID)
+					except:
+						pass
+
 				elif not sample_id is None:	## for Sample 
 					sample = Sample.objects.get(pk=sample_id)
 
@@ -500,14 +539,15 @@ def turn_on_off_software(request):
 					if not job_name is None:
 						process_SGE.set_create_sample_list_by_user(request.user, [job_name])
 
+				## set ON|OFF software 
 				is_to_run = default_parameters.set_software_to_run_by_software(software,
 						project, project_sample, sample)
 
 				## set a new default
 				data['is_to_run'] = is_to_run
 				data['message'] = "The '{}' in '{}' technology was turned '{}'.".format(
 					software.name_extended, software.technology.name,
-					"ON" if software.is_to_run else "OFF")
+					"ON" if is_to_run else "OFF")
 
 			except Software.DoesNotExist:
 				return JsonResponse(data)

settings/default_parameters.py

@@ -33,6 +33,7 @@ class DefaultParameters(object):
 	### used when the parameters are passed in other way
 	### parameters values has the META_KEY of the metakey_projectsample/metakey_project 
 	MASK_DONT_care = "dont_care"
+	MASK_not_applicable = "Not applicable"
 
 	### clean human reads
 	MASK_CLEAN_HUMAN_READS = software_names.SOFTWARE_CLEAN_HUMAN_READS_name
@@ -141,8 +142,9 @@ def get_parameters(self, software_name, user, type_of_use, project, project_samp
 						project_sample=project_sample, sample=sample)
 
 		### if only one parameter and it is don't care, return dont_care 
-		if len(list(parameters)) == 1 and list(parameters)[0].name == DefaultParameters.MASK_DONT_care:
-			return DefaultParameters.MASK_DONT_care
+		if len(list(parameters)) == 1 and list(parameters)[0].name in \
+				[DefaultParameters.MASK_not_applicable, DefaultParameters.MASK_DONT_care]:
+			return DefaultParameters.MASK_not_applicable
 
 		### parse them
 		dict_out = {}
@@ -180,6 +182,34 @@ def get_parameters(self, software_name, user, type_of_use, project, project_samp
 		if (len(return_parameter.strip()) == 0 and len(parameters) == 0): return None
 		return return_parameter.strip()
 
+	def get_list_parameters(self, software_name, user, type_of_use, project, project_sample,
+				sample, technology_name = ConstantsSettings.TECHNOLOGY_illumina):
+		"""
+		get software_name parameters, if it saved in database...
+		"""
+		try:
+			software = Software.objects.get(name=software_name, owner=user,\
+						type_of_use = type_of_use,
+						technology__name = technology_name,
+						version_parameters = self.get_software_parameters_version(software_name))
+		except Software.DoesNotExist:
+			if (type_of_use == Software.TYPE_OF_USE_global):
+				try:
+					software = Software.objects.get(name=software_name, owner=user,\
+							type_of_use=type_of_use,
+							version_parameters = self.get_software_parameters_version(software_name))
+				except Software.DoesNotExist:
+					return None
+			else: return None
+
+		## get parameters for a specific user
+		parameters = Parameter.objects.filter(software=software, project=project,
+						project_sample=project_sample, sample=sample)
+
+		### if only one parameter and it is don't care, return dont_care 
+		return list(parameters)
+
+
 	def is_software_to_run(self, software_name, user, type_of_use, project, project_sample,
 				sample, technology_name):
 		""" Test if it is necessary to run this software, By default return True """
@@ -312,6 +342,8 @@ def get_vect_parameters(self, software):
 		elif (software.name == SoftwareNames.SOFTWARE_MASK_CONSENSUS_BY_SITE_name):
 			return self.get_mask_consensus_by_site_default(software.owner, Software.TYPE_OF_USE_global,
 					ConstantsSettings.TECHNOLOGY_illumina if software.technology is None else software.technology.name)
+		elif (software.name == SoftwareNames.SOFTWARE_GENERATE_CONSENSUS_name):
+			return self.get_generate_consensus_default(software.owner)
 		elif (software.name == SoftwareNames.SOFTWARE_FREEBAYES_name):
 			return self.get_freebayes_default(software.owner, Software.TYPE_OF_USE_global,
 					ConstantsSettings.TECHNOLOGY_illumina if software.technology is None else software.technology.name)
@@ -1234,3 +1266,45 @@ def get_mask_consensus_by_site_default(self, user, type_of_use, technology_name,
 		vect_parameters.append(parameter)
 		return vect_parameters
 
+	def get_generate_consensus_default(self, user, project_sample = None):
+		"""
+		Mask consensus by site
+		"""
+		software = Software()
+		software.name = SoftwareNames.SOFTWARE_GENERATE_CONSENSUS_name
+		software.name_extended = SoftwareNames.SOFTWARE_GENERATE_CONSENSUS_name_extended
+		software.type_of_use = Software.TYPE_OF_USE_project_sample
+		software.type_of_software = Software.TYPE_SOFTWARE
+		software.version = "1.0"
+		software.version_parameters = self.get_software_parameters_version(software.name)
+		software.technology = self.get_technology(ConstantsSettings.TECHNOLOGY_generic)
+		software.can_be_on_off_in_pipeline = True		## set to True if can be ON/OFF in pipeline, otherwise always ON
+		software.is_to_run = True						## set to True if it is going to run, for example Trimmomatic can run or not
+
+		###  small description of software
+		software.help_text = ""
+
+		###  which part of pipeline is going to run
+		software.pipeline_step = self._get_pipeline(ConstantsSettings.PIPELINE_NAME_variant_detection)
+		software.owner = user
+
+		vect_parameters =  []
+
+		parameter = Parameter()
+		parameter.name = DefaultParameters.MASK_DONT_care
+		parameter.parameter = DefaultParameters.MASK_DONT_care
+		parameter.type_data = Parameter.PARAMETER_none
+		parameter.software = software
+		parameter.project_sample = project_sample
+		parameter.union_char = ""
+		parameter.can_change = False
+		parameter.is_to_run = True			### by default it's True
+		parameter.sequence_out = 1
+		parameter.range_available = ""
+		parameter.range_max = ""
+		parameter.range_min = ""
+		parameter.description = ""
+		vect_parameters.append(parameter)
+		return vect_parameters
+
+### "Generate consensus" -> it is used for set ON/OFF consensus in the AllConsensus File

settings/default_software_project_sample.py

@@ -63,8 +63,12 @@ def test_all_defaults(self, user, project, project_sample, sample):
 			self.test_default_db(SoftwareNames.SOFTWARE_MASK_CONSENSUS_BY_SITE_name,\
 						user, Software.TYPE_OF_USE_project_sample, None, project_sample, None,
 						ConstantsSettings.TECHNOLOGY_generic)
+			self.test_default_db(SoftwareNames.SOFTWARE_GENERATE_CONSENSUS_name,\
+						user, Software.TYPE_OF_USE_project_sample, None, project_sample, None,
+						ConstantsSettings.TECHNOLOGY_generic)
 
-			if (project_sample.sample.is_type_fastq_gz_sequencing()): ### illumina
+			### illumina
+			if (project_sample.sample.is_type_fastq_gz_sequencing()):
 				self.test_default_db(SoftwareNames.SOFTWARE_SNIPPY_name, user, Software.TYPE_OF_USE_project_sample, None,
 						project_sample, None, ConstantsSettings.TECHNOLOGY_illumina)
 				self.test_default_db(SoftwareNames.SOFTWARE_FREEBAYES_name, user, Software.TYPE_OF_USE_project_sample,
@@ -73,7 +77,7 @@ def test_all_defaults(self, user, project, project_sample, sample):
 						user, Software.TYPE_OF_USE_project_sample, None, project_sample, None,
 						ConstantsSettings.TECHNOLOGY_illumina)
 
-			else:
+			else:	## ONT
 				self.test_default_db(SoftwareNames.INSAFLU_PARAMETER_MASK_CONSENSUS_name,\
 						user, Software.TYPE_OF_USE_project_sample, None, project_sample, None,
 						ConstantsSettings.TECHNOLOGY_minion)
@@ -160,6 +164,10 @@ def _get_default_parameters(self, software_name, user, type_of_use, project, pro
 			if (not project_sample is None): vect_parameters = self._get_default_project(user,\
 				software_name, project_sample.project, vect_parameters, technology_name)		### base values
 			return vect_parameters
+		elif (software_name == SoftwareNames.SOFTWARE_GENERATE_CONSENSUS_name):
+			vect_parameters = self.default_parameters.get_generate_consensus_default(
+				user, project_sample)
+			return vect_parameters
 		elif (software_name == SoftwareNames.INSAFLU_PARAMETER_LIMIT_COVERAGE_ONT_name):
 			vect_parameters = self.default_parameters.get_limit_coverage_ONT_threshold_default(user, type_of_use, technology_name, project, project_sample)
 			if (not project is None): vect_parameters = self._get_default_project(user,\
@@ -1113,7 +1121,29 @@ def get_mask_consensus_single_parameter_for_project(self, project, parameter_nam
 	#####
 	#####################################################
 
+	#####################################################
+	#####
+	#####		Generate consensus
+	#####
 
+	def include_consensus(self, project_sample):
+		"""
+		get mask_consensus parameters for project and default
+		"""
+
+		### Test project_sample first
+		vect_parameters = self.default_parameters.get_list_parameters(SoftwareNames.SOFTWARE_GENERATE_CONSENSUS_name,\
+				project_sample.sample.owner,\
+				Software.TYPE_OF_USE_project_sample, None, project_sample, None, ConstantsSettings.TECHNOLOGY_generic)
+		if (not vect_parameters is None and len(vect_parameters) > 0): return vect_parameters[0].is_to_run
+		return True
+
+
+	#####
+	#####		END Generate consensus
+	#####
+	#####################################################
+
 
 	#####################################################
 	#####

settings/models.py

@@ -107,6 +107,9 @@ class Parameter(models.Model):
 	PARAMETER_null = 3		### its like only to show, not editable, Example: (TOPHRED33)
 	PARAMETER_char_list = 4
 	PARAMETER_check_box = 5	### check box
+	PARAMETER_none = 6		### This case is when some "software/procedure" doesn't have parameters at all
+							### It only has one parameter. Example: "Generate consensus"
+							### "Generate consensus" -> it is used for set ON/OFF consensus in the AllConsensus File
 
 	name = models.CharField(max_length=50, db_index=True, blank=True, null=True)
 	parameter = models.CharField(max_length=50, db_index=True, blank=True, null=True)

settings/tables.py

@@ -3,10 +3,11 @@
 
 @author: mmp
 '''
+import os
 import django_tables2 as tables
 
 from django.utils.safestring import mark_safe
-from constants.constants import Constants
+from constants.constants import Constants, TypePath
 from constants.software_names import SoftwareNames
 from django.urls import reverse
 from managing_files.models import ProjectSample
@@ -69,6 +70,10 @@ def render_select_to_run(self, value, record):
 		b_enable_options = self.b_enable_options
 		if not self.sample is None: b_enable_options = True
 
+		### check if can ON/OFF SOFTWARE_GENERATE_CONSENSUS_name (consensus make exist)
+		if not self.project_sample is None and record.name == SoftwareNames.SOFTWARE_GENERATE_CONSENSUS_name:
+			b_enable_options = os.path.exists(self.project_sample.get_consensus_file(TypePath.MEDIA_ROOT))
+
 		## need to remove # in href, otherwise still active
 		sz_href = '<a href="{}id_turn_software_on_off" {} '.format(
 			"#" if record.can_be_on_off_in_pipeline and b_enable_options else "",
@@ -184,7 +189,7 @@ def render_options(self, record):
 
 			str_links += '<a href="{}"'.format('#id_set_default_modal' if b_enable_options else '') +\
 				' id="id_default_parameter" data-toggle="modal" data-toggle="tooltip" title="{}"'.format(tooltip_reset) +\
-				'{}'.format("" if self.b_enable_options else 'onclick=\'return false;\' disable') +\
+				'{}'.format("" if b_enable_options else 'onclick=\'return false;\' disable') +\
 				' ref_name="' + record.name + '" pk="' + str(record.pk) + '" pk_proj_sample="' + str(self.project_sample.pk) +\
 				'" type_software="{}'.format('software' if record.is_software() else 'INSaFLU') +\
 				'" proj_name="' + str(self.project_sample.project.name) + '"><span ><i class="fa fa-2x fa-power-off padding-button-table ' +\