From c8f4b93ce73933d69739610498fe771b3a254252 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 20 Jan 2025 12:05:19 +0100 Subject: [PATCH 01/13] force overwrite of the default value for POSIXPath paramters --- cdci_data_analysis/analysis/instrument.py | 2 +- cdci_data_analysis/analysis/parameters.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/cdci_data_analysis/analysis/instrument.py b/cdci_data_analysis/analysis/instrument.py index f31d9e8e..9365d407 100644 --- a/cdci_data_analysis/analysis/instrument.py +++ b/cdci_data_analysis/analysis/instrument.py @@ -35,6 +35,7 @@ from urllib.parse import urlencode from cdci_data_analysis.analysis.queries import _check_is_base_query +# from .parameters import POSIXPath from ..analysis import tokenHelper, parameters from .catalog import BasicCatalog from .products import QueryOutput @@ -167,7 +168,6 @@ def set_pars_from_dic(self, arg_dic, verbose=False): for par in param_list: self.logger.info("before normalizing, set_pars_from_dic>> par: %s par.name: %s par.value: %s par_dic[par.name]: %s", par, par.name, par.value, arg_dic.get(par.name, None)) - # this is required because in some cases a parameter is set without a name (eg UserCatalog), # or they don't have to set (eg scw_list) if par.name is not None and par.name not in params_not_to_be_included: diff --git a/cdci_data_analysis/analysis/parameters.py b/cdci_data_analysis/analysis/parameters.py index 30671a4c..8dd585f7 100644 --- a/cdci_data_analysis/analysis/parameters.py +++ b/cdci_data_analysis/analysis/parameters.py @@ -209,6 +209,7 @@ def __init__(self, check_value=None, allowed_values=None, + force_default_value=False, min_value=None, max_value=None, is_optional=False, @@ -253,6 +254,7 @@ def __init__(self, self.is_optional = is_optional self._allowed_units = allowed_units self._allowed_values = allowed_values + self.force_default_value = force_default_value self._allowed_types = allowed_types self.name = name self.default_units = default_units @@ -395,6 +397,8 @@ def set_value_from_form(self, form, verbose=False): if in_dictionary is True: return self.set_par(value=v, units=u, par_format=f) + elif self.force_default_value: + return self.set_par(value=self.get_default_value(), units=u, par_format=f) else: if verbose is True: logger.debug('setting par: %s in the dictionary to its default value' % par_name) @@ -583,7 +587,7 @@ def from_owl_uri(cls, class String(Parameter): owl_uris = ("http://www.w3.org/2001/XMLSchema#str", "http://odahub.io/ontology#String") - def __init__(self, value, name_format='str', name=None, allowed_values = None, is_optional=False, extra_metadata = None): + def __init__(self, value, name_format='str', name=None, allowed_values = None, force_default_value=False, is_optional=False, extra_metadata = None): _allowed_units = ['str'] super().__init__(value=value, @@ -592,6 +596,7 @@ def __init__(self, value, name_format='str', name=None, allowed_values = None, i name=name, allowed_units=_allowed_units, allowed_values=allowed_values, + force_default_value=force_default_value, is_optional=is_optional, extra_metadata=extra_metadata) @@ -611,6 +616,12 @@ class FileReference(String): class POSIXPath(FileReference): owl_uris = FileReference.owl_uris + ("http://odahub.io/ontology#POSIXPath",) + def __init__(self, *args, **kwargs): + super().__init__(*args, force_default_value=True, **kwargs) + + def get_default_value(self): + return '' + class FileURL(FileReference): owl_uris = FileReference.owl_uris + ("http://odahub.io/ontology#FileURL",) From 0b403f4f5184c0c8769b03949eb90ce26a805126 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 20 Jan 2025 15:22:52 +0100 Subject: [PATCH 02/13] get_default_value for POSIXPath returning value if it is set --- cdci_data_analysis/analysis/parameters.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cdci_data_analysis/analysis/parameters.py b/cdci_data_analysis/analysis/parameters.py index 8dd585f7..a53bf0c2 100644 --- a/cdci_data_analysis/analysis/parameters.py +++ b/cdci_data_analysis/analysis/parameters.py @@ -620,6 +620,8 @@ def __init__(self, *args, **kwargs): super().__init__(*args, force_default_value=True, **kwargs) def get_default_value(self): + if self.value is not None and self.value != '': + return self.value return '' class FileURL(FileReference): From 344c39aff0a4ae35e3f658d5ea83824c81a88cbc Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 20 Jan 2025 15:50:34 +0100 Subject: [PATCH 03/13] if force_default_value, then force the value of the parameter to the default --- cdci_data_analysis/analysis/parameters.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cdci_data_analysis/analysis/parameters.py b/cdci_data_analysis/analysis/parameters.py index a53bf0c2..bda29bb6 100644 --- a/cdci_data_analysis/analysis/parameters.py +++ b/cdci_data_analysis/analysis/parameters.py @@ -398,7 +398,8 @@ def set_value_from_form(self, form, verbose=False): if in_dictionary is True: return self.set_par(value=v, units=u, par_format=f) elif self.force_default_value: - return self.set_par(value=self.get_default_value(), units=u, par_format=f) + self.value = self.get_default_value() + return self.value else: if verbose is True: logger.debug('setting par: %s in the dictionary to its default value' % par_name) @@ -620,8 +621,6 @@ def __init__(self, *args, **kwargs): super().__init__(*args, force_default_value=True, **kwargs) def get_default_value(self): - if self.value is not None and self.value != '': - return self.value return '' class FileURL(FileReference): From dd6a77ff5682c5d587006c7683d25b42b38e279b Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 20 Jan 2025 17:06:36 +0100 Subject: [PATCH 04/13] check overwrite_default_value condition and force the overwrite of the default value --- cdci_data_analysis/analysis/parameters.py | 31 +++++++++++++++-------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/cdci_data_analysis/analysis/parameters.py b/cdci_data_analysis/analysis/parameters.py index bda29bb6..3ee5d0a5 100644 --- a/cdci_data_analysis/analysis/parameters.py +++ b/cdci_data_analysis/analysis/parameters.py @@ -209,7 +209,8 @@ def __init__(self, check_value=None, allowed_values=None, - force_default_value=False, + overwrite_default_value=False, + overwriting_default_value=None, min_value=None, max_value=None, is_optional=False, @@ -254,7 +255,8 @@ def __init__(self, self.is_optional = is_optional self._allowed_units = allowed_units self._allowed_values = allowed_values - self.force_default_value = force_default_value + self.overwrite_default_value = overwrite_default_value + self.overwriting_default_value = overwriting_default_value self._allowed_types = allowed_types self.name = name self.default_units = default_units @@ -397,9 +399,11 @@ def set_value_from_form(self, form, verbose=False): if in_dictionary is True: return self.set_par(value=v, units=u, par_format=f) - elif self.force_default_value: - self.value = self.get_default_value() - return self.value + # If the parameter is not provided as an argument and needs to be set to a specific default value, + # we decide to overwrite it. For example, in the POSIXPath class, we "force/overwrite" the default value to '' + # if no file is provided, from which we normally generate a download URL. + elif self.overwrite_default_value and self.overwriting_default_value is not None: + return self.set_par(value=self.overwriting_default_value, units=u, par_format=f) else: if verbose is True: logger.debug('setting par: %s in the dictionary to its default value' % par_name) @@ -588,7 +592,7 @@ def from_owl_uri(cls, class String(Parameter): owl_uris = ("http://www.w3.org/2001/XMLSchema#str", "http://odahub.io/ontology#String") - def __init__(self, value, name_format='str', name=None, allowed_values = None, force_default_value=False, is_optional=False, extra_metadata = None): + def __init__(self, value, name_format='str', name=None, allowed_values = None, overwrite_default_value=False, overwriting_default_value=None, is_optional=False, extra_metadata = None): _allowed_units = ['str'] super().__init__(value=value, @@ -597,7 +601,8 @@ def __init__(self, value, name_format='str', name=None, allowed_values = None, f name=name, allowed_units=_allowed_units, allowed_values=allowed_values, - force_default_value=force_default_value, + overwrite_default_value=overwrite_default_value, + overwriting_default_value=overwriting_default_value, is_optional=is_optional, extra_metadata=extra_metadata) @@ -618,10 +623,14 @@ class POSIXPath(FileReference): owl_uris = FileReference.owl_uris + ("http://odahub.io/ontology#POSIXPath",) def __init__(self, *args, **kwargs): - super().__init__(*args, force_default_value=True, **kwargs) - - def get_default_value(self): - return '' + overwriting_default_value = '' + super().__init__(*args, + overwrite_default_value=True, + overwriting_default_value=overwriting_default_value, + **kwargs) + + def overwrite_default_value(self): + self.value = '' class FileURL(FileReference): owl_uris = FileReference.owl_uris + ("http://odahub.io/ontology#FileURL",) From 248ca741c2d2ec2d42b228b1e3eee87e7c7dd10b Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 20 Jan 2025 17:12:16 +0100 Subject: [PATCH 05/13] not needed function --- cdci_data_analysis/analysis/parameters.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/cdci_data_analysis/analysis/parameters.py b/cdci_data_analysis/analysis/parameters.py index 3ee5d0a5..cf35d382 100644 --- a/cdci_data_analysis/analysis/parameters.py +++ b/cdci_data_analysis/analysis/parameters.py @@ -629,9 +629,6 @@ def __init__(self, *args, **kwargs): overwriting_default_value=overwriting_default_value, **kwargs) - def overwrite_default_value(self): - self.value = '' - class FileURL(FileReference): owl_uris = FileReference.owl_uris + ("http://odahub.io/ontology#FileURL",) From f509c9dd5c3b6f7ec1ca1cd10da94af67b230e0d Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 20 Jan 2025 19:08:41 +0100 Subject: [PATCH 06/13] check specifically for POSIXPath --- cdci_data_analysis/analysis/instrument.py | 7 +++-- cdci_data_analysis/analysis/parameters.py | 38 ++++++++++++----------- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/cdci_data_analysis/analysis/instrument.py b/cdci_data_analysis/analysis/instrument.py index 9365d407..edbda862 100644 --- a/cdci_data_analysis/analysis/instrument.py +++ b/cdci_data_analysis/analysis/instrument.py @@ -35,7 +35,7 @@ from urllib.parse import urlencode from cdci_data_analysis.analysis.queries import _check_is_base_query -# from .parameters import POSIXPath +from .parameters import POSIXPath from ..analysis import tokenHelper, parameters from .catalog import BasicCatalog from .products import QueryOutput @@ -171,10 +171,13 @@ def set_pars_from_dic(self, arg_dic, verbose=False): # this is required because in some cases a parameter is set without a name (eg UserCatalog), # or they don't have to set (eg scw_list) if par.name is not None and par.name not in params_not_to_be_included: + if isinstance(par, POSIXPath) and par.name + '_type' in arg_dic and arg_dic[par.name + '_type'] == 'file'\ + and par.name not in arg_dic: + par.value = None + # set the value for par to a default format, # or to a default value if this is not included within the request updated_arg_dic[par.name] = par.set_value_from_form(arg_dic, verbose=verbose) - if par.units_name is not None: if par.default_units is not None: updated_arg_dic[par.units_name] = par.default_units diff --git a/cdci_data_analysis/analysis/parameters.py b/cdci_data_analysis/analysis/parameters.py index cf35d382..13afb9f6 100644 --- a/cdci_data_analysis/analysis/parameters.py +++ b/cdci_data_analysis/analysis/parameters.py @@ -209,8 +209,8 @@ def __init__(self, check_value=None, allowed_values=None, - overwrite_default_value=False, - overwriting_default_value=None, + # overwrite_default_value=False, + # overwriting_default_value=None, min_value=None, max_value=None, is_optional=False, @@ -255,8 +255,8 @@ def __init__(self, self.is_optional = is_optional self._allowed_units = allowed_units self._allowed_values = allowed_values - self.overwrite_default_value = overwrite_default_value - self.overwriting_default_value = overwriting_default_value + # self.overwrite_default_value = overwrite_default_value + # self.overwriting_default_value = overwriting_default_value self._allowed_types = allowed_types self.name = name self.default_units = default_units @@ -399,11 +399,11 @@ def set_value_from_form(self, form, verbose=False): if in_dictionary is True: return self.set_par(value=v, units=u, par_format=f) - # If the parameter is not provided as an argument and needs to be set to a specific default value, - # we decide to overwrite it. For example, in the POSIXPath class, we "force/overwrite" the default value to '' - # if no file is provided, from which we normally generate a download URL. - elif self.overwrite_default_value and self.overwriting_default_value is not None: - return self.set_par(value=self.overwriting_default_value, units=u, par_format=f) + # # If the parameter is not provided as an argument and needs to be set to a specific default value, + # # we decide to overwrite it. For example, in the POSIXPath class, we "force/overwrite" the default value to '' + # # if no file is provided, from which we normally generate a download URL. + # elif self.overwrite_default_value and self.overwriting_default_value is not None: + # return self.set_par(value=self.overwriting_default_value, units=u, par_format=f) else: if verbose is True: logger.debug('setting par: %s in the dictionary to its default value' % par_name) @@ -592,7 +592,9 @@ def from_owl_uri(cls, class String(Parameter): owl_uris = ("http://www.w3.org/2001/XMLSchema#str", "http://odahub.io/ontology#String") - def __init__(self, value, name_format='str', name=None, allowed_values = None, overwrite_default_value=False, overwriting_default_value=None, is_optional=False, extra_metadata = None): + def __init__(self, value, name_format='str', name=None, allowed_values = None, + is_optional=False, extra_metadata = None): + # overwrite_default_value=False, overwriting_default_value=None, _allowed_units = ['str'] super().__init__(value=value, @@ -601,8 +603,8 @@ def __init__(self, value, name_format='str', name=None, allowed_values = None, o name=name, allowed_units=_allowed_units, allowed_values=allowed_values, - overwrite_default_value=overwrite_default_value, - overwriting_default_value=overwriting_default_value, + # overwrite_default_value=overwrite_default_value, + # overwriting_default_value=overwriting_default_value, is_optional=is_optional, extra_metadata=extra_metadata) @@ -622,12 +624,12 @@ class FileReference(String): class POSIXPath(FileReference): owl_uris = FileReference.owl_uris + ("http://odahub.io/ontology#POSIXPath",) - def __init__(self, *args, **kwargs): - overwriting_default_value = '' - super().__init__(*args, - overwrite_default_value=True, - overwriting_default_value=overwriting_default_value, - **kwargs) + # def __init__(self, *args, **kwargs): + # overwriting_default_value = '' + # super().__init__(*args, + # overwrite_default_value=True, + # overwriting_default_value=overwriting_default_value, + # **kwargs) class FileURL(FileReference): owl_uris = FileReference.owl_uris + ("http://odahub.io/ontology#FileURL",) From 535053e5742b039ed374405aea66f2ebafe0ea2c Mon Sep 17 00:00:00 2001 From: burnout87 Date: Thu, 23 Jan 2025 12:20:20 +0100 Subject: [PATCH 07/13] removed commented code --- cdci_data_analysis/analysis/parameters.py | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/cdci_data_analysis/analysis/parameters.py b/cdci_data_analysis/analysis/parameters.py index 13afb9f6..8d73edce 100644 --- a/cdci_data_analysis/analysis/parameters.py +++ b/cdci_data_analysis/analysis/parameters.py @@ -255,8 +255,6 @@ def __init__(self, self.is_optional = is_optional self._allowed_units = allowed_units self._allowed_values = allowed_values - # self.overwrite_default_value = overwrite_default_value - # self.overwriting_default_value = overwriting_default_value self._allowed_types = allowed_types self.name = name self.default_units = default_units @@ -399,11 +397,6 @@ def set_value_from_form(self, form, verbose=False): if in_dictionary is True: return self.set_par(value=v, units=u, par_format=f) - # # If the parameter is not provided as an argument and needs to be set to a specific default value, - # # we decide to overwrite it. For example, in the POSIXPath class, we "force/overwrite" the default value to '' - # # if no file is provided, from which we normally generate a download URL. - # elif self.overwrite_default_value and self.overwriting_default_value is not None: - # return self.set_par(value=self.overwriting_default_value, units=u, par_format=f) else: if verbose is True: logger.debug('setting par: %s in the dictionary to its default value' % par_name) @@ -592,9 +585,7 @@ def from_owl_uri(cls, class String(Parameter): owl_uris = ("http://www.w3.org/2001/XMLSchema#str", "http://odahub.io/ontology#String") - def __init__(self, value, name_format='str', name=None, allowed_values = None, - is_optional=False, extra_metadata = None): - # overwrite_default_value=False, overwriting_default_value=None, + def __init__(self, value, name_format='str', name=None, allowed_values = None, is_optional=False, extra_metadata = None): _allowed_units = ['str'] super().__init__(value=value, @@ -603,8 +594,6 @@ def __init__(self, value, name_format='str', name=None, allowed_values = None, name=name, allowed_units=_allowed_units, allowed_values=allowed_values, - # overwrite_default_value=overwrite_default_value, - # overwriting_default_value=overwriting_default_value, is_optional=is_optional, extra_metadata=extra_metadata) @@ -624,13 +613,6 @@ class FileReference(String): class POSIXPath(FileReference): owl_uris = FileReference.owl_uris + ("http://odahub.io/ontology#POSIXPath",) - # def __init__(self, *args, **kwargs): - # overwriting_default_value = '' - # super().__init__(*args, - # overwrite_default_value=True, - # overwriting_default_value=overwriting_default_value, - # **kwargs) - class FileURL(FileReference): owl_uris = FileReference.owl_uris + ("http://odahub.io/ontology#FileURL",) From 7526dcd6f18de86a4d2b3cd10ff58cce07e48cec Mon Sep 17 00:00:00 2001 From: burnout87 Date: Thu, 23 Jan 2025 12:21:38 +0100 Subject: [PATCH 08/13] removed commented code --- cdci_data_analysis/analysis/parameters.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/cdci_data_analysis/analysis/parameters.py b/cdci_data_analysis/analysis/parameters.py index 8d73edce..30671a4c 100644 --- a/cdci_data_analysis/analysis/parameters.py +++ b/cdci_data_analysis/analysis/parameters.py @@ -209,8 +209,6 @@ def __init__(self, check_value=None, allowed_values=None, - # overwrite_default_value=False, - # overwriting_default_value=None, min_value=None, max_value=None, is_optional=False, From bb2547a149df4dc15834f2f6301954d35bdb6e74 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Thu, 23 Jan 2025 14:46:05 +0100 Subject: [PATCH 09/13] new dummy instrument for testing POSIX path arg --- .../dummy_plugin/dummy_plugin_conf.yml | 1 + .../empty_instrument_with_posix_path.py | 144 ++++++++++++++++++ .../plugins/dummy_plugin/exposer.py | 1 + tests/test_server_basic.py | 53 +++++++ 4 files changed, 199 insertions(+) create mode 100644 cdci_data_analysis/plugins/dummy_plugin/empty_instrument_with_posix_path.py diff --git a/cdci_data_analysis/plugins/dummy_plugin/dummy_plugin_conf.yml b/cdci_data_analysis/plugins/dummy_plugin/dummy_plugin_conf.yml index aa8a7544..4b5bfb8c 100644 --- a/cdci_data_analysis/plugins/dummy_plugin/dummy_plugin_conf.yml +++ b/cdci_data_analysis/plugins/dummy_plugin/dummy_plugin_conf.yml @@ -5,3 +5,4 @@ instruments: - empty_development_instrument - empty_async_return_progress_instrument - empty_instrument_with_conf +- empty_instrument_with_posix_path diff --git a/cdci_data_analysis/plugins/dummy_plugin/empty_instrument_with_posix_path.py b/cdci_data_analysis/plugins/dummy_plugin/empty_instrument_with_posix_path.py new file mode 100644 index 00000000..c6f33bc4 --- /dev/null +++ b/cdci_data_analysis/plugins/dummy_plugin/empty_instrument_with_posix_path.py @@ -0,0 +1,144 @@ +""" +Overview +-------- + +general info about this module + + +Classes and Inheritance Structure +---------------------------------------------- +.. inheritance-diagram:: + +Summary +--------- +.. autosummary:: + list of the module you want + +Module API +---------- +""" + +from __future__ import absolute_import, division, print_function + +__author__ = "Andrea Tramacere" + +# Standard library +# eg copy +# absolute import rg:from copy import deepcopy + +# Dependencies +# eg numpy +# absolute import eg: import numpy as np + +# Project +# relative import eg: from .mod import f + + +from cdci_data_analysis.analysis.instrument import Instrument +from cdci_data_analysis.analysis.queries import SourceQuery, InstrumentQuery, Float + +from .data_server_dispatcher import (EmptyProductQuery, + DataServerNumericQuery, + FailingProductQuery, + DataServerParametricQuery, + EchoProductQuery, + DefaultEchoProductQuery, + FileParameterQuery) + +# duplicated with jemx, but this staticmethod makes it complex. +# this all should be done commonly, for all parameters - limits are common thing +from ...analysis.exceptions import RequestNotUnderstood +from ...analysis.parameters import (SpectralBoundary, + Angle, + Energy, + Integer, + Float, + String, + StructuredParameter, + POSIXPath) + + + +class BoundaryFloat(Float): + def check_value(self): + super().check_value() + if self.value > 800: + raise RequestNotUnderstood('p value is restricted to 800 W') + + +def my_instr_factory(): + src_query = SourceQuery('src_query') + + # empty query + instr_query = InstrumentQuery(name='empty_instrument_query', + input_prod_list_name='p_list', + catalog=None, + catalog_name='user_catalog') + + empty_query = EmptyProductQuery('empty_with_posix_path_parameters_dummy_query',) + failing_query = FailingProductQuery('failing_parameters_dummy_query',) + # let's build a simple parameter to its list + p = BoundaryFloat(value=10., name='p', units='W',) + numerical_query = DataServerNumericQuery('numerical_parameters_dummy_query', + parameters_list=[p]) + + f = POSIXPath(value=None, name='dummy_POSIX_file', is_optional=True) + file_query = FileParameterQuery('file_parameters_dummy_query', + parameters_list=[p, f]) + + # let's build a simple parameter to its list + sb = SpectralBoundary(value=10., name='sb') + parametrical_query = DataServerParametricQuery('parametrical_parameters_dummy_query', + parameters_list=[sb]) + + ang = Angle(value=1., units='arcsec', default_units='arcsec', name='ang') + ang_deg = Angle(value=1., units='deg', default_units='arcsec', name='ang_deg') + energ = Energy(value=1., E_units='MeV', name='energ', units_name='energy_units') + echo_param_query = EchoProductQuery('echo_parameters_dummy_query', + parameters_list=[ang, ang_deg, energ]) + + bounded_int_param = Integer(5, name = 'bounded_int_par', min_value = 2, max_value = 8) + bounded_float_param = Float(5., name = 'bounded_float_par', min_value = 2.2, max_value = 7.7) + string_select_param = String('spam', name='string_select_par', allowed_values=('spam', 'eggs', 'ham')) + restricted_param_query = EchoProductQuery('restricted_parameters_dummy_query', + parameters_list=[bounded_int_param, + bounded_float_param, + string_select_param]) + + struct_par = StructuredParameter({'a': [1, 2]}, name='struct') + structured_echo_query = DefaultEchoProductQuery('structured_param_dummy_query', parameters_list=[struct_par]) + + optional_par0 = Float(5., name = 'optional_def_float', is_optional=True) + optional_par1 = Float(None, name = 'optional_def_none', is_optional=True) + optional_echo_query = DefaultEchoProductQuery('optional_param_dummy_query', parameters_list=[optional_par0, optional_par1]) + + # this dicts binds the product query name to the product name from frontend + # eg my_instr_image is the parameter passed by the fronted to access the + # the MyInstrMosaicQuery, and the dictionary will bind + # query_dictionary['my_instr_image'] = 'my_instr_image_query' + query_dictionary = {} + # the empty instrument does not effectively do anything and therefore support any particular query + # nor product, only a simple query that does not return anything + query_dictionary['dummy'] = 'empty_with_posix_path_parameters_dummy_query' + query_dictionary['numerical'] = 'numerical_parameters_dummy_query' + query_dictionary['file_dummy'] = 'file_parameters_dummy_query' + query_dictionary['failing'] = 'failing_parameters_dummy_query' + query_dictionary['parametrical'] = 'parametrical_parameters_dummy_query' + query_dictionary['echo'] = 'echo_parameters_dummy_query' + query_dictionary['restricted'] = 'restricted_parameters_dummy_query' + query_dictionary['structured'] = 'structured_param_dummy_query' + query_dictionary['optional'] = 'optional_param_dummy_query' + + return Instrument('empty-with-posix-path', + src_query=src_query, + instrumet_query=instr_query, + product_queries_list=[empty_query, + numerical_query, + file_query, + failing_query, + parametrical_query, + echo_param_query, + restricted_param_query, + structured_echo_query, + optional_echo_query], + query_dictionary=query_dictionary) diff --git a/cdci_data_analysis/plugins/dummy_plugin/exposer.py b/cdci_data_analysis/plugins/dummy_plugin/exposer.py index 6a45a1f1..d777e676 100644 --- a/cdci_data_analysis/plugins/dummy_plugin/exposer.py +++ b/cdci_data_analysis/plugins/dummy_plugin/exposer.py @@ -4,6 +4,7 @@ from . import empty_development_instrument from . import empty_async_return_progress_instrument from . import empty_instrument_with_conf +from . import empty_instrument_with_posix_path from . import conf_file import yaml diff --git a/tests/test_server_basic.py b/tests/test_server_basic.py index 27e59a61..d6fa16e9 100644 --- a/tests/test_server_basic.py +++ b/tests/test_server_basic.py @@ -1988,6 +1988,59 @@ def test_public_file_ownerships(dispatcher_live_fixture): assert ownerships['user_roles'] == [] +@pytest.mark.parametrize("include_file_arg", [True, False]) +def test_default_value_empty_posix_path(dispatcher_live_fixture, include_file_arg): + DispatcherJobState.remove_scratch_folders() + DispatcherJobState.empty_request_files_folders() + server = dispatcher_live_fixture + logger.info("constructed server: %s", server) + + # let's generate a valid token + token_payload = { + **default_token_payload, + "roles": "unige-hpc-full, general", + } + encoded_token = jwt.encode(token_payload, secret_key, algorithm='HS256') + + params = { + **default_params, + 'product_type': 'file_dummy', + 'query_type': "Dummy", + 'instrument': 'empty-with-posix-path', + 'dummy_POSIX_file_type': 'file', + 'p': 6., + 'token': encoded_token + } + + p_file_path = DispatcherJobState.create_p_value_file(p_value=6) + list_file = open(p_file_path) + + expected_query_status = 'done' + expected_job_status = 'done' + expected_status_code = 200 + + files = None + if include_file_arg: + files = {'dummy_POSIX_file': list_file.read()} + + jdata = ask(server, + params, + expected_query_status=expected_query_status, + expected_job_status=expected_job_status, + expected_status_code=expected_status_code, + max_time_s=150, + method='post', + files=files + ) + + list_file.close() + assert 'dummy_POSIX_file' in jdata['products']['analysis_parameters'] + if include_file_arg: + assert jdata['products']['analysis_parameters']['dummy_POSIX_file'] is not None + else: + assert jdata['products']['analysis_parameters']['dummy_POSIX_file'] is None + + def test_scws_list_file(dispatcher_live_fixture): server = dispatcher_live_fixture From 787ba325569e415c11538904cf0160adf7b76f55 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Thu, 23 Jan 2025 14:56:12 +0100 Subject: [PATCH 10/13] adapted test --- tests/test_server_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_server_basic.py b/tests/test_server_basic.py index d6fa16e9..37965200 100644 --- a/tests/test_server_basic.py +++ b/tests/test_server_basic.py @@ -161,7 +161,7 @@ def test_empty_request(dispatcher_live_fixture): assert c.status_code == 400 # parameterize this - assert sorted(jdata['installed_instruments']) == sorted(['empty', 'empty-async', 'empty-with-conf', 'empty-semi-async', 'empty-development', 'empty-async-return-progress']) or \ + assert sorted(jdata['installed_instruments']) == sorted(['empty', 'empty-async', 'empty-with-conf', 'empty-semi-async', 'empty-development', 'empty-async-return-progress', 'empty-with-posix-path']) or \ jdata['installed_instruments'] == [] assert jdata['debug_mode'] == "yes" From aa06a5dedb6fddc6cbf1b0c3881111b56b98e8fc Mon Sep 17 00:00:00 2001 From: burnout87 Date: Thu, 23 Jan 2025 15:38:52 +0100 Subject: [PATCH 11/13] adapted test --- tests/test_server_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_server_basic.py b/tests/test_server_basic.py index 37965200..6ab89ef1 100644 --- a/tests/test_server_basic.py +++ b/tests/test_server_basic.py @@ -255,7 +255,7 @@ def test_matrix_options_mode_empty_request(dispatcher_live_fixture_with_matrix_o assert c.status_code == 400 assert sorted(jdata['installed_instruments']) == sorted( - ['empty', 'empty-async', 'empty-semi-async', 'empty-with-conf', 'empty-development', 'empty-async-return-progress']) or \ + ['empty', 'empty-async', 'empty-semi-async', 'empty-with-conf', 'empty-development', 'empty-async-return-progress', 'empty-with-posix-path',]) or \ jdata['installed_instruments'] == [] # assert jdata['debug_mode'] == "no" From cc83a916e20cdec49088d56ef56e7517e2a0bb46 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Fri, 24 Jan 2025 12:18:50 +0100 Subject: [PATCH 12/13] very simple instrument and adapted test --- .../empty_instrument_with_posix_path.py | 96 ++----------------- tests/test_server_basic.py | 1 - 2 files changed, 7 insertions(+), 90 deletions(-) diff --git a/cdci_data_analysis/plugins/dummy_plugin/empty_instrument_with_posix_path.py b/cdci_data_analysis/plugins/dummy_plugin/empty_instrument_with_posix_path.py index c6f33bc4..08e7be06 100644 --- a/cdci_data_analysis/plugins/dummy_plugin/empty_instrument_with_posix_path.py +++ b/cdci_data_analysis/plugins/dummy_plugin/empty_instrument_with_posix_path.py @@ -35,110 +35,28 @@ from cdci_data_analysis.analysis.instrument import Instrument -from cdci_data_analysis.analysis.queries import SourceQuery, InstrumentQuery, Float +from cdci_data_analysis.analysis.queries import SourceQuery, InstrumentQuery -from .data_server_dispatcher import (EmptyProductQuery, - DataServerNumericQuery, - FailingProductQuery, - DataServerParametricQuery, - EchoProductQuery, - DefaultEchoProductQuery, - FileParameterQuery) - -# duplicated with jemx, but this staticmethod makes it complex. -# this all should be done commonly, for all parameters - limits are common thing -from ...analysis.exceptions import RequestNotUnderstood -from ...analysis.parameters import (SpectralBoundary, - Angle, - Energy, - Integer, - Float, - String, - StructuredParameter, - POSIXPath) - - - -class BoundaryFloat(Float): - def check_value(self): - super().check_value() - if self.value > 800: - raise RequestNotUnderstood('p value is restricted to 800 W') +from .data_server_dispatcher import FileParameterQuery +from ...analysis.parameters import POSIXPath def my_instr_factory(): src_query = SourceQuery('src_query') - # empty query instr_query = InstrumentQuery(name='empty_instrument_query', input_prod_list_name='p_list', catalog=None, catalog_name='user_catalog') - empty_query = EmptyProductQuery('empty_with_posix_path_parameters_dummy_query',) - failing_query = FailingProductQuery('failing_parameters_dummy_query',) - # let's build a simple parameter to its list - p = BoundaryFloat(value=10., name='p', units='W',) - numerical_query = DataServerNumericQuery('numerical_parameters_dummy_query', - parameters_list=[p]) - f = POSIXPath(value=None, name='dummy_POSIX_file', is_optional=True) file_query = FileParameterQuery('file_parameters_dummy_query', - parameters_list=[p, f]) - - # let's build a simple parameter to its list - sb = SpectralBoundary(value=10., name='sb') - parametrical_query = DataServerParametricQuery('parametrical_parameters_dummy_query', - parameters_list=[sb]) - - ang = Angle(value=1., units='arcsec', default_units='arcsec', name='ang') - ang_deg = Angle(value=1., units='deg', default_units='arcsec', name='ang_deg') - energ = Energy(value=1., E_units='MeV', name='energ', units_name='energy_units') - echo_param_query = EchoProductQuery('echo_parameters_dummy_query', - parameters_list=[ang, ang_deg, energ]) - - bounded_int_param = Integer(5, name = 'bounded_int_par', min_value = 2, max_value = 8) - bounded_float_param = Float(5., name = 'bounded_float_par', min_value = 2.2, max_value = 7.7) - string_select_param = String('spam', name='string_select_par', allowed_values=('spam', 'eggs', 'ham')) - restricted_param_query = EchoProductQuery('restricted_parameters_dummy_query', - parameters_list=[bounded_int_param, - bounded_float_param, - string_select_param]) - - struct_par = StructuredParameter({'a': [1, 2]}, name='struct') - structured_echo_query = DefaultEchoProductQuery('structured_param_dummy_query', parameters_list=[struct_par]) - - optional_par0 = Float(5., name = 'optional_def_float', is_optional=True) - optional_par1 = Float(None, name = 'optional_def_none', is_optional=True) - optional_echo_query = DefaultEchoProductQuery('optional_param_dummy_query', parameters_list=[optional_par0, optional_par1]) - - # this dicts binds the product query name to the product name from frontend - # eg my_instr_image is the parameter passed by the fronted to access the - # the MyInstrMosaicQuery, and the dictionary will bind - # query_dictionary['my_instr_image'] = 'my_instr_image_query' - query_dictionary = {} - # the empty instrument does not effectively do anything and therefore support any particular query - # nor product, only a simple query that does not return anything - query_dictionary['dummy'] = 'empty_with_posix_path_parameters_dummy_query' - query_dictionary['numerical'] = 'numerical_parameters_dummy_query' - query_dictionary['file_dummy'] = 'file_parameters_dummy_query' - query_dictionary['failing'] = 'failing_parameters_dummy_query' - query_dictionary['parametrical'] = 'parametrical_parameters_dummy_query' - query_dictionary['echo'] = 'echo_parameters_dummy_query' - query_dictionary['restricted'] = 'restricted_parameters_dummy_query' - query_dictionary['structured'] = 'structured_param_dummy_query' - query_dictionary['optional'] = 'optional_param_dummy_query' + parameters_list=[f]) + + query_dictionary = {'file_dummy': 'file_parameters_dummy_query'} return Instrument('empty-with-posix-path', src_query=src_query, instrumet_query=instr_query, - product_queries_list=[empty_query, - numerical_query, - file_query, - failing_query, - parametrical_query, - echo_param_query, - restricted_param_query, - structured_echo_query, - optional_echo_query], + product_queries_list=[file_query,], query_dictionary=query_dictionary) diff --git a/tests/test_server_basic.py b/tests/test_server_basic.py index 6ab89ef1..73d1a80f 100644 --- a/tests/test_server_basic.py +++ b/tests/test_server_basic.py @@ -2008,7 +2008,6 @@ def test_default_value_empty_posix_path(dispatcher_live_fixture, include_file_ar 'query_type': "Dummy", 'instrument': 'empty-with-posix-path', 'dummy_POSIX_file_type': 'file', - 'p': 6., 'token': encoded_token } From a5452db0d2817f269c09b3a7ab024645b6ad6721 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Fri, 24 Jan 2025 12:27:41 +0100 Subject: [PATCH 13/13] much more simple instrument --- .../plugins/dummy_plugin/empty_instrument_with_posix_path.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cdci_data_analysis/plugins/dummy_plugin/empty_instrument_with_posix_path.py b/cdci_data_analysis/plugins/dummy_plugin/empty_instrument_with_posix_path.py index 08e7be06..55ebf433 100644 --- a/cdci_data_analysis/plugins/dummy_plugin/empty_instrument_with_posix_path.py +++ b/cdci_data_analysis/plugins/dummy_plugin/empty_instrument_with_posix_path.py @@ -40,6 +40,7 @@ from .data_server_dispatcher import FileParameterQuery from ...analysis.parameters import POSIXPath +from .empty_instrument import BoundaryFloat def my_instr_factory(): src_query = SourceQuery('src_query') @@ -50,8 +51,9 @@ def my_instr_factory(): catalog_name='user_catalog') f = POSIXPath(value=None, name='dummy_POSIX_file', is_optional=True) + p = BoundaryFloat(value=10., name='p', units='W', ) file_query = FileParameterQuery('file_parameters_dummy_query', - parameters_list=[f]) + parameters_list=[f, p]) query_dictionary = {'file_dummy': 'file_parameters_dummy_query'}