From d0ccec2ed8e7177039d4b5f32ed851173b6eedca Mon Sep 17 00:00:00 2001 From: burnout87 Date: Thu, 11 Apr 2024 17:18:13 +0200 Subject: [PATCH 01/10] get_query_new_status function for the Job class --- cdci_data_analysis/analysis/job_manager.py | 16 +++++++++++++++ .../flask_app/dispatcher_query.py | 20 ++----------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/cdci_data_analysis/analysis/job_manager.py b/cdci_data_analysis/analysis/job_manager.py index 72d58f34..41afcbe1 100644 --- a/cdci_data_analysis/analysis/job_manager.py +++ b/cdci_data_analysis/analysis/job_manager.py @@ -248,6 +248,22 @@ def get_call_back_url(self): return url + def get_query_new_status(self): + if self.status == 'done': + query_new_status = 'done' + elif self.status == 'failed': + query_new_status = 'failed' + elif self.status == 'progress': + query_new_status = 'progress' + else: + job_monitor = self.updated_dataserver_monitor() + if job_monitor['status'] == 'progress': + query_new_status = 'progress' + else: + query_new_status = 'submitted' + self.set_submitted() + + return query_new_status class OsaJob(Job): def __init__(self, diff --git a/cdci_data_analysis/flask_app/dispatcher_query.py b/cdci_data_analysis/flask_app/dispatcher_query.py index 3c9244d0..705aca14 100644 --- a/cdci_data_analysis/flask_app/dispatcher_query.py +++ b/cdci_data_analysis/flask_app/dispatcher_query.py @@ -2095,15 +2095,7 @@ def run_query(self, off_line=False, disp_conf=None): debug_message=e.debug_message) if query_out.status_dictionary['status'] == 0: - if job.status == 'done': - query_new_status = 'done' - elif job.status == 'failed': - query_new_status = 'failed' - elif job.status == 'progress': - query_new_status = 'progress' - else: - query_new_status = 'submitted' - job.set_submitted() + query_new_status = job.get_query_new_status() if email_helper.is_email_to_send_run_query(self.logger, query_new_status, @@ -2193,15 +2185,7 @@ def run_query(self, off_line=False, disp_conf=None): self.logger.info('-----------------> job status after query: %s', job.status) if query_out.status_dictionary['status'] == 0: - if job.status == 'done': - query_new_status = 'done' - elif job.status == 'failed': - query_new_status = 'failed' - elif job.status == 'progress': - query_new_status = 'progress' - else: - query_new_status = 'submitted' - job.set_submitted() + query_new_status = job.get_query_new_status() products_url = self.generate_products_url(self.app.config.get('conf').products_url, self.par_dic) email_api_code = DispatcherAPI.set_api_code(self.par_dic, From a26eb7091b5d07aa5478b0c578d1bce6a89960a3 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Wed, 19 Jun 2024 10:10:04 +0200 Subject: [PATCH 02/10] test --- tests/test_job_management.py | 68 ++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/tests/test_job_management.py b/tests/test_job_management.py index a086019e..4e1bf1d9 100644 --- a/tests/test_job_management.py +++ b/tests/test_job_management.py @@ -515,6 +515,74 @@ def test_resubmission_job_id(dispatcher_live_fixture_no_resubmit_timeout, status assert jdata['exit_status']['job_status'] == 'ready' +@pytest.mark.not_safe_parallel +def test_resubmission_after_callback(dispatcher_live_fixture_no_resubmit_timeout): + server = dispatcher_live_fixture_no_resubmit_timeout + DispatcherJobState.remove_scratch_folders() + DataServerQuery.set_status("submitted") + logger.info("constructed server: %s", server) + + # let's generate a valid token + token_payload = { + **default_token_payload, + } + encoded_token = jwt.encode(token_payload, secret_key, algorithm='HS256') + + # these parameters define request content + base_dict_param = dict( + instrument="empty-async", + product_type="dummy-log-submit", + query_type="Real", + ) + + dict_param = dict( + query_status="new", + token=encoded_token, + **base_dict_param + ) + + c = requests.get(os.path.join(server, "run_analysis"), + dict_param + ) + + assert c.status_code == 200 + jdata = c.json() + print(json.dumps(jdata, sort_keys=True, indent=4)) + dispatcher_job_state = DispatcherJobState.from_run_analysis_response(c.json()) + time_request = jdata['time_request'] + jdata = c.json() + assert jdata['exit_status']['job_status'] == "submitted" + assert DataServerQuery.get_status() == "submitted" + + c = requests.get(os.path.join(server, "call_back"), + params=dict( + job_id=dispatcher_job_state.job_id, + session_id=dispatcher_job_state.session_id, + instrument_name="empty-async", + action="progress", + node_id='node_progress', + message='progressing', + token=encoded_token, + time_original_request=time_request + )) + assert c.status_code == 200 + jdata = dispatcher_job_state.load_job_state_record('node_progress', 'progressing') + assert jdata['status'] == "progress" + assert jdata['full_report_dict']['action'] == "progress" + + # resubmit the job after the timeout expired + time.sleep(10.5) + dict_param['job_id'] = dispatcher_job_state.job_id + dict_param['query_status'] = "progress" + c = requests.get(os.path.join(server, "run_analysis"), + dict_param + ) + + assert c.status_code == 200 + jdata = c.json() + assert jdata['exit_status']['job_status'] == "progress" + assert jdata['query_status'] == "progress" + @pytest.mark.not_safe_parallel def test_failed_resubmission(dispatcher_live_fixture_no_resubmit_timeout): server = dispatcher_live_fixture_no_resubmit_timeout From 8ffa7d1e095260f8f003c548cabd3ff9c90921c6 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Wed, 19 Jun 2024 17:37:36 +0200 Subject: [PATCH 03/10] bokeh version --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 94404d99..dec6f71d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ decorator python-logstash raven blinker -bokeh==2.4.2 +bokeh==3.4.1 json_tricks flask-restx six diff --git a/setup.py b/setup.py index d8cc4e21..56b906db 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ "python-logstash", "raven", "blinker", - "bokeh==2.4.2", + "bokeh==3.4.1", "json_tricks", "flask-restx==1.2.0", "six", From 97442b2b48cf27cf0100c7c586d8e0242615034b Mon Sep 17 00:00:00 2001 From: burnout87 Date: Thu, 20 Jun 2024 10:48:03 +0200 Subject: [PATCH 04/10] flask and werkzeug version --- requirements.txt | 2 +- setup.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index dec6f71d..94404d99 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ decorator python-logstash raven blinker -bokeh==3.4.1 +bokeh==2.4.2 json_tricks flask-restx six diff --git a/setup.py b/setup.py index 56b906db..9d8cfe8e 100644 --- a/setup.py +++ b/setup.py @@ -31,11 +31,11 @@ "python-logstash", "raven", "blinker", - "bokeh==3.4.1", + "bokeh==2.4.2", "json_tricks", - "flask-restx==1.2.0", + "flask-restx==1.3.0", "six", - "werkzeug==2.0.3", + "werkzeug==3.0.0", "python-shell-colors==0.2.1", "logging_tree", "celery", From 1ec0a2e94589b86b1d1f6fe4101d6cf7c635b77c Mon Sep 17 00:00:00 2001 From: burnout87 Date: Thu, 20 Jun 2024 10:59:49 +0200 Subject: [PATCH 05/10] flask, werkzeug, jsonschema freezing version --- requirements.txt | 2 +- setup.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 94404d99..7efde913 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ future numpy pyyaml simplejson -flask>=2.0.3 +flask==2.0.3 astropy>=5.0.1 pylogstash_context>=0.1.19 gunicorn diff --git a/setup.py b/setup.py index 9d8cfe8e..57a442ee 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ "numpy", "pyyaml", "simplejson", - "flask", + "flask==2.0.3", "astropy>=2.0.3", "gunicorn", "decorator", @@ -33,9 +33,9 @@ "blinker", "bokeh==2.4.2", "json_tricks", - "flask-restx==1.3.0", + "flask-restx==1.2.0", "six", - "werkzeug==3.0.0", + "werkzeug==2.0.3", "python-shell-colors==0.2.1", "logging_tree", "celery", @@ -49,7 +49,7 @@ "giturlparse", "sentry-sdk", "validators==0.28.3", - "jsonschema" + "jsonschema==3.2.0" ] test_req = [ From a1efd3fc9072a0a12b1d433617d8bb255f41b19d Mon Sep 17 00:00:00 2001 From: burnout87 Date: Thu, 20 Jun 2024 12:16:04 +0200 Subject: [PATCH 06/10] freezing numpy version --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7efde913..3f179c89 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ future -numpy +numpy==1.24.4 pyyaml simplejson flask==2.0.3 diff --git a/setup.py b/setup.py index 57a442ee..ab6c7c0c 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ install_req = [ 'oda_api>=1.1.31', 'pylogstash_context>=0.1.19', - "numpy", + "numpy==1.24.4", "pyyaml", "simplejson", "flask==2.0.3", From f636b0284ca8ed5e6f9e8316f6b090244e0b8db2 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Thu, 20 Jun 2024 17:04:00 +0200 Subject: [PATCH 07/10] less strict version requirement for numpy --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3f179c89..cecbd478 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ future -numpy==1.24.4 +numpy<2.0.0 pyyaml simplejson flask==2.0.3 diff --git a/setup.py b/setup.py index ab6c7c0c..4bc675bc 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ install_req = [ 'oda_api>=1.1.31', 'pylogstash_context>=0.1.19', - "numpy==1.24.4", + "numpy<2.0.0", "pyyaml", "simplejson", "flask==2.0.3", From 2f3ca23778f225697756f26eae5859d8ae2cef41 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Fri, 21 Jun 2024 09:05:46 +0200 Subject: [PATCH 08/10] looser restriction on jsonschema --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4bc675bc..de2adfba 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ "giturlparse", "sentry-sdk", "validators==0.28.3", - "jsonschema==3.2.0" + "jsonschema<=4.17.3" ] test_req = [ From bb70ce460e6606ccaba9b864eaf550d04ae6511f Mon Sep 17 00:00:00 2001 From: burnout87 Date: Fri, 21 Jun 2024 11:13:43 +0200 Subject: [PATCH 09/10] looser restriction on flask --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index cecbd478..72f0c53a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ future numpy<2.0.0 pyyaml simplejson -flask==2.0.3 +flask<=2.3.2 astropy>=5.0.1 pylogstash_context>=0.1.19 gunicorn diff --git a/setup.py b/setup.py index de2adfba..604b73e0 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ "numpy<2.0.0", "pyyaml", "simplejson", - "flask==2.0.3", + "flask<=2.3.2", "astropy>=2.0.3", "gunicorn", "decorator", From 63347bb7e6b5abc929fb86295ef52dda224df05d Mon Sep 17 00:00:00 2001 From: burnout87 Date: Fri, 21 Jun 2024 11:35:21 +0200 Subject: [PATCH 10/10] Revert "looser restriction on flask" This reverts commit bb70ce460e6606ccaba9b864eaf550d04ae6511f. --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 72f0c53a..cecbd478 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ future numpy<2.0.0 pyyaml simplejson -flask<=2.3.2 +flask==2.0.3 astropy>=5.0.1 pylogstash_context>=0.1.19 gunicorn diff --git a/setup.py b/setup.py index 604b73e0..de2adfba 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ "numpy<2.0.0", "pyyaml", "simplejson", - "flask<=2.3.2", + "flask==2.0.3", "astropy>=2.0.3", "gunicorn", "decorator",