From d9c5387a48e13c720c149347b3a2c66c6ee258a3 Mon Sep 17 00:00:00 2001 From: Nicky Hochmuth Date: Tue, 19 Dec 2023 09:38:39 +0100 Subject: [PATCH 1/3] bump all product versions to 2 --- stixcore/products/product.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stixcore/products/product.py b/stixcore/products/product.py index 23fead36..b0183de1 100644 --- a/stixcore/products/product.py +++ b/stixcore/products/product.py @@ -117,7 +117,7 @@ class BaseProduct: Base TMProduct that all other product inherit from contains the registry for the factory pattern """ - PRODUCT_PROCESSING_VERSION = 1 + PRODUCT_PROCESSING_VERSION = 2 _registry = {} From 5802e1ec1d07887c98ded9f29625ac75f6b627b2 Mon Sep 17 00:00:00 2001 From: Nicky Hochmuth Date: Fri, 8 Nov 2024 16:21:54 +0100 Subject: [PATCH 2/3] add global error detection in pipeline logger (not just main process) --- .gitignore | 1 + stixcore/io/RidLutManager.py | 2 +- stixcore/processing/pipeline.py | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 7b8deb96..730f3e88 100644 --- a/.gitignore +++ b/.gitignore @@ -164,3 +164,4 @@ stixcore/data/publish/ stixcore/data/test/products/end2end/ monitor_status.json stixcore/data/test/idb/v2.26.38/idb.sqlite +.python-version diff --git a/stixcore/io/RidLutManager.py b/stixcore/io/RidLutManager.py index 337dc574..321b6ea3 100644 --- a/stixcore/io/RidLutManager.py +++ b/stixcore/io/RidLutManager.py @@ -165,7 +165,7 @@ def read_rid_lut(cls, file, update=False): # the stix datacenter API is throttled to 2 calls per second time.sleep(0.5) except Exception: - logger.error("RID API ERROR", exc_info=True) + logger.warning("RID API ERROR", exc_info=True) rid_lut = unique(rid_lut, silent=True) ascii.write(rid_lut, file, overwrite=True, delimiter=",", quotechar='"') diff --git a/stixcore/processing/pipeline.py b/stixcore/processing/pipeline.py index 24163fec..22969ace 100644 --- a/stixcore/processing/pipeline.py +++ b/stixcore/processing/pipeline.py @@ -149,6 +149,7 @@ def emit(self, record): """Called in case of a logging event.""" self.allright = False self.error = record + self.err_file.touch() def __enter__(self): return self @@ -159,7 +160,7 @@ def __exit__(self, type, value, traceback): logging.getLogger().removeHandler(self.fh) PipelineStatus.instance.last_tm = (self.tm_file, datetime.now()) PipelineStatus.instance.current_tm = (None, datetime.now()) - if not self.allright: + if not self.allright or self.err_file.exists(): shutil.copyfile(self.log_file, self.err_file) PipelineStatus.instance.last_error = (self.tm_file, datetime.now(), self.error, self.err_file) From 0660c6a854336a0e8a993d0adc7503c658bccc7c Mon Sep 17 00:00:00 2001 From: Nicky Hochmuth Date: Mon, 11 Nov 2024 16:42:18 +0100 Subject: [PATCH 3/3] add exception handling for get_reason --- stixcore/io/RidLutManager.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/stixcore/io/RidLutManager.py b/stixcore/io/RidLutManager.py index 321b6ea3..51a5f983 100644 --- a/stixcore/io/RidLutManager.py +++ b/stixcore/io/RidLutManager.py @@ -70,9 +70,13 @@ def get_reason(self, rid): str verbal description of the request purpose """ - request = self.rid_lut.loc[rid] - reason = " ".join(np.atleast_1d(request['description'])) - return reason + try: + request = self.rid_lut.loc[rid] + reason = " ".join(np.atleast_1d(request['description'])) + return reason + except IndexError: + logger.warning("can't get request purpose: no request founds for rid: {rid}") + return "" def get_scaling_factor(self, rid): """Gets the trigger descaling factor connected to the BSD request.