diff --git a/stixcore/io/fits/processors.py b/stixcore/io/fits/processors.py index cd0030b4..977910b4 100644 --- a/stixcore/io/fits/processors.py +++ b/stixcore/io/fits/processors.py @@ -28,6 +28,12 @@ NAN = 2 ** 32 - 1 +def version_format(version): + # some very strange work around for direct use of format '{0:02d}'.format(version) + # as this is not supported by magicMoc + return f'{version:02d}' + + def set_bscale_unsigned(table_hdu): """ Set bscale value to 1 if unsigned int. @@ -73,7 +79,8 @@ def generate_filename(cls, product, *, version, date_range, status='', header=Tr product : `BaseProduct` QLProduct version : `int` - Version of this product + the version modifier for the filename + default 0 = detect from codebase. status : `str` Status of the packets Returns @@ -95,10 +102,11 @@ def generate_filename(cls, product, *, version, date_range, status='', header=Tr status = 'U' return f'solo_{product.level}_stix-{product.type}-{product.name.replace("_", "-")}' \ - f'_{date_range}_V{version:02d}{status}{user_req}{tc_control}.fits' + f'_{date_range}_V{version_format(version)}{status}{user_req}{tc_control}.fits' @classmethod - def generate_common_header(cls, filename, product, *, version=1): + def generate_common_header(cls, filename, product, *, version=0): + headers = ( # Name, Value, Comment ('FILENAME', filename, 'FITS filename'), @@ -114,7 +122,7 @@ def generate_common_header(cls, filename, product, *, version=1): ('CREATOR', 'stixcore', 'FITS creation software'), ('VERS_SW', str(stixcore.__version__), 'Version of SW that provided FITS file'), # ('VERS_CAL', '', 'Version of the calibration pack'), - ('VERSION', f'{version:02d}', 'Version of data product'), + ('VERSION', version_format(version), 'Version of data product'), ('OBSRVTRY', 'Solar Orbiter', 'Satellite name'), ('TELESCOP', 'SOLO/STIX', 'Telescope/Sensor name'), ('INSTRUME', 'STIX', 'Instrument name'), @@ -137,6 +145,8 @@ def generate_filename(cls, product, *, curtime, status=''): product : `sticore.product.Product` version : `int` + the version modifier for the filename + default 0 = detect from codebase. date_range status @@ -326,7 +336,8 @@ def generate_filename(cls, product, *, version, status=''): product : `BaseProduct` QLProduct version : `int` - Version of this product + the version modifier for the filename + default 0 = detect from codebase. status : `str` Status of the packets Returns @@ -351,7 +362,7 @@ def generate_filename(cls, product, *, version, status=''): f'_{scet_obs}_V{version:02d}{status}{addon}.fits' @classmethod - def generate_primary_header(cls, filename, product, *, version=1): + def generate_primary_header(cls, filename, product, *, version=0): """ Generate primary header cards. Parameters @@ -382,17 +393,24 @@ def generate_primary_header(cls, filename, product, *, version=1): ) return headers - def write_fits(self, product, *, version=1): + def write_fits(self, product, *, version=0): """Write or merge the product data into a FITS file. Parameters ---------- product : `LevelB` The data product to write. + version : `int` + the version modifier for the filename + default 0 = detect from codebase. Raises ------ ValueError If the data length in the header and actual data length differ """ + + if version == 0: + version = product.get_processing_version() + files = [] for prod in product.to_files(): filename = self.generate_filename(prod, version=version) @@ -457,7 +475,7 @@ def __init__(self, archive_path): """ self.archive_path = archive_path - def write_fits(self, product, path=None, *, version=1): + def write_fits(self, product, path=None, *, version=0): """ Write level 0 products into fits files. @@ -465,6 +483,10 @@ def write_fits(self, product, path=None, *, version=1): ---------- product : `stixcore.product.level0` + version : `int` + the version modifier for the filename + default 0 = detect from codebase. + Returns ------- list @@ -472,6 +494,10 @@ def write_fits(self, product, path=None, *, version=1): """ created_files = [] + + if version == 0: + version = product.get_processing_version() + for prod in product.split_to_files(): filename = self.generate_filename(product=prod, version=version, header=False) @@ -606,8 +632,9 @@ def generate_filename(product, *, version=None, status='', header=True): ---------- product : stix_parser.product.BaseProduct QLProduct - version : int - Version of this product + version : `int` + the version modifier for the filename + default 0 = detect from codebase. status : str Status of the packets @@ -627,7 +654,7 @@ def generate_filename(product, *, version=None, status='', header=True): date_range=date_range, status=status, header=header) @classmethod - def generate_primary_header(cls, filename, product, *, version=1): + def generate_primary_header(cls, filename, product, *, version=0): """ Generate primary header cards. @@ -679,7 +706,7 @@ def __init__(self, archive_path): self.archive_path = archive_path @classmethod - def generate_filename(cls, product, *, version=1, status='', header=True): + def generate_filename(cls, product, *, version=0, status='', header=True): date_range = f'{product.utc_timerange.start.strftime("%Y%m%dT%H%M%S")}-' +\ f'{product.utc_timerange.end.strftime("%Y%m%dT%H%M%S")}' @@ -689,7 +716,7 @@ def generate_filename(cls, product, *, version=1, status='', header=True): return FitsProcessor.generate_filename(product, version=version, date_range=date_range, status=status, header=header) - def generate_primary_header(self, filename, product, *, version=1): + def generate_primary_header(self, filename, product, *, version=0): # if product.level != 'L1': # raise ValueError(f"Try to crate FITS file L1 for {product.level} data product") @@ -749,7 +776,7 @@ def generate_primary_header(self, filename, product, *, version=1): return headers + data_headers + soop_headers + time_headers, ephemeris_headers - def write_fits(self, product, *, version=1): + def write_fits(self, product, *, version=0): """ Write level 0 products into fits files. @@ -757,6 +784,10 @@ def write_fits(self, product, *, version=1): ---------- product : `stixcore.product.level0` + version : `int` + the version modifier for the filename + default 0 = detect from codebase. + Returns ------- list @@ -764,6 +795,9 @@ def write_fits(self, product, *, version=1): """ created_files = [] + if version == 0: + version = product.get_processing_version() + for prod in product.split_to_files(): filename = self.generate_filename(product=prod, version=version, header=False) # start_day = np.floor((prod.obs_beg.as_float() @@ -869,7 +903,27 @@ class FitsL2Processor(FitsL1Processor): def __init__(self, archive_path): super().__init__(archive_path) - def write_fits(self, product, *, version=1): + def write_fits(self, product, *, version=0): + """ + Write level 2 products into fits files. + + Parameters + ---------- + product : `stixcore.product.level2` + + version : `int` + the version modifier for the filename + default 0 = detect from codebase. + + Returns + ------- + list + of created file as `pathlib.Path` + + """ + if version == 0: + version = product.get_processing_version() + # TODO remove writeout supression of all products but aux files if product.type == 'aux': return super().write_fits(product, version=version) @@ -877,7 +931,7 @@ def write_fits(self, product, *, version=1): logger.info(f"no writeout of L2 {product.type}-{product.name} FITS files.") return [] - def generate_primary_header(self, filename, product, *, version=1): + def generate_primary_header(self, filename, product, *, version=0): # if product.level != 'L2': # raise ValueError(f"Try to crate FITS file L2 for {product.level} data product") diff --git a/stixcore/processing/pipeline.py b/stixcore/processing/pipeline.py index f254b151..215f9850 100644 --- a/stixcore/processing/pipeline.py +++ b/stixcore/processing/pipeline.py @@ -5,6 +5,7 @@ import time import shutil import socket +import inspect import logging import smtplib import warnings @@ -19,6 +20,7 @@ from watchdog.events import FileSystemEventHandler, LoggingEventHandler from watchdog.observers import Observer +import stixcore from stixcore.config.config import CONFIG from stixcore.ephemeris.manager import Spice, SpiceKernelManager from stixcore.idb.manager import IDBManager @@ -27,6 +29,7 @@ from stixcore.processing.L1toL2 import Level2 from stixcore.processing.LBtoL0 import Level0 from stixcore.processing.TMTCtoLB import process_tmtc_to_levelbinary +from stixcore.products import Product from stixcore.soop.manager import SOOPManager from stixcore.util.logging import STX_LOGGER_DATE_FORMAT, STX_LOGGER_FORMAT, get_logger from stixcore.util.singleton import Singleton @@ -266,18 +269,37 @@ def get_singletons(): s.seek(0) return s.read() + @staticmethod + def get_version(): + s = io.StringIO() + s.write("\nPIPELINE VERSION\n\n") + s.write(f"Version: {str(stixcore.__version__)}\n") + s.write("PROCESSING VERSIONS\n\n") + for p in Product.registry: + s.write(f"Prod: {p.__name__}\n File: {inspect.getfile(p)}\n" + f" Vers: {p.get_cls_processing_version()}\n") + s.seek(0) + return s.read() + @staticmethod def log_singletons(level=logging.INFO): logger.log(level, PipelineStatus.get_singletons()) + @staticmethod + def log_version(level=logging.INFO): + logger.log(level, PipelineStatus.get_version()) + @staticmethod def log_setup(level=logging.INFO): + PipelineStatus.log_version(level=level) PipelineStatus.log_config(level=level) PipelineStatus.log_singletons(level=level) @staticmethod def get_setup(): - return PipelineStatus.get_config() + PipelineStatus.get_singletons() + return PipelineStatus.get_version() +\ + PipelineStatus.get_config() +\ + PipelineStatus.get_singletons() def status_next(self): if not self.tm_handler: diff --git a/stixcore/processing/tests/test_publish.py b/stixcore/processing/tests/test_publish.py index 1a358bbc..57c0a825 100644 --- a/stixcore/processing/tests/test_publish.py +++ b/stixcore/processing/tests/test_publish.py @@ -140,6 +140,7 @@ def test_publish_fits_to_esa_incomplete(product, out_dir): product.date_end = end product.split_to_files.return_value = [product] product.get_energies = False + product.get_processing_version.return_value = 1 files.extend(processor.write_fits(product)) @@ -147,7 +148,7 @@ def test_publish_fits_to_esa_incomplete(product, out_dir): # this was processed with predicted and flown assert fits.getval(files[0], 'SPICE_MK') ==\ "solo_ANC_soc-pred-mk_V106_20201116_001.tm, solo_ANC_soc-flown-mk_V105_20200515_001.tm" - # the filename should be mared as incomplete + # the filename should be marked as incomplete assert get_complete_file_name(files[0].name) != files[0].name assert get_incomplete_file_name(files[0].name) == files[0].name @@ -219,6 +220,7 @@ def test_fits_incomplete_switch_over(out_dir): product.date_end = end product.split_to_files.return_value = [product] product.get_energies = False + product.get_processing_version.return_value = 1 files_first.extend(processor.write_fits(product)) @@ -338,6 +340,7 @@ def test_publish_fits_to_esa(product, out_dir): product.date_beg = beg product.date_end = end product.split_to_files.return_value = [product] + product.get_processing_version.return_value = 1 product.get_energies = False data = product.data[:] # make a clone diff --git a/stixcore/products/level0/housekeepingL0.py b/stixcore/products/level0/housekeepingL0.py index 86c12ed7..c187ae92 100644 --- a/stixcore/products/level0/housekeepingL0.py +++ b/stixcore/products/level0/housekeepingL0.py @@ -82,6 +82,7 @@ class MiniReport(HKProduct): In level 0 format. """ + PRODUCT_PROCESSING_VERSION = 2 def __init__(self, *, service_type, service_subtype, ssid, control, data, idb_versions=defaultdict(SCETimeRange), **kwargs): @@ -130,6 +131,7 @@ class MaxiReport(HKProduct): In level 0 format. """ + PRODUCT_PROCESSING_VERSION = 2 def __init__(self, *, service_type, service_subtype, ssid, control, data, idb_versions=defaultdict(SCETimeRange), **kwargs): diff --git a/stixcore/products/level1/housekeepingL1.py b/stixcore/products/level1/housekeepingL1.py index d3f3322f..c283c05a 100644 --- a/stixcore/products/level1/housekeepingL1.py +++ b/stixcore/products/level1/housekeepingL1.py @@ -16,6 +16,8 @@ class MiniReport(HKProduct, L1Mixin): In level 1 format. """ + PRODUCT_PROCESSING_VERSION = 2 + def __init__(self, *, service_type, service_subtype, ssid, control, data, idb_versions=defaultdict(SCETimeRange), **kwargs): super().__init__(service_type=service_type, service_subtype=service_subtype, @@ -36,6 +38,7 @@ class MaxiReport(HKProduct, L1Mixin): In level 1 format. """ + PRODUCT_PROCESSING_VERSION = 2 def __init__(self, *, service_type, service_subtype, ssid, control, data, idb_versions=defaultdict(SCETimeRange), **kwargs): diff --git a/stixcore/products/level2/housekeepingL2.py b/stixcore/products/level2/housekeepingL2.py index 72eb79e1..fbeb1588 100644 --- a/stixcore/products/level2/housekeepingL2.py +++ b/stixcore/products/level2/housekeepingL2.py @@ -99,6 +99,7 @@ def from_level1(cls, l1product, parent='', idlprocessor=None): data['y_srf'] = 0.0 data['z_srf'] = 0.0 data['calib'] = 0.0 + data['sas_ok'] = np.byte(0) data['error'] = "" data['control_index'] = l2.data['control_index'] @@ -148,7 +149,6 @@ def __init__(self): data_f.error = "FATAL_IDL_ERROR" data = [data, data_f] catch, /cancel - continue endif @@ -171,6 +171,7 @@ def __init__(self): y_srf : hk_file.DATA.y_srf[i], $ z_srf : hk_file.DATA.z_srf[i], $ calib : hk_file.DATA.calib[i], $ + sas_ok : fix(hk_file.DATA.sas_ok[i]), $ error : hk_file.DATA.error[i], $ control_index : hk_file.DATA.control_index[i], $ parentfits : file_index $ @@ -184,24 +185,32 @@ def __init__(self): ;endif ; START ASPECT PROCESSING - help, data_f + help, data_e, /str print, n_elements(data_f) flush, -1 print,"Calibrating data..." flush, -1 ; First, substract dark currents and applies relative gains - calib_sas_data, data_f, calib_file + stx_calib_sas_data, data_f, calib_file + + ; copy result in a new object + data_calib = data_f + ; Added 2023-09-18: remove data points with some error detected during calibration + stx_remove_bad_sas_data, data_calib ; Now automatically compute global calibration correction factor and applies it ; Note: this takes a bit of time - print, "scale" - flush, -1 - auto_scale_sas_data, data_f, simu_data_file, aperfile + stx_auto_scale_sas_data, data_calib, simu_data_file, aperfile + + cal_corr_factor = data_calib[0].calib + data_f.CHA_DIODE0 *= cal_corr_factor + data_f.CHA_DIODE1 *= cal_corr_factor + data_f.CHB_DIODE0 *= cal_corr_factor + data_f.CHB_DIODE1 *= cal_corr_factor print,"Computing aspect solution..." - flush, -1 - derive_aspect_solution, data_f, simu_data_file, interpol_r=1, interpol_xy=1 + stx_derive_aspect_solution, data_f, simu_data_file, interpol_r=1, interpol_xy=1 print,"END Computing aspect solution..." flush, -1 @@ -210,7 +219,7 @@ def __init__(self): data = [data, data_f] ENDFOREACH - idlgswversion="tbd" + stx_gsw_version, version = idlgswversion undefine, hk_file, hk_files, data_e, i, di, data_f, d @@ -275,7 +284,10 @@ def postprocessing(self, result, fits_processor): data['spice_disc_size'] = (idldata['spice_disc_size'] * u.arcsec).astype(np.float32) data['y_srf'] = (idldata['y_srf'] * u.arcsec).astype(np.float32) data['z_srf'] = (idldata['z_srf'] * u.arcsec).astype(np.float32) - # TODO do calculations + data['sas_ok'] = (idldata['sas_ok']).astype(np.bool_) + data['sas_ok'].description = "0: not usable, 1: good" + data['sas_error'] = [e.decode() if hasattr(e, 'decode') else e + for e in idldata['error']] data['solo_loc_carrington_lonlat'] = np.tile(np.array([0.0, 0.0]), (n, 1)).\ astype(np.float32) * u.deg @@ -299,10 +311,10 @@ def postprocessing(self, result, fits_processor): aux = Ephemeris(control=control, data=data, idb_versions=HK.idb_versions) aux.add_additional_header_keywords( - ('STX_GSW', result.idlgswversion.decode(), + ('STX_GSW', result.idlgswversion[0].decode(), 'Version of STX-GSW that provided data')) aux.add_additional_header_keywords( - ('HISTORY', 'some data processed by STX-GSW', '')) + ('HISTORY', 'aspect data processed by STX-GSW', '')) files.extend(fits_processor.write_fits(aux)) else: logger.error("IDL ERROR") @@ -315,6 +327,7 @@ class Ephemeris(HKProduct, L2Mixin): In level 2 format. """ + PRODUCT_PROCESSING_VERSION = 2 def __init__(self, *, service_type=0, service_subtype=0, ssid=1, control, data, idb_versions=defaultdict(SCETimeRange), **kwargs): diff --git a/stixcore/products/product.py b/stixcore/products/product.py index 56802ffa..e0b997be 100644 --- a/stixcore/products/product.py +++ b/stixcore/products/product.py @@ -117,6 +117,8 @@ class BaseProduct: Base TMProduct that all other product inherit from contains the registry for the factory pattern """ + PRODUCT_PROCESSING_VERSION = 1 + _registry = {} def __init_subclass__(cls, **kwargs): @@ -134,6 +136,17 @@ def __init_subclass__(cls, **kwargs): def fits_daily_file(self): raise NotImplementedError("SubClass of BaseProduct should implement") + def get_processing_version(self): + version = self.__class__.PRODUCT_PROCESSING_VERSION\ + if hasattr(self.__class__, 'PRODUCT_PROCESSING_VERSION') else 1 + return max(version, BaseProduct.PRODUCT_PROCESSING_VERSION) + + @classmethod + def get_cls_processing_version(cls): + version = cls.PRODUCT_PROCESSING_VERSION\ + if hasattr(cls, 'PRODUCT_PROCESSING_VERSION') else 1 + return max(version, BaseProduct.PRODUCT_PROCESSING_VERSION) + class ProductFactory(BasicRegistrationFactory): def __call__(self, *args, **kwargs):