add quailty column to SAS (auxiliary fits) (i4Ds#310)

* add quailty column

* add     PRODUCT_PROCESSING_VERSION to product

* fix aspect idl call

* add pipeline and processing version to logs

* fix number formats in strings and magicMoc error

azure-pipelines.yml

@@ -46,6 +46,7 @@ jobs:
         name: py39_linux
 
       - linux: codestyle
+        default_python: '3.9'
         name: codestyle
 
 

stixcore/io/fits/processors.py

@@ -28,6 +28,12 @@
 NAN = 2 ** 32 - 1
 
 
+def version_format(version):
+    # some very strange work around for direct use of format '{0:02d}'.format(version)
+    # as this is not supported by magicMoc
+    return f'{version:02d}'
+
+
 def set_bscale_unsigned(table_hdu):
     """
     Set bscale value to 1 if unsigned int.
@@ -73,7 +79,8 @@ def generate_filename(cls, product, *, version, date_range, status='', header=Tr
         product : `BaseProduct`
             QLProduct
         version : `int`
-            Version of this product
+            the version modifier for the filename
+            default 0 = detect from codebase.
         status : `str`
             Status of the packets
         Returns
@@ -95,10 +102,11 @@ def generate_filename(cls, product, *, version, date_range, status='', header=Tr
             status = 'U'
 
         return f'solo_{product.level}_stix-{product.type}-{product.name.replace("_", "-")}' \
-               f'_{date_range}_V{version:02d}{status}{user_req}{tc_control}.fits'
+               f'_{date_range}_V{version_format(version)}{status}{user_req}{tc_control}.fits'
 
     @classmethod
-    def generate_common_header(cls, filename, product, *, version=1):
+    def generate_common_header(cls, filename, product, *, version=0):
+
         headers = (
             # Name, Value, Comment
             ('FILENAME', filename, 'FITS filename'),
@@ -114,7 +122,7 @@ def generate_common_header(cls, filename, product, *, version=1):
             ('CREATOR', 'stixcore', 'FITS creation software'),
             ('VERS_SW', str(stixcore.__version__), 'Version of SW that provided FITS file'),
             # ('VERS_CAL', '', 'Version of the calibration pack'),
-            ('VERSION', f'{version:02d}', 'Version of data product'),
+            ('VERSION', version_format(version), 'Version of data product'),
             ('OBSRVTRY', 'Solar Orbiter', 'Satellite name'),
             ('TELESCOP', 'SOLO/STIX', 'Telescope/Sensor name'),
             ('INSTRUME', 'STIX', 'Instrument name'),
@@ -137,6 +145,8 @@ def generate_filename(cls, product, *, curtime, status=''):
         product : `sticore.product.Product`
 
         version : `int`
+            the version modifier for the filename
+            default 0 = detect from codebase.
 
         date_range
         status
@@ -326,7 +336,8 @@ def generate_filename(cls, product, *, version, status=''):
         product : `BaseProduct`
             QLProduct
         version : `int`
-            Version of this product
+            the version modifier for the filename
+            default 0 = detect from codebase.
         status : `str`
             Status of the packets
         Returns
@@ -351,7 +362,7 @@ def generate_filename(cls, product, *, version, status=''):
                f'_{scet_obs}_V{version:02d}{status}{addon}.fits'
 
     @classmethod
-    def generate_primary_header(cls, filename, product, *, version=1):
+    def generate_primary_header(cls, filename, product, *, version=0):
         """
         Generate primary header cards.
         Parameters
@@ -382,17 +393,24 @@ def generate_primary_header(cls, filename, product, *, version=1):
         )
         return headers
 
-    def write_fits(self, product, *, version=1):
+    def write_fits(self, product, *, version=0):
         """Write or merge the product data into a FITS file.
         Parameters
         ----------
         product : `LevelB`
             The data product to write.
+        version : `int`
+            the version modifier for the filename
+            default 0 = detect from codebase.
         Raises
         ------
         ValueError
             If the data length in the header and actual data length differ
         """
+
+        if version == 0:
+            version = product.get_processing_version()
+
         files = []
         for prod in product.to_files():
             filename = self.generate_filename(prod, version=version)
@@ -457,21 +475,29 @@ def __init__(self, archive_path):
         """
         self.archive_path = archive_path
 
-    def write_fits(self, product, path=None, *, version=1):
+    def write_fits(self, product, path=None, *, version=0):
         """
         Write level 0 products into fits files.
 
         Parameters
         ----------
         product : `stixcore.product.level0`
 
+        version : `int`
+            the version modifier for the filename
+            default 0 = detect from codebase.
+
         Returns
         -------
         list
             of created file as `pathlib.Path`
 
         """
         created_files = []
+
+        if version == 0:
+            version = product.get_processing_version()
+
         for prod in product.split_to_files():
             filename = self.generate_filename(product=prod, version=version, header=False)
 
@@ -606,8 +632,9 @@ def generate_filename(product, *, version=None, status='', header=True):
         ----------
         product : stix_parser.product.BaseProduct
             QLProduct
-        version : int
-            Version of this product
+        version : `int`
+            the version modifier for the filename
+            default 0 = detect from codebase.
         status : str
             Status of the packets
 
@@ -627,7 +654,7 @@ def generate_filename(product, *, version=None, status='', header=True):
                                                date_range=date_range, status=status, header=header)
 
     @classmethod
-    def generate_primary_header(cls, filename, product, *, version=1):
+    def generate_primary_header(cls, filename, product, *, version=0):
         """
         Generate primary header cards.
 
@@ -679,7 +706,7 @@ def __init__(self, archive_path):
         self.archive_path = archive_path
 
     @classmethod
-    def generate_filename(cls, product, *, version=1, status='', header=True):
+    def generate_filename(cls, product, *, version=0, status='', header=True):
 
         date_range = f'{product.utc_timerange.start.strftime("%Y%m%dT%H%M%S")}-' +\
                      f'{product.utc_timerange.end.strftime("%Y%m%dT%H%M%S")}'
@@ -689,7 +716,7 @@ def generate_filename(cls, product, *, version=1, status='', header=True):
         return FitsProcessor.generate_filename(product, version=version, date_range=date_range,
                                                status=status, header=header)
 
-    def generate_primary_header(self, filename, product, *, version=1):
+    def generate_primary_header(self, filename, product, *, version=0):
         # if product.level != 'L1':
         #    raise ValueError(f"Try to crate FITS file L1 for {product.level} data product")
 
@@ -749,21 +776,28 @@ def generate_primary_header(self, filename, product, *, version=1):
 
         return headers + data_headers + soop_headers + time_headers, ephemeris_headers
 
-    def write_fits(self, product, *, version=1):
+    def write_fits(self, product, *, version=0):
         """
         Write level 0 products into fits files.
 
         Parameters
         ----------
         product : `stixcore.product.level0`
 
+        version : `int`
+            the version modifier for the filename
+            default 0 = detect from codebase.
+
         Returns
         -------
         list
             of created file as `pathlib.Path`
 
         """
         created_files = []
+        if version == 0:
+            version = product.get_processing_version()
+
         for prod in product.split_to_files():
             filename = self.generate_filename(product=prod, version=version, header=False)
             # start_day = np.floor((prod.obs_beg.as_float()
@@ -869,15 +903,35 @@ class FitsL2Processor(FitsL1Processor):
     def __init__(self, archive_path):
         super().__init__(archive_path)
 
-    def write_fits(self, product, *, version=1):
+    def write_fits(self, product, *, version=0):
+        """
+        Write level 2 products into fits files.
+
+        Parameters
+        ----------
+        product : `stixcore.product.level2`
+
+        version : `int`
+            the version modifier for the filename
+            default 0 = detect from codebase.
+
+        Returns
+        -------
+        list
+            of created file as `pathlib.Path`
+
+        """
+        if version == 0:
+            version = product.get_processing_version()
+
         # TODO remove writeout supression of all products but aux files
         if product.type == 'aux':
             return super().write_fits(product, version=version)
         else:
             logger.info(f"no writeout of L2 {product.type}-{product.name} FITS files.")
             return []
 
-    def generate_primary_header(self, filename, product, *, version=1):
+    def generate_primary_header(self, filename, product, *, version=0):
         # if product.level != 'L2':
         #    raise ValueError(f"Try to crate FITS file L2 for {product.level} data product")
 

stixcore/processing/pipeline.py

@@ -5,6 +5,7 @@
 import time
 import shutil
 import socket
+import inspect
 import logging
 import smtplib
 import warnings
@@ -19,6 +20,7 @@
 from watchdog.events import FileSystemEventHandler, LoggingEventHandler
 from watchdog.observers import Observer
 
+import stixcore
 from stixcore.config.config import CONFIG
 from stixcore.ephemeris.manager import Spice, SpiceKernelManager
 from stixcore.idb.manager import IDBManager
@@ -27,6 +29,7 @@
 from stixcore.processing.L1toL2 import Level2
 from stixcore.processing.LBtoL0 import Level0
 from stixcore.processing.TMTCtoLB import process_tmtc_to_levelbinary
+from stixcore.products import Product
 from stixcore.soop.manager import SOOPManager
 from stixcore.util.logging import STX_LOGGER_DATE_FORMAT, STX_LOGGER_FORMAT, get_logger
 from stixcore.util.singleton import Singleton
@@ -266,18 +269,37 @@ def get_singletons():
         s.seek(0)
         return s.read()
 
+    @staticmethod
+    def get_version():
+        s = io.StringIO()
+        s.write("\nPIPELINE VERSION\n\n")
+        s.write(f"Version: {str(stixcore.__version__)}\n")
+        s.write("PROCESSING VERSIONS\n\n")
+        for p in Product.registry:
+            s.write(f"Prod: {p.__name__}\n    File: {inspect.getfile(p)}\n"
+                    f"    Vers: {p.get_cls_processing_version()}\n")
+        s.seek(0)
+        return s.read()
+
     @staticmethod
     def log_singletons(level=logging.INFO):
         logger.log(level, PipelineStatus.get_singletons())
 
+    @staticmethod
+    def log_version(level=logging.INFO):
+        logger.log(level, PipelineStatus.get_version())
+
     @staticmethod
     def log_setup(level=logging.INFO):
+        PipelineStatus.log_version(level=level)
         PipelineStatus.log_config(level=level)
         PipelineStatus.log_singletons(level=level)
 
     @staticmethod
     def get_setup():
-        return PipelineStatus.get_config() + PipelineStatus.get_singletons()
+        return PipelineStatus.get_version() +\
+               PipelineStatus.get_config() +\
+               PipelineStatus.get_singletons()
 
     def status_next(self):
         if not self.tm_handler:

stixcore/processing/tests/test_publish.py

@@ -140,14 +140,15 @@ def test_publish_fits_to_esa_incomplete(product, out_dir):
         product.date_end = end
         product.split_to_files.return_value = [product]
         product.get_energies = False
+        product.get_processing_version.return_value = 1
 
         files.extend(processor.write_fits(product))
 
     assert len(files) == 3
     # this was processed with predicted and flown
     assert fits.getval(files[0], 'SPICE_MK') ==\
         "solo_ANC_soc-pred-mk_V106_20201116_001.tm, solo_ANC_soc-flown-mk_V105_20200515_001.tm"
-    # the filename should be mared as incomplete
+    # the filename should be marked as incomplete
     assert get_complete_file_name(files[0].name) != files[0].name
     assert get_incomplete_file_name(files[0].name) == files[0].name
 
@@ -219,6 +220,7 @@ def test_fits_incomplete_switch_over(out_dir):
             product.date_end = end
             product.split_to_files.return_value = [product]
             product.get_energies = False
+            product.get_processing_version.return_value = 1
 
             files_first.extend(processor.write_fits(product))
 
@@ -338,6 +340,7 @@ def test_publish_fits_to_esa(product, out_dir):
     product.date_beg = beg
     product.date_end = end
     product.split_to_files.return_value = [product]
+    product.get_processing_version.return_value = 1
     product.get_energies = False
 
     data = product.data[:]  # make a clone

stixcore/products/level0/housekeepingL0.py

@@ -82,6 +82,7 @@ class MiniReport(HKProduct):
 
     In level 0 format.
     """
+    PRODUCT_PROCESSING_VERSION = 2
 
     def __init__(self, *, service_type, service_subtype, ssid, control, data,
                  idb_versions=defaultdict(SCETimeRange), **kwargs):
@@ -130,6 +131,7 @@ class MaxiReport(HKProduct):
 
     In level 0 format.
     """
+    PRODUCT_PROCESSING_VERSION = 2
 
     def __init__(self, *, service_type, service_subtype, ssid, control, data,
                  idb_versions=defaultdict(SCETimeRange), **kwargs):

stixcore/products/level1/housekeepingL1.py

@@ -16,6 +16,8 @@ class MiniReport(HKProduct, L1Mixin):
     In level 1 format.
     """
 
+    PRODUCT_PROCESSING_VERSION = 2
+
     def __init__(self, *, service_type, service_subtype, ssid, control, data,
                  idb_versions=defaultdict(SCETimeRange), **kwargs):
         super().__init__(service_type=service_type, service_subtype=service_subtype,
@@ -36,6 +38,7 @@ class MaxiReport(HKProduct, L1Mixin):
 
         In level 1 format.
     """
+    PRODUCT_PROCESSING_VERSION = 2
 
     def __init__(self, *, service_type, service_subtype, ssid, control, data,
                  idb_versions=defaultdict(SCETimeRange), **kwargs):