(Fix) FishSOOP needs to be UPLOAD_ONLY + fix cython

aodn · May 28, 2024 · 557ccfb · 557ccfb
1 parent 4a35b66
commit 557ccfb
Show file tree

Hide file tree

Showing 5 changed files with 58 additions and 43 deletions.
diff --git a/aodndata/soop/soop_fishsoop.py b/aodndata/soop/soop_fishsoop.py
@@ -1,14 +1,11 @@
 import datetime
 import os
 import re
-import tempfile
 from datetime import datetime
 
-import matplotlib.pyplot as plt
-from aodncore.pipeline import HandlerBase, PipelineFile, PipelineFilePublishType
+from aodncore.pipeline import HandlerBase, PipelineFilePublishType, FileType
 from aodncore.util.misc import get_pattern_subgroups_from_string
-from netCDF4 import Dataset
-from numpy import ma
+from aodncore.pipeline.exceptions import InvalidFileNameError
 
 FISHSOOP_FILE_PATTERN = re.compile(r"""
                               IMOS_SOOP-FishSOOP_
@@ -20,27 +17,37 @@
                               """, re.VERBOSE)
 
 
-def dest_path_soop_fishsoop(filepath, file_pattern=FISHSOOP_FILE_PATTERN):
-    file_basename = os.path.basename(filepath)
-    if file_pattern.match(file_basename):
-        fields = get_pattern_subgroups_from_string(file_basename, file_pattern)
-        nc_time_cov_start = datetime.strptime(fields['nc_time_cov_start'], '%Y%m%dT%H%M%SZ')
+class SoopFishSoopHandler(HandlerBase):
+    def __init__(self, *args, **kwargs):
+        super(SoopFishSoopHandler, self).__init__(*args, **kwargs)
+        self.allowed_extensions = ['.nc']
 
-        qc_level = fields['qc_level']
-        serial_number = fields['serial_number']
+    def process(self):
+        nc_file = self.file_collection.filter_by_attribute_id('file_type', FileType.NETCDF)
+        nc_file[0].publish_type = PipelineFilePublishType.UPLOAD_ONLY
 
-        if qc_level == 'FV00':
-            data_mode = 'REALTIME'
-        elif qc_level == 'FV01':
-            data_mode = "DELAYED"
+    @staticmethod
+    def dest_path(filepath, file_pattern=FISHSOOP_FILE_PATTERN):
+        file_basename = os.path.basename(filepath)
+        if file_pattern.match(file_basename):
+            fields = get_pattern_subgroups_from_string(file_basename, file_pattern)
+            nc_time_cov_start = datetime.strptime(fields['nc_time_cov_start'], '%Y%m%dT%H%M%SZ')
+
+            qc_level = fields['qc_level']
+            serial_number = fields['serial_number']
+
+            if qc_level == 'FV00':
+                data_mode = 'REALTIME'
+            elif qc_level == 'FV01':
+                data_mode = "DELAYED"
+            else:
+                return ValueError(f"{qc_level} not matching expected qc_level")
+
+            return os.path.join('IMOS', 'SOOP', 'SOOP-FishSOOP', data_mode,
+                                str(nc_time_cov_start.year),
+                                str('%02d' % nc_time_cov_start.month),
+                                os.path.basename(filepath))
         else:
-            return ValueError(f"{qc_level} not matching expected qc_level")
-
-        return os.path.join('IMOS', 'SOOP', 'SOOP-FishSOOP', data_mode,
-                            str(nc_time_cov_start.year),
-                            str('%02d' % nc_time_cov_start.month),
-                            os.path.basename(filepath))
-    else:
-        raise InvalidFileNameError(
-            "file name: \"{filename}\" not matching regex to deduce dest_path".format(
-                filename=os.path.basename(filepath)))
+            raise InvalidFileNameError(
+                "file name: \"{filename}\" not matching regex to deduce dest_path".format(
+                    filename=os.path.basename(filepath)))
diff --git a/setup.py b/setup.py
@@ -24,6 +24,7 @@
         'NswOehHandler = aodndata.nsw_oeh.handler:NswOehHandler',
         'SoopAsfSstHandler = aodndata.soop.soop_asf_sst:SoopAsfSstHandler',
         'SoopBaHandler = aodndata.soop.soop_ba:SoopBaHandler',
+        'SoopFishSoopHandler = aodndata.soop.soop_fishsoop:SoopFishSoopHandler',
         'SoopTrvHandler = aodndata.soop.soop_trv:SoopTrvHandler',
         'SoopXbtDmHandler = aodndata.soop.soop_xbt_dm:SoopXbtDmHandler',
         'SoopXbtNrtHandler = aodndata.soop.soop_xbt_nrt:SoopXbtNrtHandler',
@@ -45,7 +46,6 @@
         'dest_path_cars = aodndata.csiro.cars:dest_path_cars',
         'dest_path_deakin_bathymetry = aodndata.deakin.deakin_bathymetry:dest_path_deakin_bathymetry',
         'dest_path_soop_rad_aodn = aodndata.curtin.soop_rad_aodn:dest_path_soop_rad_aodn',
-        'dest_path_soop_fishsoop = aodndata.soop.soop_fishsoop:dest_path_soop_fishsoop',
         'dest_path_oa = aodndata.csiro.ocean_acidification:dest_path_oa',
         'dest_path_srs_oc_ljco_aeronet = aodndata.srs.srs_oc_ljco_aeronet:dest_path_srs_oc_ljco_aeronet',
         'dest_path_srs_surface_waves = aodndata.srs.srs_surface_waves:dest_path_srs_surface_waves',

diff --git a/stage_environment.yml b/stage_environment.yml
@@ -13,9 +13,9 @@ channels:
   - defaults
   - pypi
 dependencies:
-  - cython
+  - cython==0.29.30
   - cf-units  # force cf-units to be installed from conda-forge. packaged failed otherwise
-  - python=3.8
+  - python=3.8.13
   - cartopy=0.20.3
   - pip
   - pip:

diff --git a/test_aodndata/soop/IMOS_SOOP-FishSOOP_TP_20240310T094642Z_FV00_786.nc b/test_aodndata/soop/IMOS_SOOP-FishSOOP_TP_20240310T094642Z_FV00_786.nc
diff --git a/test_aodndata/soop/test_fishsoop.py b/test_aodndata/soop/test_fishsoop.py
@@ -1,26 +1,34 @@
 import os
 import unittest
 
-from aodncore.pipeline.exceptions import InvalidFileNameError
-from aodndata.soop.soop_fishsoop import dest_path_soop_fishsoop
+from aodndata.soop.soop_fishsoop import SoopFishSoopHandler
+from aodncore.pipeline import PipelineFileCheckType, PipelineFilePublishType
+from aodncore.testlib import HandlerTestCase
 
 TEST_ROOT = os.path.join(os.path.dirname(__file__))
+NC_FILE_NRT = 'IMOS_SOOP-FishSOOP_TP_20240310T094642Z_FV00_786.nc'
 
 
-class TestSoopFishSoop(unittest.TestCase):
+class TestSoopFishSoop(HandlerTestCase):
+    def setUp(self):
+        self.handler_class = SoopFishSoopHandler
+        super(TestSoopFishSoop, self).setUp()
+        self.nc_file_nrt = os.path.join(TEST_ROOT, NC_FILE_NRT)
 
     def test_dest_path_soop_fishsoop(self):
-        good_nc_nrt = os.path.join(TEST_ROOT, 'IMOS_SOOP-FishSOOP_PT_20240310T094642Z_FV00_786.nc')
-        self.assertEqual(dest_path_soop_fishsoop(good_nc_nrt),
-                         'IMOS/SOOP/SOOP-FishSOOP/REALTIME/2024/03/{basename}'.format(
-                             basename=os.path.basename(good_nc_nrt)),
-        )
-
-        good_nc_dm = os.path.join(TEST_ROOT, 'IMOS_SOOP-FishSOOP_TP_20240310T094642Z_FV01_786.nc')
-        self.assertEqual(dest_path_soop_fishsoop(good_nc_dm),
-                         'IMOS/SOOP/SOOP-FishSOOP/DELAYED/2024/03/{basename}'.format(
-                             basename=os.path.basename(good_nc_dm)),
-        )
+        handler = self.run_handler(self.nc_file_nrt,
+                                   check_params={'checks': ['cf:1.6', 'imos:1.4']}
+                                   )
+
+        f = handler.file_collection[0]
+
+        self.assertEqual(f.check_type, PipelineFileCheckType.NC_COMPLIANCE_CHECK)
+        self.assertEqual(f.publish_type, PipelineFilePublishType.UPLOAD_ONLY)
+        self.assertEqual(f.dest_path,
+                         os.path.join(f"IMOS/SOOP/SOOP-FishSOOP/REALTIME/2024/03/{os.path.basename(self.nc_file_nrt)}"))
+
+        self.assertTrue(f.is_checked)
+        self.assertTrue(f.is_stored)
 
 
 if __name__ == '__main__':