From 9344fcf32557852815b3ec44b1d663daa0036cbf Mon Sep 17 00:00:00 2001 From: Praneeth Ratna <63547155+praneethratna@users.noreply.github.com> Date: Wed, 8 Nov 2023 23:21:31 +0530 Subject: [PATCH] removed the usage of parameter in parsing (#1214) --- echopype/convert/api.py | 10 +--- echopype/convert/parse_ad2cp.py | 4 +- echopype/convert/parse_azfp.py | 4 +- echopype/convert/parse_base.py | 57 ++++------------------- echopype/convert/parse_ek60.py | 13 +----- echopype/convert/parse_ek80.py | 42 +---------------- echopype/tests/convert/test_parse_base.py | 3 -- 7 files changed, 20 insertions(+), 113 deletions(-) diff --git a/echopype/convert/api.py b/echopype/convert/api.py index 29c4b63b6..82793d6ae 100644 --- a/echopype/convert/api.py +++ b/echopype/convert/api.py @@ -411,17 +411,11 @@ def open_raw( # Check file extension and existence file_chk, xml_chk = _check_file(raw_file, sonar_model, xml_path, storage_options) - # TODO: the if-else below only works for the AZFP vs EK contrast, - # but is brittle since it is abusing params by using it implicitly - if SONAR_MODELS[sonar_model]["xml"]: - params = xml_chk - else: - params = "ALL" # reserved to control if only wants to parse a certain type of datagram - # Parse raw file and organize data into groups parser = SONAR_MODELS[sonar_model]["parser"]( file_chk, - params=params, + # Currently used only for AZFP XML File + file_meta=xml_chk, storage_options=storage_options, sonar_model=sonar_model, ) diff --git a/echopype/convert/parse_ad2cp.py b/echopype/convert/parse_ad2cp.py index 6c5c796fd..5d9f7e601 100644 --- a/echopype/convert/parse_ad2cp.py +++ b/echopype/convert/parse_ad2cp.py @@ -219,7 +219,9 @@ class NoMorePackets(Exception): class ParseAd2cp(ParseBase): - def __init__(self, file, params, storage_options={}, dgram_zarr_vars={}, sonar_model="AD2CP"): + def __init__( + self, file, file_meta, storage_options={}, dgram_zarr_vars={}, sonar_model="AD2CP" + ): super().__init__(file, storage_options, sonar_model) self.config = None self.packets: List[Ad2cpDataPacket] = [] diff --git a/echopype/convert/parse_azfp.py b/echopype/convert/parse_azfp.py index ae9f928e2..425b0bc1b 100644 --- a/echopype/convert/parse_azfp.py +++ b/echopype/convert/parse_azfp.py @@ -73,12 +73,12 @@ class ParseAZFP(ParseBase): HEADER_FORMAT = ">HHHHIHHHHHHHHHHHHHHHHHHHHHHHHHHHHHBBBBHBBBBBBBBHHHHHHHHHHHHHHHHHHHH" FILE_TYPE = 64770 - def __init__(self, file, params, storage_options={}, dgram_zarr_vars={}, sonar_model="AZFP"): + def __init__(self, file, file_meta, storage_options={}, dgram_zarr_vars={}, sonar_model="AZFP"): super().__init__(file, storage_options, sonar_model) # Parent class attributes # regex pattern used to grab datetime embedded in filename self.timestamp_pattern = FILENAME_DATETIME_AZFP - self.xml_path = params + self.xml_path = file_meta # Class attributes self.parameters = defaultdict(list) diff --git a/echopype/convert/parse_base.py b/echopype/convert/parse_base.py index ac2b981d4..f5716c7b3 100644 --- a/echopype/convert/parse_base.py +++ b/echopype/convert/parse_base.py @@ -43,7 +43,7 @@ def _print_status(self): class ParseEK(ParseBase): """Class for converting data from Simrad echosounders.""" - def __init__(self, file, params, storage_options, sonar_model): + def __init__(self, file, storage_options, sonar_model): super().__init__(file, storage_options, sonar_model) # Parent class attributes @@ -59,7 +59,6 @@ def __init__(self, file, params, storage_options, sonar_model): self.ch_ids = defaultdict( list ) # Stores the channel ids for each data type (power, angle, complex) - self.data_type = self._select_datagrams(params) self.nmea = defaultdict(list) # Dictionary to store NMEA data(timestamp and string) self.mru = defaultdict(list) # Dictionary to store MRU data (heading, pitch, roll, heave) @@ -371,19 +370,8 @@ def parse_raw(self): # and in the form of floats separated by semicolons v["pulse_duration"] = [float(x) for x in v["pulse_length"].split(";")] - # If exporting to XML file (EK80/EA640 only), print a message - if "print_export_msg" in self.data_type: - if "ENV" in self.data_type: - xml_type = "environment" - elif "CONFIG" in self.data_type: - xml_type = "configuration" - logger.info(f"exporting {xml_type} XML file") - # Don't parse anything else if only the config xml is required. - if "CONFIG" in self.data_type: - return - # If not exporting to XML, print the usual converting message - else: - self._print_status() + # print the usual converting message + self._print_status() # Check if reading an ME70 file with a CON1 datagram. next_datagram = fid.peek() @@ -398,11 +386,10 @@ def parse_raw(self): # Read the rest of datagrams self._read_datagrams(fid) - if "ALL" in self.data_type: - # Convert ping time to 1D numpy array, stored in dict indexed by channel, - # this will help merge data from all channels into a cube - for ch, val in self.ping_time.items(): - self.ping_time[ch] = np.array(val, dtype="datetime64[ns]") + # Convert ping time to 1D numpy array, stored in dict indexed by channel, + # this will help merge data from all channels into a cube + for ch, val in self.ping_time.items(): + self.ping_time[ch] = np.array(val, dtype="datetime64[ns]") def _read_datagrams(self, fid): """Read all datagrams. @@ -494,25 +481,13 @@ def _read_datagrams(self, fid): num_datagrams_parsed += 1 - # Skip any datagram that the user does not want to save - if ( - not any(new_datagram["type"].startswith(dgram) for dgram in self.data_type) - and "ALL" not in self.data_type - ): - continue - # XML datagrams store environment or instrument parameters for EK80 if new_datagram["type"].startswith("XML"): - if new_datagram["subtype"] == "environment" and ( - "ENV" in self.data_type or "ALL" in self.data_type - ): + if new_datagram["subtype"] == "environment": self.environment = new_datagram["environment"] self.environment["xml"] = new_datagram["xml"] self.environment["timestamp"] = new_datagram["timestamp"] - # Don't parse anything else if only the environment xml is required. - if "ENV" in self.data_type: - break - elif new_datagram["subtype"] == "parameter" and ("ALL" in self.data_type): + elif new_datagram["subtype"] == "parameter": current_parameters = new_datagram["parameter"] # RAW0 datagrams store raw acoustic data for a channel for EK60 @@ -661,17 +636,3 @@ def pad_shorter_ping(data_list) -> np.ndarray: else: out_array = np.array(data_list) return out_array - - def _select_datagrams(self, params): - """Translates user input into specific datagrams or ALL - - Valid use cases: - # get GPS info only (EK60, EK80) - # ec.to_netcdf(data_type='GPS') - - # get configuration XML only (EK80) - # ec.to_netcdf(data_type='CONFIG') - - # get environment XML only (EK80) - # ec.to_netcdf(data_type='ENV') - """ diff --git a/echopype/convert/parse_ek60.py b/echopype/convert/parse_ek60.py index c5841ac43..b33b45fba 100644 --- a/echopype/convert/parse_ek60.py +++ b/echopype/convert/parse_ek60.py @@ -4,14 +4,5 @@ class ParseEK60(ParseEK): """Class for converting data from Simrad EK60 echosounders.""" - def __init__(self, file, params, storage_options={}, sonar_model="EK60"): - super().__init__(file, params, storage_options, sonar_model) - - def _select_datagrams(self, params): - # Translates user input into specific datagrams or ALL - if params == "ALL": - return ["ALL"] - elif params == "GPS": - return ["NME"] - else: - raise ValueError("Unknown data type", params) + def __init__(self, file, file_meta, storage_options={}, sonar_model="EK60"): + super().__init__(file, storage_options, sonar_model) diff --git a/echopype/convert/parse_ek80.py b/echopype/convert/parse_ek80.py index 895ec4b82..cbc7498e6 100644 --- a/echopype/convert/parse_ek80.py +++ b/echopype/convert/parse_ek80.py @@ -4,44 +4,6 @@ class ParseEK80(ParseEK): """Class for converting data from Simrad EK80 echosounders.""" - def __init__(self, file, params, storage_options={}, sonar_model="EK80"): - super().__init__(file, params, storage_options, sonar_model) + def __init__(self, file, file_meta, storage_options={}, sonar_model="EK80"): + super().__init__(file, storage_options, sonar_model) self.environment = {} # dictionary to store environment data - - def _select_datagrams(self, params): - """Translates user input into specific datagrams or ALL""" - - def translate_to_dgram(s): - if s == "ALL": - return ["ALL"] - # The GPS flag indicates that only the NME and MRU datagrams are parsed. - # It is kept in the list because it is used in SetGroups - # to flag that only the platform group is saved. - elif s == "GPS": - return ["NME", "MRU"] - # CONFIG flag indicates that only the configuration XML is parsed. - # The XML flag is not needed because the configuration is always - # the first datagram parsed. - elif s == "CONFIG": - return ["CONFIG"] - # XML flag indicates that XML0 datagrams should be read. - # ENV flag indicates that of the XML datagrams, - # only keep the environment datagrams - elif s == "ENV": - return ["XML", "ENV"] - # EXPORT_XML flag passed in only by the to_xml function - # Used to print the export message when writing to an xml file - elif s == "EXPORT_XML": - return ["print_export_msg"] - else: - raise ValueError("Unknown data type", params) - - # Params is a string when user sets data_type in to_netcdf/to_zarr - if isinstance(params, str): - dgrams = translate_to_dgram(params) - # Params is a list when the parse classes are called by to_xml - else: - dgrams = [] - for p in params: - dgrams += translate_to_dgram(p) - return dgrams diff --git a/echopype/tests/convert/test_parse_base.py b/echopype/tests/convert/test_parse_base.py index 160048cea..69d34eaea 100644 --- a/echopype/tests/convert/test_parse_base.py +++ b/echopype/tests/convert/test_parse_base.py @@ -42,14 +42,12 @@ def test__print_status(self): class TestParseEK: file = "./my_file.raw" storage_options = {} - params = "ALL" data_types = ["power", "angle", "complex"] raw_types = ["receive", "transmit"] def _get_parser(self, sonar_model, ping_data_dict): parser = ParseEK( file=self.file, - params=self.params, storage_options=self.storage_options, sonar_model=sonar_model, ) @@ -61,7 +59,6 @@ def _get_parser(self, sonar_model, ping_data_dict): def test_constructor(self, sonar_model): parser = ParseEK( file=self.file, - params=self.params, storage_options=self.storage_options, sonar_model=sonar_model, )