diff --git a/covjsonkit/decoder/BoundingBox.py b/covjsonkit/decoder/BoundingBox.py index 0ed8b1c..f5b9ba6 100644 --- a/covjsonkit/decoder/BoundingBox.py +++ b/covjsonkit/decoder/BoundingBox.py @@ -39,48 +39,62 @@ def to_geopandas(self): pass def to_xarray(self): - dims = ["number", "steps", "points"] + dims = ["datetimes", "number", "steps", "points"] dataarraydict = {} # Get coordinates x = [] y = [] + datetimes = [] for coord in self.get_coordinates()["composite"]["values"]: x.append(float(coord[0])) y.append(float(coord[1])) - - """ - # Get values - for parameter in self.parameters: - dataarray = xr.DataArray(self.get_values()[parameter][0], dims=dims) - dataarray.attrs["type"] = self.get_parameter_metadata(parameter)["type"] - dataarray.attrs["units"] = self.get_parameter_metadata(parameter)["unit"]["symbol"] - dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)["observedProperty"]["id"] - dataarraydict[dataarray.attrs["long_name"]] = dataarray - """ + for datetime in self.get_coordinates()["t"]["values"]: + datetimes.append(datetime) values = {} for parameter in self.parameters: - values[parameter] = [] + values[parameter] = {} + datetimes = [] numbers = [] steps = [] for coverage in self.coverages: + if "number" not in coverage["mars:metadata"]: + coverage["mars:metadata"]["number"] = 0 numbers.append(coverage["mars:metadata"]["number"]) + if "step" not in coverage["mars:metadata"]: + coverage["mars:metadata"]["step"] = 0 steps.append(coverage["mars:metadata"]["step"]) + datetimes.append(coverage["domain"]["axes"]["t"]["values"][0]) for parameter in self.parameters: - values[parameter].append(coverage["ranges"][parameter]["values"]) - + # values[parameter].append(coverage["ranges"][parameter]["values"]) + if coverage["domain"]["axes"]["t"]["values"][0] not in values[parameter]: + values[parameter][coverage["domain"]["axes"]["t"]["values"][0]] = {} + if ( + coverage["mars:metadata"]["number"] + not in values[parameter][coverage["domain"]["axes"]["t"]["values"][0]] + ): + values[parameter][coverage["domain"]["axes"]["t"]["values"][0]][ + coverage["mars:metadata"]["number"] + ] = {} + values[parameter][coverage["domain"]["axes"]["t"]["values"][0]][coverage["mars:metadata"]["number"]][ + coverage["mars:metadata"]["step"] + ] = coverage["ranges"][parameter]["values"] + + datetimes = list(set(datetimes)) numbers = list(set(numbers)) steps = list(set(steps)) new_values = {} - for parameter in self.parameters: + for parameter in values.keys(): new_values[parameter] = [] - for i, num in enumerate(numbers): + for i, datetime in enumerate(datetimes): new_values[parameter].append([]) - for j, step in enumerate(steps): - new_values[parameter][i].append(values[parameter][i * len(steps) + j]) + for j, number in enumerate(numbers): + new_values[parameter][i].append([]) + for k, step in enumerate(steps): + new_values[parameter][i][j].append(values[parameter][datetime][number][step]) for parameter in self.parameters: dataarray = xr.DataArray(new_values[parameter], dims=dims) @@ -92,6 +106,7 @@ def to_xarray(self): ds = xr.Dataset( dataarraydict, coords=dict( + datetimes=(["datetimes"], datetimes), number=(["number"], numbers), steps=(["steps"], steps), points=(["points"], list(range(0, len(x)))), diff --git a/covjsonkit/decoder/Path.py b/covjsonkit/decoder/Path.py index 8290bf6..4169d3e 100644 --- a/covjsonkit/decoder/Path.py +++ b/covjsonkit/decoder/Path.py @@ -36,7 +36,7 @@ def to_geopandas(self): pass def to_xarray(self): - dims = ["points"] + dims = ["datetimes", "number", "steps", "points"] dataarraydict = {} # Get coordinates @@ -48,9 +48,52 @@ def to_xarray(self): y.append(float(coord[2])) t.append(coord[0]) - # Get values + values = {} for parameter in self.parameters: - dataarray = xr.DataArray(self.get_values()[parameter][0], dims=dims) + values[parameter] = {} + + datetimes = [] + numbers = [] + steps = [] + for coverage in self.coverages: + if "number" not in coverage["mars:metadata"]: + coverage["mars:metadata"]["number"] = 0 + numbers.append(coverage["mars:metadata"]["number"]) + if "step" not in coverage["mars:metadata"]: + coverage["mars:metadata"]["step"] = 0 + steps.append(coverage["mars:metadata"]["step"]) + datetimes.append(coverage["mars:metadata"]["Forecast date"]) + for parameter in self.parameters: + # values[parameter].append(coverage["ranges"][parameter]["values"]) + if coverage["mars:metadata"]["Forecast date"] not in values[parameter]: + values[parameter][coverage["mars:metadata"]["Forecast date"]] = {} + if ( + coverage["mars:metadata"]["number"] + not in values[parameter][coverage["mars:metadata"]["Forecast date"]] + ): + values[parameter][coverage["mars:metadata"]["Forecast date"]][ + coverage["mars:metadata"]["number"] + ] = {} + values[parameter][coverage["mars:metadata"]["Forecast date"]][coverage["mars:metadata"]["number"]][ + coverage["mars:metadata"]["step"] + ] = coverage["ranges"][parameter]["values"] + + datetimes = list(set(datetimes)) + numbers = list(set(numbers)) + steps = list(set(steps)) + + new_values = {} + for parameter in values.keys(): + new_values[parameter] = [] + for i, datetime in enumerate(datetimes): + new_values[parameter].append([]) + for j, number in enumerate(numbers): + new_values[parameter][i].append([]) + for k, step in enumerate(steps): + new_values[parameter][i][j].append(values[parameter][datetime][number][step]) + + for parameter in self.parameters: + dataarray = xr.DataArray(new_values[parameter], dims=dims) dataarray.attrs["type"] = self.get_parameter_metadata(parameter)["type"] dataarray.attrs["units"] = self.get_parameter_metadata(parameter)["unit"]["symbol"] dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)["observedProperty"]["id"] @@ -59,10 +102,19 @@ def to_xarray(self): ds = xr.Dataset( dataarraydict, coords=dict( - points=(["points"], list(range(0, len(x)))), x=(["points"], x), y=(["points"], y), t=(["points"], t) + datetimes=(["datetimes"], datetimes), + number=(["number"], numbers), + steps=(["steps"], steps), + points=(["points"], list(range(0, len(x)))), + x=(["points"], x), + y=(["points"], y), + t=(["points"], t), ), ) for mars_metadata in self.mars_metadata[0]: ds.attrs[mars_metadata] = self.mars_metadata[0][mars_metadata] + # Add date attribute + # ds.attrs["date"] = self.get_coordinates()["t"]["values"][0] + return ds diff --git a/covjsonkit/decoder/TimeSeries.py b/covjsonkit/decoder/TimeSeries.py index ebed33f..79e75ff 100644 --- a/covjsonkit/decoder/TimeSeries.py +++ b/covjsonkit/decoder/TimeSeries.py @@ -1,5 +1,3 @@ -import datetime as dt - import xarray as xr from .decoder import Decoder @@ -40,9 +38,12 @@ def get_coordinates(self): x = domain["axes"]["x"]["values"][0] y = domain["axes"]["y"]["values"][0] z = domain["axes"]["z"]["values"][0] - fct = self.mars_metadata[ind]["date"] + fct = domain["axes"]["t"]["values"][0] ts = domain["axes"]["t"]["values"] - num = self.mars_metadata[ind]["number"] + if "number" in self.mars_metadata[ind]: + num = self.mars_metadata[ind]["number"] + else: + num = 0 for param in self.parameters: coords = [] for t in ts: @@ -57,37 +58,55 @@ def to_geopandas(self): # function to convert covjson to xarray dataset def to_xarray(self): - dims = ["x", "y", "z", "number", "t"] + dims = ["x", "y", "z", "number", "datetime", "t"] dataarraydict = {} # Get coordinates + coords = self.get_domains() + x = coords[0]["axes"]["x"]["values"] + y = coords[0]["axes"]["y"]["values"] + z = coords[0]["axes"]["z"]["values"] + steps = coords[0]["axes"]["t"]["values"] + steps = list(range(len(steps))) + + num = [] + datetime = [] + for coverage in self.covjson["coverages"]: + num.append(coverage["mars:metadata"]["number"]) + datetime.append(coverage["mars:metadata"]["Forecast date"]) + + nums = list(set(num)) + datetime = list(set(datetime)) + + param_values = {} + + for parameter in ["ssrd", "2t"]: + param_values[parameter] = [] + for i, num in enumerate(nums): + param_values[parameter].append([]) + for j, date in enumerate(datetime): + param_values[parameter][i].append([]) + for k, step in enumerate(steps): + for coverage in self.covjson["coverages"]: + if ( + coverage["mars:metadata"]["number"] == num + and coverage["mars:metadata"]["Forecast date"] == date + ): + param_values[parameter][i][j] = coverage["ranges"][parameter]["values"] + for parameter in self.parameters: - param_values = [[[self.get_values()[parameter]]]] - for ind, fc_time_vals in enumerate(self.get_values()[parameter]): - coords = self.get_coordinates()[parameter] - x = [coords[ind][0][0]] - y = [coords[ind][0][1]] - z = [coords[ind][0][2]] - - num = [int(coord[0][5]) for coord in coords] - coords_fc = coords[ind] - try: - t = [dt.datetime.strptime(coord[4], "%Y-%m-%d %H:%M:%S") for coord in coords_fc] - except ValueError: - t = [dt.datetime.strptime(coord[4], "%Y-%m-%dT%H:%M:%SZ") for coord in coords_fc] - - param_coords = {"x": x, "y": y, "z": z, "number": num, "t": t} - dataarray = xr.DataArray( - param_values, - dims=dims, - coords=param_coords, - name=parameter, - ) - - dataarray.attrs["type"] = self.get_parameter_metadata(parameter)["type"] - dataarray.attrs["units"] = self.get_parameter_metadata(parameter)["unit"]["symbol"] - dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)["observedProperty"]["id"] - dataarraydict[dataarray.attrs["long_name"]] = dataarray + param_coords = {"x": x, "y": y, "z": [z], "number": nums, "datetime": datetime, "t": steps} + dataarray = xr.DataArray( + [[[param_values[parameter]]]], + dims=dims, + coords=param_coords, + name=parameter, + ) + + dataarray.attrs["type"] = self.get_parameter_metadata(parameter)["type"] + dataarray.attrs["units"] = self.get_parameter_metadata(parameter)["unit"]["symbol"] + dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)["observedProperty"]["id"] + dataarraydict[dataarray.attrs["long_name"]] = dataarray ds = xr.Dataset(dataarraydict) for mars_metadata in self.mars_metadata[0]: diff --git a/covjsonkit/encoder/Path.py b/covjsonkit/encoder/Path.py index 3a0d9ee..7a8c3bd 100644 --- a/covjsonkit/encoder/Path.py +++ b/covjsonkit/encoder/Path.py @@ -1,8 +1,3 @@ -import json - -import pandas as pd -from covjson_pydantic.coverage import Coverage - from .encoder import Encoder @@ -10,6 +5,7 @@ class Path(Encoder): def __init__(self, type, domaintype): super().__init__(type, domaintype) self.covjson["domainType"] = "Trajectory" + self.covjson["coverages"] = [] def add_coverage(self, mars_metadata, coords, values): new_coverage = {} @@ -20,15 +16,16 @@ def add_coverage(self, mars_metadata, coords, values): self.add_mars_metadata(new_coverage, mars_metadata) self.add_domain(new_coverage, coords) self.add_range(new_coverage, values) - cov = Coverage.model_validate_json(json.dumps(new_coverage)) - self.pydantic_coverage.coverages.append(cov) + self.covjson["coverages"].append(new_coverage) + # cov = Coverage.model_validate_json(json.dumps(new_coverage)) + # self.pydantic_coverage.coverages.append(cov) def add_domain(self, coverage, coords): coverage["domain"]["type"] = "Domain" coverage["domain"]["axes"] = {} coverage["domain"]["axes"]["composite"] = {} coverage["domain"]["axes"]["composite"]["dataType"] = "tuple" - coverage["domain"]["axes"]["composite"]["coordinates"] = self.referencing + coverage["domain"]["axes"]["composite"]["coordinates"] = self.covjson["referencing"][0]["coordinates"] coverage["domain"]["axes"]["composite"]["values"] = coords["composite"] def add_range(self, coverage, values): @@ -53,7 +50,7 @@ def from_xarray(self, dataset): self.add_reference( { - "coordinates": ["x", "y", "z"], + "coordinates": ["t", "x", "y", "z"], "system": { "type": "GeographicCRS", "id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", @@ -77,32 +74,22 @@ def from_xarray(self, dataset): return self.covjson def from_polytope(self, result): - ancestors = [val.get_ancestors() for val in result.leaves] - values = [val.result for val in result.leaves] - - columns = [] - df_dict = {} - # Create empty dataframe - for feature in ancestors[0]: - columns.append(str(feature).split("=")[0]) - df_dict[str(feature).split("=")[0]] = [] - - # populate dataframe - for ancestor in ancestors: - for feature in ancestor: - df_dict[str(feature).split("=")[0]].append(str(feature).split("=")[1]) - values = [val.result for val in result.leaves] - df_dict["values"] = values - df = pd.DataFrame(df_dict) - params = df["param"].unique() + coords = {} + # coords['composite'] = [] + mars_metadata = {} + range_dict = {} + lat = 0 + param = 0 + number = [0] + step = 0 + dates = [0] - for param in params: - self.add_parameter(param) + self.walk_tree(result, lat, coords, mars_metadata, param, range_dict, number, step, dates) self.add_reference( { - "coordinates": ["t", "x", "y"], + "coordinates": ["t", "x", "y", "z"], "system": { "type": "GeographicCRS", "id": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", @@ -110,25 +97,25 @@ def from_polytope(self, result): } ) - mars_metadata = {} - mars_metadata["class"] = df["class"].unique()[0] - mars_metadata["expver"] = df["expver"].unique()[0] - mars_metadata["levtype"] = df["levtype"].unique()[0] - mars_metadata["type"] = df["type"].unique()[0] - mars_metadata["domain"] = df["domain"].unique()[0] - mars_metadata["stream"] = df["stream"].unique()[0] - - range_dict = {} - coords = {} - coords["composite"] = [] - - for param in params: - df_param = df[df["param"] == param] - range_dict[param] = df_param["values"].values.tolist() - - df_param = df[df["param"] == params[0]] - for row in df_param.iterrows(): - coords["composite"].append([row[1]["date"], row[1]["latitude"], row[1]["longitude"]]) - - self.add_coverage(mars_metadata, coords, range_dict) - return json.loads(self.get_json()) + for date in range_dict.keys(): + new_coords = [] + for val in coords[date]["composite"]: + val.insert(0, date) + new_coords.append(val) + coords[date]["composite"] = new_coords + + for num in range_dict[date].keys(): + val_dict = {} + for step in range_dict[date][num][self.parameters[0]].keys(): + val_dict[step] = {} + for para in range_dict[date][num].keys(): + for step in range_dict[date][num][para].keys(): + val_dict[step][para] = range_dict[date][num][para][step] + for step in val_dict.keys(): + mm = mars_metadata.copy() + mm["number"] = num + mm["step"] = step + self.add_coverage(mm, coords[date], val_dict[step]) + + # self.add_coverage(mars_metadata, coords, range_dict) + return self.covjson diff --git a/covjsonkit/encoder/TimeSeries.py b/covjsonkit/encoder/TimeSeries.py index 566a576..0e85fce 100644 --- a/covjsonkit/encoder/TimeSeries.py +++ b/covjsonkit/encoder/TimeSeries.py @@ -44,7 +44,9 @@ def add_range(self, coverage, values): coverage["ranges"][param]["dataType"] = "float" coverage["ranges"][param]["shape"] = [len(values[parameter])] coverage["ranges"][param]["axisNames"] = [str(param)] - coverage["ranges"][param]["values"] = values[parameter] + coverage["ranges"][param]["values"] = [ + values[parameter][val][0] for val in values[parameter].keys() + ] # values[parameter] def add_mars_metadata(self, coverage, metadata): coverage["mars:metadata"] = metadata @@ -83,29 +85,16 @@ def from_xarray(self, dataset): return self.covjson def from_polytope(self, result): - """ - ancestors = [val.get_ancestors() for val in result.leaves] - values = [val.result for val in result.leaves] - - columns = [] - df_dict = {} - # Create empty dataframe - for feature in ancestors[0]: - columns.append(str(feature).split("=")[0]) - df_dict[str(feature).split("=")[0]] = [] - - # populate dataframe - for ancestor in ancestors: - for feature in ancestor: - df_dict[str(feature).split("=")[0]].append(str(feature).split("=")[1]) - values = [val.result for val in result.leaves] - df_dict["values"] = values - df = pd.DataFrame(df_dict) - - params = df["param"].unique() + coords = {} + mars_metadata = {} + range_dict = {} + lat = 0 + param = 0 + number = [0] + step = 0 + dates = [0] - for param in params: - self.add_parameter(param) + self.walk_tree(result, lat, coords, mars_metadata, param, range_dict, number, step, dates) self.add_reference( { @@ -116,69 +105,48 @@ def from_polytope(self, result): }, } ) - steps = df["step"].unique() - mars_metadata = {} - mars_metadata["class"] = df["class"].unique()[0] - mars_metadata["expver"] = df["expver"].unique()[0] - mars_metadata["levtype"] = df["levtype"].unique()[0] - mars_metadata["type"] = df["type"].unique()[0] - mars_metadata["date"] = df["date"].unique()[0] - mars_metadata["domain"] = df["domain"].unique()[0] - mars_metadata["stream"] = df["stream"].unique()[0] - - coords = {} - coords["x"] = list(df["latitude"].unique()) - coords["y"] = list(df["longitude"].unique()) - coords["z"] = ["sfc"] - coords["t"] = [] - - # convert step into datetime - date_format = "%Y%m%dT%H%M%S" - date = pd.Timestamp(mars_metadata["date"]).strftime(date_format) - start_time = datetime.strptime(date, date_format) - for step in steps: - # add current date to list by converting it to iso format - stamp = start_time + timedelta(hours=int(step)) - coords["t"].append(stamp.isoformat() + "Z") - # increment start date by timedelta - - if "number" not in df.columns: - new_metadata = mars_metadata.copy() - range_dict = {} - for param in params: - df_param = df[df["param"] == param] - range_dict[param] = df_param["values"].values.tolist() - self.add_coverage(new_metadata, coords, range_dict) - else: - for number in df["number"].unique(): - new_metadata = mars_metadata.copy() - new_metadata["number"] = number - df_number = df[df["number"] == number] - range_dict = {} - for param in params: - df_param = df_number[df_number["param"] == param] - range_dict[param] = df_param["values"].values.tolist() - self.add_coverage(new_metadata, coords, range_dict) + coordinates = {} + for date in range_dict.keys(): + coordinates[date] = { + "x": [coords[date]["composite"][0][0]], + "y": [coords[date]["composite"][0][1]], + "z": "sfc", + } + coordinates[date]["t"] = [] + for num in range_dict[date].keys(): + for para in range_dict[date][num].keys(): + for step in range_dict[date][num][para].keys(): + date_format = "%Y%m%dT%H%M%S" + new_date = pd.Timestamp(date).strftime(date_format) + start_time = datetime.strptime(new_date, date_format) + # add current date to list by converting it to iso format + stamp = start_time + timedelta(hours=int(step)) + coordinates[date]["t"].append(stamp.isoformat() + "Z") + break + break + + for date in range_dict.keys(): + for num in range_dict[date].keys(): + mm = mars_metadata.copy() + mm["number"] = num + mm["Forecast date"] = date + del mm["step"] + self.add_coverage(mm, coordinates[date], range_dict[date][num]) - return json.loads(self.get_json()) - """ + return self.covjson + def from_polytope_step(self, result): coords = {} - coords["x"] = [] - coords["y"] = [] - coords["z"] = [] - coords["t"] = [] mars_metadata = {} range_dict = {} lat = 0 param = 0 - number = 0 + number = [0] step = 0 - long = 0 + dates = [0] - self.func(result, lat, long, coords, mars_metadata, param, range_dict, number, step) - # print(range_dict) + self.walk_tree(result, lat, coords, mars_metadata, param, range_dict, number, step, dates) self.add_reference( { @@ -189,81 +157,48 @@ def from_polytope(self, result): }, } ) - - for param in range_dict[1].keys(): - self.add_parameter(param) - - for step in range_dict[1][self.parameters[0]].keys(): - date_format = "%Y%m%dT%H%M%S" - date = pd.Timestamp(coords["t"][0]).strftime(date_format) - start_time = datetime.strptime(date, date_format) - # add current date to list by converting it to iso format - stamp = start_time + timedelta(hours=int(step)) - coords["t"].append(stamp.isoformat() + "Z") - - val_dict = {} - for num in range_dict.keys(): - val_dict[num] = {} - for para in range_dict[1].keys(): - val_dict[num][para] = [] - for para in range_dict[num].keys(): - # for step in range_dict[num][para].keys(): - for step in range_dict[num][para]: - val_dict[num][para].extend(range_dict[num][para][step]) - mm = mars_metadata.copy() - mm["number"] = num - self.add_coverage(mm, coords, val_dict[num]) + print(coords) + print(range_dict) + + coordinates = {} + for date in range_dict.keys(): + for num in range_dict[date].keys(): + for para in range_dict[date][num].keys(): + for step in range_dict[date][num][para].keys(): + for dt in range_dict.keys(): + date_format = "%Y%m%dT%H%M%S" + new_date = pd.Timestamp(dt).strftime(date_format) + start_time = datetime.strptime(new_date, date_format) + # add current date to list by converting it to iso format + stamp = start_time + timedelta(hours=int(step)) + if step not in coordinates: + coordinates[step] = { + "x": [coords[date]["composite"][0][0]], + "y": [coords[date]["composite"][0][1]], + "z": "sfc", + } + if "t" not in coordinates[step]: + coordinates[step]["t"] = [stamp.isoformat() + "Z"] + else: + coordinates[step]["t"].append(stamp.isoformat() + "Z") + break + break + break + + print(coordinates) + for date in range_dict.keys(): + for num in range_dict[date].keys(): + for param in range_dict[date][num].keys(): + step_dict = {} + for step in range_dict[date][num][param].keys(): + if step not in step_dict: + step_dict[step] = [] + step_dict[step].append(range_dict[date][num][param][step]) + for step in range_dict[date][num][param].keys(): + mm = mars_metadata.copy() + mm["number"] = num + mm["Forecast date"] = date + mm["step"] = step + self.add_coverage(mm, coordinates[step], range_dict[date][num]) return self.covjson - - def func(self, tree, lat, long, coords, mars_metadata, param, range_dict, number, step): - if len(tree.children) != 0: - # recurse while we are not a leaf - for c in tree.children: - if ( - c.axis.name != "latitude" - and c.axis.name != "longitude" - and c.axis.name != "param" - and c.axis.name != "step" - and c.axis.name != "date" - ): - mars_metadata[c.axis.name] = c.values[0] - if c.axis.name == "latitude": - lat = c.values[0] - if c.axis.name == "param": - param = c.values - for num in range_dict: - for para in param: - range_dict[num][para] = {} - if c.axis.name == "date": - coords["t"] = [str(c.values[0]) + "Z"] - mars_metadata[c.axis.name] = str(c.values[0]) + "Z" - if c.axis.name == "number": - number = c.values - for num in number: - range_dict[num] = {} - if c.axis.name == "step": - step = c.values - for num in number: - for para in param: - for s in step: - range_dict[num][para][s] = [] - - self.func(c, lat, long, coords, mars_metadata, param, range_dict, number, step) - else: - vals = len(tree.values) - tree.values = [float(val) for val in tree.values] - tree.result = [float(val) for val in tree.result] - num_intervals = int(len(tree.result) / len(number)) - para_intervals = int(num_intervals / len(param)) - - coords["x"] = [lat] - coords["y"] = [long] - coords["z"] = ["sfc"] - - for num in range_dict: - for i, para in enumerate(range_dict[num]): - for s in range_dict[num][para]: - start = ((int(num) - 1) * num_intervals) + (vals * int(s)) + ((i * para_intervals)) - end = ((int(num) - 1) * num_intervals) + ((vals) * int(s + 1)) + ((i) * (para_intervals)) - range_dict[num][para][s].extend(tree.result[start:end]) diff --git a/covjsonkit/encoder/VerticalProfile.py b/covjsonkit/encoder/VerticalProfile.py index 7251762..104f3c9 100644 --- a/covjsonkit/encoder/VerticalProfile.py +++ b/covjsonkit/encoder/VerticalProfile.py @@ -1,13 +1,11 @@ -import json - -import pandas as pd - from .encoder import Encoder class VerticalProfile(Encoder): def __init__(self, type, domaintype): super().__init__(type, domaintype) + self.covjson["domainType"] = "VerticalProfile" + self.covjson["coverages"] = [] def add_coverage(self, mars_metadata, coords, values): new_coverage = {} @@ -34,12 +32,13 @@ def add_domain(self, coverage, coords): def add_range(self, coverage, values): for parameter in self.parameters: - coverage["ranges"][parameter] = {} - coverage["ranges"][parameter]["type"] = "NdArray" - coverage["ranges"][parameter]["dataType"] = "float" - coverage["ranges"][parameter]["shape"] = [len(values[parameter])] - coverage["ranges"][parameter]["axisNames"] = ["z"] - coverage["ranges"][parameter]["values"] = values[parameter] # [values[parameter]] + param = self.convert_param_id_to_param(parameter) + coverage["ranges"][param] = {} + coverage["ranges"][param]["type"] = "NdArray" + coverage["ranges"][param]["dataType"] = "float" + coverage["ranges"][param]["shape"] = [len(values[parameter])] + coverage["ranges"][param]["axisNames"] = ["z"] + coverage["ranges"][param]["values"] = values[parameter] # [values[parameter]] def add_mars_metadata(self, coverage, metadata): coverage["mars:metadata"] = metadata @@ -84,28 +83,33 @@ def from_xarray(self, dataset): return self.covjson def from_polytope(self, result): - ancestors = [val.get_ancestors() for val in result.leaves] - values = [val.result for val in result.leaves] - - columns = [] - df_dict = {} - # Create empty dataframe - for feature in ancestors[0]: - columns.append(str(feature).split("=")[0]) - df_dict[str(feature).split("=")[0]] = [] - - # populate dataframe - for ancestor in ancestors: - for feature in ancestor: - df_dict[str(feature).split("=")[0]].append(str(feature).split("=")[1]) - values = [val.result for val in result.leaves] - df_dict["values"] = values - df = pd.DataFrame(df_dict) - - params = df["param"].unique() - - for param in params: - self.add_parameter(param) + coords = {} + # coords['x'] = [] + # coords['y'] = [] + # coords['z'] = [] + # coords['t'] = [] + mars_metadata = {} + range_dict = {} + lat = 0 + param = 0 + # number = 0 + step = 0 + long = 0 + levels = 0 + dates = 0 + + self.func( + result, + lat, + long, + coords, + mars_metadata, + param, + range_dict, + step, + levels, + dates, + ) self.add_reference( { @@ -117,37 +121,87 @@ def from_polytope(self, result): } ) - mars_metadata = {} - mars_metadata["class"] = df["class"].unique()[0] - mars_metadata["expver"] = df["expver"].unique()[0] - mars_metadata["levtype"] = df["levtype"].unique()[0] - mars_metadata["type"] = df["type"].unique()[0] - mars_metadata["date"] = df["date"].unique()[0] - mars_metadata["domain"] = df["domain"].unique()[0] - mars_metadata["stream"] = df["stream"].unique()[0] + for date in range_dict.keys(): + for param in range_dict[date].keys(): + # self.coord_length = len(range_dict[date][param]) + self.add_parameter(param) + break - coords = {} - coords["x"] = list(df["latitude"].unique()) - coords["y"] = list(df["longitude"].unique()) - coords["z"] = list(df["level"].unique()) - coords["t"] = list(df["date"].unique()) - - if "number" not in df.columns: - new_metadata = mars_metadata.copy() - range_dict = {} - for param in params: - df_param = df[df["param"] == param] - range_dict[param] = df_param["values"].values.tolist() - self.add_coverage(new_metadata, coords, range_dict) + for date in range_dict.keys(): + self.add_coverage(mars_metadata, coords[date], range_dict[date]) + + # return json.loads(self.get_json()) + return self.covjson + + def func( + self, + tree, + lat, + long, + coords, + mars_metadata, + param, + range_dict, + step, + levels, + dates, + ): + if len(tree.children) != 0: + # recurse while we are not a leaf + for c in tree.children: + if ( + c.axis.name != "latitude" + and c.axis.name != "longitude" + and c.axis.name != "param" + and c.axis.name != "step" + and c.axis.name != "date" + and c.axis.name != "levelist" + ): + mars_metadata[c.axis.name] = c.values[0] + if c.axis.name == "latitude": + lat = c.values[0] + if c.axis.name == "param": + param = c.values + for date in dates: + for para in param: + range_dict[date][para] = [] + if c.axis.name == "date": + dates = [str(date) + "Z" for date in c.values] + for date in dates: + coords[date] = {} + range_dict[date] = {} + mars_metadata[c.axis.name] = str(c.values[0]) + "Z" + if c.axis.name == "levelist": + levels = c.values + + self.func( + c, + lat, + long, + coords, + mars_metadata, + param, + range_dict, + step, + levels, + dates, + ) else: - for number in df["number"].unique(): - new_metadata = mars_metadata.copy() - new_metadata["number"] = number - df_number = df[df["number"] == number] - range_dict = {} - for param in params: - df_param = df_number[df_number["param"] == param] - range_dict[param] = df_param["values"].values.tolist() - self.add_coverage(new_metadata, coords, range_dict) - - return json.loads(self.get_json()) + tree.values = [float(val) for val in tree.values] + tree.result = [float(val) for val in tree.result] + # num_intervals = int(len(tree.result)/len(number)) + # para_intervals = int(num_intervals/len(param)) + len_paras = len(param) * len(levels) + len_levels = len(param) + + for date in dates: + + coords[date]["x"] = [lat] + coords[date]["y"] = [long] + coords[date]["z"] = list(levels) + coords[date]["t"] = date + + for i, date in enumerate(dates): + for j, level in enumerate(list(levels)): + for k, para in enumerate(param): + range_dict[date][para].append(tree.result[i * len_paras + j * len_levels + k]) diff --git a/covjsonkit/encoder/encoder.py b/covjsonkit/encoder/encoder.py index f1f857e..6655ae6 100644 --- a/covjsonkit/encoder/encoder.py +++ b/covjsonkit/encoder/encoder.py @@ -35,11 +35,13 @@ def __init__(self, type, domaintype): elif domaintype == "frame": self.domaintype = DomainType.multi_point elif domaintype == "path": - self.domaintype = DomainType.trajectory + self.domaintype = "Trajectory" - self.pydantic_coverage = CoverageCollection( - type=type, coverages=[], domainType=self.domaintype, parameters={}, referencing=[] - ) + # Trajectory not yet implemented in covjson-pydantic + if self.domaintype != "Trajectory": + self.pydantic_coverage = CoverageCollection( + type=type, coverages=[], domainType=self.domaintype, parameters={}, referencing=[] + ) self.parameters = [] def add_parameter(self, param): @@ -112,8 +114,9 @@ def walk_tree(self, tree, lat, coords, mars_metadata, param, range_dict, number, if para not in range_dict[date][num]: range_dict[date][num][para] = {} self.add_parameter(para) - if c.axis.name == "date": + if c.axis.name == "date" or c.axis.name == "time": dates = [str(date) + "Z" for date in c.values] + mars_metadata["Forecast date"] = str(c.values[0]) for date in dates: coords[date] = {} coords[date]["composite"] = [] @@ -135,7 +138,6 @@ def walk_tree(self, tree, lat, coords, mars_metadata, param, range_dict, number, self.walk_tree(c, lat, coords, mars_metadata, param, range_dict, number, step, dates) else: - # vals = len(tree.values) tree.values = [float(val) for val in tree.values] if all(val is None for val in tree.result): range_dict.pop(dates[0], None) diff --git a/tests/test_decoder_time_series.py b/tests/test_decoder_time_series.py index 8b91613..bf19fe9 100644 --- a/tests/test_decoder_time_series.py +++ b/tests/test_decoder_time_series.py @@ -260,26 +260,26 @@ def test_timeseries_coordinates(self): coordinates = { "t": [ [ - [3, 7, 1, "20170101", "2017-01-01 00:00:00", "0"], - [3, 7, 1, "20170101", "2017-01-01 06:00:00", "0"], - [3, 7, 1, "20170101", "2017-01-01 12:00:00", "0"], + [3, 7, 1, "2017-01-01 00:00:00", "2017-01-01 00:00:00", "0"], + [3, 7, 1, "2017-01-01 00:00:00", "2017-01-01 06:00:00", "0"], + [3, 7, 1, "2017-01-01 00:00:00", "2017-01-01 12:00:00", "0"], ], [ - [3, 7, 1, "20170102", "2017-01-02 00:00:00", "0"], - [3, 7, 1, "20170102", "2017-01-02 06:00:00", "0"], - [3, 7, 1, "20170102", "2017-01-02 12:00:00", "0"], + [3, 7, 1, "2017-01-02 00:00:00", "2017-01-02 00:00:00", "0"], + [3, 7, 1, "2017-01-02 00:00:00", "2017-01-02 06:00:00", "0"], + [3, 7, 1, "2017-01-02 00:00:00", "2017-01-02 12:00:00", "0"], ], ], "p": [ [ - [3, 7, 1, "20170101", "2017-01-01 00:00:00", "0"], - [3, 7, 1, "20170101", "2017-01-01 06:00:00", "0"], - [3, 7, 1, "20170101", "2017-01-01 12:00:00", "0"], + [3, 7, 1, "2017-01-01 00:00:00", "2017-01-01 00:00:00", "0"], + [3, 7, 1, "2017-01-01 00:00:00", "2017-01-01 06:00:00", "0"], + [3, 7, 1, "2017-01-01 00:00:00", "2017-01-01 12:00:00", "0"], ], [ - [3, 7, 1, "20170102", "2017-01-02 00:00:00", "0"], - [3, 7, 1, "20170102", "2017-01-02 06:00:00", "0"], - [3, 7, 1, "20170102", "2017-01-02 12:00:00", "0"], + [3, 7, 1, "2017-01-02 00:00:00", "2017-01-02 00:00:00", "0"], + [3, 7, 1, "2017-01-02 00:00:00", "2017-01-02 06:00:00", "0"], + [3, 7, 1, "2017-01-02 00:00:00", "2017-01-02 12:00:00", "0"], ], ], } diff --git a/tests/test_encoder_time_series.py b/tests/test_encoder_time_series.py index 74096d3..1b0c50b 100644 --- a/tests/test_encoder_time_series.py +++ b/tests/test_encoder_time_series.py @@ -223,7 +223,7 @@ def test_add_coverage(self): "t": timestamps, } coords.append(coord) - value = {"2t": [random.uniform(230, 270) for _ in range(0, len(timestamps))]} + value = {"2t": {0: [random.uniform(230, 270) for _ in range(0, len(timestamps))]}} values.append(value) encoder.add_coverage(metadata, coord, value)