Skip to content

Commit

Permalink
checkpointing, charts, updating info
Browse files Browse the repository at this point in the history
  • Loading branch information
@kauevestena
kauevestena committed Oct 15, 2024
1 parent 2f7109c commit 57d6273
Show file tree
Hide file tree
Showing 5 changed files with 151 additions and 129 deletions.
26 changes: 19 additions & 7 deletions dashboard/statistics_funcs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ def create_double_scatter_bar(
yh2="smoothness",
hcolor=None,
fontsize=24,
tooltip_fields: list = ["element_type", "id"],
# tooltip_fields: list = ["element_type", "id"], # deprecated, as there's generally too much data
):

# preselect only the needed columns:
Expand All @@ -190,14 +190,13 @@ def create_double_scatter_bar(
The color for the histograms. If None, defaults to 'lightseagreen'.
fontsize : int
The font size for the title.
tooltip_fields : list
A list of column names to be included in the tooltip.
Returns
-------
An Altair chart object.
"""
needed_columns = tooltip_fields + [xs, ys, xh, yh1, yh2, hcolor, scolor]
needed_columns = [xs, ys, xh, yh1, yh2, hcolor, scolor]

# remove None, and any one with "()":
needed_columns = [column for column in needed_columns if column]
Expand All @@ -224,7 +223,7 @@ def create_double_scatter_bar(
x=xs,
y=ys,
color=scolor,
tooltip=alt.Tooltip(tooltip_fields),
tooltip=alt.Tooltip(["count()"], title="count:"),
)
.properties(
width=600,
Expand All @@ -239,7 +238,7 @@ def create_double_scatter_bar(
.encode(
x=xh,
color=hcolor,
tooltip=alt.Tooltip(tooltip_fields),
tooltip=alt.Tooltip(["count()"], title="count:"),
)
.properties(
width=300,
Expand All @@ -253,7 +252,20 @@ def create_double_scatter_bar(
# if hcolor:
# hist_base.encode(color=hcolor)

hist = hist_base.encode(y=yh1) | hist_base.encode(y=yh2)
# extra encoding rules
encoding_base = {}

# if hcolor:
# encoding_base["color"] = hcolor

encoding_h1 = encoding_base.copy()
encoding_h1["y"] = yh1

encoding_h2 = encoding_base.copy()
encoding_h2["y"] = yh2

# define the histograms
hist = hist_base.encode(**encoding_h1) | hist_base.encode(**encoding_h2)

return (scatter & hist).configure_title(fontSize=fontsize, align="center")

Expand Down
54 changes: 28 additions & 26 deletions dashboard/statistics_specs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,8 @@
"xh": "count()",
"yh1": "surface",
"yh2": "smoothness",
"hcolor": "length(km)",
"hcolor": "age",
"fontsize": 24,
"tooltip_fields": ["id"],
},
"title": "Surface x Smoothness",
},
Expand Down Expand Up @@ -103,8 +102,9 @@
"column": "length(km)",
"boxplot_title": "Sidewalks Length (km)",
"tooltip_fields": ["id"],
"maxbins": 33,
},
"title": "Sidewalks Length (km)",
"title": "Length (km)",
},
"sidewalks_n_revs": {
"function": create_linked_boxplot_histogram,
Expand Down Expand Up @@ -178,10 +178,20 @@
"yh2": "smoothness",
"hcolor": "crossing",
"fontsize": 24,
"tooltip_fields": ["id"],
},
"title": "Surface x Smoothness",
},
"crossings_length": {
"function": create_linked_boxplot_histogram,
"params": {
"df": gdfs_dict["crossings"],
"column": "length(km)",
"boxplot_title": "Crossings Length (km)",
"tooltip_fields": ["id"],
"maxbins": 33,
},
"title": "Length (km)",
},
"crossings_age": {
"function": create_linked_boxplot_histogram,
"params": {
Expand All @@ -192,24 +202,6 @@
},
"title": "Update Age",
},
# "crossings_yr_moth_update": {
# "function": create_barchart,
# "params": {
# "input_df": updating_dicts["crossings"],
# "fieldname": "year_month",
# "title": "Year and Month Of Update (Crossings)",
# },
# "title": "Year and Month Of Update",
# },
# "crossings_number_revisions": {
# "function": create_barchart,
# "params": {
# "input_df": updating_dicts["crossings"],
# "fieldname": "n_revs",
# "title": "Number of Revisions (crossings)",
# },
# "title": "Number Of Revisions",
# },
"crossings_n_revs": {
"function": create_linked_boxplot_histogram,
"params": {
Expand All @@ -235,7 +227,6 @@
"yh2": "tactile_paving",
"hcolor": "wheelchair",
"fontsize": 24,
"tooltip_fields": ["id"],
},
"title": "Surface x Smoothness",
},
Expand Down Expand Up @@ -354,13 +345,12 @@
"title": "Surface x Smoothness (other_footways)",
"xs": "surface",
"ys": "smoothness",
"scolor": None,
"scolor": oswm_footway_fieldname,
"xh": "count()",
"yh1": "surface",
"yh2": "smoothness",
"hcolor": "crossing",
"hcolor": oswm_footway_fieldname,
"fontsize": 24,
"tooltip_fields": ["id"],
},
"title": "Surface x Smoothness",
},
Expand All @@ -373,6 +363,18 @@
# },
# "title": "Year and Month Of Update",
# },
"other_footways_length": {
"function": create_linked_boxplot_histogram,
"params": {
"df": gdfs_dict["other_footways"],
"column": "length(km)",
"boxplot_title": "Other Footways Length (km)",
"color_field": oswm_footway_fieldname,
"tooltip_fields": ["id"],
"maxbins": 33,
},
"title": "Length (km)",
},
"other_footways_age": {
"function": create_linked_boxplot_histogram,
"params": {
Expand Down
95 changes: 0 additions & 95 deletions functions.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -522,101 +522,6 @@ def parse_datetime_str(inputstr, format="ymdhms"):
return datetime.strptime(inputstr, format_dict[format])


def get_datetime_last_update(
featureid,
featuretype="way",
onlylast=True,
return_parsed=True,
return_special_tuple=True,
):
# TODO: use osmapi!

h_url = get_feature_history_url(featureid, featuretype)

try:
response = requests.get(h_url)
except:
if onlylast:
if return_parsed and return_special_tuple:
return [None] * 4 # 4 Nones

return ""
else:
return []

if response.status_code == 200:
tree = ElementTree.fromstring(response.content)

element_list = tree.findall(featuretype)

if element_list:
date_rec = [element.attrib["timestamp"][:-1] for element in element_list]

if onlylast:
if return_parsed:
if return_special_tuple:
# parsed = datetime.strptime(date_rec[-1],'%Y-%m-%dT%H:%M:%S')
parsed = parse_datetime_str(date_rec[-1])
return len(date_rec), parsed.day, parsed.month, parsed.year

else:
# return datetime.strptime(date_rec[-1],'%Y-%m-%dT%H:%M:%S')
return parse_datetime_str(date_rec[-1])

else:
return date_rec[-1]

else:
if return_parsed:
return [parse_datetime_str(record) for record in date_rec]

else:
return date_rec

else:
if onlylast:
return ""
else:
return []

else:
print("bad request, check feature id/type")
if onlylast:
return ""
else:
return []


class GetDatetimeLastUpdate:
import osmapi

api = osmapi.OsmApi()

def __init__(self):
pass

def get_datetime_last_update_way(self, featureid):
try:
res = self.api.WayGet(featureid)
dt = res["timestamp"] # date time object
return res["version"], dt.day, dt.month, dt.year
except:
return -1, default_missing_day, default_missing_month, default_missing_year

def get_datetime_last_update_node(self, featureid):
try:
res = self.api.NodeGet(featureid)
dt = res["timestamp"] # date time object
return res["version"], dt.day, dt.month, dt.year
except:
return -1, default_missing_day, default_missing_month, default_missing_year


def get_datetime_last_update_node(featureid):
# all default options
return get_datetime_last_update(featureid, featuretype="node")


def print_relevant_columnamesV2(
input_df, not_include=("score", "geometry", "type", "id"), outfilepath=None
):
Expand Down
2 changes: 1 addition & 1 deletion getting_feature_versioning_data.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# from constants import *
from functions import *
from versioning_funcs import *
import geopandas as gpd
import pandas as pd
from tqdm import tqdm
Expand Down
103 changes: 103 additions & 0 deletions versioning_funcs.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
from functions import *
import osmapi


class GetDatetimeLastUpdate:

default_return = (
-1,
default_missing_day,
default_missing_month,
default_missing_year,
)

def __init__(self):
self.api = osmapi.OsmApi()

def get_datetime_last_update_way(self, featureid):
try:
res = self.api.WayGet(featureid)
dt = res["timestamp"] # date time object
return res["version"], dt.day, dt.month, dt.year
except Exception as e:
print(e)
return self.default_return

def get_datetime_last_update_node(self, featureid):
try:
res = self.api.NodeGet(featureid)
dt = res["timestamp"] # date time object
return res["version"], dt.day, dt.month, dt.year
except Exception as e:
print(e)
return self.default_return


# def get_datetime_last_update(
# featureid,
# featuretype="way",
# onlylast=True,
# return_parsed=True,
# return_special_tuple=True,
# ):
# # TODO: use osmapi!

# h_url = get_feature_history_url(featureid, featuretype)

# try:
# response = requests.get(h_url)
# except:
# if onlylast:
# if return_parsed and return_special_tuple:
# return [None] * 4 # 4 Nones

# return ""
# else:
# return []

# if response.status_code == 200:
# tree = ElementTree.fromstring(response.content)

# element_list = tree.findall(featuretype)

# if element_list:
# date_rec = [element.attrib["timestamp"][:-1] for element in element_list]

# if onlylast:
# if return_parsed:
# if return_special_tuple:
# # parsed = datetime.strptime(date_rec[-1],'%Y-%m-%dT%H:%M:%S')
# parsed = parse_datetime_str(date_rec[-1])
# return len(date_rec), parsed.day, parsed.month, parsed.year

# else:
# # return datetime.strptime(date_rec[-1],'%Y-%m-%dT%H:%M:%S')
# return parse_datetime_str(date_rec[-1])

# else:
# return date_rec[-1]

# else:
# if return_parsed:
# return [parse_datetime_str(record) for record in date_rec]

# else:
# return date_rec

# else:
# if onlylast:
# return ""
# else:
# return []

# else:
# print("bad request, check feature id/type")
# if onlylast:
# return ""
# else:
# return []


# def get_datetime_last_update_node(featureid):
# # all default options
# return get_datetime_last_update(featureid, featuretype="node")

0 comments on commit 57d6273

Please sign in to comment.