From e7dababefbd513df6c4b56a90a966f996c308c21 Mon Sep 17 00:00:00 2001 From: mashiro210 <63339688+mashiro210@users.noreply.github.com> Date: Mon, 28 Nov 2022 15:15:46 +0900 Subject: [PATCH] Add files via upload --- getDataFromInfluxDBCloud.py | 84 +++++++++++++++ main_v.1.1.py | 201 ++++++++++++++++++++++++++++++++++++ visualize.py | 102 ++++++++++++++++++ 3 files changed, 387 insertions(+) create mode 100644 getDataFromInfluxDBCloud.py create mode 100644 main_v.1.1.py create mode 100644 visualize.py diff --git a/getDataFromInfluxDBCloud.py b/getDataFromInfluxDBCloud.py new file mode 100644 index 0000000..655585f --- /dev/null +++ b/getDataFromInfluxDBCloud.py @@ -0,0 +1,84 @@ +# Author: Mashiro +# Last update: 2022/10/12 +# Description: module for getting data from InfluxDB cloud + +# load molues +from datetime import datetime +from dateutil.tz import gettz + +import pandas as pd +import influxdb_client + +# define functions for converting timezone + +def convertTimeZoneFromUTC(timestampUTC, timeZone = 'Asia/Tokyo'): + convertedDatetime = timestampUTC.astimezone(gettz(timeZone)) + convertedTimestamp = datetime.strftime(convertedDatetime, '%Y-%m-%d %H:%M') + + return convertedTimestamp + +def convertTimeZoneToUTC(timestamp): + convertedDatetime = timestamp.astimezone(gettz('UTC')) + convertedTimestamp = datetime.strftime(convertedDatetime, '%Y-%m-%d %H:%M') + + return convertedTimestamp + +# define function for generating params + +def generateParams(startDate, startHour, startMin, stopDate, stopHour, stopMin, + token = "t8fA0ToMKRTTj4qgcSpHzaknTARp8l9lONCFQiysVxzRwMsIaLrnHV_rTgUO3S4kz0nAq6fsSKH4SElOZN559w==", + org = "mashiro.jst.midori@gmail.com", + url = "https://europe-west1-1.gcp.cloud2.influxdata.com", + bucket = "kumamoto_weather_station"): + + '''all args require str''' + + start = startDate + 'T' + startHour + ':' + startMin + ':00Z' + start = datetime.strptime(start, '%Y-%m-%dT%H:%M:%SZ') + start = datetime.strptime(convertTimeZoneToUTC(start), '%Y-%m-%d %H:%M') + stop = stopDate + 'T' + stopHour + ':' + stopMin + ':00Z' + stop = datetime.strptime(stop, '%Y-%m-%dT%H:%M:%SZ') + stop = datetime.strptime(convertTimeZoneToUTC(stop), '%Y-%m-%d %H:%M') + + params = dict(token = token, + org = org, + url = url, + bucket = bucket, + start = start, + stop = stop) + + return params + +# define function for getting data from cloud + +def getData(params): + + # API setting + client = influxdb_client.InfluxDBClient(url = params['url'], + org = params['org'], + token = params['token']) + + queryAPI = client.query_api() + + # query setting at InfluxDB side + query = ''' + from(bucket: _bucket) + |> range(start: _start, stop: _stop) + |> filter(fn: (r) => r["_measurement"] == "sensor_data") + ''' + + # get data from InfluxDB Cloud via API + tables = queryAPI.query(query = query, org = params['org'], + params = {'_bucket': params['bucket'], + '_start': params['start'], + '_stop': params['stop']}) + + # convert data to Pandas dataframe + df = pd.read_json(tables.to_json()) + + # extract target columns and convert timezone UTC to JST + dfNow = df.filter(items = ['_time', '_field', '_value'], axis = 'columns') + dfNow.columns = ['time_UTC', 'weatherVariable', 'value'] + dfNow['time_JST'] = dfNow['time_UTC'].map(convertTimeZoneFromUTC) # add new column on dataframe + + return dfNow \ No newline at end of file diff --git a/main_v.1.1.py b/main_v.1.1.py new file mode 100644 index 0000000..87a003b --- /dev/null +++ b/main_v.1.1.py @@ -0,0 +1,201 @@ +# Author: Mashiro +# Last update: 2022/11/28 +# Description: application for visualizing data collected by IoT sensors + +# load modules +import datetime +import time + +import flet +from flet import (dropdown, Page, Dropdown, + ElevatedButton, Row, Image, + Text, UserControl, icons, + FilePickerResultEvent, Column, FilePicker) + +# load my modules +import getDataFromInfluxDBCloud +import visualize + +# drop down menues +class ddMenu(UserControl): + def build(self): + + today = datetime.date.today() + self.dateList = [dropdown.Option(str(today))] + + for i in range(1, 30): + date = str(today - datetime.timedelta(days = i)) + self.dateList.append(dropdown.Option(date)) + + + weatherVariables = ['wind_direction', 'wind_speed', 'TSR', + 'rain_snow', 'pressure', 'PM10', 'PM2_5', + 'wind_direction_angle', 'CO2', 'rain_gauge', + 'HUM', 'illumination', 'wind_speed_level', 'TEM'] + + self.ddStartDate = Dropdown(label = 'Start Date', + options = list(reversed(self.dateList)), + width = 300) + + self.ddStartHour = Dropdown(label = 'Hour', + options = [dropdown.Option(j) for j in + ['0' + str(i) if len(str(i)) == 1 + else str(i) for i in range(0, 24)]]) + + self.ddStartMin = Dropdown(label = 'Min', + options = [dropdown.Option(j) for j in + ['0' + str(i) if len(str(i)) == 1 + else str(i) for i in range(0, 60)]]) + + self.ddEndDate = Dropdown(label = 'End Date', options = self.dateList) + + self.ddEndHour = Dropdown(label = 'Hour', + options = [dropdown.Option(j) for j in + ['0' + str(i) if len(str(i)) == 1 + else str(i) for i in range(0, 24)]]) + + self.ddEndMin = Dropdown(label = 'Min', + options = [dropdown.Option(j) for j in + ['0' + str(i) if len(str(i)) == 1 + else str(i) for i in range(0, 60)]]) + + self.ddWeatherVariables = Dropdown(label = 'Weather Variables', + options = [dropdown.Option(i) for i in weatherVariables]) + + self.ddVisualize = Dropdown(label = 'Select format', + options = [dropdown.Option('Raw Data'), + dropdown.Option('Figure')]) + + duration = ['2w', '1w', + '3d', '2d', '1d', + '12h', '6h', '3h', '2h', '1h', + '30m'] + + self.ddSmooth = Dropdown(label = 'option', + options = [dropdown.Option(i) for i in duration]) + + return Column([Row(controls = [self.ddStartDate, self.ddStartHour, self.ddStartMin]), + Row(controls = [self.ddEndDate, self.ddEndHour, self.ddEndMin]), + Row(controls = [self.ddWeatherVariables, self.ddVisualize, self.ddSmooth])]) + + +def main(page: Page): + + formatHist = [] + userRequestedParam = ddMenu() + + + def saveFileResult(e: FilePickerResultEvent): + saveFilePath.value = e.path if e.path else 'File saving was cancelled!' + saveFilePath.update() + + + def execute(e): + + # check blanks are filled + if isinstance(userRequestedParam.ddVisualize.value, str): + if userRequestedParam.ddVisualize.value == 'Raw Data': + filled = (isinstance(userRequestedParam.ddStartDate.value, str) + and isinstance(userRequestedParam.ddStartHour.value, str) + and isinstance(userRequestedParam.ddStartMin.value, str) + and isinstance(userRequestedParam.ddEndDate.value, str) + and isinstance(userRequestedParam.ddEndHour.value, str) + and isinstance(userRequestedParam.ddEndMin.value, str)) + + elif userRequestedParam.ddVisualize.value == 'Figure': + filled = (isinstance(userRequestedParam.ddStartDate.value, str) + and isinstance(userRequestedParam.ddStartHour.value, str) + and isinstance(userRequestedParam.ddStartMin.value, str) + and isinstance(userRequestedParam.ddEndDate.value, str) + and isinstance(userRequestedParam.ddEndHour.value, str) + and isinstance(userRequestedParam.ddEndMin.value, str) + and isinstance(userRequestedParam.ddWeatherVariables.value, str)) + + else: + filled = False + + # after filled check + if filled: + status.value = 'Getting data from cloud server' ; status.update() + + requestParamForInflux = getDataFromInfluxDBCloud.generateParams(startDate = userRequestedParam.ddStartDate.value, + startHour = userRequestedParam.ddStartHour.value, + startMin = userRequestedParam.ddStartMin.value, + stopDate = userRequestedParam.ddEndDate.value, + stopHour = userRequestedParam.ddEndHour.value, + stopMin = userRequestedParam.ddEndMin.value) + # check how many times execute botton clicked + if execBotton.data > 1: + if lastRequestParamForInflux == requestParamForInflux: + sensorData = getDataFromInfluxDBCloud.getData(lastRequestParamForInflux) + else: + sensorData = getDataFromInfluxDBCloud.getData(requestParamForInflux) + + else: + sensorData = getDataFromInfluxDBCloud.getData(requestParamForInflux) + lastRequestParamForInflux = requestParamForInflux + + if userRequestedParam.ddVisualize.value == 'Raw Data': + formatHist.append('R') + status.value = 'Click Save file botton and Enter file name' ; status.update() + + while saveFilePath.value is None: + time.sleep(0.25) + + if saveFilePath.value[-4:] != '.csv': + saveFilePath.value = saveFilePath.value + '.csv' + + sensorData.to_csv(saveFilePath.value, index = False) + + status.value = 'File saving was successfully done!' ; status.update() + saveFilePath.value = '' ; saveFilePath.update() + + elif userRequestedParam.ddVisualize.value == 'Figure': + formatHist.append('F') + status.value = 'Now visualizing' ; status.update() + selectedVariable = userRequestedParam.ddWeatherVariables.value + sensorDataTemp = sensorData.query('weatherVariable == @selectedVariable') + if isinstance(userRequestedParam.ddSmooth.value, str): + sensorDataTemp = visualize.calcSmoothedValue(df = sensorDataTemp, + unit = userRequestedParam.ddSmooth.value) + plotData = visualize.drawPlot(x = sensorDataTemp.index, y = sensorDataTemp['value'], + weatherVariable = userRequestedParam.ddWeatherVariables.value, + smoothed = userRequestedParam.ddSmooth.value) + + else: + plotData = visualize.drawPlot(x = sensorDataTemp['time_JST'], y = sensorDataTemp['value'], + weatherVariable = userRequestedParam.ddWeatherVariables.value) + + if formatHist.count('F') == 1: + status.value = 'Done!' ; status.update() + plot = Image(src_base64 = plotData) + page.add(plot) + else: + status.value = 'Done!' ; status.update() + page.controls[-1].src_base64 = plotData + page.update() + + else: + pass + + else: + status.value = 'Error: fill all blanks!' + + status.update() + + status = Text(size = 20, color = 'red', text_align = 'center') + execBotton = ElevatedButton(text = 'Execute', on_click = execute, data = 0) + saveFilePath = Text() + saveFileDialog = FilePicker(on_result = saveFileResult) + fileSaveBotton = ElevatedButton(text = 'Save file', icon = icons.SAVE, + on_click = lambda _: saveFileDialog.save_file(), + disabled = page.web) + + page.scroll = 'auto' + + page.add(userRequestedParam, + Row(controls = [execBotton, fileSaveBotton, status]), + Row(controls = [saveFileDialog, saveFilePath])) + + +flet.app(target = main) diff --git a/visualize.py b/visualize.py new file mode 100644 index 0000000..66f5987 --- /dev/null +++ b/visualize.py @@ -0,0 +1,102 @@ +# Author: Mashiro +# Last update: 2022/11/07 +# Description: module for visualizing + +# load modules +import base64 +from io import BytesIO + +import matplotlib +matplotlib.use('svg') +import matplotlib.pyplot as plt +import pandas as pd + +# define function for deciding unit +def decideUnit(x): + + if x == 'wind_direction_angle': + return 'Degree' + elif x == 'wind_speed': + return 'm/s' + elif x == 'TSR': + return 'W/m^2' + elif x == 'pressure': + return 'hPa' + elif x == 'PM10' or x == 'PM2_5': + return 'µg/m^3' + elif x == 'CO2': + return 'ppm' + elif x == 'rain_gauge': + return 'mm' + elif x == 'HUM': + return '%RH' + elif x == 'illumination': + return 'Lux' + elif x == 'wind_speed_level': + return 'UnitLess' + elif x == 'TEM': + return 'Celsius_degree' + elif x == 'rain_snow' or x == 'wind_direction_angle': + return 'nounit' + +# define function for drawing plot + +def drawPlot(x, y, weatherVariable, smoothed = None): + plt.rcParams["figure.subplot.bottom"] = 0.20 + # draw plot + plt.plot(x, y) + + if smoothed is not None: + plt.title('Sensor Data; {}'.format(weatherVariable) + + ' (' + + 'smoothed: ' + + smoothed + +')') + else: + plt.title('Sensor Data; {}'.format(weatherVariable)+ ' (raw)') + plt.xlabel('Date') + plt.xticks(rotation = 45) + plt.ylabel(weatherVariable + ' (' + decideUnit(weatherVariable) + ')') + + ofs = BytesIO() + plt.savefig(ofs, format = 'png') + data = ofs.getvalue() + plt.clf() ; plt.close() + + base64Data = base64.b64encode(data).decode() + + return base64Data + +def drawPlot2(x, y, weatherVariable, smoothed = None): + fig, ax = plt.subplots() + + # draw plot + ax.plot(x, y) + + if smoothed is not None: + ax.set_title('Sensor Data; {}'.format(weatherVariable) + + ' (' + + 'smoothed: ' + + smoothed + +')') + else: + ax.set_title('Sensor Data; {}'.format(weatherVariable)+ ' (raw)') + ax.set_xlabel('Date') + fig.autofmt_xdate(rotation = 45) + ax.set_ylabel(weatherVariable + ' (' + decideUnit(weatherVariable) + ')') + + return fig + +# define function for calculating smoothed average value + +def calcSmoothedValue(df, unit): + durationDict = {'2w': '2W', '1w': '1W', + '3d': '3D', '2d': '2D', '1d': '1D', + '12h': '12H', '6h': '6H', '3h': '3H', '2h': '2H', '1h': '1H', + '30m': '30T'} + + value = df.filter(items = ['value']) + value = value.astype(float) + value.index = pd.to_datetime(df['time_JST']) + + return value.resample(rule = durationDict[unit]).mean() \ No newline at end of file