Add files via upload

getDataFromInfluxDBCloud.py

@@ -0,0 +1,84 @@
+# Author: Mashiro
+# Last update: 2022/10/12
+# Description: module for getting data from InfluxDB cloud
+
+# load molues
+from datetime import datetime
+from dateutil.tz import gettz
+
+import pandas as pd
+import influxdb_client
+
+# define functions for converting timezone
+
+def convertTimeZoneFromUTC(timestampUTC, timeZone = 'Asia/Tokyo'):
+    convertedDatetime = timestampUTC.astimezone(gettz(timeZone))
+    convertedTimestamp = datetime.strftime(convertedDatetime, '%Y-%m-%d %H:%M')
+
+    return convertedTimestamp
+
+def convertTimeZoneToUTC(timestamp):
+    convertedDatetime = timestamp.astimezone(gettz('UTC'))
+    convertedTimestamp = datetime.strftime(convertedDatetime, '%Y-%m-%d %H:%M')
+
+    return convertedTimestamp
+
+# define function for generating params
+
+def generateParams(startDate, startHour, startMin, stopDate, stopHour, stopMin,
+                   token = "t8fA0ToMKRTTj4qgcSpHzaknTARp8l9lONCFQiysVxzRwMsIaLrnHV_rTgUO3S4kz0nAq6fsSKH4SElOZN559w==",
+                   org = "[email protected]",
+                   url = "https://europe-west1-1.gcp.cloud2.influxdata.com",
+                   bucket = "kumamoto_weather_station"):
+
+    '''all args require str'''
+
+    start = startDate + 'T' + startHour + ':' + startMin + ':00Z'
+    start = datetime.strptime(start, '%Y-%m-%dT%H:%M:%SZ')
+    start = datetime.strptime(convertTimeZoneToUTC(start), '%Y-%m-%d %H:%M')
+    stop = stopDate + 'T' + stopHour + ':' + stopMin + ':00Z'
+    stop = datetime.strptime(stop, '%Y-%m-%dT%H:%M:%SZ')
+    stop = datetime.strptime(convertTimeZoneToUTC(stop), '%Y-%m-%d %H:%M')
+
+    params = dict(token = token,
+                  org = org,
+                  url = url,
+                  bucket = bucket,
+                  start = start,
+                  stop = stop)
+
+    return params
+
+# define function for getting data from cloud
+
+def getData(params):
+
+    # API setting
+    client = influxdb_client.InfluxDBClient(url = params['url'],
+                                            org = params['org'],
+                                            token = params['token'])
+
+    queryAPI = client.query_api()
+
+    # query setting at InfluxDB side
+    query = '''
+            from(bucket: _bucket)
+                |> range(start: _start, stop: _stop)
+                |> filter(fn: (r) => r["_measurement"] == "sensor_data")
+            '''
+
+    # get data from InfluxDB Cloud via API
+    tables = queryAPI.query(query = query, org = params['org'],
+                            params = {'_bucket': params['bucket'],
+                                    '_start': params['start'],
+                                    '_stop': params['stop']})
+
+    # convert data to Pandas dataframe
+    df = pd.read_json(tables.to_json())
+
+    # extract target columns and convert timezone UTC to JST
+    dfNow = df.filter(items = ['_time', '_field', '_value'], axis = 'columns')
+    dfNow.columns = ['time_UTC', 'weatherVariable', 'value']
+    dfNow['time_JST'] = dfNow['time_UTC'].map(convertTimeZoneFromUTC) # add new column on dataframe
+
+    return dfNow

main_v.1.1.py

@@ -0,0 +1,201 @@
+# Author: Mashiro
+# Last update: 2022/11/28
+# Description: application for visualizing data collected by IoT sensors
+
+# load modules
+import datetime
+import time
+
+import flet
+from flet import (dropdown, Page, Dropdown,
+                  ElevatedButton, Row, Image,
+                  Text, UserControl, icons,
+                  FilePickerResultEvent, Column, FilePicker)
+
+# load my modules
+import getDataFromInfluxDBCloud
+import visualize
+
+# drop down menues
+class ddMenu(UserControl):
+    def build(self):
+
+        today = datetime.date.today()
+        self.dateList = [dropdown.Option(str(today))]
+
+        for i in range(1, 30):
+            date = str(today - datetime.timedelta(days = i))
+            self.dateList.append(dropdown.Option(date))
+
+
+        weatherVariables = ['wind_direction', 'wind_speed', 'TSR',
+                            'rain_snow', 'pressure', 'PM10', 'PM2_5',
+                            'wind_direction_angle', 'CO2', 'rain_gauge',
+                            'HUM', 'illumination', 'wind_speed_level', 'TEM']
+
+        self.ddStartDate = Dropdown(label = 'Start Date',
+                                    options = list(reversed(self.dateList)),
+                                    width = 300)
+
+        self.ddStartHour = Dropdown(label = 'Hour',
+                                    options = [dropdown.Option(j) for j in
+                                               ['0' + str(i) if len(str(i)) == 1
+                                                else str(i) for i in range(0, 24)]])
+
+        self.ddStartMin = Dropdown(label = 'Min',
+                                   options = [dropdown.Option(j) for j in
+                                              ['0' + str(i) if len(str(i)) == 1
+                                               else str(i) for i in range(0, 60)]])
+
+        self.ddEndDate = Dropdown(label = 'End Date', options = self.dateList)
+
+        self.ddEndHour = Dropdown(label = 'Hour',
+                                  options = [dropdown.Option(j) for j in
+                                             ['0' + str(i) if len(str(i)) == 1
+                                              else str(i) for i in range(0, 24)]])
+
+        self.ddEndMin = Dropdown(label = 'Min',
+                                 options = [dropdown.Option(j) for j in
+                                            ['0' + str(i) if len(str(i)) == 1
+                                             else str(i) for i in range(0, 60)]])
+
+        self.ddWeatherVariables = Dropdown(label = 'Weather Variables',
+                                           options = [dropdown.Option(i) for i in weatherVariables])
+
+        self.ddVisualize = Dropdown(label = 'Select format',
+                                    options = [dropdown.Option('Raw Data'),
+                                               dropdown.Option('Figure')])
+
+        duration = ['2w', '1w',
+                    '3d', '2d', '1d',
+                    '12h', '6h', '3h', '2h', '1h',
+                    '30m']
+
+        self.ddSmooth = Dropdown(label = 'option',
+                                 options = [dropdown.Option(i) for i in duration])
+
+        return Column([Row(controls = [self.ddStartDate, self.ddStartHour, self.ddStartMin]),
+                       Row(controls = [self.ddEndDate, self.ddEndHour, self.ddEndMin]),
+                       Row(controls = [self.ddWeatherVariables, self.ddVisualize, self.ddSmooth])])
+
+
+def main(page: Page):
+
+    formatHist = []
+    userRequestedParam = ddMenu()
+
+
+    def saveFileResult(e: FilePickerResultEvent):
+        saveFilePath.value = e.path if e.path else 'File saving was cancelled!'
+        saveFilePath.update()
+
+
+    def execute(e):
+
+        # check blanks are filled
+        if isinstance(userRequestedParam.ddVisualize.value, str):
+            if userRequestedParam.ddVisualize.value == 'Raw Data':
+                filled = (isinstance(userRequestedParam.ddStartDate.value, str)
+                          and isinstance(userRequestedParam.ddStartHour.value, str)
+                          and isinstance(userRequestedParam.ddStartMin.value, str)
+                          and isinstance(userRequestedParam.ddEndDate.value, str)
+                          and isinstance(userRequestedParam.ddEndHour.value, str)
+                          and isinstance(userRequestedParam.ddEndMin.value, str))
+
+            elif userRequestedParam.ddVisualize.value == 'Figure':
+                filled = (isinstance(userRequestedParam.ddStartDate.value, str)
+                          and isinstance(userRequestedParam.ddStartHour.value, str)
+                          and isinstance(userRequestedParam.ddStartMin.value, str)
+                          and isinstance(userRequestedParam.ddEndDate.value, str)
+                          and isinstance(userRequestedParam.ddEndHour.value, str)
+                          and isinstance(userRequestedParam.ddEndMin.value, str)
+                          and isinstance(userRequestedParam.ddWeatherVariables.value, str))
+
+        else:
+            filled = False
+
+        # after filled check
+        if filled:
+            status.value = 'Getting data from cloud server' ; status.update()
+
+            requestParamForInflux = getDataFromInfluxDBCloud.generateParams(startDate = userRequestedParam.ddStartDate.value,
+                                                                            startHour = userRequestedParam.ddStartHour.value,
+                                                                            startMin = userRequestedParam.ddStartMin.value,
+                                                                            stopDate = userRequestedParam.ddEndDate.value,
+                                                                            stopHour = userRequestedParam.ddEndHour.value,
+                                                                            stopMin = userRequestedParam.ddEndMin.value)
+            # check how many times execute botton clicked
+            if execBotton.data > 1:
+                if lastRequestParamForInflux == requestParamForInflux:
+                    sensorData = getDataFromInfluxDBCloud.getData(lastRequestParamForInflux)
+                else:
+                    sensorData = getDataFromInfluxDBCloud.getData(requestParamForInflux)
+
+            else:
+                sensorData = getDataFromInfluxDBCloud.getData(requestParamForInflux)
+                lastRequestParamForInflux = requestParamForInflux
+
+            if userRequestedParam.ddVisualize.value == 'Raw Data':
+                formatHist.append('R')
+                status.value = 'Click Save file botton and Enter file name' ; status.update()
+
+                while saveFilePath.value is None:
+                    time.sleep(0.25)
+
+                if saveFilePath.value[-4:] != '.csv':
+                    saveFilePath.value = saveFilePath.value + '.csv'
+
+                sensorData.to_csv(saveFilePath.value, index = False)
+
+                status.value = 'File saving was successfully done!' ; status.update()
+                saveFilePath.value = '' ; saveFilePath.update()
+
+            elif userRequestedParam.ddVisualize.value == 'Figure':
+                formatHist.append('F')
+                status.value = 'Now visualizing' ; status.update()
+                selectedVariable = userRequestedParam.ddWeatherVariables.value
+                sensorDataTemp = sensorData.query('weatherVariable == @selectedVariable')
+                if isinstance(userRequestedParam.ddSmooth.value, str):
+                    sensorDataTemp = visualize.calcSmoothedValue(df = sensorDataTemp,
+                                                                 unit = userRequestedParam.ddSmooth.value)
+                    plotData = visualize.drawPlot(x = sensorDataTemp.index, y = sensorDataTemp['value'],
+                                                  weatherVariable =  userRequestedParam.ddWeatherVariables.value,
+                                                  smoothed = userRequestedParam.ddSmooth.value)
+
+                else:
+                    plotData = visualize.drawPlot(x = sensorDataTemp['time_JST'], y = sensorDataTemp['value'],
+                                                  weatherVariable =  userRequestedParam.ddWeatherVariables.value)
+
+                if formatHist.count('F') == 1:
+                    status.value = 'Done!' ; status.update()
+                    plot = Image(src_base64 = plotData)
+                    page.add(plot)
+                else:
+                    status.value = 'Done!' ; status.update()
+                    page.controls[-1].src_base64 = plotData
+                    page.update()
+
+            else:
+                pass
+
+        else:
+            status.value = 'Error: fill all blanks!'
+
+        status.update()
+
+    status = Text(size = 20, color = 'red', text_align = 'center')
+    execBotton = ElevatedButton(text = 'Execute', on_click = execute, data = 0)
+    saveFilePath = Text()
+    saveFileDialog = FilePicker(on_result = saveFileResult)
+    fileSaveBotton = ElevatedButton(text = 'Save file', icon = icons.SAVE,
+                                    on_click = lambda _: saveFileDialog.save_file(),
+                                    disabled = page.web)
+
+    page.scroll = 'auto'
+
+    page.add(userRequestedParam,
+             Row(controls = [execBotton, fileSaveBotton, status]),
+             Row(controls = [saveFileDialog, saveFilePath]))
+
+
+flet.app(target = main)

visualize.py

@@ -0,0 +1,102 @@
+# Author: Mashiro
+# Last update: 2022/11/07
+# Description: module for visualizing
+
+# load modules
+import base64
+from io import BytesIO
+
+import matplotlib
+matplotlib.use('svg')
+import matplotlib.pyplot as plt
+import pandas as pd
+
+# define function for deciding unit
+def decideUnit(x):
+
+    if x == 'wind_direction_angle':
+        return 'Degree'
+    elif x == 'wind_speed':
+        return 'm/s'
+    elif x == 'TSR':
+        return 'W/m^2'
+    elif x == 'pressure':
+        return 'hPa'
+    elif x == 'PM10' or x == 'PM2_5':
+        return 'µg/m^3'
+    elif x == 'CO2':
+        return 'ppm'
+    elif x == 'rain_gauge':
+        return 'mm'
+    elif x == 'HUM':
+        return '%RH'
+    elif x == 'illumination':
+        return 'Lux'
+    elif x == 'wind_speed_level':
+        return 'UnitLess'
+    elif x == 'TEM':
+        return 'Celsius_degree'
+    elif x == 'rain_snow' or x == 'wind_direction_angle':
+        return 'nounit'
+
+# define function for drawing plot
+
+def drawPlot(x, y, weatherVariable, smoothed = None):
+    plt.rcParams["figure.subplot.bottom"] = 0.20
+    # draw plot
+    plt.plot(x, y)
+
+    if smoothed is not None:
+        plt.title('Sensor Data; {}'.format(weatherVariable)
+                  + ' ('
+                  + 'smoothed: '
+                  + smoothed
+                  +')')
+    else:
+        plt.title('Sensor Data; {}'.format(weatherVariable)+ ' (raw)')
+    plt.xlabel('Date')
+    plt.xticks(rotation = 45)
+    plt.ylabel(weatherVariable + ' (' + decideUnit(weatherVariable) + ')')
+
+    ofs = BytesIO()
+    plt.savefig(ofs, format = 'png')
+    data = ofs.getvalue()
+    plt.clf() ; plt.close()
+
+    base64Data = base64.b64encode(data).decode()
+
+    return base64Data
+
+def drawPlot2(x, y, weatherVariable, smoothed = None):
+    fig, ax = plt.subplots()
+
+    # draw plot
+    ax.plot(x, y)
+
+    if smoothed is not None:
+        ax.set_title('Sensor Data; {}'.format(weatherVariable)
+                  + ' ('
+                  + 'smoothed: '
+                  + smoothed
+                  +')')
+    else:
+        ax.set_title('Sensor Data; {}'.format(weatherVariable)+ ' (raw)')
+    ax.set_xlabel('Date')
+    fig.autofmt_xdate(rotation = 45)
+    ax.set_ylabel(weatherVariable + ' (' + decideUnit(weatherVariable) + ')')
+
+    return fig
+
+# define function for calculating smoothed average value
+
+def calcSmoothedValue(df, unit):
+    durationDict = {'2w': '2W', '1w': '1W',
+                    '3d': '3D', '2d': '2D', '1d': '1D',
+                    '12h': '12H', '6h': '6H', '3h': '3H', '2h': '2H', '1h': '1H',
+                    '30m': '30T'}
+
+    value = df.filter(items = ['value'])
+    value = value.astype(float)
+    value.index = pd.to_datetime(df['time_JST'])
+
+    return value.resample(rule = durationDict[unit]).mean()