diff --git a/VisualCrossing/api.py b/VisualCrossing/api.py index ffe4be7..decca59 100644 --- a/VisualCrossing/api.py +++ b/VisualCrossing/api.py @@ -55,7 +55,7 @@ def _date_formatter(self, value : str or dt.date) -> str: raise ValueError(f"Start/End Value is Accepted as either `str` or `dt.date`, got ({type(value)}).") if _is_string: - value = dt.datetime.strptime("2023-01-13", "%Y-%m-%d").date() + value = dt.datetime.strptime(value, "%Y-%m-%d").date() else: value = str(value) @@ -154,11 +154,11 @@ def response(self, formatting : str = "json", onfail : str = "pass", verbose : b for location in TQ(self.locations): _resolved_uri = self.URI.format( location = urllib.parse.quote(location), # iterate over all locations - start = self.start, end = self.start, key = self.key + start = self.start, end = self.end, key = self.key ) try: - r = requests.get(_resolved_uri, verify = False) + r = requests.get(_resolved_uri, verify = True) data = r.json() responses.append(data) diff --git a/assets/db.sql b/assets/db.sql new file mode 100644 index 0000000..8bed8ee --- /dev/null +++ b/assets/db.sql @@ -0,0 +1,69 @@ +CREATE TABLE `tblDailyWeather` ( + _id INTEGER PRIMARY KEY AUTO_INCREMENT, + + /* Primary Identifier Fileds */ + EffectiveDate DATE NOT NULL, + CityUUID VARCHAR(36) NOT NULL, + ResolvedLocation VARCHAR(128) NOT NULL COMMENT 'Send the Resolved Address as Provided by Visual Crossing API', + + /* Daily Temperature Information */ + TempAvg DECIMAL(5, 2) DEFAULT NULL, + TempMin DECIMAL(5, 2) DEFAULT NULL, + TempMax DECIMAL(5, 2) DEFAULT NULL, + + FeelsLikeAvg DECIMAL(5, 2) DEFAULT NULL, + FeelsLikeMin DECIMAL(5, 2) DEFAULT NULL, + FeelsLikeMax DECIMAL(5, 2) DEFAULT NULL, + + DewTemp DECIMAL(5, 2) DEFAULT NULL, + HumidityPercent DECIMAL(5, 2) DEFAULT NULL, + + PrecipAmt DECIMAL(5, 2) DEFAULT NULL COMMENT 'Amount of Liquid Precipitation (or equivalent) Predicted to Fall in the Period', + PrecipProb DECIMAL(5, 2) DEFAULT NULL COMMENT 'Forecast Only, the likelihood of measurable precipitation ranging from (0 - 100)%', + PrecipType VARCHAR(128) DEFAULT NULL COMMENT 'CSV Value indicating the type(s) of expected precipitation expected or occured.', + PrecipCover DECIMAL(5, 2) DEFAULT NULL COMMENT 'The proportion fo hours where there was non-zero precipitation.', + + SnowCover DECIMAL(5, 2) DEFAULT NULL, + SnowDepth DECIMAL(5, 2) DEFAULT NULL, + + WindDir DECIMAL(5, 2) DEFAULT NULL, + WindGust DECIMAL(5, 2) DEFAULT NULL COMMENT 'Instantaneous speed, may be empty representing significantly lower wind speed.', + WindSpeed DECIMAL(5, 2) DEFAULT NULL, + WindSpeedAvg DECIMAL(5, 2) DEFAULT NULL, + WindSpeedMin DECIMAL(5, 2) DEFAULT NULL, + WindSpeedMax DECIMAL(5, 2) DEFAULT NULL, + + AtmPressure DECIMAL(5, 2) DEFAULT NULL COMMENT 'Sea level atmospheric pressure in millibars.', + CloudCoverage DECIMAL(5, 2) DEFAULT NULL COMMENT 'Percentage of sky covered in cloud raning from (0 - 100)%', + VisibilityDist DECIMAL(5, 2) DEFAULT NULL, + + UVIndex DECIMAL(5, 2) DEFAULT NULL COMMENT 'A value between 0 to 10, where 0 = lowest exposure and 10 is the highest exposure for that hour/day.', + SevereRisk DECIMAL(5, 2) DEFAULT NULL COMMENT 'Forecast Only, A value between 0 to 100 representing convective storms.', + + SolarEnergy DECIMAL(5, 2) DEFAULT NULL COMMENT 'Solar Energy in MJ/m^2', + SolarRadiation DECIMAL(5, 2) DEFAULT NULL COMMENT 'Solar Radion in W/m^2', + + SunRise DATETIME DEFAULT NULL, + SunSet DATETIME DEFAULT NULL, + MoonPhase DECIMAL(5, 2) DEFAULT NULL COMMENT 'Value range from 0.0 (new moon) to 0.5 (full moon) to 1.0 (next new moon).', + + WeatherIcon VARCHAR(12) DEFAULT NULL, + WeatherCond VARCHAR(16) DEFAULT NULL, + WeatherDesc VARCHAR(64) DEFAULT NULL, + + _source VARCHAR(12) NOT NULL COMMENT 'Type of weather data used - obs, fcst, histfcst, or stats.', + _stations VARCHAR(512) DEFAULT NULL COMMENT 'CSV values of weather statiosn names which was used to buold the data.', + + UNIQUE KEY `ck_weather_record` (EffectiveDate, CityUUID) +); + + +CREATE TABLE `dbo.mwVCWeatherStations` ( + VCWeatherStationID VARCHAR(16) PRIMARY KEY COMMENT 'Station ID as represented in Visual Crossing API', + + StationName VARCHAR(32) UNIQUE NOT NULL, + StationLatitude DECIMAL(18, 6) NULL, + StationLongitude DECIMAL(18, 6) NULL, + StationQuality DECIMAL(5, 2) NULL, + StationContribution DECIMAL(5, 2) NULL +); diff --git a/examples/Getting Started with Visual Crossing.ipynb b/examples/Getting Started with Visual Crossing.ipynb index 2218453..afb456c 100644 --- a/examples/Getting Started with Visual Crossing.ipynb +++ b/examples/Getting Started with Visual Crossing.ipynb @@ -1,17 +1,25 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "

Visual Crossing API

" + ] + }, { "cell_type": "code", "execution_count": 1, "metadata": { "ExecuteTime": { - "end_time": "2021-11-14T17:41:33.837659Z", - "start_time": "2021-11-14T17:41:33.827287Z" + "end_time": "2024-01-31T19:07:29.244778Z", + "start_time": "2024-01-31T19:07:29.232288Z" } }, "outputs": [], "source": [ - "import os" + "import os\n", + "import sys" ] }, { @@ -19,8 +27,22 @@ "execution_count": 2, "metadata": { "ExecuteTime": { - "end_time": "2021-10-19T10:56:05.636550Z", - "start_time": "2021-10-19T10:56:05.315187Z" + "end_time": "2024-01-31T19:07:29.260290Z", + "start_time": "2024-01-31T19:07:29.246293Z" + } + }, + "outputs": [], + "source": [ + "import json" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "ExecuteTime": { + "end_time": "2024-01-31T19:07:29.734029Z", + "start_time": "2024-01-31T19:07:29.263291Z" } }, "outputs": [], @@ -31,30 +53,140 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": { "ExecuteTime": { - "end_time": "2021-10-19T10:56:05.659671Z", - "start_time": "2021-10-19T10:56:05.639110Z" + "end_time": "2024-01-31T19:07:29.750009Z", + "start_time": "2024-01-31T19:07:29.737000Z" + } + }, + "outputs": [], + "source": [ + "from tqdm import tqdm as TQ" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "ExecuteTime": { + "end_time": "2024-01-31T19:07:30.641145Z", + "start_time": "2024-01-31T19:07:29.751001Z" + } + }, + "outputs": [], + "source": [ + "import ndprdconfig\n", + "testDB = ndprdconfig.DBConnection(instance = \"ap-south-1-neuralnod-dev/test\")\n", + "nxprdsys = ndprdconfig.DBConnection(instance = \"ap-south-1-neuralnod-dev\")\n", + "\n", + "engine_testdb = testDB.connect()\n", + "engine_nxprdsys = nxprdsys.connect()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "ExecuteTime": { + "end_time": "2024-01-31T19:07:30.841472Z", + "start_time": "2024-01-31T19:07:30.642243Z" } }, "outputs": [ { "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
CityNameCityLocation
CityUUID
27FBAC74-3AC7-4B8F-B44E-2FD21FE9BEFBRajahmundry(17.00517, 81.77784)
\n", + "
" + ], "text/plain": [ - "True" + " CityName CityLocation\n", + "CityUUID \n", + "27FBAC74-3AC7-4B8F-B44E-2FD21FE9BEFB Rajahmundry (17.00517, 81.77784)" ] }, - "execution_count": 3, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "# setting the environment\n", - "# if required, pip install python-dotenv\n", - "from dotenv import load_dotenv # Python 3.6+\n", - "load_dotenv(verbose = True) # configure .env File or set Environment Variables" + "mwCities = \"\"\"\n", + "SELECT\n", + " CityUUID\n", + " , CityName\n", + " , CONCAT(CityLatitude, ',', CityLongitude) AS CityLocation\n", + "FROM `dbo.mwCities`\n", + "WHERE _get_weather_data = 1\n", + "\"\"\"\n", + "\n", + "mwCities = pd.read_sql(mwCities, engine_nxprdsys, index_col = \"CityUUID\")\n", + "mwCities[\"CityLocation\"] = mwCities[\"CityLocation\"].apply(eval) # sql statement format ensures tuple conversion\n", + "\n", + "mwCities.sample()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "ExecuteTime": { + "end_time": "2024-01-31T19:07:30.888014Z", + "start_time": "2024-01-31T19:07:30.842472Z" + } + }, + "outputs": [], + "source": [ + "engine_nxprdsys.close()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "ExecuteTime": { + "end_time": "2024-01-31T19:07:30.904050Z", + "start_time": "2024-01-31T19:07:30.889724Z" + } + }, + "outputs": [], + "source": [ + "sys.path.append(\"E:\\\\VisualCrossing\")" ] }, { @@ -70,36 +202,28 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 9, "metadata": { "ExecuteTime": { - "end_time": "2021-10-19T10:56:05.741333Z", - "start_time": "2021-10-19T10:56:05.664416Z" + "end_time": "2024-01-31T19:07:31.057743Z", + "start_time": "2024-01-31T19:07:30.906051Z" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0.1.0-beta\n" - ] - } - ], + "outputs": [], "source": [ "import VisualCrossing as vc\n", "\n", "# get installed version\n", - "print(vc.__version__)" + "# print(vc.__version__)" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 16, "metadata": { "ExecuteTime": { - "end_time": "2021-10-19T10:25:29.303119Z", - "start_time": "2021-10-19T10:25:29.290595Z" + "end_time": "2024-01-31T15:14:12.234125Z", + "start_time": "2024-01-31T15:14:12.219139Z" }, "collapsed": true, "deletable": false, @@ -115,115 +239,86 @@ "text": [ "Help on class API in module VisualCrossing.api:\n", "\n", - "class API(VisualCrossing.base)\n", - " | API(date: str, APIKey: str, location: str, **kwargs)\n", + "class API(base)\n", + " | API(start: str, locations: list, key: str, **kwargs) -> None\n", " | \n", " | A python wrapper to fetch weather data from VisualCrossing\n", " | \n", - " | The function is devised keeping in mind - minimal coding approach, and recurrent job-scheduling\n", - " | for any person. All the keyword arguments that the API accepts are same as that available\n", - " | in the Visual Crossing API Documentation (check README for more information). The API is configured\n", - " | with default values, which is easier to understand and retreive. However, the data requires certain\n", - " | required arguments defined below. The same can also be obtained from `config.json` file, but is not\n", - " | recomended.\n", - " | \n", - " | :param date: Date for which weather data is required. Pass the date in `YYYY-MM-DD` format,\n", - " | or pass `FORCAST` to get the forecasted data for a particular date.\n", - " | \n", - " | :param APIKey: All VisualCrossing API calls are made with an API key associated with your billing/free\n", - " | account. Get yourself a new Key, if you do not have one. It is advised that the key\n", - " | is not written to a configuration file, and suitable `environment variables` should\n", - " | be called/defined. However, there is a dedicated function :func:`get_key_from_config`\n", - " | which can be used to get `APIKey` from configuration file, which should be defined\n", - " | under `__api_key__`. The key can also be written into file either passing `key` or\n", - " | `__api_key__` as a keyword argument to :func:`generate_config`. # TODO\n", - " | \n", - " | :param location: Location of which the weather is required. Defaults to either a place name (like `india`),\n", - " | or, you can directly pass the coordinates of the particular place as (like `(long, lat)`),\n", - " | or, you can also pass a list/set of locations (either name or coordinates.) # TODO\n", - " | \n", - " | :Keyword Arguments:\n", - " | * *endDate* (``str``) -- When end date is defined, the api fetches data for a given date\n", - " | range which starts from :param:`date` to `endDate`, else only singe day data is fetched.\n", - " | By default, only a single day data is fetched by setting start and end date as :param:`date`.\n", + " | The function is devised keeping in mind - minimal coding approach, and\n", + " | recurrent job-scheduling for any person. All the keyword arguments that\n", + " | the API accepts are same as that available in the Visual Crossing API\n", + " | Documentation (check README for more information). The API is configured\n", + " | with default values, which is easier to understand and retreive. However,\n", + " | the data requires certain required arguments defined below. The same can\n", + " | also be obtained from `config.json` file, but is not recommended.\n", " | \n", - " | * *unitGroup* (``str``) -- specify unit group, defaults to metric.\n", - " | \n", - " | * *contentType* (``str``) -- specify content type, defaults to csv.\n", - " | \n", - " | * *aggregateHours* (``str``) -- specify aggregate hours, defaults to 24 (daily).\n", + " | :type start: str or dt.date\n", + " | :param start: The starting and ending data to fetch the data from the\n", + " | API. The date is to be either in `str(\"%Y-%m-%d\")` format or an\n", + " | instance of the `dt.date()` class.\n", + " | \n", + " | :type locations: list\n", + " | :param locations: A list of location, which can be either a list of place\n", + " | name with ISO2 parameter like: `[\"Delhi,IN\", \"Kolkata,IN\", ...]` or\n", + " | can also pass the location as `latitude,longitude` combination like:\n", + " | `[\"28.704059,77.102490\", \"22.986757,87.854976\", ...]`. The Visual\n", + " | Crossing API is designed to fetch data for one single location,\n", + " | however, this libraru sypports multiple location input and create an\n", + " | iterable object on the data response.\n", + " | \n", + " | :type key: str\n", + " | :param key: Obtain an API key from https://www.visualcrossing.com/ and\n", + " | pass the value as a string. Typically, the key is a 25 digit\n", + " | alphanumeric key in all caps.\n", + " | \n", + " | Keyword Arguments\n", + " | -----------------\n", + " | * **`end`** (`str` or `dt.date`): The end date, which is optional, and\n", + " | the format is either `str(\"%Y-%m-%d\")` or an instance of the\n", + " | `dt.date()` class.\n", " | \n", " | Method resolution order:\n", " | API\n", - " | VisualCrossing.base\n", + " | base\n", " | builtins.object\n", " | \n", " | Methods defined here:\n", " | \n", - " | BaseURL(self, location: str) -> str\n", - " | Base URL for fetching weather data. Check Visual-Crossing documentation for information.\n", - " | \n", - " | __init__(self, date: str, APIKey: str, location: str, **kwargs)\n", + " | __init__(self, start: str, locations: list, key: str, **kwargs) -> None\n", " | Initialize self. See help(type(self)) for accurate signature.\n", " | \n", - " | get(self, **kwargs)\n", - " | Fetch API Data and Return in desirable Format\n", - " | \n", - " | queryType(self) -> str\n", - " | Set data query type to either `historical` or `forecast` as per given date choice\n", - " | \n", - " | ----------------------------------------------------------------------\n", - " | Methods inherited from VisualCrossing.base:\n", - " | \n", - " | __args_default__(self) -> dict\n", - " | Defines a Dictionary of Default Values for Keyword Arguments (or Attributes)\n", - " | \n", - " | __get_args_default__(self, args: str)\n", - " | Get the Default Value associated with a Keyword Argument\n", - " | \n", - " | __set_attr__(self, response: dict, **kwargs)\n", - " | \n", - " | generate_config(self, defaultSettings: bool = True, fileName: str = 'config.json', overwrite: bool = False, keepBackup: bool = True, **kwargs) -> bool\n", - " | Generate configuration file at `__homepath__` when executed\n", - " | \n", - " | The configuration file can be generated with default settings as defined at\n", - " | :func:`__args_default__` else, user is requested to pass all necessary settings\n", - " | in a correct format (as required by API) to the function, setting `key` as the\n", - " | attribute name, and `value` as the desired value. Users are also advised not to\n", - " | save the `API_KEY` in the configuration file (for security purpose), and to use\n", - " | :func:`_generate_key` to save the key file in an encrypted format.\n", - " | \n", - " | :param defaultSettings: Should you wish to save the configuration file with\n", - " | the default settings. If set to `False` then user is\n", - " | requested to pass all necessary attributes (`key`) and\n", - " | their values. Defaults to `True`.\n", + " | response(self, formatting: str = 'json', onfail: str = 'pass', verbose: bool = True) -> object\n", + " | Get the API Response from the Visual Crossing API\n", " | \n", - " | :param fileName: Output file name (with extension - `json`). Defaults to\n", - " | `config.json`.\n", + " | Get the API response, and format the data based on the required\n", + " | formatting, which is by default `json` which is rendered as `dict`\n", + " | in python.\n", " | \n", - " | :param overwrite: Overwrite existing configuration file, if exists (same filename).\n", - " | Defaults to `False`.\n", - " | \n", - " | :param keepBackup: If same file name exists, then setting the parameter to `True` will\n", - " | create a backup of the file with the following format\n", - " | `..json` where `UUID` is a randomly generated\n", - " | 7-charecters long name. Defaults to `True`.\n", - " | \n", - " | Accepts n-Keyword Arguments, which are all default settings that can be used to initialize\n", - " | the API.\n", + " | :type formatting: str\n", + " | :param formatting: Final output format of all the response in a\n", + " | consolidated format. The output format can be any of the following:\n", + " | `[json, csv]`. Defaults to `json`. # TODO other formatting.\n", + " | \n", + " | :type onfail: str\n", + " | :param onfail: When a set of date, locations or any other iterable\n", + " | format is passed, the query may fail - server unavailable, calls\n", + " | exhausted, etc. The parameter can be set to either `pass` to\n", + " | return only the fetched/parsed records, while setting value to\n", + " | `fail` the code fails if any parsing is failed. Defaults to\n", + " | `pass`. # TODO other failing criteria\n", " | \n", - " | get_key_from_config(self)\n", + " | ----------------------------------------------------------------------\n", + " | Methods inherited from base:\n", " | \n", - " | set_config(self, file: str, **kwargs)\n", + " | write_json(self, values: dict, fname: str, **kwargs) -> bool\n", " | \n", " | ----------------------------------------------------------------------\n", - " | Readonly properties inherited from VisualCrossing.base:\n", + " | Readonly properties inherited from base:\n", " | \n", - " | __optional_args__\n", - " | Get List of all the Optional Keyword Arguments Accepted by the API\n", + " | URI\n", " | \n", " | ----------------------------------------------------------------------\n", - " | Data descriptors inherited from VisualCrossing.base:\n", + " | Data descriptors inherited from base:\n", " | \n", " | __dict__\n", " | dictionary for instance variables (if defined)\n", @@ -242,20 +337,6 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "

The API can be configured with a configuration which is located at HOME/.visualCrossing/config.json, and an example file is provided in the directory. The file is set, such that same setting can be reused, or if required you can keep pre-defined templates to be used regularly.

\n", - "\n", - "All the attributes required, can be defined like:\n", - "\n", - "```json\n", - "...\n", - " \"attributes\": {\n", - " \"unitGroup\": \"metric\",\n", - " \"contentType\": \"dataframe\",\n", - " \"aggregateHours\": 24\n", - " },\n", - "...\n", - "```\n", - "\n", "The API requires three minimal options:\n", "1. Date for which the data is required, in `YYYY-MM-DD` format,\n", "2. API Key which can be obtained from [website](https://www.visualcrossing.com/weather-api), and\n", @@ -275,167 +356,301 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 10, "metadata": { "ExecuteTime": { - "end_time": "2021-10-19T10:56:05.749405Z", - "start_time": "2021-10-19T10:56:05.744026Z" + "end_time": "2024-01-31T19:07:31.073605Z", + "start_time": "2024-01-31T19:07:31.061603Z" } }, "outputs": [], "source": [ - "locations = (\"india\", \"tokyo\")\n", - "api = vc.API(\"2021-09-25\", APIKey = os.getenv(\"__api_key__\"), location = locations, contentType = \"dataframe\") # default parameters are set from `config.json`" + "locations = {idx : f\"{row.CityLocation[0]},{row.CityLocation[1]}\" for idx, row in mwCities.iterrows()}\n", + "# api = vc.API(\"2021-09-25\", APIKey = os.getenv(\"__api_key__\"), location = locations, contentType = \"dataframe\") # default parameters are set from `config.json`" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 11, + "metadata": { + "ExecuteTime": { + "end_time": "2024-01-31T19:07:31.088695Z", + "start_time": "2024-01-31T19:07:31.074602Z" + } + }, + "outputs": [], + "source": [ + "api = vc.API(start = \"1975-01-01\", locations = locations.values(), key = \"\", end = \"1975-12-31\")" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "ExecuteTime": { + "end_time": "2024-01-31T19:12:31.008599Z", + "start_time": "2024-01-31T19:07:31.089694Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 162/162 [04:59<00:00, 1.85s/it]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total Query Cost: 1,084,627\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "responses = api.response()" + ] + }, + { + "cell_type": "code", + "execution_count": 13, "metadata": { "ExecuteTime": { - "end_time": "2021-10-19T10:56:08.273752Z", - "start_time": "2021-10-19T10:56:05.753306Z" + "end_time": "2024-01-31T19:12:31.023456Z", + "start_time": "2024-01-31T19:12:31.009459Z" } }, + "outputs": [], + "source": [ + "def exists(station_id : str, engine : object) -> bool:\n", + " # check if a station exists\n", + " statement = f\"SELECT * FROM `dbo.mwVCWeatherStations` WHERE VCWeatherStationID = '{station_id}';\"\n", + " records = engine.execute(statement).fetchall()\n", + " return True if records else False" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "ExecuteTime": { + "end_time": "2024-01-31T19:12:39.673182Z", + "start_time": "2024-01-31T19:12:31.025458Z" + }, + "code_folding": [] + }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "../VisualCrossing/api.py:160: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n", - " total = np.array([self._get_df_converted_data(loc, ssl_verify) for loc in self._location])\n" + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 162/162 [00:08<00:00, 18.79it/s]\n" ] + } + ], + "source": [ + "for response in TQ(responses):\n", + " try:\n", + " for station, value in response[\"stations\"].items():\n", + " insert_statement = \"INSERT INTO `dbo.mwVCWeatherStations` VALUES ('{VCWeatherStationID}', '{StationName}', {StationLatitude:.6f}, {StationLongitude:.6f}, {StationQuality:.2f}, {StationContribution:.2f});\"\n", + " insert_statement = insert_statement.format(\n", + " VCWeatherStationID = station,\n", + " StationName = value[\"name\"],\n", + " StationLatitude = value[\"latitude\"],\n", + " StationLongitude = value[\"longitude\"],\n", + " StationQuality = value[\"quality\"],\n", + " StationContribution = value[\"contribution\"]\n", + " )\n", + "\n", + " if not exists(station, engine = engine_testdb):\n", + " engine_testdb.execute(insert_statement)\n", + " except Exception as err:\n", + " pass # print(\"Failed to Fetch Stations for Resolved Address = \", response[\"resolvedAddress\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "ExecuteTime": { + "end_time": "2024-01-31T19:12:39.689098Z", + "start_time": "2024-01-31T19:12:39.675094Z" }, + "code_folding": [ + 0 + ] + }, + "outputs": [], + "source": [ + "rename_final_columns = {\n", + " \"datetime\" : \"EffectiveDate\",\n", + " \"tempmax\" : \"TempMax\",\n", + " \"tempmin\" : \"TempMin\",\n", + " \"temp\" : \"TempAvg\",\n", + " \"feelslikemax\" : \"FeelsLikeMax\",\n", + " \"feelslikemin\" : \"FeelsLikeMin\",\n", + " \"feelslike\" : \"FeelsLikeAvg\",\n", + " \"dew\" : \"DewTemp\",\n", + " \"humidity\" : \"HumidityPercent\",\n", + " \"precip\" : \"PrecipAmt\",\n", + " \"precipprob\" : \"PrecipProb\",\n", + " \"precipcover\" : \"PrecipCover\",\n", + " \"preciptype\" : \"PrecipType\",\n", + " \"snow\" : \"SnowCover\",\n", + " \"snowdepth\" : \"SnowDepth\",\n", + " \"windgust\" : \"WindGust\",\n", + " \"windspeed\" : \"WindSpeed\",\n", + " \"winddir\" : \"WindDir\",\n", + " \"pressure\" : \"AtmPressure\",\n", + " \"cloudcover\" : \"CloudCoverage\",\n", + " \"visibility\" : \"VisibilityDist\",\n", + " \"solarradiation\" : \"SolarRadiation\",\n", + " \"solarenergy\" : \"SolarEnergy\",\n", + " \"sunrise\" : \"SunRise\",\n", + " \"sunset\" : \"SunSet\",\n", + " \"moonphase\" : \"MoonPhase\",\n", + " \"conditions\" : \"WeatherCond\",\n", + " \"description\" : \"WeatherDesc\",\n", + " \"icon\" : \"WeatherIcon\",\n", + " \"stations\" : \"_stations\",\n", + " \"source\" : \"_source\",\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "ExecuteTime": { + "end_time": "2024-01-31T19:12:41.833742Z", + "start_time": "2024-01-31T19:12:39.689821Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 162/162 [00:01<00:00, 86.73it/s]\n" + ] + } + ], + "source": [ + "tblDailyWeather = []\n", + "for response, (CityUUID, _) in TQ(zip(responses, locations.items()), total = len(responses)):\n", + " ResolvedLocation = response[\"resolvedAddress\"]\n", + " tblDailyWeather_ = pd.DataFrame(response[\"days\"])\n", + " \n", + " for existing_col_name, final_col_name in rename_final_columns.items():\n", + " try:\n", + " tblDailyWeather_.rename(columns = {existing_col_name : final_col_name}, inplace = True)\n", + " except:\n", + " pass\n", + " \n", + " tblDailyWeather_ = tblDailyWeather_[[col for col in tblDailyWeather_.columns if col in list(rename_final_columns.values())]]\n", + " tblDailyWeather_[\"CityUUID\"] = CityUUID\n", + " tblDailyWeather_[\"ResolvedLocation\"] = ResolvedLocation\n", + " \n", + " if \"_stations\" in tblDailyWeather_.columns:\n", + " tblDailyWeather_[\"_stations\"] = tblDailyWeather_[\"_stations\"].apply(lambda x : \",\".join(x) if x else None)\n", + " else:\n", + " tblDailyWeather_[\"_stations\"] = None\n", + " \n", + " if \"PrecipType\" in tblDailyWeather_.columns:\n", + " tblDailyWeather_[\"PrecipType\"] = tblDailyWeather_[\"PrecipType\"].apply(lambda x : \",\".join(x) if x else None)\n", + " else:\n", + " tblDailyWeather_[\"PrecipType\"] = None\n", + " \n", + " tblDailyWeather.append(tblDailyWeather_)\n", + "\n", + "tblDailyWeather = pd.concat(tblDailyWeather)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "ExecuteTime": { + "end_time": "2024-01-31T19:12:42.802880Z", + "start_time": "2024-01-31T19:12:41.837744Z" + } + }, + "outputs": [], + "source": [ + "tblDailyWeather[\"SunRise\"] = pd.to_datetime(tblDailyWeather[[\"EffectiveDate\", \"SunRise\"]].apply(lambda x : f\"{x[0]} {x[1]}\", axis = 1), format = \"%Y-%m-%d\")\n", + "tblDailyWeather[\"SunSet\"] = pd.to_datetime(tblDailyWeather[[\"EffectiveDate\", \"SunSet\"]].apply(lambda x : f\"{x[0]} {x[1]}\", axis = 1), format = \"%Y-%m-%d\")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "ExecuteTime": { + "end_time": "2024-01-31T19:12:44.735755Z", + "start_time": "2024-01-31T19:12:42.804699Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 33/33 [00:01<00:00, 17.23it/s]\n" + ] + } + ], + "source": [ + "for column in TQ(tblDailyWeather.columns):\n", + " tblDailyWeather[column] = tblDailyWeather[column].apply(lambda x : np.nan if str(x).strip().replace(\" \", \" \") == \"\" else x)\n", + " tblDailyWeather[column].fillna(value = np.nan, inplace = True)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "ExecuteTime": { + "end_time": "2024-01-31T19:13:00.521860Z", + "start_time": "2024-01-31T19:12:44.736626Z" + } + }, + "outputs": [ { "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
AddressDate timeMinimum TemperatureMaximum TemperatureTemperatureDew PointRelative HumidityHeat IndexWind SpeedWind Gust...VisibilityCloud CoverSea Level PressureWeather TypeLatitudeLongitudeResolved AddressNameInfoConditions
0india09/25/202125.532.528.725.784.7540.99.4NaN...3.269.51006.3Mist, Thunderstorm Without Precipitation, Smok...28.631077.2172IndiaIndiaNaNRain, Partially cloudy
0tokyo09/25/202120.424.122.217.273.29NaN18.1NaN...18.825.41024.0Rain, Light Rain, Sky Coverage Increasing, Smo...35.6841139.8090Tokyo, JapanTokyo, JapanNaNPartially cloudy
\n", - "

2 rows × 25 columns

\n", - "
" - ], "text/plain": [ - " Address Date time Minimum Temperature Maximum Temperature Temperature \\\n", - "0 india 09/25/2021 25.5 32.5 28.7 \n", - "0 tokyo 09/25/2021 20.4 24.1 22.2 \n", - "\n", - " Dew Point Relative Humidity Heat Index Wind Speed Wind Gust ... \\\n", - "0 25.7 84.75 40.9 9.4 NaN ... \n", - "0 17.2 73.29 NaN 18.1 NaN ... \n", - "\n", - " Visibility Cloud Cover Sea Level Pressure \\\n", - "0 3.2 69.5 1006.3 \n", - "0 18.8 25.4 1024.0 \n", - "\n", - " Weather Type Latitude Longitude \\\n", - "0 Mist, Thunderstorm Without Precipitation, Smok... 28.6310 77.2172 \n", - "0 Rain, Light Rain, Sky Coverage Increasing, Smo... 35.6841 139.8090 \n", - "\n", - " Resolved Address Name Info Conditions \n", - "0 India India NaN Rain, Partially cloudy \n", - "0 Tokyo, Japan Tokyo, Japan NaN Partially cloudy \n", - "\n", - "[2 rows x 25 columns]" + "59130" ] }, - "execution_count": 6, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "data = api.get(ssl_verify = True)\n", - "data.head()" + "tblDailyWeather.to_sql(\"tblDailyWeather\", engine_testdb, index = False, if_exists = \"append\")" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "ExecuteTime": { + "end_time": "2024-01-31T19:15:21.059925Z", + "start_time": "2024-01-31T19:13:00.522861Z" + } + }, + "outputs": [], + "source": [ + "with open(\"responses.json\", \"w\") as f:\n", + " json.dump(responses, f, indent = 2)" ] } ], @@ -456,7 +671,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.8" + "version": "3.9.13" }, "latex_envs": { "LaTeX_envs_menu_present": true, diff --git a/examples/[REFERENCE] Database Design.ipynb b/examples/[REFERENCE] Database Design.ipynb index ca73d4d..37654d3 100644 --- a/examples/[REFERENCE] Database Design.ipynb +++ b/examples/[REFERENCE] Database Design.ipynb @@ -93,7 +93,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.8" + "version": "3.9.13" }, "latex_envs": { "LaTeX_envs_menu_present": true, @@ -125,6 +125,35 @@ "toc_position": {}, "toc_section_display": true, "toc_window_display": false + }, + "varInspector": { + "cols": { + "lenName": 16, + "lenType": 16, + "lenVar": 40 + }, + "kernels_config": { + "python": { + "delete_cmd_postfix": "", + "delete_cmd_prefix": "del ", + "library": "var_list.py", + "varRefreshCmd": "print(var_dic_list())" + }, + "r": { + "delete_cmd_postfix": ") ", + "delete_cmd_prefix": "rm(", + "library": "var_list.r", + "varRefreshCmd": "cat(var_dic_list()) " + } + }, + "types_to_exclude": [ + "module", + "function", + "builtin_function_or_method", + "instance", + "_Feature" + ], + "window_display": false } }, "nbformat": 4,