diff --git a/Example_get_series_exp.ipynb b/Example_get_series_exp.ipynb index 53b6a51..ef3c10b 100644 --- a/Example_get_series_exp.ipynb +++ b/Example_get_series_exp.ipynb @@ -8,9 +8,482 @@ "outputs": [], "source": [ "# !pip install evdspy -U \n", - "from evdspy import get_series_exp" + "from evdspy import get_series_exp, get_series " ] }, + { + "cell_type": "code", + "execution_count": 12, + "id": "0c370f8a-5883-43ce-9e47-d338e0ceedf2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requesting... will be running now...\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
TP_DK_USD_A_YTLTP_DK_EUR_A_YTLTarih
0NaNNaN01-01-2000
1NaNNaN02-01-2000
20.5400980.54209603-01-2000
30.5407930.54566004-01-2000
40.5358150.55296105-01-2000
............
8975NaNNaN28-07-2024
897632.95860035.77320029-07-2024
897732.95090035.71100030-07-2024
897833.03040035.76120031-07-2024
897933.08350035.81450001-08-2024
\n", + "

8980 rows × 3 columns

\n", + "
" + ], + "text/plain": [ + " TP_DK_USD_A_YTL TP_DK_EUR_A_YTL Tarih\n", + "0 NaN NaN 01-01-2000\n", + "1 NaN NaN 02-01-2000\n", + "2 0.540098 0.542096 03-01-2000\n", + "3 0.540793 0.545660 04-01-2000\n", + "4 0.535815 0.552961 05-01-2000\n", + "... ... ... ...\n", + "8975 NaN NaN 28-07-2024\n", + "8976 32.958600 35.773200 29-07-2024\n", + "8977 32.950900 35.711000 30-07-2024\n", + "8978 33.030400 35.761200 31-07-2024\n", + "8979 33.083500 35.814500 01-08-2024\n", + "\n", + "[8980 rows x 3 columns]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ID = \"\"\"\n", + "\n", + "TP_DK_usd_a.YTL\n", + "TP_DK_eur_a.YTL\n", + "\n", + "\n", + "\"\"\" \n", + "\n", + "df = get_series(ID , cache = True )\n", + "\n", + "df\n" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "339c0542-2c34-4443-b436-dbe03c6d4225", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n" + ] + }, + { + "data": { + "text/html": [ + "
Could not convert some columns to float type...\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[3;38;5;199;40mCould not convert some columns to float type\u001b[0m\u001b[3;33;40m...\u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/html": [ + "
Could not convert some columns to float type...\n",
+       "
\n" + ], + "text/plain": [ + "\u001b[3;36;40mCould not convert some columns to float type\u001b[0m\u001b[3;33;40m...\u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
SERIE_CODEDATAGROUP_CODESERIE_NAMESERIE_NAME_ENGFREQUENCY_STRDEFAULT_AGG_METHOD_STRDEFAULT_AGG_METHODTAGTAG_ENGDATASOURCEDATASOURCE_ENGMETADATA_LINKMETADATA_LINK_ENGREV_POL_LINKREV_POL_LINK_ENGAPP_CHA_LINKAPP_CHA_LINK_ENGSTART_DATEEND_DATE
0TP.DK.USD.A.YTLbie_dkdovytl(USD) ABD Doları (Döviz Alış)(USD) US Dollar (Buying)GÜNLÜKORTALAMAavgKurlar, Döviz, Kurları, GünlükExchange, Rates, Daily,TCMBCBRT02-01-195001-08-2024
0TP.DK.EUR.A.YTLbie_dkdovytl(EUR) Euro (Döviz Alış)(EUR) Euro (Buying)GÜNLÜKORTALAMAavgKurlar, Döviz, Kurları, GünlükExchange, Rates, Daily,TCMBCBRT04-01-199901-08-2024
\n", + "
" + ], + "text/plain": [ + " SERIE_CODE DATAGROUP_CODE SERIE_NAME \\\n", + "0 TP.DK.USD.A.YTL bie_dkdovytl (USD) ABD Doları (Döviz Alış) \n", + "0 TP.DK.EUR.A.YTL bie_dkdovytl (EUR) Euro (Döviz Alış) \n", + "\n", + " SERIE_NAME_ENG FREQUENCY_STR DEFAULT_AGG_METHOD_STR \\\n", + "0 (USD) US Dollar (Buying) GÜNLÜK ORTALAMA \n", + "0 (EUR) Euro (Buying) GÜNLÜK ORTALAMA \n", + "\n", + " DEFAULT_AGG_METHOD TAG TAG_ENG \\\n", + "0 avg Kurlar, Döviz, Kurları, Günlük Exchange, Rates, Daily, \n", + "0 avg Kurlar, Döviz, Kurları, Günlük Exchange, Rates, Daily, \n", + "\n", + " DATASOURCE DATASOURCE_ENG METADATA_LINK METADATA_LINK_ENG REV_POL_LINK \\\n", + "0 TCMB CBRT \n", + "0 TCMB CBRT \n", + "\n", + " REV_POL_LINK_ENG APP_CHA_LINK APP_CHA_LINK_ENG START_DATE END_DATE \n", + "0 02-01-1950 01-08-2024 \n", + "0 04-01-1999 01-08-2024 " + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ID = \"\"\"\n", + "\n", + "TP_DK_usd_a.YTL\n", + "TP_DK_eur_a.YTL\n", + "\n", + "\n", + "\"\"\" \n", + "\n", + "result = get_series_exp(ID , cache = True )\n", + "\n", + "result \n", + "# print(result )\n", + "\n", + "result.metadata" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "f70b1cf2-7dd9-42ca-9009-0d825f4fcaf2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " writing file : [out.xlsx] \n" + ] + } + ], + "source": [ + "result.to_excel(\"out.xlsx\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "68592e47-a808-4136-9faf-4591287aa71f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
TP_DK_USD_A_YTLTarih
0NaN01-01-2000
1NaN02-01-2000
20.54009803-01-2000
30.54079304-01-2000
40.53581505-01-2000
.........
8975NaN28-07-2024
897632.95860029-07-2024
897732.95090030-07-2024
897833.03040031-07-2024
897933.08350001-08-2024
\n", + "

8980 rows × 2 columns

\n", + "
" + ], + "text/plain": [ + " TP_DK_USD_A_YTL Tarih\n", + "0 NaN 01-01-2000\n", + "1 NaN 02-01-2000\n", + "2 0.540098 03-01-2000\n", + "3 0.540793 04-01-2000\n", + "4 0.535815 05-01-2000\n", + "... ... ...\n", + "8975 NaN 28-07-2024\n", + "8976 32.958600 29-07-2024\n", + "8977 32.950900 30-07-2024\n", + "8978 33.030400 31-07-2024\n", + "8979 33.083500 01-08-2024\n", + "\n", + "[8980 rows x 2 columns]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24b15c75-82d0-45ab-959b-f00bbb7860f1", + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "code", "execution_count": 4, diff --git a/evdspy/EVDSlocal/console/proxy_for_menu.py b/evdspy/EVDSlocal/console/proxy_for_menu.py new file mode 100644 index 0000000..fa22009 --- /dev/null +++ b/evdspy/EVDSlocal/console/proxy_for_menu.py @@ -0,0 +1,21 @@ +import os + + +def get_proxies_env() -> dict: + from ..config.dotenv import load_env + load_env() + a = os.getenv("PROXY_http") + b = os.getenv("PROXY_https") + + if a is None and b is None: + return False + if a and not b: + return {"http": a} + if b and not a: + return {"https": b} + return { + "http": a, + "https": b + } + +# check_proxy() diff --git a/evdspy/EVDSlocal/index_requests/user_requests/Proxy_manager.py b/evdspy/EVDSlocal/index_requests/user_requests/Proxy_manager.py index 5694e18..c38a4b7 100644 --- a/evdspy/EVDSlocal/index_requests/user_requests/Proxy_manager.py +++ b/evdspy/EVDSlocal/index_requests/user_requests/Proxy_manager.py @@ -2,6 +2,8 @@ from dataclasses import dataclass from typing import Optional, Any +from evdspy.EVDSlocal.console.proxy_for_menu import get_proxies_env + @dataclass class ProxyManager: @@ -9,9 +11,13 @@ class ProxyManager: proxies: Optional[dict[Any, Any]] = None def get_proxies(self) -> Optional[dict[Any, Any]]: + env_proxy = get_proxies_env() if self.proxies is None: if self.proxy is None: - proxies = None + if env_proxy: + proxies = env_proxy + else: + proxies = None else: proxies = self.get_proxies_helper() else: diff --git a/evdspy/EVDSlocal/requests_/ev_request.py b/evdspy/EVDSlocal/requests_/ev_request.py index 7a082d8..7a548a3 100644 --- a/evdspy/EVDSlocal/requests_/ev_request.py +++ b/evdspy/EVDSlocal/requests_/ev_request.py @@ -1,7 +1,7 @@ - # ------------------------------------------------------------------------------ -import requests +import requests from evdspy.EVDSlocal.components.url_class import URLClass +from evdspy.EVDSlocal.console.proxy_for_menu import get_proxies_env from evdspy.EVDSlocal.utils.utils_general import * from evdspy.EVDSlocal.requests_.my_cache import MyCache, save_pickle_for_test, lru_cache_patched, load_test_pickle from evdspy.EVDSlocal.components.options_class import Options @@ -17,11 +17,14 @@ from dataclasses import dataclass from evdspy.EVDSlocal.requests_.mock_req import * from evdspy.EVDSlocal.requests_.real_requests import * -from typing import Optional -from dataclasses import dataclass , field +from typing import Optional +from dataclasses import dataclass, field + # ------------------------------------------------------------------------------ m_cache = MyCache() CANCEL_REQUEST_TEMP = config.cancel_request_temp + + def do_first_true_order(funcs, preds, url): conds = zip(funcs, preds) for func, pred in conds: @@ -29,6 +32,8 @@ def do_first_true_order(funcs, preds, url): v = func(url) return v return False + + def decide_request_for_test_real(url: str, proxies=None): """ Finally ready to make request """ if config.current_mode_is_test and not config.temp_cancel_mock_request: @@ -42,6 +47,8 @@ def decide_request_for_test_real(url: str, proxies=None): if proxies is None: return requests.get(url) return requests.get(url, proxies=proxies) + + @dataclass class EVRequest: options_: Options @@ -51,6 +58,7 @@ class EVRequest: last_url_checked: str = "NoneURL" URL_Instance: field(default_factory=URLClass) = field(default_factory=URLClass) force_no_cache = False + # report : post init def __post_init__(self): # self.URL_Instance = field(default_factory=URLClass) # None @@ -61,33 +69,49 @@ def __post_init__(self): request by using cache results... """ self.report = Report("stats_requests.txt") + def get_request_with_proxy(self, url: str, proxy: str): proxies = { - 'http': proxy, - 'https': proxy, + 'http': proxy, + 'https': proxy, } return decide_request_for_test_real(url, proxies=proxies) + def get_proxies(self): + env_proxies = get_proxies_env() if self.args.proxy is None: + if env_proxies: + return env_proxies return None proxy = self.args.proxy proxies = { - 'http': proxy, - 'https': proxy, + 'http': proxy, + 'https': proxy, } return proxies + def proxy_from_file(self, url: str): print("using proxy from file") return self.get_request_with_proxy(url, self.args.proxy) + def proxy_from_cmd_line(self, url: str): print("using proxy from commandline") return self.get_request_with_proxy(url, self.args.proxy) + + def proxy_from_env(self, url: str): + print("using proxy from .env file ") + env_proxies = get_proxies_env() + if env_proxies: + return self.get_request_with_proxy(url, env_proxies) + return self.proxy_from_cmd_line(url) + def check_any_apikey(self): if not config.current_mode_is_test: return True if not ApikeyClass().get_valid_api_key(): raise ApiKeyNotSetError return True + def check_if_request_ok(self, status_code: int): corrects = (200,) if status_code in corrects: @@ -102,11 +126,12 @@ def check_if_request_ok(self, status_code: int): # raise ExceptionClass(self.__doc__) else: print( - f"Program was checking request status code and noticed " - f"{status_code} not included in `REQUEST_ERROR_CODES`" + f"Program was checking request status code and noticed " + f"{status_code} not included in `REQUEST_ERROR_CODES`" ) self.report.add(f"{status_code} returned. ") return False + def post_urls(self, url: str) -> tuple: # url = url.translate({ord(c): None for c in string.whitespace}) url = URL_temizle(url) # whitespace etc. @@ -114,20 +139,24 @@ def post_urls(self, url: str) -> tuple: self.report.add(f"{no_apikey_url} will be requested. ") self.last_url_checked = url return (url, no_apikey_url) + def get_request_alternatives(self, url): result = do_first_true_order( - funcs=[ - self.proxy_from_cmd_line, - self.proxy_from_file, - decide_request_for_test_real - ], - preds=[ - self.args.proxy, - self.credentials.proxy, - True], - url=url + funcs=[ + self.proxy_from_env, + self.proxy_from_cmd_line, + self.proxy_from_file, + decide_request_for_test_real + ], + preds=[ + get_proxies_env(), + self.args.proxy, + self.credentials.proxy, + True], + url=url ) return result + def check_result(self, result): if any( ( @@ -139,6 +168,7 @@ def check_result(self, result): print(f"request returned an error code {result.status_code} , url : {self.no_apikey_url}") return False return True + def get_request_before_cache(self, url: str): """ decorated version of this function needs to be @@ -152,6 +182,7 @@ def get_request_before_cache(self, url: str): if not self.check_any_apikey(): return False return self.get_request(url) + def get_request(self, url: str): """ first will check cache if there is cache @@ -161,18 +192,22 @@ def get_request(self, url: str): # return self.get_request_by_checking_NO_CACHE(url) return self.get_request_by_checking_cache(url) # return self.get_request_helper(url ) + @lru_cache_patched @m_cache.cache def get_request_by_checking_cache(self, url: str): """ """ return self.get_request_common(url) + def get_request_by_checking_NO_CACHE(self, url: str): """ """ return self.get_request_common(url) + def get_request_w_param(self, url: str, proxies=None): return RealRequestWithParam(url, proxies).request() + def get_request_common(self, url: str): """ first will check cache if there is cache @@ -194,4 +229,4 @@ def get_request_common(self, url: str): """ request looks solid we are ready to save """ if result and self.check_if_request_ok(result.status_code): save_pickle_for_test(result) - return result \ No newline at end of file + return result diff --git a/evdspy/EVDSlocal/requests_/test_reg_result.pickle b/evdspy/EVDSlocal/requests_/test_reg_result.pickle index 78cacab..61da9d5 100644 Binary files a/evdspy/EVDSlocal/requests_/test_reg_result.pickle and b/evdspy/EVDSlocal/requests_/test_reg_result.pickle differ