From b8da35a59546d737bf540359d66c298937000134 Mon Sep 17 00:00:00 2001 From: davidlatte Date: Wed, 11 Dec 2024 11:10:00 -0800 Subject: [PATCH 1/4] backtest: When running a long backtest that goes across Daylight Savings transitions, lumibot is reporting the start time at 10:30am instead of 9:30am --- lumibot/backtesting/backtesting_broker.py | 8 +++++++- .../interactive_brokers_rest_data.py | 8 +++++--- lumibot/data_sources/tradier_data.py | 6 +++--- lumibot/strategies/_strategy.py | 17 +++++++++-------- lumibot/tools/thetadata_helper.py | 4 ++-- 5 files changed, 26 insertions(+), 17 deletions(-) diff --git a/lumibot/backtesting/backtesting_broker.py b/lumibot/backtesting/backtesting_broker.py index 6047de8cd..0b879a6df 100644 --- a/lumibot/backtesting/backtesting_broker.py +++ b/lumibot/backtesting/backtesting_broker.py @@ -4,7 +4,7 @@ from decimal import Decimal from functools import wraps -import pandas as pd +import pytz from lumibot.brokers import Broker from lumibot.data_sources import DataSourceBacktesting @@ -88,6 +88,8 @@ def get_historical_account_value(self): def _update_datetime(self, update_dt, cash=None, portfolio_value=None): """Works with either timedelta or datetime input and updates the datetime of the broker""" + tz = self.datetime.tzinfo + is_pytz = isinstance(tz, (pytz.tzinfo.StaticTzInfo, pytz.tzinfo.DstTzInfo)) if isinstance(update_dt, timedelta): new_datetime = self.datetime + update_dt @@ -95,6 +97,10 @@ def _update_datetime(self, update_dt, cash=None, portfolio_value=None): new_datetime = self.datetime + timedelta(seconds=update_dt) else: new_datetime = update_dt + + # This is needed to handle Daylight Savings Time changes + new_datetime = tz.normalize(new_datetime) if is_pytz else new_datetime + self.data_source._update_datetime(new_datetime, cash=cash, portfolio_value=portfolio_value) if self.option_source: self.option_source._update_datetime(new_datetime, cash=cash, portfolio_value=portfolio_value) diff --git a/lumibot/data_sources/interactive_brokers_rest_data.py b/lumibot/data_sources/interactive_brokers_rest_data.py index 9edfcc4ad..5c7942c1a 100644 --- a/lumibot/data_sources/interactive_brokers_rest_data.py +++ b/lumibot/data_sources/interactive_brokers_rest_data.py @@ -1,8 +1,10 @@ import logging from termcolor import colored -from ..entities import Asset, Bars +from lumibot import LUMIBOT_DEFAULT_PYTZ +from ..entities import Asset, Bars from .data_source import DataSource + import subprocess import os import time @@ -813,7 +815,7 @@ def get_historical_prices( # Convert timestamp to datetime and set as index df["timestamp"] = pd.to_datetime(df["timestamp"], unit="ms") df["timestamp"] = ( - df["timestamp"].dt.tz_localize("UTC").dt.tz_convert("America/New_York") + df["timestamp"].dt.tz_localize("UTC").dt.tz_convert(LUMIBOT_DEFAULT_PYTZ) ) df.set_index("timestamp", inplace=True) @@ -1078,4 +1080,4 @@ def get_quote(self, asset, quote=None, exchange=None): else: result["ask"] = None - return result \ No newline at end of file + return result diff --git a/lumibot/data_sources/tradier_data.py b/lumibot/data_sources/tradier_data.py index 67ed0b851..d4d6a98c0 100644 --- a/lumibot/data_sources/tradier_data.py +++ b/lumibot/data_sources/tradier_data.py @@ -3,8 +3,8 @@ from datetime import datetime, date, timedelta import pandas as pd -import pytz +from lumibot import LUMIBOT_DEFAULT_PYTZ from lumibot.entities import Asset, Bars from lumibot.tools.helpers import create_options_symbol, parse_timestep_qty_and_unit, get_trading_days from lumiwealth_tradier import Tradier @@ -189,7 +189,7 @@ def get_historical_prices( end_date = datetime.now() # Use pytz to get the US/Eastern timezone - eastern = pytz.timezone("US/Eastern") + eastern = LUMIBOT_DEFAULT_PYTZ # Convert datetime object to US/Eastern timezone end_date = end_date.astimezone(eastern) @@ -242,7 +242,7 @@ def get_historical_prices( # if type of index is date, convert it to timestamp with timezone info of "America/New_York" if isinstance(df.index[0], date): - df.index = pd.to_datetime(df.index).tz_localize("America/New_York") + df.index = pd.to_datetime(df.index).tz_localize(LUMIBOT_DEFAULT_PYTZ) # Convert the dataframe to a Bars object bars = Bars(df, self.SOURCE, asset, raw=df, quote=quote) diff --git a/lumibot/strategies/_strategy.py b/lumibot/strategies/_strategy.py index 93af0da47..7d804b18b 100644 --- a/lumibot/strategies/_strategy.py +++ b/lumibot/strategies/_strategy.py @@ -21,6 +21,7 @@ from sqlalchemy import create_engine, inspect, text import pandas as pd +from lumibot import LUMIBOT_DEFAULT_PYTZ from ..backtesting import BacktestingBroker, PolygonDataBacktesting, ThetaDataBacktesting from ..entities import Asset, Position, Order from ..tools import ( @@ -1791,7 +1792,7 @@ def send_account_summary_to_discord(self): cash = self.get_cash() # # Get the datetime - now = pd.Timestamp(datetime.datetime.now()).tz_localize("America/New_York") + now = pd.Timestamp(datetime.datetime.now()).tz_localize(LUMIBOT_DEFAULT_PYTZ) # Get the returns returns_text, stats_df = self.calculate_returns() @@ -1820,7 +1821,7 @@ def get_stats_from_database(self, stats_table_name, retries=5, delay=5): self.logger.info(f"Table {stats_table_name} does not exist. Creating it now.") # Get the current time in New York - ny_tz = pytz.timezone("America/New_York") + ny_tz = LUMIBOT_DEFAULT_PYTZ now = datetime.datetime.now(ny_tz) # Create an empty stats dataframe @@ -1884,7 +1885,7 @@ def backup_variables_to_db(self): self.db_engine = create_engine(self.db_connection_str) # Get the current time in New York - ny_tz = pytz.timezone("America/New_York") + ny_tz = LUMIBOT_DEFAULT_PYTZ now = datetime.datetime.now(ny_tz) if not inspect(self.db_engine).has_table(self.backup_table_name): @@ -2008,7 +2009,7 @@ def calculate_returns(self): # Calculate the return over the past 24 hours, 7 days, and 30 days using the stats dataframe # Get the current time in New York - ny_tz = pytz.timezone("America/New_York") + ny_tz = LUMIBOT_DEFAULT_PYTZ # Get the datetime now = datetime.datetime.now(ny_tz) @@ -2025,11 +2026,11 @@ def calculate_returns(self): # Check if the datetime column is timezone-aware if stats_df['datetime'].dt.tz is None: # If the datetime is timezone-naive, directly localize it to "America/New_York" - stats_df["datetime"] = stats_df["datetime"].dt.tz_localize("America/New_York", ambiguous='infer') + stats_df["datetime"] = stats_df["datetime"].dt.tz_localize(LUMIBOT_DEFAULT_PYTZ, ambiguous='infer') else: # If the datetime is already timezone-aware, first remove timezone and then localize stats_df["datetime"] = stats_df["datetime"].dt.tz_localize(None) - stats_df["datetime"] = stats_df["datetime"].dt.tz_localize("America/New_York", ambiguous='infer') + stats_df["datetime"] = stats_df["datetime"].dt.tz_localize(LUMIBOT_DEFAULT_PYTZ, ambiguous='infer') # Get the stats stats_new = pd.DataFrame( @@ -2049,7 +2050,7 @@ def calculate_returns(self): stats_df = pd.concat([stats_df, stats_new]) # # Convert the datetime column to eastern time - stats_df["datetime"] = stats_df["datetime"].dt.tz_convert("America/New_York") + stats_df["datetime"] = stats_df["datetime"].dt.tz_convert(LUMIBOT_DEFAULT_PYTZ) # Remove any duplicate rows stats_df = stats_df[~stats_df["datetime"].duplicated(keep="last")] @@ -2160,4 +2161,4 @@ def calculate_returns(self): return results_text, stats_df else: - return "Not enough data to calculate returns", stats_df \ No newline at end of file + return "Not enough data to calculate returns", stats_df diff --git a/lumibot/tools/thetadata_helper.py b/lumibot/tools/thetadata_helper.py index 45a0a78e2..78179da86 100644 --- a/lumibot/tools/thetadata_helper.py +++ b/lumibot/tools/thetadata_helper.py @@ -8,7 +8,7 @@ import pandas as pd import pandas_market_calendars as mcal import requests -from lumibot import LUMIBOT_CACHE_FOLDER +from lumibot import LUMIBOT_CACHE_FOLDER, LUMIBOT_DEFAULT_PYTZ from lumibot.entities import Asset from thetadata import ThetaClient from tqdm import tqdm @@ -295,7 +295,7 @@ def update_df(df_all, result): ], } """ - ny_tz = pytz.timezone('America/New_York') + ny_tz = LUMIBOT_DEFAULT_PYTZ df = pd.DataFrame(result) if not df.empty: if "datetime" not in df.index.names: From 2bd6634e0e2d46b014751a1fcb77a4b11c4c9295 Mon Sep 17 00:00:00 2001 From: davidlatte Date: Wed, 11 Dec 2024 11:10:00 -0800 Subject: [PATCH 2/4] backtest: When running a long backtest that goes across Daylight Savings transitions, lumibot is reporting the start time at 10:30am instead of 9:30am --- lumibot/backtesting/backtesting_broker.py | 8 +++++++- .../interactive_brokers_rest_data.py | 8 +++++--- lumibot/data_sources/tradier_data.py | 2 ++ lumibot/strategies/_strategy.py | 17 +++++++++-------- lumibot/tools/thetadata_helper.py | 4 ++-- 5 files changed, 25 insertions(+), 14 deletions(-) diff --git a/lumibot/backtesting/backtesting_broker.py b/lumibot/backtesting/backtesting_broker.py index 6047de8cd..0b879a6df 100644 --- a/lumibot/backtesting/backtesting_broker.py +++ b/lumibot/backtesting/backtesting_broker.py @@ -4,7 +4,7 @@ from decimal import Decimal from functools import wraps -import pandas as pd +import pytz from lumibot.brokers import Broker from lumibot.data_sources import DataSourceBacktesting @@ -88,6 +88,8 @@ def get_historical_account_value(self): def _update_datetime(self, update_dt, cash=None, portfolio_value=None): """Works with either timedelta or datetime input and updates the datetime of the broker""" + tz = self.datetime.tzinfo + is_pytz = isinstance(tz, (pytz.tzinfo.StaticTzInfo, pytz.tzinfo.DstTzInfo)) if isinstance(update_dt, timedelta): new_datetime = self.datetime + update_dt @@ -95,6 +97,10 @@ def _update_datetime(self, update_dt, cash=None, portfolio_value=None): new_datetime = self.datetime + timedelta(seconds=update_dt) else: new_datetime = update_dt + + # This is needed to handle Daylight Savings Time changes + new_datetime = tz.normalize(new_datetime) if is_pytz else new_datetime + self.data_source._update_datetime(new_datetime, cash=cash, portfolio_value=portfolio_value) if self.option_source: self.option_source._update_datetime(new_datetime, cash=cash, portfolio_value=portfolio_value) diff --git a/lumibot/data_sources/interactive_brokers_rest_data.py b/lumibot/data_sources/interactive_brokers_rest_data.py index bb89b3217..124d07ea0 100644 --- a/lumibot/data_sources/interactive_brokers_rest_data.py +++ b/lumibot/data_sources/interactive_brokers_rest_data.py @@ -1,8 +1,10 @@ import logging from termcolor import colored -from ..entities import Asset, Bars +from lumibot import LUMIBOT_DEFAULT_PYTZ +from ..entities import Asset, Bars from .data_source import DataSource + import subprocess import os import time @@ -817,7 +819,7 @@ def get_historical_prices( # Convert timestamp to datetime and set as index df["timestamp"] = pd.to_datetime(df["timestamp"], unit="ms") df["timestamp"] = ( - df["timestamp"].dt.tz_localize("UTC").dt.tz_convert("America/New_York") + df["timestamp"].dt.tz_localize("UTC").dt.tz_convert(LUMIBOT_DEFAULT_PYTZ) ) df.set_index("timestamp", inplace=True) @@ -1082,4 +1084,4 @@ def get_quote(self, asset, quote=None, exchange=None): else: result["ask"] = None - return result \ No newline at end of file + return result diff --git a/lumibot/data_sources/tradier_data.py b/lumibot/data_sources/tradier_data.py index 2b58d3595..aca4d3f12 100644 --- a/lumibot/data_sources/tradier_data.py +++ b/lumibot/data_sources/tradier_data.py @@ -12,9 +12,11 @@ from .data_source import DataSource + class TradierAPIError(Exception): pass + class TradierData(DataSource): MIN_TIMESTEP = "minute" diff --git a/lumibot/strategies/_strategy.py b/lumibot/strategies/_strategy.py index 93af0da47..7d804b18b 100644 --- a/lumibot/strategies/_strategy.py +++ b/lumibot/strategies/_strategy.py @@ -21,6 +21,7 @@ from sqlalchemy import create_engine, inspect, text import pandas as pd +from lumibot import LUMIBOT_DEFAULT_PYTZ from ..backtesting import BacktestingBroker, PolygonDataBacktesting, ThetaDataBacktesting from ..entities import Asset, Position, Order from ..tools import ( @@ -1791,7 +1792,7 @@ def send_account_summary_to_discord(self): cash = self.get_cash() # # Get the datetime - now = pd.Timestamp(datetime.datetime.now()).tz_localize("America/New_York") + now = pd.Timestamp(datetime.datetime.now()).tz_localize(LUMIBOT_DEFAULT_PYTZ) # Get the returns returns_text, stats_df = self.calculate_returns() @@ -1820,7 +1821,7 @@ def get_stats_from_database(self, stats_table_name, retries=5, delay=5): self.logger.info(f"Table {stats_table_name} does not exist. Creating it now.") # Get the current time in New York - ny_tz = pytz.timezone("America/New_York") + ny_tz = LUMIBOT_DEFAULT_PYTZ now = datetime.datetime.now(ny_tz) # Create an empty stats dataframe @@ -1884,7 +1885,7 @@ def backup_variables_to_db(self): self.db_engine = create_engine(self.db_connection_str) # Get the current time in New York - ny_tz = pytz.timezone("America/New_York") + ny_tz = LUMIBOT_DEFAULT_PYTZ now = datetime.datetime.now(ny_tz) if not inspect(self.db_engine).has_table(self.backup_table_name): @@ -2008,7 +2009,7 @@ def calculate_returns(self): # Calculate the return over the past 24 hours, 7 days, and 30 days using the stats dataframe # Get the current time in New York - ny_tz = pytz.timezone("America/New_York") + ny_tz = LUMIBOT_DEFAULT_PYTZ # Get the datetime now = datetime.datetime.now(ny_tz) @@ -2025,11 +2026,11 @@ def calculate_returns(self): # Check if the datetime column is timezone-aware if stats_df['datetime'].dt.tz is None: # If the datetime is timezone-naive, directly localize it to "America/New_York" - stats_df["datetime"] = stats_df["datetime"].dt.tz_localize("America/New_York", ambiguous='infer') + stats_df["datetime"] = stats_df["datetime"].dt.tz_localize(LUMIBOT_DEFAULT_PYTZ, ambiguous='infer') else: # If the datetime is already timezone-aware, first remove timezone and then localize stats_df["datetime"] = stats_df["datetime"].dt.tz_localize(None) - stats_df["datetime"] = stats_df["datetime"].dt.tz_localize("America/New_York", ambiguous='infer') + stats_df["datetime"] = stats_df["datetime"].dt.tz_localize(LUMIBOT_DEFAULT_PYTZ, ambiguous='infer') # Get the stats stats_new = pd.DataFrame( @@ -2049,7 +2050,7 @@ def calculate_returns(self): stats_df = pd.concat([stats_df, stats_new]) # # Convert the datetime column to eastern time - stats_df["datetime"] = stats_df["datetime"].dt.tz_convert("America/New_York") + stats_df["datetime"] = stats_df["datetime"].dt.tz_convert(LUMIBOT_DEFAULT_PYTZ) # Remove any duplicate rows stats_df = stats_df[~stats_df["datetime"].duplicated(keep="last")] @@ -2160,4 +2161,4 @@ def calculate_returns(self): return results_text, stats_df else: - return "Not enough data to calculate returns", stats_df \ No newline at end of file + return "Not enough data to calculate returns", stats_df diff --git a/lumibot/tools/thetadata_helper.py b/lumibot/tools/thetadata_helper.py index 45a0a78e2..78179da86 100644 --- a/lumibot/tools/thetadata_helper.py +++ b/lumibot/tools/thetadata_helper.py @@ -8,7 +8,7 @@ import pandas as pd import pandas_market_calendars as mcal import requests -from lumibot import LUMIBOT_CACHE_FOLDER +from lumibot import LUMIBOT_CACHE_FOLDER, LUMIBOT_DEFAULT_PYTZ from lumibot.entities import Asset from thetadata import ThetaClient from tqdm import tqdm @@ -295,7 +295,7 @@ def update_df(df_all, result): ], } """ - ny_tz = pytz.timezone('America/New_York') + ny_tz = LUMIBOT_DEFAULT_PYTZ df = pd.DataFrame(result) if not df.empty: if "datetime" not in df.index.names: From 397363caa75ae2ef4c2291f56c299b5edae1c046 Mon Sep 17 00:00:00 2001 From: davidlatte Date: Wed, 11 Dec 2024 20:21:46 -0800 Subject: [PATCH 3/4] unittest: skipping ThetaData tests that aren't ready yet. --- tests/backtest/test_thetadata.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/backtest/test_thetadata.py b/tests/backtest/test_thetadata.py index 5ec21b4a2..a2a138cdd 100644 --- a/tests/backtest/test_thetadata.py +++ b/tests/backtest/test_thetadata.py @@ -316,10 +316,12 @@ def verify_backtest_results(self, theta_strat_obj): ) assert "fill" not in theta_strat_obj.order_time_tracker[stoploss_order_id] - @pytest.mark.skipif( - secrets_not_found, - reason="Skipping test because ThetaData API credentials not found in environment variables", - ) + # @pytest.mark.skipif( + # secrets_not_found, + # reason="Skipping test because ThetaData API credentials not found in environment variables", + # ) + @pytest.skip("Skipping test because ThetaData API credentials not found in Github Pipeline " + "environment variables") def test_thetadata_restclient(self): """ Test ThetaDataBacktesting with Lumibot Backtesting and real API calls to ThetaData. Using the Amazon stock From 6cb4be2afdd2ba3319522d4bd7111b6ca0813053 Mon Sep 17 00:00:00 2001 From: davidlatte Date: Wed, 11 Dec 2024 20:29:03 -0800 Subject: [PATCH 4/4] unittest: skipping ThetaData tests that aren't ready yet. --- tests/backtest/test_thetadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/backtest/test_thetadata.py b/tests/backtest/test_thetadata.py index a2a138cdd..f2a633d74 100644 --- a/tests/backtest/test_thetadata.py +++ b/tests/backtest/test_thetadata.py @@ -320,8 +320,8 @@ def verify_backtest_results(self, theta_strat_obj): # secrets_not_found, # reason="Skipping test because ThetaData API credentials not found in environment variables", # ) - @pytest.skip("Skipping test because ThetaData API credentials not found in Github Pipeline " - "environment variables") + @pytest.mark.skip("Skipping test because ThetaData API credentials not found in Github Pipeline " + "environment variables") def test_thetadata_restclient(self): """ Test ThetaDataBacktesting with Lumibot Backtesting and real API calls to ThetaData. Using the Amazon stock