Skip to content

Commit

Permalink
backtest: When running a long backtest that goes across Daylight Savi…
Browse files Browse the repository at this point in the history
…ngs transitions, lumibot is reporting the start time at 10:30am instead of 9:30am
davidlatte committed Dec 11, 2024
1 parent 9239262 commit b8da35a
Showing 5 changed files with 26 additions and 17 deletions.
8 changes: 7 additions & 1 deletion lumibot/backtesting/backtesting_broker.py
Original file line number Diff line number Diff line change
@@ -4,7 +4,7 @@
from decimal import Decimal
from functools import wraps

import pandas as pd
import pytz

from lumibot.brokers import Broker
from lumibot.data_sources import DataSourceBacktesting
@@ -88,13 +88,19 @@ def get_historical_account_value(self):
def _update_datetime(self, update_dt, cash=None, portfolio_value=None):
"""Works with either timedelta or datetime input
and updates the datetime of the broker"""
tz = self.datetime.tzinfo
is_pytz = isinstance(tz, (pytz.tzinfo.StaticTzInfo, pytz.tzinfo.DstTzInfo))

if isinstance(update_dt, timedelta):
new_datetime = self.datetime + update_dt
elif isinstance(update_dt, int) or isinstance(update_dt, float):
new_datetime = self.datetime + timedelta(seconds=update_dt)
else:
new_datetime = update_dt

# This is needed to handle Daylight Savings Time changes
new_datetime = tz.normalize(new_datetime) if is_pytz else new_datetime

self.data_source._update_datetime(new_datetime, cash=cash, portfolio_value=portfolio_value)
if self.option_source:
self.option_source._update_datetime(new_datetime, cash=cash, portfolio_value=portfolio_value)
8 changes: 5 additions & 3 deletions lumibot/data_sources/interactive_brokers_rest_data.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
import logging
from termcolor import colored
from ..entities import Asset, Bars

from lumibot import LUMIBOT_DEFAULT_PYTZ
from ..entities import Asset, Bars
from .data_source import DataSource

import subprocess
import os
import time
@@ -813,7 +815,7 @@ def get_historical_prices(
# Convert timestamp to datetime and set as index
df["timestamp"] = pd.to_datetime(df["timestamp"], unit="ms")
df["timestamp"] = (
df["timestamp"].dt.tz_localize("UTC").dt.tz_convert("America/New_York")
df["timestamp"].dt.tz_localize("UTC").dt.tz_convert(LUMIBOT_DEFAULT_PYTZ)
)
df.set_index("timestamp", inplace=True)

@@ -1078,4 +1080,4 @@ def get_quote(self, asset, quote=None, exchange=None):
else:
result["ask"] = None

return result
return result
6 changes: 3 additions & 3 deletions lumibot/data_sources/tradier_data.py
Original file line number Diff line number Diff line change
@@ -3,8 +3,8 @@
from datetime import datetime, date, timedelta

import pandas as pd
import pytz

from lumibot import LUMIBOT_DEFAULT_PYTZ
from lumibot.entities import Asset, Bars
from lumibot.tools.helpers import create_options_symbol, parse_timestep_qty_and_unit, get_trading_days
from lumiwealth_tradier import Tradier
@@ -189,7 +189,7 @@ def get_historical_prices(
end_date = datetime.now()

# Use pytz to get the US/Eastern timezone
eastern = pytz.timezone("US/Eastern")
eastern = LUMIBOT_DEFAULT_PYTZ

# Convert datetime object to US/Eastern timezone
end_date = end_date.astimezone(eastern)
@@ -242,7 +242,7 @@ def get_historical_prices(

# if type of index is date, convert it to timestamp with timezone info of "America/New_York"
if isinstance(df.index[0], date):
df.index = pd.to_datetime(df.index).tz_localize("America/New_York")
df.index = pd.to_datetime(df.index).tz_localize(LUMIBOT_DEFAULT_PYTZ)

# Convert the dataframe to a Bars object
bars = Bars(df, self.SOURCE, asset, raw=df, quote=quote)
17 changes: 9 additions & 8 deletions lumibot/strategies/_strategy.py
Original file line number Diff line number Diff line change
@@ -21,6 +21,7 @@
from sqlalchemy import create_engine, inspect, text

import pandas as pd
from lumibot import LUMIBOT_DEFAULT_PYTZ
from ..backtesting import BacktestingBroker, PolygonDataBacktesting, ThetaDataBacktesting
from ..entities import Asset, Position, Order
from ..tools import (
@@ -1791,7 +1792,7 @@ def send_account_summary_to_discord(self):
cash = self.get_cash()

# # Get the datetime
now = pd.Timestamp(datetime.datetime.now()).tz_localize("America/New_York")
now = pd.Timestamp(datetime.datetime.now()).tz_localize(LUMIBOT_DEFAULT_PYTZ)

# Get the returns
returns_text, stats_df = self.calculate_returns()
@@ -1820,7 +1821,7 @@ def get_stats_from_database(self, stats_table_name, retries=5, delay=5):
self.logger.info(f"Table {stats_table_name} does not exist. Creating it now.")

# Get the current time in New York
ny_tz = pytz.timezone("America/New_York")
ny_tz = LUMIBOT_DEFAULT_PYTZ
now = datetime.datetime.now(ny_tz)

# Create an empty stats dataframe
@@ -1884,7 +1885,7 @@ def backup_variables_to_db(self):
self.db_engine = create_engine(self.db_connection_str)

# Get the current time in New York
ny_tz = pytz.timezone("America/New_York")
ny_tz = LUMIBOT_DEFAULT_PYTZ
now = datetime.datetime.now(ny_tz)

if not inspect(self.db_engine).has_table(self.backup_table_name):
@@ -2008,7 +2009,7 @@ def calculate_returns(self):
# Calculate the return over the past 24 hours, 7 days, and 30 days using the stats dataframe

# Get the current time in New York
ny_tz = pytz.timezone("America/New_York")
ny_tz = LUMIBOT_DEFAULT_PYTZ

# Get the datetime
now = datetime.datetime.now(ny_tz)
@@ -2025,11 +2026,11 @@ def calculate_returns(self):
# Check if the datetime column is timezone-aware
if stats_df['datetime'].dt.tz is None:
# If the datetime is timezone-naive, directly localize it to "America/New_York"
stats_df["datetime"] = stats_df["datetime"].dt.tz_localize("America/New_York", ambiguous='infer')
stats_df["datetime"] = stats_df["datetime"].dt.tz_localize(LUMIBOT_DEFAULT_PYTZ, ambiguous='infer')
else:
# If the datetime is already timezone-aware, first remove timezone and then localize
stats_df["datetime"] = stats_df["datetime"].dt.tz_localize(None)
stats_df["datetime"] = stats_df["datetime"].dt.tz_localize("America/New_York", ambiguous='infer')
stats_df["datetime"] = stats_df["datetime"].dt.tz_localize(LUMIBOT_DEFAULT_PYTZ, ambiguous='infer')

# Get the stats
stats_new = pd.DataFrame(
@@ -2049,7 +2050,7 @@ def calculate_returns(self):
stats_df = pd.concat([stats_df, stats_new])

# # Convert the datetime column to eastern time
stats_df["datetime"] = stats_df["datetime"].dt.tz_convert("America/New_York")
stats_df["datetime"] = stats_df["datetime"].dt.tz_convert(LUMIBOT_DEFAULT_PYTZ)

# Remove any duplicate rows
stats_df = stats_df[~stats_df["datetime"].duplicated(keep="last")]
@@ -2160,4 +2161,4 @@ def calculate_returns(self):
return results_text, stats_df

else:
return "Not enough data to calculate returns", stats_df
return "Not enough data to calculate returns", stats_df
4 changes: 2 additions & 2 deletions lumibot/tools/thetadata_helper.py
Original file line number Diff line number Diff line change
@@ -8,7 +8,7 @@
import pandas as pd
import pandas_market_calendars as mcal
import requests
from lumibot import LUMIBOT_CACHE_FOLDER
from lumibot import LUMIBOT_CACHE_FOLDER, LUMIBOT_DEFAULT_PYTZ
from lumibot.entities import Asset
from thetadata import ThetaClient
from tqdm import tqdm
@@ -295,7 +295,7 @@ def update_df(df_all, result):
],
}
"""
ny_tz = pytz.timezone('America/New_York')
ny_tz = LUMIBOT_DEFAULT_PYTZ
df = pd.DataFrame(result)
if not df.empty:
if "datetime" not in df.index.names:

0 comments on commit b8da35a

Please sign in to comment.