diff --git a/lumibot/components/drift_rebalancer_logic.py b/lumibot/components/drift_rebalancer_logic.py index fdeec0b68..2d54719d9 100644 --- a/lumibot/components/drift_rebalancer_logic.py +++ b/lumibot/components/drift_rebalancer_logic.py @@ -132,16 +132,6 @@ def __init__( def calculate(self, target_weights: Dict[str, Decimal]) -> pd.DataFrame: - if self.drift_type == DriftType.ABSOLUTE: - # The absolute value of all the weights are less than the drift_threshold - # then we will never trigger a rebalance. - - if all([abs(weight) < self.drift_threshold for weight in target_weights.values()]): - self.strategy.logger.warning( - f"All target weights are less than the drift_threshold: {self.drift_threshold}. " - f"No rebalance will be triggered." - ) - self.df = pd.DataFrame({ "symbol": target_weights.keys(), "is_quote_asset": False, @@ -222,19 +212,23 @@ def _calculate_drift_row(self, row: pd.Series) -> Decimal: return Decimal(0) elif row["current_weight"] == Decimal(0) and row["target_weight"] == Decimal(0): - # Should nothing change? + # Do nothing return Decimal(0) elif row["current_quantity"] > Decimal(0) and row["target_weight"] == Decimal(0): - # Should we sell everything + # Sell everything return Decimal(-1) + elif row["current_quantity"] < Decimal(0) and row["target_weight"] == Decimal(0): + # Cover our short position + return Decimal(1) + elif row["current_quantity"] == Decimal(0) and row["target_weight"] > Decimal(0): # We don't have any of this asset, but we want to buy some. return Decimal(1) elif row["current_quantity"] == Decimal(0) and row["target_weight"] == Decimal(-1): - # Should we short everything we have + # Short everything we have return Decimal(-1) elif row["current_quantity"] == Decimal(0) and row["target_weight"] < Decimal(0): @@ -352,7 +346,17 @@ def _rebalance(self, df: pd.DataFrame = None) -> None: # Execute buys for index, row in df.iterrows(): - if row["drift"] > 0: + if row["drift"] == 1 and row['current_quantity'] < 0 and self.shorting: + # Cover our short position + symbol = row["symbol"] + quantity = abs(row["current_quantity"]) + last_price = Decimal(self.strategy.get_last_price(symbol)) + limit_price = self.calculate_limit_price(last_price=last_price, side="buy") + order = self.place_order(symbol=symbol, quantity=quantity, limit_price=limit_price, side="buy") + buy_orders.append(order) + cash_position -= quantity * limit_price + + elif row["drift"] > 0: symbol = row["symbol"] last_price = Decimal(self.strategy.get_last_price(symbol)) limit_price = self.calculate_limit_price(last_price=last_price, side="buy") diff --git a/lumibot/data_sources/alpaca_data.py b/lumibot/data_sources/alpaca_data.py index 5ff0d8551..953815254 100644 --- a/lumibot/data_sources/alpaca_data.py +++ b/lumibot/data_sources/alpaca_data.py @@ -243,10 +243,9 @@ def get_barset_from_api(self, asset, freq, limit=None, end=None, start=None, quo loop_limit = limit elif str(freq) == "1Day": - loop_limit = limit * 1.5 # number almost perfect for normal weeks where only weekends are off - - # Add 3 days to the start date to make sure we get enough data on extra long weekends (like Thanksgiving) - loop_limit += 3 + weeks_requested = limit // 5 # Full trading week is 5 days + extra_padding_days = weeks_requested * 3 # to account for 3day weekends + loop_limit = max(5, limit + extra_padding_days) # Get at least 5 days df = [] # to use len(df) below without an error diff --git a/lumibot/data_sources/pandas_data.py b/lumibot/data_sources/pandas_data.py index 1c83e5568..e3c544c79 100644 --- a/lumibot/data_sources/pandas_data.py +++ b/lumibot/data_sources/pandas_data.py @@ -412,8 +412,12 @@ def get_start_datetime_and_ts_unit(self, length, timestep, start_dt=None, start_ # Convert timestep string to timedelta and get start datetime td, ts_unit = self.convert_timestep_str_to_timedelta(timestep) - # Multiply td by length to get the end datetime - td *= length + if ts_unit == "day": + weeks_requested = length // 5 # Full trading week is 5 days + extra_padding_days = weeks_requested * 3 # to account for 3day weekends + td = timedelta(days=length + extra_padding_days) + else: + td *= length if start_dt is not None: start_datetime = start_dt - td diff --git a/lumibot/data_sources/yahoo_data.py b/lumibot/data_sources/yahoo_data.py index 2d079112e..1c9907495 100644 --- a/lumibot/data_sources/yahoo_data.py +++ b/lumibot/data_sources/yahoo_data.py @@ -92,6 +92,9 @@ def _pull_source_symbol_bars( end = self._datetime.replace(second=59, microsecond=999999) if timeshift: + # Ensure timeshift is a timedelta object + if isinstance(timeshift, int): + timeshift = timedelta(days=timeshift) end = end - timeshift end = self.to_default_timezone(end) diff --git a/lumibot/tools/polygon_helper.py b/lumibot/tools/polygon_helper.py index 469203f40..e27f95a42 100644 --- a/lumibot/tools/polygon_helper.py +++ b/lumibot/tools/polygon_helper.py @@ -411,19 +411,16 @@ def get_missing_dates(df_all, asset, start, end): dates = pd.Series(df_all.index.date).unique() missing_dates = sorted(set(trading_dates) - set(dates)) - # TODO: This code works AFAIK, But when i enable it the tests for "test_polygon_missing_day_caching" and - # i don't know why nor how to fix this code or the tests. So im leaving it disabled for now. If you have problems - # with NANs in cached polygon data, you can try to enable this code and fix the tests. - - # # Find any dates with nan values in the df_all DataFrame - # missing_dates += df_all[df_all.isnull().all(axis=1)].index.date.tolist() - # - # # make sure the dates are unique - # missing_dates = list(set(missing_dates)) - # missing_dates.sort() - # - # # finally, filter out any dates that are not in start/end range (inclusive) - # missing_dates = [d for d in missing_dates if start.date() <= d <= end.date()] + # Find any dates with nan values in the df_all DataFrame. This happens for some infrequently traded assets, but + # it is difficult to know if the data is actually missing or if it is just infrequent trading, query for it again. + missing_dates += df_all[df_all.isnull().all(axis=1)].index.date.tolist() + + # make sure the dates are unique + missing_dates = list(set(missing_dates)) + missing_dates.sort() + + # finally, filter out any dates that are not in start/end range (inclusive) + missing_dates = [d for d in missing_dates if start.date() <= d <= end.date()] return missing_dates diff --git a/tests/backtest/test_example_strategies.py b/tests/backtest/test_example_strategies.py index 930a640d7..749390659 100644 --- a/tests/backtest/test_example_strategies.py +++ b/tests/backtest/test_example_strategies.py @@ -17,9 +17,7 @@ # Global parameters # API Key for testing Polygon.io -POLYGON_API_KEY = os.environ.get("POLYGON_API_KEY") -POLYGON_IS_PAID_SUBSCRIPTION = os.getenv("POLYGON_IS_PAID_SUBSCRIPTION", "true").lower() not in {'false', '0', 'f', 'n', 'no'} - +from lumibot.credentials import POLYGON_CONFIG class TestExampleStrategies: def test_stock_bracket(self): @@ -208,7 +206,14 @@ def test_limit_and_trailing_stops(self): assert round(results["total_return"] * 100, 1) >= 0.7 assert round(results["max_drawdown"]["drawdown"] * 100, 1) <= 0.2 - @pytest.mark.skipif(POLYGON_API_KEY == '', reason="This test requires a Polygon.io API key") + @pytest.mark.skipif( + not POLYGON_CONFIG["API_KEY"], + reason="This test requires a Polygon.io API key" + ) + @pytest.mark.skipif( + POLYGON_CONFIG['API_KEY'] == '', + reason="This test requires a Polygon.io API key" + ) def test_options_hold_to_expiry(self): """ Test the example strategy OptionsHoldToExpiry by running a backtest and checking that the strategy object is @@ -227,7 +232,7 @@ def test_options_hold_to_expiry(self): show_plot=False, show_tearsheet=False, save_tearsheet=False, - polygon_api_key=POLYGON_API_KEY, + polygon_api_key=POLYGON_CONFIG["API_KEY"], ) trades_df = strat_obj.broker._trade_event_log_df diff --git a/tests/backtest/test_polygon.py b/tests/backtest/test_polygon.py index 3feb59e6b..7690aa370 100644 --- a/tests/backtest/test_polygon.py +++ b/tests/backtest/test_polygon.py @@ -20,8 +20,7 @@ from datetime import timedelta # Global parameters -# API Key for testing Polygon.io -from lumibot.credentials import POLYGON_API_KEY +from lumibot.credentials import POLYGON_CONFIG class PolygonBacktestStrat(Strategy): @@ -204,7 +203,18 @@ def verify_backtest_results(self, poly_strat_obj): ) assert "fill" not in poly_strat_obj.order_time_tracker[stoploss_order_id] - @pytest.mark.skipif(POLYGON_API_KEY == '', reason="This test requires a Polygon.io API key") + @pytest.mark.skipif( + not POLYGON_CONFIG["API_KEY"], + reason="This test requires a Polygon.io API key" + ) + @pytest.mark.skipif( + POLYGON_CONFIG['API_KEY'] == '', + reason="This test requires a Polygon.io API key" + ) + @pytest.mark.skipif( + not POLYGON_CONFIG["IS_PAID_SUBSCRIPTION"], + reason="This test requires a paid Polygon.io API key" + ) def test_polygon_restclient(self): """ Test Polygon REST Client with Lumibot Backtesting and real API calls to Polygon. Using the Amazon stock @@ -219,7 +229,7 @@ def test_polygon_restclient(self): data_source = PolygonDataBacktesting( datetime_start=backtesting_start, datetime_end=backtesting_end, - api_key=POLYGON_API_KEY, + api_key=POLYGON_CONFIG['API_KEY'], ) broker = BacktestingBroker(data_source=data_source) poly_strat_obj = PolygonBacktestStrat( @@ -232,7 +242,18 @@ def test_polygon_restclient(self): assert results self.verify_backtest_results(poly_strat_obj) - @pytest.mark.skipif(POLYGON_API_KEY == '', reason="This test requires a Polygon.io API key") + @pytest.mark.skipif( + not POLYGON_CONFIG["API_KEY"], + reason="This test requires a Polygon.io API key" + ) + @pytest.mark.skipif( + POLYGON_CONFIG['API_KEY'] == '', + reason="This test requires a Polygon.io API key" + ) + @pytest.mark.skipif( + not POLYGON_CONFIG["IS_PAID_SUBSCRIPTION"], + reason="This test requires a paid Polygon.io API key" + ) def test_intraday_daterange(self): tzinfo = pytz.timezone("America/New_York") backtesting_start = datetime.datetime(2024, 2, 7).astimezone(tzinfo) @@ -241,7 +262,7 @@ def test_intraday_daterange(self): data_source = PolygonDataBacktesting( datetime_start=backtesting_start, datetime_end=backtesting_end, - api_key=POLYGON_API_KEY, + api_key=POLYGON_CONFIG['API_KEY'], ) broker = BacktestingBroker(data_source=data_source) poly_strat_obj = PolygonBacktestStrat( @@ -256,7 +277,18 @@ def test_intraday_daterange(self): # Assert the end datetime is before the market open of the next trading day. assert broker.datetime == datetime.datetime.fromisoformat("2024-02-12 08:30:00-05:00") - @pytest.mark.skipif(POLYGON_API_KEY == '', reason="This test requires a Polygon.io API key") + @pytest.mark.skipif( + not POLYGON_CONFIG["API_KEY"], + reason="This test requires a Polygon.io API key" + ) + @pytest.mark.skipif( + POLYGON_CONFIG['API_KEY'] == '', + reason="This test requires a Polygon.io API key" + ) + @pytest.mark.skipif( + not POLYGON_CONFIG["IS_PAID_SUBSCRIPTION"], + reason="This test requires a paid Polygon.io API key" + ) def test_polygon_legacy_backtest(self): """ Do the same backtest as test_polygon_restclient() but using the legacy backtest() function call instead of @@ -283,7 +315,18 @@ def test_polygon_legacy_backtest(self): assert results self.verify_backtest_results(poly_strat_obj) - @pytest.mark.skipif(POLYGON_API_KEY == '', reason="This test requires a Polygon.io API key") + @pytest.mark.skipif( + not POLYGON_CONFIG["API_KEY"], + reason="This test requires a Polygon.io API key" + ) + @pytest.mark.skipif( + POLYGON_CONFIG['API_KEY'] == '', + reason="This test requires a Polygon.io API key" + ) + @pytest.mark.skipif( + not POLYGON_CONFIG["IS_PAID_SUBSCRIPTION"], + reason="This test requires a paid Polygon.io API key" + ) def test_polygon_legacy_backtest2(self): """Test that the legacy backtest() function call works without returning the startegy object""" # Parameters: True = Live Trading | False = Backtest @@ -300,7 +343,7 @@ def test_polygon_legacy_backtest2(self): show_plot=False, show_tearsheet=False, save_tearsheet=False, - polygon_api_key=POLYGON_API_KEY, # Testing the legacy parameter name while DeprecationWarning is active + polygon_api_key=POLYGON_CONFIG['API_KEY'], # Testing the legacy parameter name while DeprecationWarning is active ) assert results @@ -336,8 +379,9 @@ def test_pull_source_symbol_bars_with_api_call(self, polygon_data_backtesting, m mocked_get_price_data.assert_called_once() call_args = mocked_get_price_data.call_args + extra_padding_days = (length // 5) * 3 expected_start_date = polygon_data_backtesting.datetime_start - \ - datetime.timedelta(days=length) - START_BUFFER + datetime.timedelta(days=length + extra_padding_days) - START_BUFFER assert call_args[0][0] == polygon_data_backtesting._api_key assert call_args[0][1] == asset @@ -349,14 +393,25 @@ def test_pull_source_symbol_bars_with_api_call(self, polygon_data_backtesting, m class TestPolygonDataSource: - @pytest.mark.skipif(POLYGON_API_KEY == '', reason="This test requires a Polygon.io API key") + @pytest.mark.skipif( + not POLYGON_CONFIG["API_KEY"], + reason="This test requires a Polygon.io API key" + ) + @pytest.mark.skipif( + POLYGON_CONFIG['API_KEY'] == '', + reason="This test requires a Polygon.io API key" + ) + @pytest.mark.skipif( + not POLYGON_CONFIG["IS_PAID_SUBSCRIPTION"], + reason="This test requires a paid Polygon.io API key" + ) def test_get_historical_prices(self): tzinfo = pytz.timezone("America/New_York") start = datetime.datetime(2024, 2, 5).astimezone(tzinfo) end = datetime.datetime(2024, 2, 10).astimezone(tzinfo) data_source = PolygonDataBacktesting( - start, end, api_key=POLYGON_API_KEY + start, end, api_key=POLYGON_CONFIG['API_KEY'] ) data_source._datetime = datetime.datetime(2024, 2, 7, 10).astimezone(tzinfo) # This call will set make the data source use minute bars. diff --git a/tests/test_drift_rebalancer.py b/tests/test_drift_rebalancer.py index f1483126c..4f806c6d9 100644 --- a/tests/test_drift_rebalancer.py +++ b/tests/test_drift_rebalancer.py @@ -880,7 +880,7 @@ def test_selling_part_of_a_holding_with_market_order(self): assert strategy.orders[0].quantity == Decimal("5") assert strategy.orders[0].type == Order.OrderType.MARKET - def test_selling_short_doesnt_create_and_order_when_shorting_is_disabled(self): + def test_selling_short_doesnt_create_an_order_when_shorting_is_disabled(self): strategy = MockStrategyWithOrderLogic( broker=self.backtesting_broker, order_type=Order.OrderType.LIMIT @@ -898,7 +898,7 @@ def test_selling_short_doesnt_create_and_order_when_shorting_is_disabled(self): strategy.order_logic.rebalance(drift_df=df) assert len(strategy.orders) == 0 - def test_selling_small_short_position_creates_and_order_when_shorting_is_enabled(self): + def test_selling_small_short_position_creates_an_order_when_shorting_is_enabled(self): strategy = MockStrategyWithOrderLogic( broker=self.backtesting_broker, order_type=Order.OrderType.LIMIT, @@ -971,6 +971,76 @@ def test_selling_a_100_percent_short_position_creates_an_order_when_shorting_is_ assert strategy.orders[0].side == "sell" assert strategy.orders[0].type == Order.OrderType.LIMIT + def test_covering_a_100_percent_short_position_creates_the_right_quantity(self): + strategy = MockStrategyWithOrderLogic( + broker=self.backtesting_broker, + order_type=Order.OrderType.LIMIT, + shorting=True + ) + df = pd.DataFrame([ + { + "symbol": "AAPL", + "is_quote_asset": False, + "current_quantity": Decimal("-10"), + "current_value": Decimal("-1000"), + "current_weight": Decimal("-1.0"), + "target_weight": Decimal("0"), + "target_value": Decimal("0.0"), + "drift": Decimal("1") + }, + { + "symbol": "USD", + "is_quote_asset": True, + "current_quantity": Decimal("2000"), + "current_value": Decimal("2000"), + "current_weight": Decimal("2.0"), + "target_weight": Decimal("0.0"), + "target_value": Decimal("0"), + "drift": Decimal("0") + } + ]) + + strategy.order_logic.rebalance(drift_df=df) + assert len(strategy.orders) == 1 + assert strategy.orders[0].quantity == Decimal("10") + assert strategy.orders[0].side == "buy" + assert strategy.orders[0].type == Order.OrderType.LIMIT + + def test_covering_a_short_position_creates_the_right_quantity(self): + strategy = MockStrategyWithOrderLogic( + broker=self.backtesting_broker, + order_type=Order.OrderType.LIMIT, + shorting=True + ) + df = pd.DataFrame([ + { + "symbol": "AAPL", + "is_quote_asset": False, + "current_quantity": Decimal("-5"), + "current_value": Decimal("-500"), + "current_weight": Decimal("-0.5"), + "target_weight": Decimal("0"), + "target_value": Decimal("0.0"), + "drift": Decimal("1") + }, + { + "symbol": "USD", + "is_quote_asset": True, + "current_quantity": Decimal("1500"), + "current_value": Decimal("1500"), + "current_weight": Decimal("1.5"), + "target_weight": Decimal("0.0"), + "target_value": Decimal("0"), + "drift": Decimal("0") + } + ]) + + strategy.order_logic.rebalance(drift_df=df) + assert len(strategy.orders) == 1 + assert strategy.orders[0].quantity == Decimal("5") + assert strategy.orders[0].side == "buy" + assert strategy.orders[0].type == Order.OrderType.LIMIT + def test_buying_something_when_we_have_enough_money_and_there_is_slippage(self): strategy = MockStrategyWithOrderLogic( broker=self.backtesting_broker, diff --git a/tests/test_get_historical_prices.py b/tests/test_get_historical_prices.py index 4a19037c0..39b777eb9 100644 --- a/tests/test_get_historical_prices.py +++ b/tests/test_get_historical_prices.py @@ -16,9 +16,7 @@ from lumibot.tools import get_trading_days # Global parameters -# API Key for testing Polygon.io -from lumibot.credentials import POLYGON_API_KEY -from lumibot.credentials import TRADIER_CONFIG, ALPACA_CONFIG +from lumibot.credentials import TRADIER_CONFIG, ALPACA_CONFIG, POLYGON_CONFIG logger = logging.getLogger(__name__) @@ -59,22 +57,79 @@ class TestDatasourceBacktestingGetHistoricalPricesDailyData: @classmethod def setup_class(cls): pass - + + # noinspection PyMethodMayBeStatic + def get_mlk_day(self, year): + # Start from January 1st of the given year + mlk_date = datetime(year, 1, 1) + # Find the first Monday of January + while mlk_date.weekday() != 0: # 0 = Monday + mlk_date += timedelta(days=1) + # Add 14 days to get to the third Monday + mlk_date += timedelta(days=14) + return mlk_date + + # noinspection PyMethodMayBeStatic + def get_first_trading_day_after_long_weekend(self, year): + # Martin Luther King Jr. Day is observed on the third Monday of January each year. + mlk_date = self.get_mlk_day(year) + first_trading_day = mlk_date + timedelta(days=1) + return first_trading_day + # noinspection PyMethodMayBeStatic def check_date_of_last_bar_is_date_of_last_trading_date_before_backtest_start( self, bars: Bars, backtesting_start: datetime ): # The current behavior of the backtesting data sources is to return the data for the - # last trading day before now. In this case, "now" is the backtesting_start date. - # So based on the backtesting_start date, the last bar should be the bar from the previous trading day. - previous_trading_day_date = get_trading_days( + # last trading day before now. + # To simulate this, we set backtesting_start date to what we want "now" to be. + # So based on the backtesting_start date, the last bar should be the bar from the previous trading day + # before the backtesting_start date. + + # Get trading days around the backtesting_start date + trading_days = get_trading_days( market="NYSE", start_date=backtesting_start - timedelta(days=5), - end_date=backtesting_start - timedelta(days=1) - ).index[-1].date() + end_date=backtesting_start + timedelta(days=5) + ) + + # find the index of the backtesting_start date in the trading_days + backtesting_start_index = trading_days.index.get_loc(backtesting_start) + + # get the date of the last trading day before the backtesting_start date + previous_trading_day_date = trading_days.index[backtesting_start_index - 1].date() assert bars.df.index[-1].date() == previous_trading_day_date + # noinspection PyMethodMayBeStatic + def check_date_of_last_bar_is_date_of_first_trading_date_on_or_after_backtest_start( + self, + bars: Bars, + backtesting_start: datetime + ): + # The backtesting broker needs to look into the future to fill orders. + # To simulate this, we set backtesting_start date to what we want "now" to be. + # So the first bar should be the backtesting_start date, or if the + # backtesting_start date is not a trading day, the first trading day after the backtesting_start date. + + # Get trading days around the backtesting_start date + trading_days = get_trading_days( + market="NYSE", + start_date=backtesting_start - timedelta(days=5), + end_date=backtesting_start + timedelta(days=5) + ) + + # Check if backtesting_start is in trading_days + if backtesting_start in trading_days.index: + backtesting_start_index = trading_days.index.get_loc(backtesting_start) + else: + # Find the first trading date after backtesting_start + backtesting_start_index = trading_days.index.get_indexer([backtesting_start], method='bfill')[0] + + # get the date of the first trading day on or after the backtesting_start date + first_trading_day_date = trading_days.index[backtesting_start_index].date() + assert bars.df.index[0].date() == first_trading_day_date + # noinspection PyMethodMayBeStatic def check_dividends_and_adjusted_returns(self, bars): assert "dividend" in bars.df.columns @@ -117,7 +172,14 @@ def check_dividends_and_adjusted_returns(self, bars): rtol=0 ) - def test_pandas_backtesting_data_source_get_historical_prices_daily_bars(self, pandas_data_fixture): + def test_get_first_trading_day_after_long_weekend(self): + first_trading_day_after_mlk = self.get_first_trading_day_after_long_weekend(2019) + assert first_trading_day_after_mlk == datetime(2019, 1, 22) + + first_trading_day_after_mlk = self.get_first_trading_day_after_long_weekend(2023) + assert first_trading_day_after_mlk == datetime(2023, 1, 17) + + def test_pandas_backtesting_data_source_get_historical_prices_daily_bars_dividends_and_adj_returns(self, pandas_data_fixture): """ This tests that the pandas data_source calculates adjusted returns for bars and that they are calculated correctly. It assumes that it is provided split adjusted OHLCV and dividend data. @@ -131,22 +193,144 @@ def test_pandas_backtesting_data_source_get_historical_prices_daily_bars(self, p ) bars = data_source.get_historical_prices(asset=self.asset, length=self.length, timestep=self.timestep) check_bars(bars=bars, length=self.length) - self.check_date_of_last_bar_is_date_of_last_trading_date_before_backtest_start(bars, backtesting_start=backtesting_start) + self.check_date_of_last_bar_is_date_of_last_trading_date_before_backtest_start( + bars, + backtesting_start=backtesting_start + ) self.check_dividends_and_adjusted_returns(bars) - @pytest.mark.skip(reason="This test exposes a possible bug in data.py that we have not investigated yet.") - @pytest.mark.skipif(POLYGON_API_KEY == '', reason="This test requires a Polygon.io API key") - def test_polygon_backtesting_data_source_get_historical_prices_daily_bars(self): - backtesting_end = datetime.now() - timedelta(days=1) - backtesting_start = backtesting_end - timedelta(days=self.length * 2 + 5) + def test_pandas_backtesting_data_source_get_historical_prices_daily_bars_for_backtesting_broker( + self, + pandas_data_fixture + ): + # Test getting 2 bars into the future (which is what the backtesting does when trying to fill orders + # for the next trading day) + backtesting_start = datetime(2019, 3, 26) + backtesting_end = datetime(2019, 4, 25) + length = 2 + data_source = PandasData( + datetime_start=backtesting_start, + datetime_end=backtesting_end, + pandas_data=pandas_data_fixture + ) + timeshift = -length # negative length gets future bars + bars = data_source.get_historical_prices( + asset=self.asset, + length=length, + timeshift=timeshift, + timestep=self.timestep + ) + check_bars(bars=bars, length=length) + self.check_date_of_last_bar_is_date_of_first_trading_date_on_or_after_backtest_start( + bars, + backtesting_start=backtesting_start + ) + + def test_pandas_backtesting_data_source_get_historical_prices_daily_bars_over_long_weekend( + self, + pandas_data_fixture + ): + # Get MLK day in 2019 + mlk_day = self.get_mlk_day(2019) + + # First trading day after MLK day + backtesting_start = mlk_day + timedelta(days=1) + backtesting_end = datetime(2019, 2, 22) + + # get 10 bars starting from backtesting_start (going back in time) + length = 10 + data_source = PandasData( + datetime_start=backtesting_start, + datetime_end=backtesting_end, + pandas_data=pandas_data_fixture + ) + bars = data_source.get_historical_prices(asset=self.asset, length=length, timestep=self.timestep) + check_bars(bars=bars, length=length) + self.check_date_of_last_bar_is_date_of_last_trading_date_before_backtest_start( + bars, + backtesting_start=backtesting_start + ) + + @pytest.mark.skipif( + not POLYGON_CONFIG["API_KEY"], + reason="This test requires a Polygon.io API key" + ) + @pytest.mark.skipif( + POLYGON_CONFIG['API_KEY'] == '', + reason="This test requires a Polygon.io API key" + ) + @pytest.mark.skipif( + not POLYGON_CONFIG["IS_PAID_SUBSCRIPTION"], + reason="This test requires a paid Polygon.io API key" + ) + def test_polygon_backtesting_data_source_get_historical_prices_daily_bars_for_backtesting_broker(self): + # Test getting 2 bars into the future (which is what the backtesting does when trying to fill orders + # for the next trading day) + last_year = datetime.now().year - 1 + + # Get MLK day last year which is a non-trading monday + mlk_day = self.get_mlk_day(last_year) + + # First trading day after MLK day + backtesting_start = mlk_day + timedelta(days=1) + backtesting_end = datetime(last_year, 2, 22) + data_source = PolygonDataBacktesting( - backtesting_start, backtesting_end, api_key=POLYGON_API_KEY + backtesting_start, backtesting_end, api_key=POLYGON_CONFIG["API_KEY"] + ) + + length = 2 + timeshift = -length # negative length gets future bars + bars = data_source.get_historical_prices( + asset=self.asset, + length=length, + timeshift=timeshift, + timestep=self.timestep + ) + + check_bars(bars=bars, length=length) + self.check_date_of_last_bar_is_date_of_first_trading_date_on_or_after_backtest_start( + bars, + backtesting_start=backtesting_start + ) + + @pytest.mark.skipif( + not POLYGON_CONFIG["API_KEY"], + reason="This test requires a Polygon.io API key" + ) + @pytest.mark.skipif( + POLYGON_CONFIG['API_KEY'] == '', + reason="This test requires a Polygon.io API key" + ) + @pytest.mark.skipif( + not POLYGON_CONFIG["IS_PAID_SUBSCRIPTION"], + reason="This test requires a paid Polygon.io API key" + ) + def test_polygon_backtesting_data_source_get_historical_prices_daily_bars_over_long_weekend(self): + # Get MLK day for last year + last_year = datetime.now().year - 1 + mlk_day = self.get_mlk_day(last_year) + + # First trading day after MLK day + backtesting_start = mlk_day + timedelta(days=1) + backtesting_end = datetime(last_year, 2, 22) + + # get 10 bars starting from backtesting_start (going back in time) + length = 10 + data_source = PolygonDataBacktesting( + backtesting_start, backtesting_end, api_key=POLYGON_CONFIG["API_KEY"] ) bars = data_source.get_historical_prices(asset=self.asset, length=self.length, timestep=self.timestep) check_bars(bars=bars, length=self.length) - self.check_date_of_last_bar_is_date_of_last_trading_date_before_backtest_start(bars, backtesting_start=backtesting_start) + self.check_date_of_last_bar_is_date_of_last_trading_date_before_backtest_start( + bars, + backtesting_start=backtesting_start + ) - def test_yahoo_backtesting_data_source_get_historical_prices_daily_bars(self, pandas_data_fixture): + def test_yahoo_backtesting_data_source_get_historical_prices_daily_bars_dividends_and_adj_returns( + self, + pandas_data_fixture + ): """ This tests that the yahoo data_source calculates adjusted returns for bars and that they are calculated correctly. It assumes that it is provided split adjusted OHLCV and dividend data. @@ -161,10 +345,67 @@ def test_yahoo_backtesting_data_source_get_historical_prices_daily_bars(self, pa bars = data_source.get_historical_prices(asset=self.asset, length=self.length, timestep=self.timestep) check_bars(bars=bars, length=self.length) self.check_dividends_and_adjusted_returns(bars) - self.check_date_of_last_bar_is_date_of_last_trading_date_before_backtest_start(bars, backtesting_start=backtesting_start) + self.check_date_of_last_bar_is_date_of_last_trading_date_before_backtest_start( + bars, + backtesting_start=backtesting_start + ) + + def test_yahoo_backtesting_data_source_get_historical_prices_daily_bars_for_backtesting_broker( + self, + pandas_data_fixture + ): + # Test getting 2 bars into the future (which is what the backtesting does when trying to fill orders + # for the next trading day) + backtesting_start = datetime(2019, 3, 25) + backtesting_end = datetime(2019, 4, 25) + data_source = YahooDataBacktesting( + datetime_start=backtesting_start, + datetime_end=backtesting_end, + pandas_data=pandas_data_fixture + ) + length = 2 + timeshift = -length # negative length gets future bars + bars = data_source.get_historical_prices( + asset=self.asset, + length=length, + timeshift=timeshift, + timestep=self.timestep + ) -# @pytest.mark.skip() + check_bars(bars=bars, length=length) + self.check_date_of_last_bar_is_date_of_first_trading_date_on_or_after_backtest_start( + bars, + backtesting_start=backtesting_start + ) + + def test_yahoo_backtesting_data_source_get_historical_prices_daily_bars_over_long_weekend( + self, + pandas_data_fixture + ): + # Get MLK day in 2019 + mlk_day = self.get_mlk_day(2019) + + # First trading day after MLK day + backtesting_start = mlk_day + timedelta(days=1) + backtesting_end = datetime(2019, 2, 22) + + # get 10 bars starting from backtesting_start (going back in time) + length = 10 + data_source = YahooDataBacktesting( + datetime_start=backtesting_start, + datetime_end=backtesting_end, + pandas_data=pandas_data_fixture + ) + bars = data_source.get_historical_prices(asset=self.asset, length=self.length, timestep=self.timestep) + check_bars(bars=bars, length=self.length) + self.check_date_of_last_bar_is_date_of_last_trading_date_before_backtest_start( + bars, + backtesting_start=backtesting_start + ) + + +@pytest.mark.skip() class TestDatasourceGetHistoricalPricesDailyData: """These tests check the daily Bars returned from get_historical_prices for live data sources.""" @@ -202,8 +443,10 @@ def check_date_of_last_bar_is_correct_for_live_data_sources(self, bars): # if it's not a trading day, the last bar the bar should from the last trading day assert bars.df.index[-1].date() == self.trading_days.index[-1].date() - # @pytest.mark.skip() - @pytest.mark.skipif(not ALPACA_CONFIG['API_KEY'], reason="This test requires an alpaca API key") + @pytest.mark.skipif( + not ALPACA_CONFIG['API_KEY'], + reason="This test requires an alpaca API key" + ) @pytest.mark.skipif( ALPACA_CONFIG['API_KEY'] == '', reason="This test requires an alpaca API key" @@ -225,7 +468,6 @@ def test_alpaca_data_source_get_historical_prices_daily_bars(self): check_bars(bars=bars, length=1, check_timezone=False) self.check_date_of_last_bar_is_correct_for_live_data_sources(bars) - # @pytest.mark.skip() @pytest.mark.skipif(not TRADIER_CONFIG['ACCESS_TOKEN'], reason="No Tradier credentials provided.") def test_tradier_data_source_get_historical_prices_daily_bars(self): data_source = TradierData( diff --git a/tests/test_pandas_data.py b/tests/test_pandas_data.py index 1045f95d5..6da78e3ad 100644 --- a/tests/test_pandas_data.py +++ b/tests/test_pandas_data.py @@ -1,3 +1,7 @@ +from datetime import datetime, timedelta + +from lumibot.data_sources import PandasData + from tests.fixtures import pandas_data_fixture @@ -18,3 +22,19 @@ def test_spy_has_dividends(self, pandas_data_fixture): ] assert spy.df.columns.tolist() == expected_columns + def test_get_start_datetime_and_ts_unit(self): + start = datetime(2023, 3, 25) + end = datetime(2023, 4, 5) + data_source = PandasData(datetime_start=start, datetime_end=end, pandas_data={}) + length = 30 + timestep = '1day' + start_datetime, ts_unit = data_source.get_start_datetime_and_ts_unit( + length, + timestep, + start, + start_buffer=timedelta(days=0) # just test our math + ) + extra_padding_days = (length // 5) * 3 + expected_datetime = datetime(2023, 3, 25) - timedelta(days=length + extra_padding_days) + assert start_datetime == expected_datetime + diff --git a/tests/test_polygon_helper.py b/tests/test_polygon_helper.py index 8d1387f39..7af65bc1c 100644 --- a/tests/test_polygon_helper.py +++ b/tests/test_polygon_helper.py @@ -480,10 +480,5 @@ def test_polygon_missing_day_caching(self, mocker, tmpdir, timespan, force_cache assert mock_polyclient.create().get_aggs.call_count == 3 assert expected_cachefile.exists() assert len(df) == 7 - df = ph.get_price_data_from_polygon(api_key, asset, start_date, end_date, timespan, force_cache_update=force_cache_update) - assert len(df) == 7 - if force_cache_update: - assert mock_polyclient.create().get_aggs.call_count == 2 * 3 - else: - assert mock_polyclient.create().get_aggs.call_count == 3 + expected_cachefile.unlink()