Merge pull request #450 from Lumiwealth/dev

update master

.github/workflows/cicd.yaml

@@ -32,3 +32,6 @@ jobs:
           coverage run
           coverage report
           coverage html
+      - name: Python Coverage Comment
+          uses: py-cov-action/[email protected]
+

docs/_sources/index.rst.txt

@@ -97,7 +97,7 @@ Table of Contents
    :maxdepth: 3
 
    Home <self>
-   GitHUb <https://github.com/Lumiwealth/lumibot>
+   GitHub <https://github.com/Lumiwealth/lumibot>
    Community/Forum <https://discord.gg/v6asVjTCvh>
    getting_started
    lifecycle_methods

docsrc/_build/html/_sources/index.rst.txt

@@ -97,7 +97,7 @@ Table of Contents
    :maxdepth: 3
 
    Home <self>
-   GitHUb <https://github.com/Lumiwealth/lumibot>
+   GitHub <https://github.com/Lumiwealth/lumibot>
    Community/Forum <https://discord.gg/v6asVjTCvh>
    getting_started
    lifecycle_methods

docsrc/index.rst

@@ -97,7 +97,7 @@ Table of Contents
    :maxdepth: 3
 
    Home <self>
-   GitHUb <https://github.com/Lumiwealth/lumibot>
+   GitHub <https://github.com/Lumiwealth/lumibot>
    Community/Forum <https://discord.gg/v6asVjTCvh>
    getting_started
    lifecycle_methods

lumibot/backtesting/backtesting_broker.py

@@ -358,6 +358,14 @@ def _process_cash_settlement(self, order, price, quantity):
 
     def submit_order(self, order):
         """Submit an order for an asset"""
+        # NOTE: This code is to address Tradier API requirements, they want is as "to_open" or "to_close" instead of just "buy" or "sell"
+        # If the order has a "buy_to_open" or "buy_to_close" side, then we should change it to "buy"
+        if order.side in ["buy_to_open", "buy_to_close"]:
+            order.side = "buy"
+        # If the order has a "sell_to_open" or "sell_to_close" side, then we should change it to "sell"
+        if order.side in ["sell_to_open", "sell_to_close"]:
+            order.side = "sell"
+
         order.update_raw(order)
         self.stream.dispatch(
             self.NEW_ORDER,
@@ -396,11 +404,15 @@ def cash_settle_options_contract(self, position, strategy):
             logging.error(f"Cannot cash settle non-option contract {position.asset}")
             return
 
-        # Create a stock asset for the underlying asset
-        underlying_asset = Asset(
-            symbol=position.asset.symbol,
-            asset_type="stock",
-        )
+        # First check if the option asset has an underlying asset
+        if position.asset.underlying_asset is None:
+            # Create a stock asset for the underlying asset
+            underlying_asset = Asset(
+                symbol=position.asset.symbol,
+                asset_type="stock",
+            )
+        else:
+            underlying_asset = position.asset.underlying_asset
 
         # Get the price of the underlying asset
         underlying_price = self.get_last_price(underlying_asset)
@@ -560,11 +572,11 @@ def process_pending_orders(self, strategy):
                 )
 
                 dt = ohlc.df.index[-1]
-                open = ohlc.df.open[-1]
-                high = ohlc.df.high[-1]
-                low = ohlc.df.low[-1]
-                close = ohlc.df.close[-1]
-                volume = ohlc.df.volume[-1]
+                open = ohlc.df['open'].iloc[-1]
+                high = ohlc.df['high'].iloc[-1]
+                low = ohlc.df['low'].iloc[-1]
+                close = ohlc.df['close'].iloc[-1]
+                volume = ohlc.df['volume'].iloc[-1]
 
             # Get the OHLCV data for the asset if we're using the PANDAS data source
             elif self.data_source.SOURCE == "PANDAS":

lumibot/backtesting/polygon_backtesting.py

@@ -51,7 +51,7 @@ def _enforce_storage_limit(pandas_data: OrderedDict):
             storage_used -= mu
             logging.info(f"Storage limit exceeded. Evicted LRU data: {k} used {mu:,} bytes")
 
-    def _update_pandas_data(self, asset, quote, length, timestep, start_dt=None, update_data_store=False):
+    def _update_pandas_data(self, asset, quote, length, timestep, start_dt=None):
         """
         Get asset data and update the self.pandas_data dictionary.
 
@@ -67,10 +67,6 @@ def _update_pandas_data(self, asset, quote, length, timestep, start_dt=None, upd
             The timestep to use. For example, "1minute" or "1hour" or "1day".
         start_dt : datetime
             The start datetime to use. If None, the current self.start_datetime will be used.
-        update_data_store : bool
-            If True, the data will also be added to the self._data_store dictionary.
-            That update will not include the adjustments made by PandasData.load_data.
-            See https://github.com/Lumiwealth/lumibot/issues/391 and its PR for further discussion.
         """
         search_asset = asset
         asset_separated = asset
@@ -202,11 +198,6 @@ def _update_pandas_data(self, asset, quote, length, timestep, start_dt=None, upd
         self.pandas_data.update(pandas_data_update)
         if PolygonDataBacktesting.MAX_STORAGE_BYTES:
             self._enforce_storage_limit(self.pandas_data)
-        if update_data_store:
-            # TODO: Why do we have both self.pandas_data and self._data_store?
-            self._data_store.update(pandas_data_update)
-            if PolygonDataBacktesting.MAX_STORAGE_BYTES:
-                self._enforce_storage_limit(self._data_store)
 
     def _pull_source_symbol_bars(
         self,
@@ -220,10 +211,9 @@ def _pull_source_symbol_bars(
     ):
         # Get the current datetime and calculate the start datetime
         current_dt = self.get_datetime()
-        start_dt, ts_unit = self.get_start_datetime_and_ts_unit(length, timestep, current_dt, start_buffer=START_BUFFER)
 
         # Get data from Polygon
-        self._update_pandas_data(asset, quote, length, timestep, start_dt)
+        self._update_pandas_data(asset, quote, length, timestep, current_dt)
 
         return super()._pull_source_symbol_bars(
             asset, length, timestep, timeshift, quote, exchange, include_after_hours
@@ -255,7 +245,7 @@ def get_historical_prices_between_dates(
     def get_last_price(self, asset, timestep="minute", quote=None, exchange=None, **kwargs):
         try:
             dt = self.get_datetime()
-            self._update_pandas_data(asset, quote, 1, timestep, dt, update_data_store=True)
+            self._update_pandas_data(asset, quote, 1, timestep, dt)
         except Exception as e:
             print(f"Error get_last_price from Polygon: {e}")
             print(f"Error get_last_price from Polygon: {asset=} {quote=} {timestep=} {dt=} {e}")

lumibot/brokers/broker.py

@@ -1028,10 +1028,10 @@ def _process_trade_event(self, stored_order, type_event, price=None, filled_quan
             )
 
         if filled_quantity is not None:
-            error = ValueError(f"filled_quantity must be a positive integer, received {filled_quantity} instead")
+            error = ValueError(f"filled_quantity must be a positive integer or float, received {filled_quantity} instead")
             try:
-                if not isinstance(filled_quantity, Decimal):
-                    filled_quantity = Decimal(filled_quantity)
+                if not isinstance(filled_quantity, float):
+                    filled_quantity = float(filled_quantity)
                 if filled_quantity < 0:
                     raise error
             except ValueError:
@@ -1041,8 +1041,6 @@ def _process_trade_event(self, stored_order, type_event, price=None, filled_quan
             error = ValueError("price must be a positive float, received %r instead" % price)
             try:
                 price = float(price)
-                if price < 0:
-                    raise error
             except ValueError:
                 raise error
 

lumibot/brokers/tradier.py

@@ -368,6 +368,7 @@ def _parse_broker_order(self, response: dict, strategy_name: str, strategy_objec
             date_created=response["create_date"],
         )
         order.status = response["status"]
+        order.avg_fill_price = response.get("avg_fill_price", order.avg_fill_price)
         order.update_raw(response)  # This marks order as 'transmitted'
         return order
 
@@ -426,7 +427,7 @@ def _lumi_side2tradier(self, order: Order) -> str:
 
         # Stoploss and limit orders are always used to close positions, even if they are submitted "before" the
         # position is technically open (i.e. buy and stoploss order are submitted simultaneously)
-        if order.type in [Order.OrderType.STOP, Order.OrderType.LIMIT, Order.OrderType.TRAIL]:
+        if order.type in [Order.OrderType.STOP, Order.OrderType.TRAIL]:
             side = side.replace("to_open", "to_close")
 
         # Check if the side is a valid Tradier side

lumibot/data_sources/pandas_data.py

@@ -24,11 +24,10 @@ def __init__(self, *args, pandas_data=None, auto_adjust=True, **kwargs):
         self.name = "pandas"
         self.pandas_data = self._set_pandas_data_keys(pandas_data)
         self.auto_adjust = auto_adjust
-        self._data_store = OrderedDict()
+        self._data_store = self.pandas_data
         self._date_index = None
         self._date_supply = None
         self._timestep = "minute"
-        self._expiries_exist = False
 
     @staticmethod
     def _set_pandas_data_keys(pandas_data):
@@ -65,9 +64,6 @@ def _get_new_pandas_data_key(data):
 
     def load_data(self):
         self._data_store = self.pandas_data
-        self._expiries_exist = (
-            len([v.asset.expiration for v in self._data_store.values() if v.asset.expiration is not None]) > 0
-        )
         self._date_index = self.update_date_index()
 
         if len(self._data_store.values()) > 0:
@@ -217,7 +213,7 @@ def find_asset_in_data_store(self, asset, quote=None):
             asset = (asset, quote)
             if asset in self._data_store:
                 return asset
-        elif isinstance(asset, Asset) and asset.asset_type in ["option", "future", "stock"]:
+        elif isinstance(asset, Asset) and asset.asset_type in ["option", "future", "stock", "index"]:
             asset = (asset, Asset("USD", "forex"))
             if asset in self._data_store:
                 return asset

lumibot/data_sources/yahoo_data.py

@@ -13,7 +13,9 @@ class YahooData(DataSourceBacktesting):
     SOURCE = "YAHOO"
     MIN_TIMESTEP = "day"
     TIMESTEP_MAPPING = [
-        {"timestep": "day", "representations": ["1D", "day"]},
+        {"timestep": "day", "representations": ["1d", "day"]},
+        {"timestep": "15 minutes", "representations": ["15m", "15 minutes"]},
+        {"timestep": "minute", "representations": ["1m", "1 minute"]},
     ]
 
     def __init__(self, *args, auto_adjust=True, **kwargs):
@@ -64,12 +66,13 @@ def _pull_source_symbol_bars(
         if quote is not None:
             logging.warning(f"quote is not implemented for YahooData, but {quote} was passed as the quote")
 
-        self._parse_source_timestep(timestep, reverse=True)
+        interval = self._parse_source_timestep(timestep, reverse=True)
         if asset in self._data_store:
             data = self._data_store[asset]
         else:
             data = YahooHelper.get_symbol_data(
                 asset.symbol,
+                interval=interval,
                 auto_adjust=self.auto_adjust,
                 last_needed_datetime=self.datetime_end,
             )
@@ -79,11 +82,13 @@ def _pull_source_symbol_bars(
                 return None
             data = self._append_data(asset, data)
 
-        # Get the last minute of self._datetime to get the current bar
-        dt = self._datetime.replace(hour=23, minute=59, second=59, microsecond=999999)
+        if timestep == "day":
+            # Get the last minute of self._datetime to get the current bar
+            dt = self._datetime.replace(hour=23, minute=59, second=59, microsecond=999999)
+            end = dt - timedelta(days=1)
+        else:
+            end = self._datetime.replace(second=59, microsecond=999999)
 
-        # End should be yesterday because otherwise you can see the future
-        end = dt - timedelta(days=1)
         if timeshift:
             end = end - timeshift
 
@@ -101,11 +106,11 @@ def _pull_source_bars(
         if quote is not None:
             logging.warning(f"quote is not implemented for YahooData, but {quote} was passed as the quote")
 
-        self._parse_source_timestep(timestep, reverse=True)
+        interval = self._parse_source_timestep(timestep, reverse=True)
         missing_assets = [asset.symbol for asset in assets if asset not in self._data_store]
 
         if missing_assets:
-            dfs = YahooHelper.get_symbols_data(missing_assets, auto_adjust=self.auto_adjust)
+            dfs = YahooHelper.get_symbols_data(missing_assets, interval=interval, auto_adjust=self.auto_adjust)
             for symbol, df in dfs.items():
                 self._append_data(symbol, df)
 

lumibot/entities/asset.py

@@ -29,6 +29,8 @@ class Asset:
     multiplier : int
         Price multiplier.
         default : 1
+    underlying_asset : Asset
+        Underlying asset for options.
 
     Attributes
     ----------
@@ -118,6 +120,7 @@ class AssetType:
     right: str = None
     multiplier: int = 1
     precision: str = None
+    underlying_asset: "Asset" = None
 
     # Pull the asset types from the AssetType class
     _asset_types: list = [v for k, v in AssetType.__dict__.items() if not k.startswith("__")]
@@ -134,12 +137,18 @@ def __init__(
         right: str = None,
         multiplier: int = 1,
         precision: str = None,
+        underlying_asset: "Asset" = None,
     ):
         self.symbol = symbol
         self.asset_type = asset_type
         self.strike = strike
         self.multiplier = multiplier
         self.precision = precision
+        self.underlying_asset = underlying_asset
+
+        # If the underlying asset is set but the symbol is not, set the symbol to the underlying asset symbol
+        if self.underlying_asset is not None and self.symbol is None:
+            self.symbol = self.underlying_asset.symbol
 
         # If the expiration is a datetime object, convert it to date
         if isinstance(expiration, datetime):

lumibot/entities/data.py

@@ -496,6 +496,7 @@ def get_bars(self, dt, length=1, timestep=MIN_TIMESTEP, timeshift=0):
         """
         # Parse the timestep
         quantity, timestep = parse_timestep_qty_and_unit(timestep)
+        num_periods = length
 
         if timestep == "minute" and self.timestep == "day":
             raise ValueError("You are requesting minute data from a daily data source. This is not supported.")
@@ -530,6 +531,14 @@ def get_bars(self, dt, length=1, timestep=MIN_TIMESTEP, timeshift=0):
         # Drop any rows that have NaN values (this can happen if the data is not complete, eg. weekends)
         df_result = df_result.dropna()
 
+        # Remove partial day data from the current day, which can happen if the data is in minute timestep.
+        if timestep == "day":
+            df_result = df_result[df_result.index < dt.replace(hour=0, minute=0, second=0, microsecond=0)]
+
+        # The original df_result may include more rows when timestep is day and self.timestep is minute.
+        # In this case, we only want to return the last n rows.
+        df_result = df_result.tail(n=num_periods)
+
         return df_result
 
     def get_bars_between_dates(self, timestep=MIN_TIMESTEP, exchange=None, start_date=None, end_date=None):

lumibot/entities/order.py

@@ -307,9 +307,6 @@ def __init__(
 
         self.quantity = quantity
 
-        # setting the side
-        if side not in [BUY, SELL]:
-            raise ValueError("Side must be either sell or buy, got %r instead" % side)
         self.side = side
 
         self._set_type(
@@ -552,7 +549,7 @@ def quantity(self, value):
                 value = Decimal(str(value))
 
         quantity = Decimal(value)
-        self._quantity = check_quantity(quantity, "Order quantity must be a positive Decimal")
+        self._quantity = quantity
 
     def __hash__(self):
         return hash(self.identifier)

lumibot/example_strategies/options_hold_to_expiry.py

@@ -47,14 +47,19 @@ def on_trading_iteration(self):
                 right="call",
             )
 
-            # Bracket order
+            # Create order
             order = self.create_order(
                 asset,
                 10,
-                "buy",
+                "buy_to_open",
             )
+
+            # Submit order
             self.submit_order(order)
 
+            # Log a message
+            self.log_message(f"Bought {order.quantity} of {asset}")
+
 
 if __name__ == "__main__":
     is_live = False