Merge remote-tracking branch 'origin/dev' into send-data-to-cloud

lumibot/backtesting/backtesting_broker.py

@@ -98,7 +98,7 @@ def _update_datetime(self, update_dt, cash=None, portfolio_value=None):
         self.data_source._update_datetime(new_datetime, cash=cash, portfolio_value=portfolio_value)
         if self.option_source:
             self.option_source._update_datetime(new_datetime, cash=cash, portfolio_value=portfolio_value)
-        logger.debug(f"Current backtesting datetime {self.datetime}")
+        logger.info(f"Current backtesting datetime {self.datetime}")
 
     # =========Clock functions=====================
 

lumibot/brokers/interactive_brokers_rest.py

@@ -6,6 +6,7 @@
 import datetime
 from decimal import Decimal
 from math import gcd
+import re
 
 TYPE_MAP = dict(
     stock="STK",
@@ -408,17 +409,92 @@ def _pull_positions(self, strategy) -> list[Position]:
             if asset_class == Asset.AssetType.STOCK:
                 asset = Asset(symbol=symbol, asset_type=asset_class)
             elif asset_class == Asset.AssetType.OPTION:
-                expiry = position["expiry"]
-                strike = position["strike"]
-                right = position["putOrCall"]
-                # If asset class is option, create an option asset
-                asset = Asset(
-                    symbol=symbol,
-                    asset_type=asset_class,
-                    expiration=expiry,
-                    strike=strike,
-                    right=right,
-                )
+                # Example contract_desc: 'SPY    NOV2024 562 P [SPY   241105P00562000 100]'
+                # This example format includes:
+                #   - An underlying symbol at the beginning (e.g., "SPY")
+                #   - Expiry and strike in human-readable format (e.g., "NOV2024 562 P")
+                #   - Option details within square brackets (e.g., "[SPY   241105P00562000 100]"),
+                #     where "241105P00562000" holds the expiry (YYMMDD), option type (C/P), and strike price
+
+                contract_desc = position.get("contractDesc", "").strip()
+
+                if not contract_desc:
+                    logging.error("Empty contract description for option. Skipping this position.")
+                    continue  # Skip processing this position as contract_desc is missing
+
+                try:
+                    # Locate the square brackets and extract the option details part
+                    start_idx = contract_desc.find('[')
+                    end_idx = contract_desc.find(']', start_idx)
+
+                    if start_idx == -1 or end_idx == -1:
+                        logging.error(f"Brackets not found in contract description '{contract_desc}'. Expected format like '[SPY   241105P00562000 100]'.")
+                        continue  # Skip if brackets are missing
+
+                    # Extract content within brackets and find the critical pattern (e.g., "241105P00562000")
+                    bracket_content = contract_desc[start_idx + 1:end_idx].strip()
+                    # Search for 6 digits, followed by 'C' or 'P', followed by 8 digits for strike
+                    details_match = re.search(r'\d{6}[CP]\d{8}', bracket_content)
+
+                    if not details_match:
+                        logging.error(f"Expected option pattern not found in contract '{contract_desc}'.")
+                        continue  # Skip if pattern does not match
+
+                    contract_details = details_match.group(0)
+
+                    # Parse components from the details
+                    expiry_raw = contract_details[:6]      # First six digits (YYMMDD format)
+                    right_raw = contract_details[6]        # Seventh character (C or P)
+                    strike_raw = contract_details[7:]      # Remaining characters (strike price)
+
+                    # Check if expiry is in the correct format and convert to date
+                    try:
+                        expiry = datetime.datetime.strptime(expiry_raw, "%y%m%d").date()
+                    except ValueError as ve:
+                        logging.error(f"Invalid expiry format '{expiry_raw}' in contract '{contract_desc}': {ve}")
+                        continue  # Skip this position due to invalid expiry format
+
+                    # Convert strike to a float, assuming it’s in thousandths (e.g., "00562000" to "562.00")
+                    try:
+                        strike = round(float(strike_raw) / 1000, 2)
+                    except ValueError as ve:
+                        logging.error(f"Invalid strike price '{strike_raw}' in contract '{contract_desc}': {ve}")
+                        continue  # Skip this position due to invalid strike price
+
+                    # Validate the option type (right) as either C or P
+                    if right_raw.upper() not in ["C", "P"]:
+                        logging.error(f"Invalid option type '{right_raw}' in contract '{contract_desc}'. Expected 'C' or 'P'.")
+                        continue  # Skip if option type is not valid
+
+                    # Determine the option right type
+                    right = Asset.OptionRight.CALL if right_raw.upper() == "C" else Asset.OptionRight.PUT
+
+                    # Extract the underlying symbol, assumed to be the first word in contract_desc
+                    underlying_asset_raw = contract_desc.split()[0]
+
+                    # Ensure underlying symbol is alphanumeric and non-empty
+                    if not underlying_asset_raw.isalnum():
+                        logging.error(f"Invalid underlying asset symbol '{underlying_asset_raw}' in '{contract_desc}'.")
+                        continue
+
+                    # Create the underlying asset object
+                    underlying_asset = Asset(
+                        symbol=underlying_asset_raw,
+                        asset_type=Asset.AssetType.STOCK
+                    )
+
+                    # Create the option asset object
+                    asset = Asset(
+                        symbol=symbol,
+                        asset_type=asset_class,
+                        expiration=expiry,
+                        strike=strike,
+                        right=right,
+                        underlying_asset=underlying_asset,
+                    )
+
+                except Exception as e:
+                    logging.error(f"Error processing contract '{contract_desc}': {e}")
             elif asset_class == Asset.AssetType.FUTURE:
                 #contract_details = self.data_source.get_contract_details(position['conid'])
                 expiry = position["expiry"]

lumibot/entities/order.py

@@ -893,6 +893,10 @@ def to_dict(self):
             if isinstance(value, datetime.datetime):
                 order_dict[key] = value.isoformat()
 
+            # If it is a Decimal object, convert it to a float
+            elif isinstance(value, Decimal):
+                order_dict[key] = float(value)
+
             # Recursively handle objects that have their own to_dict method (like asset, quote, etc.)
             elif hasattr(value, "to_dict"):
                 order_dict[key] = value.to_dict()

lumibot/entities/position.py

@@ -168,11 +168,11 @@ def to_dict(self):
         return {
             "strategy": self.strategy,
             "asset": self.asset.to_dict(),
-            "quantity": self.quantity,
+            "quantity": float(self.quantity),
             "orders": [order.to_dict() for order in self.orders],
             "hold": self.hold,
-            "available": self.available,
-            "avg_fill_price": self.avg_fill_price,
+            "available": float(self.available) if self.available else None,
+            "avg_fill_price": float(self.avg_fill_price) if self.avg_fill_price else None,
         }
 
     @classmethod

lumibot/strategies/_strategy.py

@@ -884,7 +884,7 @@ def run_backtest(
         save_logfile=False,
         use_quote_data=False,
         show_progress_bar=True,
-        quiet_logs=True,
+        quiet_logs=False,
         trader_class=Trader,
         **kwargs,
     ):
@@ -1515,4 +1515,4 @@ def backtest(
             trader_class=trader_class,
             **kwargs,
         )
-        return results
+        return results

lumibot/strategies/strategy.py

@@ -9,6 +9,8 @@
 from typing import Union
 from sqlalchemy import create_engine, inspect, text, bindparam
 from sqlalchemy.exc import OperationalError
+import traceback
+import math
 
 import jsonpickle
 import matplotlib
@@ -4613,11 +4615,28 @@ def send_update_to_cloud(self):
             "orders": [order.to_dict() for order in orders],
         }
 
+        # Helper function to recursively replace NaN in dictionaries
+        def replace_nan(value):
+            if isinstance(value, float) and math.isnan(value):
+                return None  # or 0 if you prefer
+            elif isinstance(value, dict):
+                return {k: replace_nan(v) for k, v in value.items()}
+            elif isinstance(value, list):
+                return [replace_nan(v) for v in value]
+            else:
+                return value
+
+        # Apply to your data dictionary
+        data = replace_nan(data)
+
         try:
             # Send the data to the cloud
-             response = requests.post(LUMIWEALTH_URL, headers=headers, json=data)
+            json_data = json.dumps(data)
+            response = requests.post(LUMIWEALTH_URL, headers=headers, data=json_data)
         except Exception as e:
-            self.logger.error(f"Failed to send update to the cloud. Error: {e}")
+            self.logger.error(f"Failed to send update to the cloud because of lumibot error. Error: {e}")
+            # Add the traceback to the log
+            self.logger.error(traceback.format_exc())
             return False
 
         # Check if the message was sent successfully
@@ -4626,7 +4645,7 @@ def send_update_to_cloud(self):
             return True
         else:
             self.logger.error(
-                f"Failed to send update to the cloud. Status code: {response.status_code}, message: {response.text}"
+                f"Failed to send update to the cloud because of cloud error. Status code: {response.status_code}, message: {response.text}"
             )
             return False
 

lumibot/traders/trader.py

@@ -25,7 +25,7 @@ def __init__(self, logfile="", backtest=False, debug=False, strategies=None, qui
         strategies: list
             A list of strategies to run. If not specified, you must add strategies using trader.add_strategy(strategy)
         quiet_logs: bool
-            Whether to quiet noisy logs by setting the log level to ERROR. Defaults to True.
+            Whether to quiet backtest logs by setting the log level to ERROR. Defaults to False.
         """
         # Check if the logfile is a valid path
         if logfile:
@@ -35,8 +35,11 @@ def __init__(self, logfile="", backtest=False, debug=False, strategies=None, qui
         # Setting debug and _logfile parameters and setting global log format
         self.debug = debug
         self.backtest = backtest
-        self.log_format = logging.Formatter("%(asctime)s | %(name)s | %(levelname)s | %(message)s")
-        self.quiet_logs = quiet_logs
+        std_format = "%(asctime)s: %(levelname)s: %(message)s"
+        debug_format = "%(asctime)s: %(name)s: %(levelname)s: %(message)s"
+        log_format = std_format if not self.debug else debug_format
+        self.log_format = logging.Formatter(log_format)
+        self.quiet_logs = quiet_logs  # Turns off all logging execpt for error messages in backtesting
 
         if logfile:
             self.logfile = Path(logfile)
@@ -172,11 +175,6 @@ def _set_logger(self):
         logging.getLogger("apscheduler.scheduler").setLevel(logging.ERROR)
         logging.getLogger("apscheduler.executors.default").setLevel(logging.ERROR)
         logging.getLogger("lumibot.data_sources.yahoo_data").setLevel(logging.ERROR)
-
-        if self.quiet_logs:
-            logging.getLogger("asyncio").setLevel(logging.ERROR)
-            logging.getLogger("lumibot.backtesting.backtesting_broker").setLevel(logging.ERROR)
-
         logger = logging.getLogger()
 
         for handler in logger.handlers:
@@ -189,12 +187,18 @@ def _set_logger(self):
 
         if self.debug:
             logger.setLevel(logging.DEBUG)
-        elif self.quiet_logs:
-            logger.setLevel(logging.ERROR)
-            for handler in logger.handlers:
-                if handler.__class__.__name__ == "StreamHandler":
-                    handler.setLevel(logging.ERROR)
+        elif self.is_backtest_broker:
+            logger.setLevel(logging.INFO)
+
+            # Quiet logs turns off all backtesting logging except for error messages
+            if self.quiet_logs:
+                logger.setLevel(logging.ERROR)
+
+            # Ensure console has minimal logging to keep things clean during backtesting
+            stream_handler.setLevel(logging.ERROR)
+
         else:
+            # Live trades should always have full logging.
             logger.setLevel(logging.INFO)
 
         # Setting file logging

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="lumibot",
-    version="3.8.2",
+    version="3.8.5",
     author="Robert Grzesik",
     author_email="[email protected]",
     description="Backtesting and Trading Library, Made by Lumiwealth",

tests/test_logging.py

@@ -13,52 +13,6 @@ def test_logging(self, caplog):
         logger.info("This is an info message")
         assert "This is an info message" in caplog.text
 
-    def test_backtest_produces_no_logs_by_default(self, caplog):
-        caplog.set_level(logging.INFO)
-        backtesting_start = datetime.datetime(2023, 1, 2)
-        backtesting_end = datetime.datetime(2023, 1, 4)
-
-        LifecycleLogger.backtest(
-            YahooDataBacktesting,
-            backtesting_start,
-            backtesting_end,
-            parameters={"sleeptime": "1D", "market": "NYSE"},
-            show_plot=False,
-            save_tearsheet=False,
-            show_tearsheet=False,
-            show_indicators=False,
-            save_logfile=False,
-        )
-        # count that this contains 3 new lines. Its an easy proxy for the number of log messages and avoids
-        # the issue where the datetime is always gonna be different.
-        assert caplog.text.count("\n") == 3
-        assert "Starting backtest...\n" in caplog.text
-        assert "Backtesting starting...\n" in caplog.text
-        assert "Backtesting finished\n" in caplog.text
-
-    def test_run_backtest_produces_no_logs_by_default(self, caplog):
-        caplog.set_level(logging.INFO)
-        backtesting_start = datetime.datetime(2023, 1, 2)
-        backtesting_end = datetime.datetime(2023, 1, 4)
-
-        LifecycleLogger.run_backtest(
-            YahooDataBacktesting,
-            backtesting_start,
-            backtesting_end,
-            parameters={"sleeptime": "1D", "market": "NYSE"},
-            show_plot=False,
-            save_tearsheet=False,
-            show_tearsheet=False,
-            show_indicators=False,
-            save_logfile=False,
-        )
-        # count that this contains 3 new lines. Its an easy proxy for the number of log messages and avoids
-        # the issue where the datetime is always gonna be different.
-        assert caplog.text.count("\n") == 3
-        assert "Starting backtest...\n" in caplog.text
-        assert "Backtesting starting...\n" in caplog.text
-        assert "Backtesting finished\n" in caplog.text
-
     def test_backtest_produces_no_logs_when_quiet_logs_is_true(self, caplog):
         caplog.set_level(logging.INFO)
         backtesting_start = datetime.datetime(2023, 1, 2)
@@ -79,10 +33,11 @@ def test_backtest_produces_no_logs_when_quiet_logs_is_true(self, caplog):
         )
         # count that this contains 3 new lines. Its an easy proxy for the number of log messages and avoids
         # the issue where the datetime is always gonna be different.
-        assert caplog.text.count("\n") == 3
+        assert caplog.text.count("\n") == 4
         assert "Starting backtest...\n" in caplog.text
         assert "Backtesting starting...\n" in caplog.text
         assert "Backtesting finished\n" in caplog.text
+        assert "Backtest took " in caplog.text
 
     def test_backtest_produces_logs_when_quiet_logs_is_false(self, caplog):
         caplog.set_level(logging.INFO)
@@ -103,12 +58,12 @@ def test_backtest_produces_logs_when_quiet_logs_is_false(self, caplog):
             quiet_logs=False,
         )
 
-        assert caplog.text.count("\n") == 9
+        assert caplog.text.count("\n") >= 9
         assert "Starting backtest...\n" in caplog.text
         assert "Backtesting starting...\n" in caplog.text
         assert "before_market_opens called\n" in caplog.text
         assert "before_starting_trading called\n" in caplog.text
         assert "on_trading_iteration called\n" in caplog.text
         assert "before_market_closes called\n" in caplog.text
         assert "after_market_closes called\n" in caplog.text
-        assert "Backtesting finished\n" in caplog.text
+        assert "Backtesting finished\n" in caplog.text