diff --git a/lumibot/backtesting/backtesting_broker.py b/lumibot/backtesting/backtesting_broker.py index 82a6a8b37..78f7a495b 100644 --- a/lumibot/backtesting/backtesting_broker.py +++ b/lumibot/backtesting/backtesting_broker.py @@ -98,7 +98,7 @@ def _update_datetime(self, update_dt, cash=None, portfolio_value=None): self.data_source._update_datetime(new_datetime, cash=cash, portfolio_value=portfolio_value) if self.option_source: self.option_source._update_datetime(new_datetime, cash=cash, portfolio_value=portfolio_value) - logger.debug(f"Current backtesting datetime {self.datetime}") + logger.info(f"Current backtesting datetime {self.datetime}") # =========Clock functions===================== diff --git a/lumibot/brokers/interactive_brokers_rest.py b/lumibot/brokers/interactive_brokers_rest.py index 16a00090b..ab94b9b23 100644 --- a/lumibot/brokers/interactive_brokers_rest.py +++ b/lumibot/brokers/interactive_brokers_rest.py @@ -6,6 +6,7 @@ import datetime from decimal import Decimal from math import gcd +import re TYPE_MAP = dict( stock="STK", @@ -408,17 +409,92 @@ def _pull_positions(self, strategy) -> list[Position]: if asset_class == Asset.AssetType.STOCK: asset = Asset(symbol=symbol, asset_type=asset_class) elif asset_class == Asset.AssetType.OPTION: - expiry = position["expiry"] - strike = position["strike"] - right = position["putOrCall"] - # If asset class is option, create an option asset - asset = Asset( - symbol=symbol, - asset_type=asset_class, - expiration=expiry, - strike=strike, - right=right, - ) + # Example contract_desc: 'SPY NOV2024 562 P [SPY 241105P00562000 100]' + # This example format includes: + # - An underlying symbol at the beginning (e.g., "SPY") + # - Expiry and strike in human-readable format (e.g., "NOV2024 562 P") + # - Option details within square brackets (e.g., "[SPY 241105P00562000 100]"), + # where "241105P00562000" holds the expiry (YYMMDD), option type (C/P), and strike price + + contract_desc = position.get("contractDesc", "").strip() + + if not contract_desc: + logging.error("Empty contract description for option. Skipping this position.") + continue # Skip processing this position as contract_desc is missing + + try: + # Locate the square brackets and extract the option details part + start_idx = contract_desc.find('[') + end_idx = contract_desc.find(']', start_idx) + + if start_idx == -1 or end_idx == -1: + logging.error(f"Brackets not found in contract description '{contract_desc}'. Expected format like '[SPY 241105P00562000 100]'.") + continue # Skip if brackets are missing + + # Extract content within brackets and find the critical pattern (e.g., "241105P00562000") + bracket_content = contract_desc[start_idx + 1:end_idx].strip() + # Search for 6 digits, followed by 'C' or 'P', followed by 8 digits for strike + details_match = re.search(r'\d{6}[CP]\d{8}', bracket_content) + + if not details_match: + logging.error(f"Expected option pattern not found in contract '{contract_desc}'.") + continue # Skip if pattern does not match + + contract_details = details_match.group(0) + + # Parse components from the details + expiry_raw = contract_details[:6] # First six digits (YYMMDD format) + right_raw = contract_details[6] # Seventh character (C or P) + strike_raw = contract_details[7:] # Remaining characters (strike price) + + # Check if expiry is in the correct format and convert to date + try: + expiry = datetime.datetime.strptime(expiry_raw, "%y%m%d").date() + except ValueError as ve: + logging.error(f"Invalid expiry format '{expiry_raw}' in contract '{contract_desc}': {ve}") + continue # Skip this position due to invalid expiry format + + # Convert strike to a float, assuming it’s in thousandths (e.g., "00562000" to "562.00") + try: + strike = round(float(strike_raw) / 1000, 2) + except ValueError as ve: + logging.error(f"Invalid strike price '{strike_raw}' in contract '{contract_desc}': {ve}") + continue # Skip this position due to invalid strike price + + # Validate the option type (right) as either C or P + if right_raw.upper() not in ["C", "P"]: + logging.error(f"Invalid option type '{right_raw}' in contract '{contract_desc}'. Expected 'C' or 'P'.") + continue # Skip if option type is not valid + + # Determine the option right type + right = Asset.OptionRight.CALL if right_raw.upper() == "C" else Asset.OptionRight.PUT + + # Extract the underlying symbol, assumed to be the first word in contract_desc + underlying_asset_raw = contract_desc.split()[0] + + # Ensure underlying symbol is alphanumeric and non-empty + if not underlying_asset_raw.isalnum(): + logging.error(f"Invalid underlying asset symbol '{underlying_asset_raw}' in '{contract_desc}'.") + continue + + # Create the underlying asset object + underlying_asset = Asset( + symbol=underlying_asset_raw, + asset_type=Asset.AssetType.STOCK + ) + + # Create the option asset object + asset = Asset( + symbol=symbol, + asset_type=asset_class, + expiration=expiry, + strike=strike, + right=right, + underlying_asset=underlying_asset, + ) + + except Exception as e: + logging.error(f"Error processing contract '{contract_desc}': {e}") elif asset_class == Asset.AssetType.FUTURE: #contract_details = self.data_source.get_contract_details(position['conid']) expiry = position["expiry"] diff --git a/lumibot/entities/order.py b/lumibot/entities/order.py index 44d142686..ab57d233a 100644 --- a/lumibot/entities/order.py +++ b/lumibot/entities/order.py @@ -893,6 +893,10 @@ def to_dict(self): if isinstance(value, datetime.datetime): order_dict[key] = value.isoformat() + # If it is a Decimal object, convert it to a float + elif isinstance(value, Decimal): + order_dict[key] = float(value) + # Recursively handle objects that have their own to_dict method (like asset, quote, etc.) elif hasattr(value, "to_dict"): order_dict[key] = value.to_dict() diff --git a/lumibot/entities/position.py b/lumibot/entities/position.py index d64924fff..0ec710f58 100644 --- a/lumibot/entities/position.py +++ b/lumibot/entities/position.py @@ -168,11 +168,11 @@ def to_dict(self): return { "strategy": self.strategy, "asset": self.asset.to_dict(), - "quantity": self.quantity, + "quantity": float(self.quantity), "orders": [order.to_dict() for order in self.orders], "hold": self.hold, - "available": self.available, - "avg_fill_price": self.avg_fill_price, + "available": float(self.available) if self.available else None, + "avg_fill_price": float(self.avg_fill_price) if self.avg_fill_price else None, } @classmethod diff --git a/lumibot/strategies/_strategy.py b/lumibot/strategies/_strategy.py index 1888eed18..711657ef9 100644 --- a/lumibot/strategies/_strategy.py +++ b/lumibot/strategies/_strategy.py @@ -884,7 +884,7 @@ def run_backtest( save_logfile=False, use_quote_data=False, show_progress_bar=True, - quiet_logs=True, + quiet_logs=False, trader_class=Trader, **kwargs, ): @@ -1515,4 +1515,4 @@ def backtest( trader_class=trader_class, **kwargs, ) - return results \ No newline at end of file + return results diff --git a/lumibot/strategies/strategy.py b/lumibot/strategies/strategy.py index b980b16a7..54da45cd8 100644 --- a/lumibot/strategies/strategy.py +++ b/lumibot/strategies/strategy.py @@ -9,6 +9,8 @@ from typing import Union from sqlalchemy import create_engine, inspect, text, bindparam from sqlalchemy.exc import OperationalError +import traceback +import math import jsonpickle import matplotlib @@ -4613,11 +4615,28 @@ def send_update_to_cloud(self): "orders": [order.to_dict() for order in orders], } + # Helper function to recursively replace NaN in dictionaries + def replace_nan(value): + if isinstance(value, float) and math.isnan(value): + return None # or 0 if you prefer + elif isinstance(value, dict): + return {k: replace_nan(v) for k, v in value.items()} + elif isinstance(value, list): + return [replace_nan(v) for v in value] + else: + return value + + # Apply to your data dictionary + data = replace_nan(data) + try: # Send the data to the cloud - response = requests.post(LUMIWEALTH_URL, headers=headers, json=data) + json_data = json.dumps(data) + response = requests.post(LUMIWEALTH_URL, headers=headers, data=json_data) except Exception as e: - self.logger.error(f"Failed to send update to the cloud. Error: {e}") + self.logger.error(f"Failed to send update to the cloud because of lumibot error. Error: {e}") + # Add the traceback to the log + self.logger.error(traceback.format_exc()) return False # Check if the message was sent successfully @@ -4626,7 +4645,7 @@ def send_update_to_cloud(self): return True else: self.logger.error( - f"Failed to send update to the cloud. Status code: {response.status_code}, message: {response.text}" + f"Failed to send update to the cloud because of cloud error. Status code: {response.status_code}, message: {response.text}" ) return False diff --git a/lumibot/traders/trader.py b/lumibot/traders/trader.py index b079951b5..b0f066597 100644 --- a/lumibot/traders/trader.py +++ b/lumibot/traders/trader.py @@ -25,7 +25,7 @@ def __init__(self, logfile="", backtest=False, debug=False, strategies=None, qui strategies: list A list of strategies to run. If not specified, you must add strategies using trader.add_strategy(strategy) quiet_logs: bool - Whether to quiet noisy logs by setting the log level to ERROR. Defaults to True. + Whether to quiet backtest logs by setting the log level to ERROR. Defaults to False. """ # Check if the logfile is a valid path if logfile: @@ -35,8 +35,11 @@ def __init__(self, logfile="", backtest=False, debug=False, strategies=None, qui # Setting debug and _logfile parameters and setting global log format self.debug = debug self.backtest = backtest - self.log_format = logging.Formatter("%(asctime)s | %(name)s | %(levelname)s | %(message)s") - self.quiet_logs = quiet_logs + std_format = "%(asctime)s: %(levelname)s: %(message)s" + debug_format = "%(asctime)s: %(name)s: %(levelname)s: %(message)s" + log_format = std_format if not self.debug else debug_format + self.log_format = logging.Formatter(log_format) + self.quiet_logs = quiet_logs # Turns off all logging execpt for error messages in backtesting if logfile: self.logfile = Path(logfile) @@ -172,11 +175,6 @@ def _set_logger(self): logging.getLogger("apscheduler.scheduler").setLevel(logging.ERROR) logging.getLogger("apscheduler.executors.default").setLevel(logging.ERROR) logging.getLogger("lumibot.data_sources.yahoo_data").setLevel(logging.ERROR) - - if self.quiet_logs: - logging.getLogger("asyncio").setLevel(logging.ERROR) - logging.getLogger("lumibot.backtesting.backtesting_broker").setLevel(logging.ERROR) - logger = logging.getLogger() for handler in logger.handlers: @@ -189,12 +187,18 @@ def _set_logger(self): if self.debug: logger.setLevel(logging.DEBUG) - elif self.quiet_logs: - logger.setLevel(logging.ERROR) - for handler in logger.handlers: - if handler.__class__.__name__ == "StreamHandler": - handler.setLevel(logging.ERROR) + elif self.is_backtest_broker: + logger.setLevel(logging.INFO) + + # Quiet logs turns off all backtesting logging except for error messages + if self.quiet_logs: + logger.setLevel(logging.ERROR) + + # Ensure console has minimal logging to keep things clean during backtesting + stream_handler.setLevel(logging.ERROR) + else: + # Live trades should always have full logging. logger.setLevel(logging.INFO) # Setting file logging diff --git a/setup.py b/setup.py index d5246abc1..81c3ffa66 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="lumibot", - version="3.8.2", + version="3.8.5", author="Robert Grzesik", author_email="rob@lumiwealth.com", description="Backtesting and Trading Library, Made by Lumiwealth", diff --git a/tests/test_logging.py b/tests/test_logging.py index c335ff697..b835548c0 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -13,52 +13,6 @@ def test_logging(self, caplog): logger.info("This is an info message") assert "This is an info message" in caplog.text - def test_backtest_produces_no_logs_by_default(self, caplog): - caplog.set_level(logging.INFO) - backtesting_start = datetime.datetime(2023, 1, 2) - backtesting_end = datetime.datetime(2023, 1, 4) - - LifecycleLogger.backtest( - YahooDataBacktesting, - backtesting_start, - backtesting_end, - parameters={"sleeptime": "1D", "market": "NYSE"}, - show_plot=False, - save_tearsheet=False, - show_tearsheet=False, - show_indicators=False, - save_logfile=False, - ) - # count that this contains 3 new lines. Its an easy proxy for the number of log messages and avoids - # the issue where the datetime is always gonna be different. - assert caplog.text.count("\n") == 3 - assert "Starting backtest...\n" in caplog.text - assert "Backtesting starting...\n" in caplog.text - assert "Backtesting finished\n" in caplog.text - - def test_run_backtest_produces_no_logs_by_default(self, caplog): - caplog.set_level(logging.INFO) - backtesting_start = datetime.datetime(2023, 1, 2) - backtesting_end = datetime.datetime(2023, 1, 4) - - LifecycleLogger.run_backtest( - YahooDataBacktesting, - backtesting_start, - backtesting_end, - parameters={"sleeptime": "1D", "market": "NYSE"}, - show_plot=False, - save_tearsheet=False, - show_tearsheet=False, - show_indicators=False, - save_logfile=False, - ) - # count that this contains 3 new lines. Its an easy proxy for the number of log messages and avoids - # the issue where the datetime is always gonna be different. - assert caplog.text.count("\n") == 3 - assert "Starting backtest...\n" in caplog.text - assert "Backtesting starting...\n" in caplog.text - assert "Backtesting finished\n" in caplog.text - def test_backtest_produces_no_logs_when_quiet_logs_is_true(self, caplog): caplog.set_level(logging.INFO) backtesting_start = datetime.datetime(2023, 1, 2) @@ -79,10 +33,11 @@ def test_backtest_produces_no_logs_when_quiet_logs_is_true(self, caplog): ) # count that this contains 3 new lines. Its an easy proxy for the number of log messages and avoids # the issue where the datetime is always gonna be different. - assert caplog.text.count("\n") == 3 + assert caplog.text.count("\n") == 4 assert "Starting backtest...\n" in caplog.text assert "Backtesting starting...\n" in caplog.text assert "Backtesting finished\n" in caplog.text + assert "Backtest took " in caplog.text def test_backtest_produces_logs_when_quiet_logs_is_false(self, caplog): caplog.set_level(logging.INFO) @@ -103,7 +58,7 @@ def test_backtest_produces_logs_when_quiet_logs_is_false(self, caplog): quiet_logs=False, ) - assert caplog.text.count("\n") == 9 + assert caplog.text.count("\n") >= 9 assert "Starting backtest...\n" in caplog.text assert "Backtesting starting...\n" in caplog.text assert "before_market_opens called\n" in caplog.text @@ -111,4 +66,4 @@ def test_backtest_produces_logs_when_quiet_logs_is_false(self, caplog): assert "on_trading_iteration called\n" in caplog.text assert "before_market_closes called\n" in caplog.text assert "after_market_closes called\n" in caplog.text - assert "Backtesting finished\n" in caplog.text \ No newline at end of file + assert "Backtesting finished\n" in caplog.text