Deephaven Python v2 (#39)

* Deephaven 2 type annotations.

* Ported table writer.

* Ported times.

* Ported PythonFunction.

* Ported PythonFunction.

* Ported NULL_DOUBLE.

* Ported PythonFunction.

* Ported Table import.

* Ported dtypes.

* Ported dtypes.

* Ported StringSet.

* Ported DateTime.

* Ported Table import.

* Debugging

* Porting move_columns_up

* Porting natural_join

* Porting rename_columns, drop_columns

* Porting select_distinct

* Porting last_by

* Porting update

* Porting column names

* Adding time parsing debugging.

* Fix date time parsing

* Improved time parsing error messages.

* Ported example scripts.

* Make thread tracing more robust to threads disappearing.

* Updated readme.

README.md

@@ -144,8 +144,9 @@ See [Access your file system with Docker data volumes](https://deephaven.io/core
 
 Follow these steps to run a [Deephaven](https://deephaven.io) plus [Interactive Brokers](https://interactivebrokers.com) system. 
 
-`<deephaven_version>` is the version of [Deephaven](https://deephaven.io) to run (e.g., `0.10.0`).  A list of available versions 
-can be found on the [Deephaven Releases GitHub page](https://github.com/deephaven/deephaven-core/releases).
+`<deephaven_version>` is the version of [Deephaven](https://deephaven.io) to run (e.g., `0.11.0`).  A list of available versions 
+can be found on the [Deephaven Releases GitHub page](https://github.com/deephaven/deephaven-core/releases).  
+Version `0.11.0` or higher must be used.
 
 **Windows users need to run the commands in WSL.**
 
@@ -371,7 +372,7 @@ Market data can be requested from the client using:
 ```python
 from ibapi.contract import Contract
 
-from deephaven import DateTimeUtils as dtu
+from deephaven.time import to_datetime
 
 contract = Contract()
 contract.symbol = "GOOG"
@@ -382,8 +383,8 @@ contract.exchange = "SMART"
 rc = client.get_registered_contract(contract)
 print(contract)
 
-start = dtu.convertDateTime("2021-01-01T00:00:00 NY")
-end = dtu.convertDateTime("2021-01-10T00:00:00 NY")
+start = to_datetime("2021-01-01T00:00:00 NY")
+end = to_datetime("2021-01-10T00:00:00 NY")
 client.request_news_historical(rc, start=start, end=end)
 
 client.request_news_article(provider_code="BRFUPDN", article_id="BRFUPDN$107d53ea")
@@ -485,8 +486,8 @@ bars_realtime = client.tables["bars_realtime"]
 
 bars_dia = bars_realtime.where("Symbol=`DIA`")
 bars_spy = bars_realtime.where("Symbol=`SPY`")
-bars_joined = bars_dia.view("Timestamp", "TimestampEnd", "Dia=Close") \
-    .naturalJoin(bars_spy, "TimestampEnd", "Spy=Close") \
+bars_joined = bars_dia.view(["Timestamp", "TimestampEnd", "Dia=Close"]) \
+    .natural_join(bars_spy, on="TimestampEnd", joins="Spy=Close") \
     .update("Ratio = Dia/Spy")
 ```
 
@@ -518,9 +519,10 @@ client.request_tick_data_realtime(rc, dhib.TickDataType.BID_ASK)
 
 ticks_bid_ask = client.tables["ticks_bid_ask"]
 
-from deephaven import Plot
-plot_aapl = Plot.plot("Bid",  ticks_bid_ask, "ReceiveTime", "BidPrice") \
-    .plot("Ask",  ticks_bid_ask, "ReceiveTime", "AskPrice") \
+from deephaven.plot import Figure
+
+plot_aapl = Figure().plot_xy("Bid",  t=ticks_bid_ask, x="ReceiveTime", y="BidPrice") \
+    .plot_xy("Ask",  t=ticks_bid_ask, x="ReceiveTime", y="AskPrice") \
     .show()
 ```
 
@@ -561,6 +563,7 @@ A discussion of available logging levels can be found in the [Python `logging` m
 
 If you can not solve your problems through either the `errors` table or through logging, you can try:
 
+* [deephaven-ib API Documentation](https://deephaven-examples.github.io/deephaven-ib/)
 * [Interactive Brokers Support](https://www.interactivebrokers.com/en/support/individuals.php)
 * [Gitter: A relaxed chat room about all things Deephaven](https://gitter.im/deephaven/deephaven)
 * [Deephaven Community Slack](https://http://deephavencommunity.slack.com/)

examples/example_all_functionality.py

@@ -1,6 +1,6 @@
 from typing import Dict
 
-from deephaven import DateTimeUtils as dtu
+from deephaven.time import to_datetime
 from ibapi.contract import Contract
 from ibapi.order import Order
 
@@ -12,6 +12,7 @@
 
 print("==============================================================================================================")
 print("==== Create a client and connect.")
+print("==== ** Accept the connection in TWS **")
 print("==============================================================================================================")
 
 client = dhib.IbSessionTws(host="host.docker.internal", port=7497, client_id=0, download_short_rates=False, read_only=False)
@@ -22,6 +23,7 @@
 
 print("==============================================================================================================")
 print("==== Get registered contracts for all contract types.")
+print("==== See https://interactivebrokers.github.io/tws-api/basic_contracts.html for details on supported contract types.")
 print("==============================================================================================================")
 
 
@@ -88,22 +90,22 @@ def get_contracts() -> Dict[str, Contract]:
     contract.secType = "FUT"
     contract.exchange = "GLOBEX"
     contract.currency = "USD"
-    contract.lastTradeDateOrContractMonth = "202203"
+    contract.lastTradeDateOrContractMonth = "202206"
     rst["future_1"] = contract
 
     contract = Contract()
     contract.secType = "FUT"
     contract.exchange = "GLOBEX"
     contract.currency = "USD"
-    contract.localSymbol = "ESH2"
+    contract.localSymbol = "MESZ2"
     rst["future_2"] = contract
 
     contract = Contract()
     contract.symbol = "DAX"
     contract.secType = "FUT"
     contract.exchange = "DTB"
     contract.currency = "EUR"
-    contract.lastTradeDateOrContractMonth = "202203"
+    contract.lastTradeDateOrContractMonth = "202206"
     contract.multiplier = "5"
     rst["future_3"] = contract
 
@@ -126,7 +128,7 @@ def get_contracts() -> Dict[str, Contract]:
     contract.secType = "OPT"
     contract.exchange = "BOX"
     contract.currency = "USD"
-    contract.lastTradeDateOrContractMonth = "20220318"
+    contract.lastTradeDateOrContractMonth = "20230120"
     contract.strike = 2800
     contract.right = "C"
     contract.multiplier = "100"
@@ -159,7 +161,7 @@ def get_contracts() -> Dict[str, Contract]:
     contract.secType = "FOP"
     contract.exchange = "GLOBEX"
     contract.currency = "USD"
-    contract.lastTradeDateOrContractMonth = "202203"
+    contract.lastTradeDateOrContractMonth = "202206"
     contract.strike = 4700
     contract.right = "C"
     contract.multiplier = "50"
@@ -213,12 +215,12 @@ def get_contracts() -> Dict[str, Contract]:
 
     # Dutch warrants and structured products
 
-    contract = Contract()
-    contract.localSymbol = "PJ07S"
-    contract.secType = "IOPT"
-    contract.exchange = "SBF"
-    contract.currency = "EUR"
-    rst["dutchwarrant_1"] = contract
+    # contract = Contract()
+    # contract.localSymbol = "B881G"
+    # contract.secType = "IOPT"
+    # contract.exchange = "SBF"
+    # contract.currency = "EUR"
+    # rst["dutchwarrant_1"] = contract
 
     return rst
 
@@ -257,8 +259,8 @@ def get_contracts() -> Dict[str, Contract]:
 rc = client.get_registered_contract(contract)
 print(contract)
 
-start = dtu.convertDateTime("2021-01-01T00:00:00 NY")
-end = dtu.convertDateTime("2021-01-10T00:00:00 NY")
+start = to_datetime("2021-01-01T00:00:00 NY")
+end = to_datetime("2021-01-10T00:00:00 NY")
 client.request_news_historical(rc, start=start, end=end)
 
 client.request_news_article(provider_code="BRFUPDN", article_id="BRFUPDN$107d53ea")
@@ -316,7 +318,7 @@ def get_contracts() -> Dict[str, Contract]:
 rc = client.get_registered_contract(contract)
 print(contract)
 
-now = dtu.convertDateTime("2021-01-01T00:00:00 NY")
+now = to_datetime("2021-01-01T00:00:00 NY")
 
 client.request_tick_data_historical(rc, dhib.TickDataType.MIDPOINT, 100, start=now)
 client.request_tick_data_historical(rc, dhib.TickDataType.MIDPOINT, 100, end=now)
@@ -390,7 +392,7 @@ def get_contracts() -> Dict[str, Contract]:
 contract.secType = "OPT"
 contract.exchange = "BOX"
 contract.currency = "USD"
-contract.lastTradeDateOrContractMonth = "20220318"
+contract.lastTradeDateOrContractMonth = "20230120"
 contract.strike = 2800
 contract.right = "C"
 contract.multiplier = "100"

examples/example_market_data.py

@@ -2,6 +2,10 @@
 
 import deephaven_ib as dhib
 
+print("==============================================================================================================")
+print("==== ** Accept the connection in TWS **")
+print("==============================================================================================================")
+
 client = dhib.IbSessionTws(host="host.docker.internal", port=7497, download_short_rates=False)
 client.connect()
 

examples/example_overview_image.py

@@ -0,0 +1,54 @@
+import deephaven_ib as dhib
+
+print("==============================================================================================================")
+print("==== ** Accept the connection in TWS **")
+print("==============================================================================================================")
+
+client = dhib.IbSessionTws(host="host.docker.internal", port=7497)
+client.connect()
+
+from ibapi.contract import Contract
+
+c1 = Contract()
+c1.symbol = 'DIA'
+c1.secType = 'STK'
+c1.exchange = 'SMART'
+c1.currency = 'USD'
+
+rc1 = client.get_registered_contract(c1)
+print(rc1)
+
+c2 = Contract()
+c2.symbol = 'SPY'
+c2.secType = 'STK'
+c2.exchange = 'SMART'
+c2.currency = 'USD'
+
+rc2 = client.get_registered_contract(c2)
+print(rc2)
+
+client.set_market_data_type(dhib.MarketDataType.REAL_TIME)
+client.request_market_data(rc1)
+client.request_market_data(rc2)
+client.request_bars_realtime(rc1, bar_type=dhib.BarDataType.MIDPOINT)
+client.request_bars_realtime(rc2, bar_type=dhib.BarDataType.MIDPOINT)
+
+bars_realtime = client.tables["bars_realtime"]
+
+bars_dia = bars_realtime.where("Symbol=`DIA`")
+bars_spy = bars_realtime.where("Symbol=`SPY`")
+bars_joined = bars_dia.view(["Timestamp", "TimestampEnd", "Dia=Close"]) \
+    .natural_join(bars_spy, on="TimestampEnd", joins="Spy=Close") \
+    .update("Ratio = Dia/Spy")
+
+from deephaven.plot import Figure
+
+plot_prices = Figure().plot_xy("DIA", t=bars_dia, x="TimestampEnd", y="Close") \
+    .x_twin() \
+    .plot_xy("SPY", t=bars_dia, x="TimestampEnd", y="Close") \
+    .show()
+
+plot_ratio = Figure().plot_xy("Ratio", t=bars_joined, x="TimestampEnd", y="Ratio") \
+    .show()
+
+

examples/example_query_and_plot.py

@@ -1,5 +1,9 @@
 import deephaven_ib as dhib
 
+print("==============================================================================================================")
+print("==== ** Accept the connection in TWS **")
+print("==============================================================================================================")
+
 client = dhib.IbSessionTws(host="host.docker.internal", port=7497)
 client.connect()
 
@@ -33,18 +37,18 @@
 
 bars_dia = bars_realtime.where("Symbol=`DIA`")
 bars_spy = bars_realtime.where("Symbol=`SPY`")
-bars_joined = bars_dia.view("Timestamp", "TimestampEnd", "Dia=Close") \
-    .naturalJoin(bars_spy, "TimestampEnd", "Spy=Close") \
+bars_joined = bars_dia.view(["Timestamp", "TimestampEnd", "Dia=Close"]) \
+    .natural_join(bars_spy, on="TimestampEnd", joins="Spy=Close") \
     .update("Ratio = Dia/Spy")
 
-from deephaven import Plot
+from deephaven.plot import Figure
 
-plot_prices = Plot.plot("DIA", bars_dia, "TimestampEnd", "Close") \
-    .twinX() \
-    .plot("SPY", bars_dia, "TimestampEnd", "Close") \
+plot_prices = Figure().plot_xy("DIA", t=bars_dia, x="TimestampEnd", y="Close") \
+    .x_twin() \
+    .plot_xy("SPY", t=bars_spy, x="TimestampEnd", y="Close") \
     .show()
 
-plot_ratio = Plot.plot("Ratio", bars_joined, "TimestampEnd", "Ratio") \
+plot_ratio = Figure().plot_xy("Ratio", t=bars_joined, x="TimestampEnd", y="Ratio") \
     .show()