Release 1.198.0

See release notes.

.github/workflows/build.yml

@@ -104,13 +104,26 @@ jobs:
       #     make install-talib
       #     poetry run pip install setuptools numpy==1.26.4 ta-lib
 
-      - name: Setup cached pre-commit
+      - name: Set up cached pre-commit
         id: cached-pre-commit
         uses: actions/cache@v4
         with:
           path: ~/.cache/pre-commit
           key: ${{ runner.os }}-${{ env.PYTHON_VERSION }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}
 
+      - name: Set up cached cargo
+        id: cached-cargo
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-cargo-
+
       - name: Set poetry cache-dir
         run: echo "POETRY_CACHE_DIR=$(poetry config cache-dir)" >> $GITHUB_ENV
 
@@ -139,7 +152,13 @@ jobs:
 
       - name: Run nautilus_core cargo tests (Linux)
         run: |
-          cargo install cargo-nextest
+          if ! command -v cargo-nextest &> /dev/null
+          then
+              echo "cargo-nextest not found, installing..."
+              cargo install cargo-nextest
+          else
+              echo "cargo-nextest is already installed"
+          fi
           make cargo-test
 
       - name: Run tests (Linux)
@@ -205,13 +224,26 @@ jobs:
       - name: Install build dependencies
         run: python -m pip install --upgrade pip setuptools wheel pre-commit msgspec
 
-      - name: Setup cached pre-commit
+      - name: Set up cached pre-commit
         id: cached-pre-commit
         uses: actions/cache@v4
         with:
           path: ~/.cache/pre-commit
           key: ${{ runner.os }}-${{ matrix.python-version }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}
 
+      - name: Set up cached cargo
+        id: cached-cargo
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-cargo-
+
       - name: Set poetry cache-dir
         run: echo "POETRY_CACHE_DIR=$(poetry config cache-dir)" >> $GITHUB_ENV
 
@@ -297,13 +329,26 @@ jobs:
       - name: Install build dependencies
         run: python -m pip install --upgrade pip setuptools wheel pre-commit msgspec
 
-      - name: Setup cached pre-commit
+      - name: Set up cached pre-commit
         id: cached-pre-commit
         uses: actions/cache@v4
         with:
           path: ~/.cache/pre-commit
           key: ${{ runner.os }}-${{ env.PYTHON_VERSION }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}
 
+      - name: Set up cached cargo
+        id: cached-cargo
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-cargo-
+
       - name: Set poetry cache-dir
         run: echo "POETRY_CACHE_DIR=$(poetry config cache-dir)" >> $GITHUB_ENV
 
@@ -321,7 +366,13 @@ jobs:
 
       - name: Run nautilus_core cargo tests (macOS)
         run: |
-          cargo install cargo-nextest
+          if ! command -v cargo-nextest &> /dev/null
+          then
+              echo "cargo-nextest not found, installing..."
+              cargo install cargo-nextest
+          else
+              echo "cargo-nextest is already installed"
+          fi
           make cargo-test
 
       - name: Run tests (macOS)

.github/workflows/coverage.yml

@@ -91,7 +91,7 @@ jobs:
       #     make install-talib
       #     poetry run pip install setuptools numpy==1.26.4 ta-lib
 
-      - name: Setup cached pre-commit
+      - name: Set up cached pre-commit
         id: cached-pre-commit
         uses: actions/cache@v4
         with:

.pre-commit-config.yaml

@@ -74,7 +74,7 @@ repos:
         types: [python]
 
   - repo: https://github.com/psf/black
-    rev: 24.4.2
+    rev: 24.8.0
     hooks:
       - id: black
         types_or: [python, pyi]
@@ -83,7 +83,7 @@ repos:
         exclude: "docs/_pygments/monokai.py"
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.5.5
+    rev: v0.5.7
     hooks:
       - id: ruff
         args: ["--fix"]

Makefile

@@ -167,4 +167,4 @@ install-talib:
 
 .PHONY: install-cli
 install-cli:
-	(cd nautilus_core && cargo install --path cli --bin nautilus)
+	(cd nautilus_core && cargo install --path cli --bin nautilus --force)

README.md

@@ -14,9 +14,9 @@
 
 | Platform           | Rust    | Python |
 | :----------------- | :------ | :----- |
-| `Linux (x86_64)`   | 1.80.0+ | 3.10+  |
-| `macOS (arm64)`    | 1.80.0+ | 3.10+  |
-| `Windows (x86_64)` | 1.80.0+ | 3.10+  |
+| `Linux (x86_64)`   | 1.80.1+ | 3.10+  |
+| `macOS (arm64)`    | 1.80.1+ | 3.10+  |
+| `Windows (x86_64)` | 1.80.1+ | 3.10+  |
 
 [![](https://dcbadge.limes.pink/api/server/AUWVs3XaCS)](https://discord.gg/AUWVs3XaCS)
 
@@ -127,7 +127,7 @@ The following integrations are currently supported:
 | [Binance](https://binance.com)                            | `BINANCE`             | Crypto Exchange (CEX)   | ![status](https://img.shields.io/badge/stable-green)    | [Guide](https://nautilustrader.io/docs/latest/integrations/binance.html)   |
 | [Binance US](https://binance.us)                          | `BINANCE`             | Crypto Exchange (CEX)   | ![status](https://img.shields.io/badge/stable-green)    | [Guide](https://nautilustrader.io/docs/latest/integrations/binance.html)   |
 | [Binance Futures](https://www.binance.com/en/futures)     | `BINANCE`             | Crypto Exchange (CEX)   | ![status](https://img.shields.io/badge/stable-green)    | [Guide](https://nautilustrader.io/docs/latest/integrations/binance.html)   |
-| [Bybit](https://www.bybit.com)                            | `BYBIT`               | Crypto Exchange (CEX)   | ![status](https://img.shields.io/badge/beta-yellow)     | [Guide](https://nautilustrader.io/docs/latest/integrations/bybit.html)     |
+| [Bybit](https://www.bybit.com)                            | `BYBIT`               | Crypto Exchange (CEX)   | ![status](https://img.shields.io/badge/stable-green)    | [Guide](https://nautilustrader.io/docs/latest/integrations/bybit.html)     |
 | [Databento](https://databento.com)                        | `DATABENTO`           | Data Provider           | ![status](https://img.shields.io/badge/beta-yellow)     | [Guide](https://nautilustrader.io/docs/latest/integrations/databento.html) |
 | [Interactive Brokers](https://www.interactivebrokers.com) | `INTERACTIVE_BROKERS` | Brokerage (multi-venue) | ![status](https://img.shields.io/badge/stable-green)    | [Guide](https://nautilustrader.io/docs/latest/integrations/ib.html)        |
 

RELEASES.md

@@ -1,3 +1,24 @@
+# NautilusTrader 1.198.0 Beta
+
+Released on 9th August 2024 (UTC).
+
+### Enhancements
+- Added `@customdata` decorator to reduce need for boiler plate implementing custom data types (#1828), thanks @faysou
+- Added timeout for HTTP client in Rust (#1835), thanks @davidsblom
+- Added catalog conversion function of streamed data to backtest data (#1834), thanks @faysou
+- Upgraded Cython to 3.0.11
+
+### Breaking Changes
+None
+
+### Fixes
+- Fixed creation of `instrumend_id` folder when writing PyO3 bars in catalog (#1832), thanks @faysou
+- Fixed `StreamingFeatherWriter` handling of `include_types` option (#1833), thanks @faysou
+- Fixed `BybitExecutionClient` position reports error handling and logging
+- Fixed `BybitExecutionClient` order report handling to correctly process external orders
+
+---
+
 # NautilusTrader 1.197.0 Beta
 
 Released on 2nd August 2024 (UTC).

build.py

@@ -172,7 +172,9 @@ def _build_extensions() -> list[Extension]:
             "OleAut32.lib",
             "Pdh.lib",
             "PowrProf.lib",
+            "Propsys.lib",
             "Psapi.lib",
+            "runtimeobject.lib",
             "schannel.lib",
             "secur32.lib",
             "Shell32.lib",

docs/concepts/advanced/custom_data.md

@@ -104,13 +104,16 @@ def on_data(self, data: Data) -> None:
 ## Option Greeks example
 
 This example demonstrates how to create a custom data type for option Greeks, specifically the delta.
-By following these steps, you can create custom data types, subscribe to them, publish them, and store 
-them in the `Cache` for efficient retrieval.
+By following these steps, you can create custom data types, subscribe to them, publish them, and store
+them in the `Cache` or `ParquetDataCatalog` for efficient retrieval.
 
 ```python
 import msgspec
-from nautilus_trader.core.data import Data, DataType
+from nautilus_trader.core.data import Data
+from nautilus_trader.model.data import DataType
 from nautilus_trader.serialization.base import register_serializable_type
+from nautilus_trader.serialization.arrow.serializer import register_arrow
+import pyarrow as pa
 
 from nautilus_trader.model.identifiers import InstrumentId
 from nautilus_trader.core.datetime import dt_to_unix_nanos, unix_nanos_to_dt, format_iso8601
@@ -121,15 +124,19 @@ def unix_nanos_to_str(unix_nanos):
 
 
 class GreeksData(Data):
-    def __init__(self, instrument_id: InstrumentId, ts_event: int, ts_init: int, delta: float):
+    def __init__(
+        self, instrument_id: InstrumentId = InstrumentId.from_str("ES.GLBX"),
+        ts_event: int = 0,
+        ts_init: int = 0,
+        delta: float = 0.0,
+    ) -> None:
         self.instrument_id = instrument_id
         self._ts_event = ts_event
         self._ts_init = ts_init
-
         self.delta = delta
 
     def __repr__(self):
-        return (f"GreeksData(instrument_id={self.instrument_id}, ts_event={unix_nanos_to_str(self._ts_event)}, ts_init={unix_nanos_to_str(self._ts_init)}, delta={self.delta:.2f})")
+        return (f"GreeksData(ts_init={unix_nanos_to_str(self._ts_init)}, instrument_id={self.instrument_id}, delta={self.delta:.2f})")
 
     @property
     def ts_event(self):
@@ -144,20 +151,37 @@ class GreeksData(Data):
             "instrument_id": self.instrument_id.value,
             "ts_event": self._ts_event,
             "ts_init": self._ts_init,
-
-            "delta": self.delta
+            "delta": self.delta,
         }
 
-    def to_bytes(self):
-        return msgspec.msgpack.encode(self.to_dict())
-
     @classmethod
     def from_dict(cls, data: dict):
         return GreeksData(InstrumentId.from_str(data["instrument_id"]), data["ts_event"], data["ts_init"], data["delta"])
 
+    def to_bytes(self):
+        return msgspec.msgpack.encode(self.to_dict())
+
     @classmethod
     def from_bytes(cls, data: bytes):
         return cls.from_dict(msgspec.msgpack.decode(data))
+
+    def to_catalog(self):
+        return pa.RecordBatch.from_pylist([self.to_dict()], schema=GreeksData.schema())
+
+    @classmethod
+    def from_catalog(cls, table: pa.Table):
+        return [GreeksData.from_dict(d) for d in table.to_pylist()]
+
+    @classmethod
+    def schema(cls):
+        return pa.schema(
+            {
+                "instrument_id": pa.string(),
+                "ts_event": pa.int64(),
+                "ts_init": pa.int64(),
+                "delta": pa.float64(),
+            }
+        )
 ```
 
 ### Publishing and receiving data
@@ -178,17 +202,58 @@ def on_data(self, data):
         print("Data", data)
 ```
 
-### Writing and reading data
+### Writing and reading data using the cache
 
 Here is an example of writing and reading data using the `Cache` from an actor (which includes strategies):
 
 ```python
 def greeks_key(instrument_id: InstrumentId):
     return f"{instrument_id}_GREEKS"
 
-def cache_greeks(self, instrument_id: InstrumentId, greeks_data: GreeksData):
-    self.cache.add(greeks_key(instrument_id), greeks_data.to_bytes())
+def cache_greeks(self, greeks_data: GreeksData):
+    self.cache.add(greeks_key(greeks_data.instrument_id), greeks_data.to_bytes())
 
 def greeks_from_cache(self, instrument_id: InstrumentId):
     return GreeksData.from_bytes(self.cache.get(greeks_key(instrument_id)))
 ```
+
+### Writing and reading data using a catalog
+
+For streaming custom data to feather files or writing it to parquet files in a catalog (`register_arrow` needs to be used):
+
+```python
+register_arrow(GreeksData, GreeksData.schema(), GreeksData.to_catalog, GreeksData.from_catalog)
+
+from nautilus_trader.persistence.catalog import ParquetDataCatalog
+catalog = ParquetDataCatalog('.')
+
+catalog.write_data([GreeksData()])
+```
+
+## Creating a custom data class automatically
+
+The `@customdataclass` decorator enables the creation of a custom data class with default
+implementations for all the features described above.
+
+Each method can also be overridden if needed. Here is an example of its usage:
+
+```python
+from nautilus_trader.model.custom import customdataclass
+
+
+@customdataclass
+class GreeksTestData(Data):
+    instrument_id: InstrumentId = InstrumentId.from_str("ES.GLBX")
+    delta: float = 0.0
+
+    def __repr__(self):
+        return (f"GreeksData(instrument_id={self.instrument_id}, delta={self.delta:.2f}, ts_event={unix_nanos_to_str(self._ts_event)}, ts_init={unix_nanos_to_str(self._ts_init)})")
+
+
+GreeksTestData(
+    instrument_id=InstrumentId.from_str("CL.GLBX"),
+    delta=1000.0,
+    ts_event=1,
+    ts_init=2,
+)
+```