rakurai excanges should automatically work on sim

Too long for Windows to clone externals (at least for the Windows Server
box running jenkins)

.vscode/launch.json

@@ -183,10 +183,11 @@
       "windows": {
         "program": "PubSubServer.exe",
       },
+      "args": [
+        "--config",
+        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
+      ],
       "cwd": "build/Debug/bin/net8.0",
-      "env": {
-        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-      },
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -213,7 +214,7 @@
       "type": "coreclr",
       "request": "launch",
       "program": "",
-      "preLaunchTask": "Build Spark Sdk (Debug)",
+      "preLaunchTask": "Build Api Server (Debug)",
       "windows": {
         "program": "Bts.Spark.Sdk.Demo.exe",
       },
@@ -285,7 +286,7 @@
       "windows": {
         "program": "Spark.exe",
         "args": [
-          "--Spark:ConfigServer",
+          "--sparkd",
           "${input:sparkConfigServer}"
         ],
         "cwd": "build/Debug/bin/net8.0-windows",
@@ -448,15 +449,14 @@
       "default": "Local",
     },
     {
-      "id": "pubSubServerEnvironment",
+      "id": "pubSubServerConfig",
       "type": "pickString",
-      "description": "PubSubServer Configuration",
+      "description": "PubSub Configuration",
       "options": [
-        "Development",
-        "EchoServer",
-        "Local",
+        "PubSubServer",
+        "SimpleEchoClient",
       ],
-      "default": "Development",
+      "default": "PubSubServer",
     },
     {
       "id": "sdkDemoEnvironment",

Spark.Api.Server.sln

@@ -51,6 +51,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\B
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{BC277499-8215-4F9F-B9AB-07B6391110BF}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "client", "client", "{70E3DCF5-A3C3-45A0-8F18-B01B2D446532}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_command_bridge", "src\client\spark_command_bridge\spark_command_bridge.csproj", "{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -571,6 +575,30 @@ Global
 		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
 		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
 		{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Debug|x64.ActiveCfg = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Debug|x64.Build.0 = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Debug|Any CPU.Build.0 = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Debug|x86.ActiveCfg = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Debug|x86.Build.0 = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.MinSizeRel|x64.ActiveCfg = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.MinSizeRel|x64.Build.0 = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.MinSizeRel|Any CPU.ActiveCfg = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.MinSizeRel|Any CPU.Build.0 = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.MinSizeRel|x86.ActiveCfg = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.MinSizeRel|x86.Build.0 = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Release|x64.ActiveCfg = Release|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Release|x64.Build.0 = Release|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Release|Any CPU.ActiveCfg = Release|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Release|Any CPU.Build.0 = Release|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Release|x86.ActiveCfg = Release|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Release|x86.Build.0 = Release|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.RelWithDebInfo|x64.Build.0 = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.RelWithDebInfo|Any CPU.Build.0 = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.RelWithDebInfo|x86.ActiveCfg = Debug|x64
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.RelWithDebInfo|x86.Build.0 = Debug|x64
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -592,6 +620,8 @@ Global
 		{385C4948-E080-56A6-E29E-8B0F4913175C} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
 		{43FB297B-7A4D-4835-A878-E6DF345B580C} = {385C4948-E080-56A6-E29E-8B0F4913175C}
 		{BC277499-8215-4F9F-B9AB-07B6391110BF} = {385C4948-E080-56A6-E29E-8B0F4913175C}
+		{70E3DCF5-A3C3-45A0-8F18-B01B2D446532} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
+		{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED} = {70E3DCF5-A3C3-45A0-8F18-B01B2D446532}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}

cmake/InitPythonModule.cmake

@@ -40,7 +40,13 @@ function(init_python_module)
   add_custom_command(
     OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
     DEPENDS ${_name} ${module_proj}
-    COMMAND PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR} pybind11-stubgen --numpy-array-use-type-var --ignore-invalid-expressions '.*' ${_name}_module
+    COMMAND
+      ${CMAKE_COMMAND} -E env
+      "PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}"
+      ${Python3_EXECUTABLE} -m pybind11_stubgen
+      --numpy-array-use-type-var
+      --ignore-invalid-expressions '.*'
+      ${_name}_module
     COMMENT "Generating py stubs"
   )
   add_custom_target(

compose.yaml

@@ -19,6 +19,9 @@ services:
       - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
       - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
       - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+      - 8008:8008
+      - 8080:8080
+      - 9443:9443
     entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
     deploy:
       resources:
@@ -72,6 +75,9 @@ services:
       - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
       - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
       - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+      - 8008:8008
+      - 8080:8080
+      - 9443:9443
     entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
     deploy:
       restart_policy:
@@ -81,8 +87,6 @@ services:
         limits:
           memory: "${SPARKD__MEMORY:-4gb}"
   sparkd-multicast:
-    environment:
-      - TZ=America/Chicago
     volumes:
       - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
       - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
@@ -95,7 +99,7 @@ services:
         - CONFIG_DIR=${SPARKD__CONFIG_DIR}
         - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
         - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
-    shm_size: "${SPARKD__SHM_SIZE:-8192mb}"
+    shm_size: "${SPARKD__SHM_SIZE:-2048mb}"
     env_file:
       - .env
       - ${SPARKD__CONFIG_DIR}/.env
@@ -107,38 +111,7 @@ services:
         delay: 3s
       resources:
         limits:
-          memory: "${SPARKD__MEMORY:-16gb}"
-  
-  
-  sparkd-multicast-cert:
-    environment:
-      - TZ=America/Chicago
-    volumes:
-      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
-      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
-      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
-    build:
-      dockerfile: src/app_sparkd/Dockerfile
-      args:
-        - IMAGE=${SPARKD__IMAGE}
-        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
-        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
-        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
-        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
-    shm_size: "${SPARKD__SHM_SIZE:-8192mb}"
-    env_file:
-      - .env
-      - ${SPARKD__CONFIG_DIR}/.env
-    network_mode: "host"
-    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
-    deploy:
-      restart_policy:
-        condition: any
-        delay: 3s
-      resources:
-        limits:
-          memory: "${SPARKD__MEMORY:-16gb}"
-  
+          memory: "${SPARKD__MEMORY:-4gb}"
   sparkd-tail:
     image: bluetradingsystems/linux_dotnet8
     volumes:

doc/sparkd-security-id-lifecycle.md

@@ -1,942 +0,0 @@
-# Security ID handling in `sparkd`
-
-This document traces how security identifiers are created, translated, persisted,
-indexed, and published by the `sparkd` market-data paths. It focuses on the
-legacy CME MDP3/OnixS pipeline and the newer generalized market-data pipeline.
-
-## Executive summary
-
-`sparkd` uses two identifiers that must not be confused:
-
-| Identifier | Meaning | Typical field | Lifetime |
-|---|---|---|---|
-| Exchange/feed ID | The venue's identifier used to route incoming definitions, books, and trades | `FixedSecurity::numeric_id`, `ExchangeKey`, or `domain::Security::security_common.exchange_sec_id` | Feed/session-specific unless the source guarantees stability |
-| Spark security key | Spark's internal non-zero identifier used in shared memory, APIs, orders, positions, and downstream services | `FixedSecurity::key`, `api::Security::id()`, `SecurityKey`; historically also called `security_id` | Intended to be stable across restarts |
-
-`invalid_security_key` is `0`; a security with key `0` is not yet publicly
-discoverable through the shared-memory consumer path.
-
-The two principal paths differ:
-
-- **MDP3/OnixS:** the definition callback sets only the CME ID in
-  `numeric_id`. `MDProducer::add_security()` allocates the shared-memory entry
-  with key `0`. `SecurityIDRecorder`, driven by a 200 ms dispatcher timer,
-  resolves or inserts a stable key in `key_ids.sqlite`; its callback finally
-  calls `MDProducerHandle::set_key()`.
-- **Generalized/newer pipeline:** `GeneralMarketDataListener::on_securities()`
-  currently uses the ID already present on the incoming domain security
-  (`use_given_sec_key == true`). Live adapters normally obtain that ID
-  synchronously from `IDGenerator` first. `_copy_security_common()` then copies
-  it to both `FixedSecurity::key` and `numeric_id`, so
-  `MDProducer::add_security()` publishes it synchronously. Only the listener's
-  own fallback `IDGenerator` call is bypassed by the current constant.
-
-## End-to-end diagrams
-
-### CME MDP3 / OnixS
-
-```mermaid
-sequenceDiagram
-    participant FE as OnixS feed-engine thread
-    participant L as MDP3Listener
-    participant P as MDProducer / shared memory
-    participant Q as SecurityIDRecorder queues
-    participant D as sparkd dispatcher thread
-    participant DB as key_ids.sqlite
-
-    FE->>L: instrument definition (CME SecurityID)
-    L->>L: set_security_common() - numeric_id = CME SecurityID, key remains 0
-    L->>P: add_security(FixedSecurity)
-    P->>P: allocate hot + cold entries with key 0
-    P->>Q: ensure_key(handle)
-    L->>L: index[CME SecurityID] = handle
-    Note over FE,L: Books can route by CME ID while the Spark key is pending
-    D->>Q: every 200 ms - work_spreads(), work_handles(), work_db()
-    Q->>DB: find (namespace, descriptor), or insert a new row
-    DB-->>Q: SQLite rowid = Spark security key
-    Q->>P: handle.set_key(rowid)
-    P->>P: hot.key first, then cold security.key, then increment security_count
-    Note over P: The security is now public to shared-memory consumers
-```
-
-### Generalized/newer pipeline (current behavior)
-
-```mermaid
-sequenceDiagram
-    participant A as Venue adapter / plugin
-    participant DB as IDGenerator / key_ids.sqlite
-    participant G as GeneralMarketDataListener
-    participant I as GeneralSecurityIndex
-    participant P as MDProducer / shared memory
-
-    A->>DB: get_or_add(venue, type, adapter key)
-    DB-->>A: SQLite rowid as domain SecurityID
-    A->>G: on_securities(domain Security list with ID)
-    G->>G: _find_sec() - use incoming security_common.id
-    G->>G: _copy_security_common() - key = id, numeric_id = id
-    G->>P: add_security(FixedSecurity with non-zero key)
-    P->>P: set_key() during add_security()
-    G->>I: index[id] = handle
-    Note over G,P: No asynchronous ID assignment in the current configuration
-```
-
-## Data-flow charts
-
-The sequence diagrams above emphasize timing and thread transitions. These
-flowcharts emphasize which identifier is carried through each representation
-and where data is retained.
-
-### MDP3 / legacy data flow
-
-```mermaid
-flowchart TD
-    CME["CME MDP3 packets<br/>CME SecurityID"] --> ONIXS["OnixS feed engine<br/>decode definition and updates"]
-
-    ONIXS --> DEF["MDP3Listener definition callback"]
-    DEF --> FIXED["FixedSecurity<br/>numeric_id = CME SecurityID<br/>key = 0"]
-
-    FIXED --> PRODUCER["MDProducer::add_security"]
-    PRODUCER --> PENDING["Hot and cold shared memory<br/>allocated but not public<br/>key = 0"]
-    PRODUCER --> QUEUE["SecurityIDRecorder<br/>pending_handle queue"]
-
-    FIXED --> CMEINDEX["CMESecurityIndex<br/>CME SecurityID to handle"]
-
-    QUEUE --> TIMER["Dispatcher timer<br/>every 200 ms"]
-    TIMER --> LEGS{"Spread legs<br/>all resolved?"}
-    LEGS -- No --> UNRESOLVED["unresolved_legs<br/>exchange IDs awaiting keys"]
-    UNRESOLVED --> CMEINDEX
-    CMEINDEX --> LEGS
-    LEGS -- Yes or outright --> LOGICAL["Logical identity<br/>venue + type + security_desc"]
-
-    LOGICAL --> SELECT["SQLite SELECT<br/>key_ids.sqlite"]
-    SELECT -- Existing row --> ROWID["SQLite rowid<br/>Spark security key"]
-    SELECT -- Missing row --> INSERT["Batched SQLite INSERT"]
-    INSERT --> ROWID
-
-    ROWID --> SETKEY["MDProducerHandle::set_key"]
-    SETKEY --> HOT["Write hot-entry key"]
-    HOT --> COLD["Write cold security key"]
-    COLD --> COUNT["Increment security_count"]
-    COUNT --> PUBLIC["Public security<br/>visible to MDConsumer"]
-
-    ONIXS --> UPDATE["Book, trade, status, or RFQ<br/>carrying CME SecurityID"]
-    UPDATE --> CMEINDEX
-    CMEINDEX --> HANDLE["MDProducerHandle"]
-    HANDLE --> LIVE["Update live shared-memory data"]
-    PUBLIC --> LIVE
-```
-
-The CME index is usable before the Spark key is assigned, so feed updates can
-reach the internal handle while the security is still hidden from consumers.
-The persistent rowid crosses the publication boundary only at `set_key()`.
-
-### Generalized/newer data flow
-
-```mermaid
-flowchart TD
-    FEED["Venue feed, Databento,<br/>Python plugin, or replay"] --> ADAPTER["Source adapter<br/>build domain::Security"]
-
-    ADAPTER --> IDTYPE{"Live definition or<br/>recorded/pre-keyed input?"}
-    IDTYPE -- Live --> IDINPUT["Logical identity<br/>venue + type + adapter-chosen name"]
-    IDINPUT --> MUTEX["IDGenerator global mutex"]
-    MUTEX --> SELECT["Synchronous SQLite SELECT<br/>key_ids.sqlite"]
-    SELECT -- Existing row --> ROWID["SQLite rowid"]
-    SELECT -- Missing row --> INSERT["Synchronous SQLite INSERT<br/>autocommit"]
-    INSERT --> ROWID
-    ROWID --> DOMAINID["domain::SecurityID<br/>used as Spark ID"]
-    IDTYPE -- Recorded or pre-keyed --> DOMAINID
-
-    DOMAINID --> DOMAINSEC["SecurityCommon::id = Spark ID"]
-    DOMAINSEC --> LISTENER["GeneralMarketDataListener::on_securities"]
-    LISTENER --> CCMAP["cc_sec_id_map<br/>incoming ID to Spark ID"]
-    LISTENER --> CONVERT["_copy_security_common<br/>FixedSecurity.key = ID<br/>numeric_id = ID"]
-
-    CONVERT --> PRODUCER["MDProducer::add_security"]
-    PRODUCER --> SETKEY["Immediate set_key<br/>no recorder timer"]
-    SETKEY --> SHM["Hot and cold shared memory<br/>immediately public"]
-    CONVERT --> GENERALINDEX["GeneralSecurityIndex<br/>ID to handle"]
-
-    FEED --> EVENT["Book, trade, or statistics<br/>with exchange/source identity"]
-    EVENT --> EVENTMAP["Adapter map<br/>source identity to domain::SecurityID"]
-    EVENTMAP --> EVENTID["Event carries domain::SecurityID"]
-    EVENT --> PYHOT["Python bridge only<br/>synchronous IDGenerator lookup"]
-    PYHOT --> EVENTID
-    PYHOT -. uses same mutex and SQLite path .-> MUTEX
-    EVENTID --> CCMAP
-    CCMAP --> GENERALINDEX
-    GENERALINDEX --> HANDLE["MDProducerHandle"]
-    HANDLE --> SHMUPDATE["Update book, trade,<br/>statistics, or status"]
-    SHM --> SHMUPDATE
-
-```
-
-In the normal newer live path, persistence occurs before the definition enters
-the generalized listener. The DB call therefore adds synchronous adapter-side
-latency, but publication needs no later timer. Most adapters use an in-memory
-map for subsequent events; the dotted Python path is the documented exception.
-
-## The publication boundary
-
-`MDProducerHandle::set_key()` is the definitive publication point for the
-legacy shared-memory representation. Its write ordering is intentional:
-
-1. Write the key into the hot book entry (`HotEntry` or `HotDepthEntry`).
-2. Write the key into the cold security entry.
-3. Increment the cold header's public `security_count`.
-
-The cold security key is written last because consumers treat a non-zero cold
-key as visibility. At that moment the corresponding hot entry must already be
-addressable. `set_key()` only permits the transition from `0` to one key; it is
-not a key-update mechanism.
-
-## Identifier representations
-
-The names are historically inconsistent. In particular, many protobuf fields
-named `security_id` contain the **Spark key**, not the exchange's ID.
-
-| Layer | Exchange/feed identifier | Spark identifier |
-|---|---|---|
-| MDP3 wire/OnixS | `securityId()` | not present |
-| Legacy fixed security | `FixedSecurity::numeric_id` (`uint64_t`) | `FixedSecurity::key` / `security_key` (`uint32_t`) |
-| Legacy API security | venue-specific `exchange_id` | top-level `Security.id()` |
-| New domain model | `SecurityCommon::exchange_sec_id` (string) is descriptive; `SecurityCommon::id` is also used to correlate source events | `domain::SecurityID` (`uint64_t`), currently copied into the Spark key |
-| Shared-memory hot book | none | `HotEntry::key` or `HotDepthEntry::key` |
-| Shared-memory cold metadata | `FixedSecurity::numeric_id` | `FixedSecurity::key` |
-| CME runtime lookup | `CMESecurityIndex` key | `SecurityData::handle.key()` |
-| General runtime lookup | incoming/canonical `domain::SecurityID` | `GeneralSecurityIndex` key and `handle.key()` (currently the same value) |
-
-The generalized model declares `domain::SecurityID` as 64-bit, but the legacy
-shared-memory `security_key` remains 32-bit. `_copy_security_common()` assigns a
-64-bit value to both `numeric_id` and the 32-bit `key`. This is safe only while
-allocated IDs fit in 32 bits; unlike the legacy lookup path, this assignment has
-no explicit checked conversion.
-
-## Which parts use exchange IDs, descriptors, or Spark security IDs?
-
-The three identifier classes serve different jobs:
-
-- **Exchange/external ID:** routes data inside a venue or vendor feed adapter.
-- **Descriptor/natural key:** recognizes a definition logically at the
-  persistence, discovery, or human-facing boundary.
-- **Spark security ID/key:** joins the instrument to books, orders, fills,
-  positions, and services after publication.
-
-The core downstream system does not repeatedly identify a security from its
-description. The description is primarily used to obtain a Spark key; after
-that boundary, most joins use the numeric Spark key.
-
-### Component-level usage matrix
-
-| Component or data path | Exchange/external ID | Descriptor or name | Spark security ID/key |
-|---|---|---|---|
-| CME MDP3 definition decoder | Reads `securityId()` and stores it in `FixedSecurity::numeric_id` | Copies definition symbol/tag 55 into `security_desc` | Leaves `FixedSecurity::key` at `0` initially |
-| `CMESecurityIndex` | Primary map key; CME ID resolves to a handle | Not used for update routing | Returns `handle.key()` when the Spark key has been assigned |
-| MDP3 books, trades, status, statistics, and RFQs | Incoming CME ID selects the handle through `CMESecurityIndex` | Not used to locate each update | The selected handle writes keyed shared-memory/trade data after publication |
-| MDP3 spread definitions | Each leg initially contains `legSecurityId()` as `exchange_key` | Parent `security_desc` identifies the spread's DB row | Each leg's Spark key and then the parent Spark key must be resolved before publication |
-| Legacy ICE OnixS | Uses ICE `marketId` as `numeric_id` and index key | ICE security description becomes the logical lookup name | Uses the same recorder-assigned Spark key pattern as MDP3 |
-| `SecurityIDRecorder` | Does not allocate from the external ID | Uses `(venue, security type, securities::descriptor())` to find/create a row | Treats the SQLite rowid as the resulting Spark key |
-| Native generalized adapters | Use exchange strings/numeric IDs to parse definitions and correlate updates | Usually pass a description or exchange instrument name to `IDGenerator` | Put the returned ID into `SecurityCommon::id` and later events |
-| Databento adapter | Uses `msg.hd.instrument_id` for feed correlation and its in-memory map | Uses the decimal instrument ID string—not the human description—as the `IDGenerator` name | Caches `instrument_id -> domain::SecurityID` and places that ID on books/trades |
-| Python bridge | Reads `exchange_security_id` from definitions, books, and trades | Uses that external-ID string as the `IDGenerator` name | Resolves it to `domain::SecurityID`; currently performs this lookup for every event |
-| `IDGenerator` | Does not interpret whether `name` came from an exchange ID | Uses exactly `(venue, type, caller-supplied name)` | Returns the SQLite rowid as a 64-bit ID |
-| `GeneralMarketDataListener` | Does not route on `exchange_sec_id` directly | Copies `sec.desc()` into fixed metadata; its disabled fallback allocator would use `sec.desc()` | Uses incoming `SecurityCommon::id`, maps events through `cc_sec_id_map`, and copies the value into both `key` and `numeric_id` |
-| `GeneralSecurityIndex` | No separate external-ID concept in the current use-given path | Not used | Maps the generalized ID/Spark key to `MDProducerHandle` |
-| `MDProducer` / shared memory | Retains `numeric_id`; in legacy feeds this is the exchange ID, but in the generalized path it is currently the Spark ID | Stores `security_desc` in the cold definition | Uses the key in hot/cold entries; non-zero key is the visibility boundary |
-| `MDConsumer` | Not normally used to find live books | Can enumerate/filter converted security definitions | Finds handles and book data by `security_key` |
-| `SecurityCache` / `securities.sqlite` | Preserves venue-specific exchange data inside the serialized security blob | Maintains descriptor/name lookup indexes and supports venue/type/name queries | Primary persisted definition key is `api::Security.id()` |
-| MBO processor and MBO service | May retain exchange **order** IDs, which are separate from security exchange IDs | Not used to find the instrument | Indexes snapshots and requests by `SecurityKey`; MBO protobuf `security_id` means Spark key |
-| Order manager and algorithms | Execution gateways eventually use venue-specific fields from the resolved security to encode orders | Descriptions appear in logs and errors, not as the primary join | Order definitions, subscriptions, book events, native orders, spread legs, and fills use `security_key` / `Security.id()` |
-| Position, PNL, and risk services | Not used as their position join | May resolve metadata for display/reporting | Positions, fills, overrides, and PNL handles are keyed by Spark security ID |
-| Relay and market-data recordings | Serialized security definitions may preserve venue exchange IDs | Preserve full descriptors in recorded definitions | Books, updates, trades, and replay correlation preserve the recorded Spark key |
-| Synthetic security service | No venue exchange ID; synthetics use `VENUE_BTS` | Synthetic name is the recorder's logical descriptor | Recorder assigns the SQLite rowid and synthetic metadata is keyed by it |
-
-### Exchange/external IDs: where they stop
-
-In MDP3 the separation is explicit:
-
-```text
-CME SecurityID
-    -> FixedSecurity.numeric_id
-    -> CMESecurityIndex lookup
-    -> MDProducerHandle
-    -> Spark security_key for downstream publication
-```
-
-The external CME ID continues to route feed messages but does not become the
-order/position key. This permits the same downstream Spark key to survive a
-restart as long as the logical descriptor mapping remains in
-`key_ids.sqlite`, even if the feed must rediscover the CME routing ID.
-
-The generalized path collapses more of these roles:
-
-```text
-exchange/vendor identity
-    -> adapter calls IDGenerator
-    -> domain::SecurityID
-    -> FixedSecurity.key and FixedSecurity.numeric_id
-    -> GeneralSecurityIndex and downstream Spark key
-```
-
-`SecurityCommon::exchange_sec_id` may still contain the original textual
-external identifier, but `GeneralMarketDataListener` does not use it to route
-events and does not put it in `numeric_id`. The adapter must translate later
-events to `domain::SecurityID` before invoking the listener.
-
-### Descriptor/name users
-
-Descriptors are used in three main places:
-
-1. **Key allocation:** `SecurityIDRecorder` uses the legacy venue-specific
-   descriptor, while `IDGenerator` uses its caller-provided `name`.
-2. **Definition lookup and discovery:** `SecurityCache` supports
-   venue/type/name lookup; symbol filters and administrative tools use security
-   metadata to select or display instruments.
-3. **Diagnostics:** logs include symbols and descriptions so humans can
-   understand a numeric key.
-
-Descriptions generally do not drive the high-rate book, trade, order, fill, or
-position paths. Those paths use numeric IDs after definition resolution.
-
-It is important that “descriptor” does not mean one universal canonical field:
-
-- Legacy CME uses Globex `security_desc` from tag 55.
-- Many generalized adapters use `sec.desc()` or an exchange instrument name.
-- Databento uses its numeric instrument ID converted to a string.
-- Python uses `exchange_security_id`.
-
-Therefore, two adapters may describe the same economic contract but request
-different logical rows and receive different Spark IDs.
-
-### Spark security-ID users
-
-Once a security is published, its Spark key is the dominant system identifier:
-
-```mermaid
-flowchart LR
-    KEY["Spark security key"] --> SHM["Hot/cold shared memory"]
-    KEY --> MDC["MDConsumer and book subscriptions"]
-    KEY --> MBO["MBO snapshots and queries"]
-    KEY --> OMS["Orders, algorithms, and spread legs"]
-    KEY --> FILLS["Fills and execution records"]
-    KEY --> POS["Positions, PNL, and risk"]
-    KEY --> API["Services and client APIs"]
-    KEY --> REC["Relay and market-data recordings"]
-    KEY --> SECDB["Serialized security definitions"]
-```
-
-Fields named `security_id`, `security_key`, or top-level `Security.id()` in
-these downstream areas almost always mean the Spark key. They should not be
-compared directly with CME `SecurityID`, Databento `instrument_id`, or another
-venue/vendor identifier without going through the relevant adapter/index
-mapping.
-
-## Persistence and allocation
-
-### The authoritative key mapping: `key_ids.sqlite`
-
-Both allocators use `SqliteStore("key_ids")`. With the normal configuration the
-file is:
-
-```text
-<SystemConfig.data_path>/databases/key_ids/key_ids.sqlite
-```
-
-The database contains one table and two indexes:
-
-```sql
-CREATE TABLE store (ns TEXT, key TEXT, value TEXT);
-CREATE UNIQUE INDEX key_index ON store(ns, key);
-CREATE INDEX ns_index ON store(ns);
-```
-
-The SQLite `rowid` is the generated Spark key. `ns` is normally
-`<VENUE>.<SECURITY_TYPE>` (for example `GLBX.FUTURE`), and `key` is the security
-descriptor chosen by the caller. `value` is normally empty. Therefore ID
-stability depends on all three of these remaining stable:
-
-- the configured `data_path` and preservation of this database;
-- the venue/type namespace;
-- the descriptor supplied by the feed adapter.
-
-The two allocator implementations share the file and convention:
-
-- `SecurityIDRecorder` (legacy): derives the namespace from legacy API enums and
-  uses `securities::descriptor()`. For CME this is the Globex
-  `security_desc`, populated from the definition's symbol/tag 55.
-- `IDGenerator` (newer): derives the namespace from new domain enums and uses
-  the caller's `name`. Most adapters pass `sec.desc()` or an exchange instrument
-  identifier; Databento deliberately passes the decimal instrument ID, and the
-  Python bridge uses the exchange security ID.
-
-### Logical-row examples
-
-The following names are illustrative. Production uses the exact string emitted
-by the selected feed/adapter, including its case, spacing, and weekly-series
-notation.
-
-#### Legacy `SecurityIDRecorder`
-
-For CME, `set_security_common()` copies the MDP3 definition's symbol/tag 55 into
-Globex `security_desc`. After converting the fixed security to
-`api::Security`, `securities::descriptor()` returns that same
-`glbx.security_desc()` string.
-
-| Instrument | Legacy enums | Example descriptor from `security_desc` | Row searched/inserted in `store` |
-|---|---|---|---|
-| Future | `VENUE_GLBX`, `SECURITYTYPE_FUTURE` | `ESZ6` | `ns='GLBX.FUTURE', key='ESZ6'` |
-| Standard option | `VENUE_GLBX`, `SECURITYTYPE_OPTION` | `ESZ6 C6500` | `ns='GLBX.OPTION', key='ESZ6 C6500'` |
-| Weekly option | `VENUE_GLBX`, `SECURITYTYPE_OPTION` | `ESZ6-W1 C6500` | `ns='GLBX.OPTION', key='ESZ6-W1 C6500'` |
-| Spot from a legacy-compatible venue path | `VENUE_COINBASE`, `SECURITYTYPE_SPOT` | `BTC-USD` | `ns='COINBASE.SPOT', key='BTC-USD'` |
-
-A weekly option does not get a special namespace. Standard and weekly options
-are both `SECURITYTYPE_OPTION`, so both live in `GLBX.OPTION`; the exact
-`security_desc` distinguishes the weekly series. For example, if the database
-assigned rowids `42001` through `42004` to the rows above, those integers—not
-the descriptor strings—would be the Spark security keys.
-
-Conceptually, legacy allocation is:
-
-```text
-future:        lookup_or_insert("GLBX.FUTURE",  "ESZ6")
-option:        lookup_or_insert("GLBX.OPTION",  "ESZ6 C6500")
-weekly option: lookup_or_insert("GLBX.OPTION",  "ESZ6-W1 C6500")
-spot:          lookup_or_insert("COINBASE.SPOT", "BTC-USD")
-```
-
-#### Newer `IDGenerator`
-
-The namespace result is similar, but the third argument is chosen by the
-adapter. Examples using a description-oriented adapter are:
-
-```cpp
-get_or_add(VENUE_GLBX, SECURITY_TYPE_FUTURE, "ESZ6");
-// store: ns="GLBX.FUTURE", key="ESZ6"
-
-get_or_add(VENUE_GLBX, SECURITY_TYPE_OPTION, "ESZ6 C6500");
-// store: ns="GLBX.OPTION", key="ESZ6 C6500"
-
-get_or_add(VENUE_GLBX, SECURITY_TYPE_OPTION, "ESZ6-W1 C6500");
-// store: ns="GLBX.OPTION", key="ESZ6-W1 C6500"
-
-get_or_add(VENUE_COINBASE, SECURITY_TYPE_SPOT, "BTC-USD");
-// store: ns="COINBASE.SPOT", key="BTC-USD"
-```
-
-Databento intentionally produces a different logical row even when the human
-description is `ESZ6`. If its instrument ID were `12345678`, it would call the
-equivalent of:
-
-```cpp
-get_or_add(VENUE_GLBX, SECURITY_TYPE_FUTURE, "12345678");
-// store: ns="GLBX.FUTURE", key="12345678"
-```
-
-Likewise, the Python bridge uses `exchange_security_id`. A spot definition with
-`exchange_security_id="BTC-USD"` produces the `COINBASE.SPOT / BTC-USD` row,
-while a provider that sends `exchange_security_id="BTCUSD"` produces a
-different row and therefore a different Spark key.
-
-These examples show why the tuple is only logical when the adapter's naming
-contract is also known. `GLBX.FUTURE / ESZ6` and `GLBX.FUTURE / 12345678` may
-describe the same contract to a human but are distinct rows to `key_ids.sqlite`.
-
-Both enum-prefix removals currently produce the same namespace spelling (for
-example, `GLBX.OPTION`), so both mechanisms allocate from one global SQLite
-rowid space. Live rows cannot share a rowid. The table does not declare
-`AUTOINCREMENT`, however, so deleting rows—especially the highest row—can allow
-SQLite to reuse a number later. Security-key rows should therefore be treated
-as append-only operational data.
-
-`SqliteStore` uses WAL mode and `PRAGMA synchronous = 0`. This favors startup
-and feed latency over maximum durability against a host/power failure. The
-unique `(ns, key)` index prevents two different rows for the same logical
-security.
-
-### Other places the key is stored
-
-| Location | Purpose | Authoritative for allocation? |
-|---|---|---|
-| Hot and cold shared-memory segments | Live publication and fast consumer lookup | No; rebuilt at process startup |
-| `CMESecurityIndex` / `GeneralSecurityIndex` and adapter maps | In-process routing from an incoming ID to a producer handle | No; in-memory only |
-| `<data_path>/databases/md/securities.sqlite` | Serialized legacy `api::Security` definitions keyed by Spark key; used by security cache and MDP3 Sunday preload | No; it stores metadata after a key exists |
-| Relay/market-data recordings | Preserve the key embedded in recorded securities, books, and trades for deterministic replay | No; replay normally reuses the recorded value |
-| Orders, fills, positions, APIs | Reference a security by the published Spark key | No |
-
-The `SecurityIDRecorder::work_db()` comment says a published security is also
-recorded in the security database, but that function does **not** write
-`securities.sqlite` in the current code. Full security definitions are persisted
-separately by `SecurityCache::flush_to_disk()` through
-`db::insert_or_replace()`.
-
-## CME MDP3 / OnixS pipeline in detail
-
-### 1. Construction and thread setup
-
-`MDP3Main::PImpl` owns one `CMESecurityIndex`, one `SecurityIDRecorder`, and one
-`MDProducer`. It constructs the recorder with:
-
-- a lookup callback that maps `ExchangeKey` through the CME index to the
-  handle's current Spark key; and
-- `Dispatcher::current()`, which drives a 200 ms recorder timer.
-
-In the current `MarketDataGateway` route, `OnixsGlbxSubscription` creates a
-dedicated `dispatcher_thread`, constructs and starts `MDP3Main` on that thread,
-then calls that thread's dispatcher `run()`. The older `md run onixs` command
-uses its command/dispatcher thread similarly.
-
-The OnixS feed engine has its own configured thread pool. Instrument and market
-data callbacks therefore originate on OnixS feed-engine threads, while recorder
-database work and final key assignment execute on the MDP3 dispatcher thread.
-
-| MDP3 stage | Executing thread | DB call? |
-|---|---|---|
-| Decode definition and set CME `numeric_id` | OnixS feed-engine callback thread | No |
-| Allocate hot/cold entry and queue handle | Same OnixS callback thread | No |
-| Add CME-ID index entry | Same OnixS callback thread | No |
-| Resolve spread legs | MDP3 dispatcher thread, on 200 ms timer | No |
-| Select/insert Spark key | MDP3 dispatcher thread, on 200 ms timer | Yes |
-| `handle.set_key()` and public visibility | MDP3 dispatcher thread, in DB callback | No additional call |
-
-### 2. Definition callback: exchange ID is set, Spark key is not
-
-For futures, options, and spreads, `MDP3Listener::set_security_common()` does:
-
-```cpp
-fsec.numeric_id = def.securityId();
-```
-
-It also copies symbol/descriptor/venue and pricing fields. A newly constructed
-`FixedSecurity` already has `key == invalid_security_key` (`0`), and this helper
-does not change it.
-
-For a spread, every leg starts with:
-
-```cpp
-leg.exchange_key = ExchangeKey(definition_leg.legSecurityId());
-leg.key = invalid_security_key;
-```
-
-Thus both the parent and legs initially carry exchange-routing identity, but no
-Spark identity.
-
-### 3. Shared-memory allocation and exchange indexing
-
-`MDP3Listener::add_security()` performs these operations in order:
-
-1. `_producer.add_security(security)` allocates a hot entry and a variable-sized
-   cold entry and copies the security into the cold segment.
-2. Because the key is `0`, `MDProducer` queues the handle through
-   `SecurityIDRecorder::ensure_key()`.
-3. `_index.add_security(security.numeric_id, handle)` immediately creates the
-   CME-ID-to-handle mapping.
-4. The initial exchange status is written through the handle.
-
-This ordering means incoming books can already be routed by CME ID while the
-Spark key is pending. Shared-memory consumers cannot discover the security yet.
-Also, `MDProducerHandle::add_trade()` only appends to the public trade queue
-when `key() != 0`, so a trade received in this short window updates the cold
-last-trade state but is not added to the keyed trade log.
-
-### 4. Recorder tick: spreads, handles, then database
-
-Every 200 ms the recorder calls, in this exact order:
-
-1. `work_spreads()` tries to fill previously unresolved spread leg keys by
-   asking `CMESecurityIndex` for each leg exchange ID.
-2. `work_handles()` drains new handles. Outrights are converted to an API
-   security and queued for DB lookup immediately. A spread without all leg keys
-   is retained in `pending_handle` and its leg pointers are registered in
-   `unresolved_legs`.
-3. `work_db()` resolves queued `(venue, type, descriptor)` requests.
-
-Because spread work precedes new-handle work, a newly seen unresolved spread
-normally waits until at least the next timer tick to resolve legs. It waits
-longer if any outright leg has not yet received its Spark key. A spread's own
-key is not requested until every leg key is non-zero.
-
-### 5. Lookup or generation
-
-`work_db()` first queries `(ns, key)`:
-
-- Existing row: use its `rowid` as the key and invoke the callback.
-- Missing row: batch it into `insert_many_resolve_ids()`. SQLite assigns a new
-  rowid inside a transaction, the row object receives that ID, and the callback
-  runs after the transaction completes.
-
-New inserts are throttled to roughly 200 per timer pass (the current `> 200`
-check can admit 201) to avoid delaying heartbeats during startup. The queues are
-TBB concurrent queues, but all SQLite access is serialized onto the recorder's
-dispatcher thread. `check_access()` asserts this single-thread rule after the
-first work pass records the dispatcher thread ID.
-
-### 6. Final assignment and visibility
-
-The queued callback captures a copy of `MDProducerHandle` and calls
-`handle.set_key(id)` on the dispatcher thread. `set_key()` writes hot, then cold,
-then increments `security_count`. From this point:
-
-- shared-memory scans report the security;
-- `CMESecurityIndex::get_key(CME_ID)` returns the Spark key;
-- trades and RFQs can be emitted with a non-zero key;
-- resolved spreads can copy this key into their leg structures.
-
-### 7. Sunday preload is a fast path
-
-For configured symbols on Sunday, `MDP3Main::prepare_listener()` loads stored
-definitions from `securities.sqlite`. It converts them with
-`set_security_key=true`, so both the security key and spread leg keys come from
-the serialized API security. `MDProducer::add_security()` calls `set_key()`
-immediately, and no recorder lookup is required for that preloaded handle. The
-CME exchange ID is still restored into `numeric_id`, allowing the normal CME
-index to be built.
-
-## Generalized/newer pipeline in detail
-
-### 1. Topology and ownership
-
-`Run::run()` starts a detached thread named `MarketDataGateway`. On that thread,
-`run_md_gwy()` obtains the process-wide `IDGenerator`, constructs
-`MarketDataGateway`, and starts the configured providers. For each `(venue,
-source)` returned by a provider, the gateway creates an `MdSink` containing:
-
-- a venue-local `GeneralSecurityIndex`;
-- a default `MDProducer` (not wired to a `SecurityIDRecorder`); and
-- a `GeneralMarketDataListener` connecting the source to that producer/index.
-
-The source decides its callback thread. Some sources call the listener while
-starting; others own network or replay threads. The generalized listener calls
-the producer inline on that source thread. The process-wide `IDGenerator`
-serializes its own calls with a mutex, but the listener's transient
-`cc_sec_id_map` is an ordinary `unordered_map`, so each source/listener pair is
-expected to serialize security/event callbacks.
-
-| Generalized stage | Executing thread | DB call? |
-|---|---|---|
-| Build/parse domain definition | Source/adaptor thread | No |
-| `IDGenerator::get_or_add()` | Same source/adaptor thread | Yes, synchronously |
-| `GeneralMarketDataListener::on_securities()` | Same source callback thread | No in current use-given mode |
-| Copy ID to `FixedSecurity::key` and `numeric_id` | Same source callback thread | No |
-| `MDProducer::add_security()` / `set_key()` | Same source callback thread | No additional call |
-| Add `GeneralSecurityIndex` entry | Same source callback thread | No |
-
-There are two important compatibility exceptions inside the provider framework:
-
-- `OnixsGlbxSubscription::start()` ignores the supplied generalized listener
-  and starts `MDP3Main`; CME/OnixS therefore still follows the asynchronous
-  legacy flow documented above.
-- `EmumbaDeribitSubscription::start()` also ignores that listener and uses
-  `EmMDListener`, which allocates via `IDGenerator` and writes through its own
-  producer/index. Its assignment is still synchronous and pre-populated, so it
-  resembles the newer ID flow even though it does not pass through
-  `GeneralMarketDataListener`.
-
-### 2. How a new ID is constructed
-
-For a normal live generalized adapter, ID creation happens **in the adapter,
-before** `GeneralMarketDataListener::on_securities()`:
-
-```text
-logical identity = (domain venue, domain security type, adapter-chosen name)
-Spark ID         = SQLite rowid for that identity in key_ids.sqlite
-```
-
-`IDGenerator::get_or_add(venue, type, name)` formats a namespace by stripping
-the enum prefixes:
-
-```text
-VENUE_TAIFEX + SECURITY_TYPE_FUTURE -> TAIFEX.FUTURE
-```
-
-It then calls `retrieve_one(ns, name)`. If the row exists, it returns that
-row's `rowid`. If absent, it inserts `(ns, name, "")`, and SQLite assigns the
-next rowid. That integer is wrapped in `domain::SecurityID` and placed in
-`SecurityCommon::id`.
-
-The third identity component is adapter-specific, and this matters when trying
-to reproduce a mapping:
-
-| Source/adaptor family | Typical `name` sent to `IDGenerator` |
-|---|---|
-| Binance, Coinbase, Deribit, OKX and similar native/CCAPI paths | Usually `sec.desc()` / exchange instrument name |
-| Taifex | product/instrument ID string |
-| Databento | `std::to_string(msg.hd.instrument_id)`, not the human-readable description |
-| Python bridge | `exchange_security_id` (and it asserts Python supplied ID is zero for definitions) |
-| Emumba Deribit | extracted Emumba security descriptor |
-| Replay | no new allocation for a recorded definition; it reuses the recorded `SecurityCommon::id` |
-
-So “security name” alone is not a universal logical key. Reproduction requires
-the exact venue enum, exact security-type enum, and the exact adapter-specific
-name used at insertion time.
-
-### 3. What `GeneralMarketDataListener` does with that ID
-
-`on_securities()` first calls `_find_sec(sec, true)`. Its current, compile-time
-behavior is controlled by:
-
-```cpp
-const bool use_given_sec_key = true;
-```
-
-Consequently, for a newly seen incoming `domain::SecurityID` it does **not**
-call the DB. It copies `security_common.id.value()` into its transient
-`cc_sec_id_map`. `_add_future()`, `_add_option()`, `_add_spread()`, or
-`_add_spot()` then calls `_copy_security_common()`, which sets:
-
-```cpp
-to.key = incoming_id;
-to.numeric_id = incoming_id;
-```
-
-This collapses the generalized source/correlation ID and Spark shared-memory
-key into the same number. The source's text `exchange_sec_id` remains on the
-domain security but is not copied into `FixedSecurity::numeric_id` by this
-path.
-
-`_commit_security()` then:
-
-1. calls `MDProducer::add_security()` with an already non-zero key;
-2. causes `MDProducerHandle::set_key()` to run immediately during that call;
-3. inserts the handle in `GeneralSecurityIndex` under `numeric_id` (the same
-   number); and
-4. returns with the security already visible in shared memory.
-
-Because the sink's producer has no recorder, supplying `id == 0` would allocate
-the shared-memory entry but never assign or publish its key. Generalized live
-sources must therefore provide a valid non-zero ID before `on_securities()`.
-
-The listener contains an alternative branch for `use_given_sec_key == false`.
-That branch would call `IDGenerator::get_or_add(me->venue, sec.type(),
-sec.desc())` itself. It is currently disabled, and `_generate_security_key()` is
-also effectively unused. Turning the flag off is not a mechanical change: an
-adapter such as Databento allocates using numeric instrument ID while the
-listener branch would use `sec.desc()`, potentially creating a second key for
-the same instrument.
-
-There is also a spread failure edge case. `_find_sec(sec, true)` records the
-parent ID in `cc_sec_id_map` before `_add_spread()` resolves its legs. If any leg
-ID is absent, `_add_spread()` logs and returns without committing the spread. A
-later repeat of the parent definition is then classified as a duplicate and
-ignored, so definition ordering is a hard requirement in the current code.
-
-### 4. How later books, trades, and statistics find the security
-
-Every later domain event carries a `domain::SecurityID`:
-
-- `Book::security_id()`;
-- `TradeRecord::sec_id`; or
-- `SecurityMarketDataStatistics::security_id`.
-
-The listener first uses `cc_sec_id_map` to translate that event ID to the value
-stored in `GeneralSecurityIndex`, then obtains the `MDProducerHandle`. With
-`use_given_sec_key == true`, this mapping is numerically identity-to-identity,
-but it still serves as the admission check that the definition was seen.
-
-Most high-throughput adapters avoid SQLite on market-data events:
-
-- Databento allocates on the definition and caches
-  `instrument_id -> domain::SecurityID` in `sec_id_map` for books/trades.
-- Taifex keeps the resolved ID in its security-details mapper.
-- Other native subscriptions maintain comparable descriptor/ID maps.
-
-The Python bridge is a notable exception: `to_bts_security_()`,
-`to_bts_trade_()`, and `to_book_()` all call `get_sec_id_()`, which calls
-`IDGenerator::get_id(..., add_if_new=true)`. As written, that performs at least
-one SQLite `SELECT` for each converted Python definition, trade, or book. It
-returns the existing ID after the first insert, but it is still a DB call and
-takes the global `IDGenerator` mutex on every event.
-
-### 5. Persistence timing and latency
-
-The newer allocator is synchronous:
-
-- Existing security: mutex acquisition plus one SQLite `SELECT` on the caller's
-  thread.
-- New security: mutex acquisition, one `SELECT`, then one SQLite `INSERT`
-  (SQLite autocommit), also on the caller's thread.
-- Listener publication: no further DB call under the current
-  `use_given_sec_key` setting.
-
-The ID therefore exists in `key_ids.sqlite` before the adapter sends the
-definition to the listener. There is no intentional 200 ms publication delay,
-but the source callback/startup path directly pays DB and mutex latency. Bulk
-definition discovery performs one lookup and possibly one insert per security;
-`IDGenerator` does not batch those operations.
-
-WAL mode helps concurrency, while `synchronous=0` reduces fsync cost. It does
-not eliminate SQL parsing/stepping, mutex contention, page-cache misses, or
-occasional file-system work. The impact is usually a startup/definition cost
-when adapters cache the result, but it becomes hot-path latency for code such as
-the current Python bridge that resolves every event through the DB.
-
-`GeneralMarketDataListener` itself does not persist the full contract definition
-to `securities.sqlite`; that remains the separate security-cache/metadata
-persistence path described earlier.
-
-## Comparison: MDP3 recorder vs generalized allocator
-
-| Property | MDP3 / `SecurityIDRecorder` | Generalized / `IDGenerator` |
-|---|---|---|
-| Logical input | venue + type + legacy `security_desc` | venue + type + adapter-chosen name |
-| Numeric output | SQLite rowid, checked/read as 32-bit `security_key` | SQLite rowid returned as 64-bit `domain::SecurityID`, then narrowed into 32-bit shared-memory key |
-| Allocation point | After definition is copied to shared memory | Normally inside adapter before listener sees definition |
-| DB execution | Asynchronous on dispatcher timer | Synchronous on caller/source thread |
-| Existing ID cost | One deferred `SELECT` | One immediate `SELECT` per call |
-| New ID cost | Deferred, batched transaction (about 200/pass) | Immediate `SELECT` + individual autocommit `INSERT` |
-| Time until public | Usually 0–200+ ms; spreads may take extra ticks | Immediate after synchronous lookup/insert completes |
-| Feed callback blocking | Queue push only for ID work | Yes, if the adapter invokes `IDGenerator` there |
-| Spread legs | Parent waits asynchronously until all leg keys resolve by exchange ID | Definition processing requires every leg's incoming ID already exist in `cc_sec_id_map`; otherwise the spread is aborted |
-| Runtime ID relationship | Exchange ID and Spark key remain distinct | Incoming/canonical ID, `numeric_id`, index ID, and Spark key are currently the same number |
-| Persistence database | `key_ids.sqlite` | The same `key_ids.sqlite` |
-| Batching | Yes for new rows | No |
-| Failure behavior | Security has an internal handle with key 0 while normal queued work is pending; DB work does not block the feed callback | DB errors/zero results occur in the synchronous source path; a zero ID can leave an unpublishable entry, with no pending-key state machine |
-
-Neither algorithm generates a UUID or a content hash. Both ultimately use an
-opaque, database-assigned SQLite rowid. It is normally increasing and looks like
-a sequence, but this schema does not request SQLite `AUTOINCREMENT`. The tuple
-used to *look up* the row is logical; the resulting number is not derivable from
-that tuple.
-
-## Can another system reconstruct the same keys?
-
-Not from `(venue, type, name)` alone. Another system can reconstruct the
-**logical lookup tuple**, but it cannot calculate the numeric Spark key because
-the key depends on the historical insertion order of every row in
-`key_ids.sqlite`.
-
-For example, two empty systems can insert the exact same `GLBX.FUTURE / ESZ6`
-row and receive different IDs if other securities were inserted before it on
-one system. They are sequential opaque IDs, not deterministic hashes and not
-UUIDs.
-
-Ways to preserve cross-system identity are:
-
-1. Copy/replicate `key_ids.sqlite` (including rowids) and keep descriptor rules
-   identical. This preserves existing numeric keys but requires coordinated
-   allocation for new ones.
-2. Export an explicit mapping of `(namespace, adapter key) -> Spark key`, then
-   import rows with their original rowids before either system allocates new
-   securities.
-3. Treat a stable external identity (venue + type + exchange instrument ID, or
-   another canonical instrument identifier) as the cross-system key and carry a
-   translation table to each system's local Spark key.
-4. Introduce a deterministic ID scheme only as a versioned migration. A hash can
-   be computed independently, but collision handling, descriptor
-   canonicalization, aliases/renames, the existing 32-bit ABI, and compatibility
-   with orders/recordings/databases all need explicit design.
-
-Copying only `securities.sqlite` is insufficient as an allocator strategy. It
-contains security blobs keyed by already-assigned IDs, but `key_ids.sqlite` is
-the mapping used to find or allocate those IDs. Conversely, copying only
-`key_ids.sqlite` preserves the numbers but not all contract metadata.
-
-## Other assignment paths in `sparkd`
-
-The two main pipelines reduce to four ways a producer handle can acquire a key:
-
-1. **Already keyed input.** `MDProducer::add_security()` sees a non-zero
-   `FixedSecurity::key` and calls `set_key()` immediately. Generalized feeds,
-   Sunday preload, relay playback, and other replay paths use this mechanism.
-2. **Recorder-backed live input.** The input key is zero and the producer owns a
-   `SecurityIDRecorder`; `ensure_key()` queues asynchronous assignment. CME
-   MDP3 and legacy OnixS ICE use this mechanism. ICE uses `marketId` as
-   `numeric_id` and otherwise follows the same recorder/index design.
-3. **Service assignment.** `SyntheticSecurityService` calls
-   `SecurityIDRecorder::assign_key()` directly on a background dispatcher. The
-   callback receives a copied `api::Security` with its top-level `id` set, then
-   publishes it to synthetic subscribers. It uses the same `key_ids.sqlite`
-   rowid allocator.
-4. **Unkeyed input with no recorder.** The entry is allocated with key zero and
-   never becomes public. This is not a valid production assignment path, but it
-   is the failure mode if a generalized source omits its ID.
-
-Relay and replay deserve care when reasoning about persistence:
-
-- The generalized `MDReplayer` forwards the recorded domain security ID and
-  emits later books/statistics with the recorded `sec_key`, so the listener's
-  current use-given mode preserves the recording's numeric identity.
-- The legacy relay producer converts a recorded `api::Security` with
-  `set_security_key=true`, so its shared-memory publication also preserves the
-  embedded key.
-- The legacy relay recorder additionally calls `bulk_insert(..., "REPLAY: ")`
-  for logical mappings. Those rows use a `REPLAY: ` namespace prefix; this is a
-  separate mapping population step, not a deterministic derivation of the key.
-
-## Operational implications
-
-- Back up `key_ids.sqlite` and `securities.sqlite` together if numeric ID and
-  definition continuity matter.
-- Do not delete/rebuild `key_ids.sqlite` on one host and assume IDs will match
-  another host, even when all instruments are identical.
-- Before moving a feed between adapters, compare the exact `name` each adapter
-  supplies to `IDGenerator`. Matching venue/type is insufficient.
-- Cache generated IDs in the source after definition processing. Avoid calling
-  `IDGenerator` from book/trade hot paths; the Python bridge currently violates
-  this guideline.
-- Keep generated rowids below the 32-bit shared-memory ceiling until the
-  `security_key` ABI is widened or explicit bounds checking is added to the
-  generalized path.
-- Ensure spread legs are announced before generalized spread parents.
-- Treat `synchronous=0` as a durability decision. A copied DB should be taken
-  with SQLite-aware backup/checkpoint handling rather than by casually copying
-  active WAL files.
-
-Useful inspection queries (run against the configured DB path) are:
-
-```sql
-SELECT rowid AS spark_key, ns, key
-FROM store
-ORDER BY rowid;
-
-SELECT rowid AS spark_key, ns, key
-FROM store
-WHERE ns = 'GLBX.FUTURE' AND key = '<exact-adapter-key>';
-```
-
-## Source map
-
-- [`SecurityIDRecorder.cpp`](../src/lib_spark/spark/md/SecurityIDRecorder.cpp):
-  legacy queues, 200 ms work ordering, logical row construction, DB callbacks,
-  and spread-leg resolution.
-- [`IDGenerator.cpp`](../src/lib_spark/spark/persistence/IDGenerator.cpp):
-  synchronous generalized lookup/allocation and process-wide mutex.
-- [`SqliteStore.cpp`](../src/lib_spark/spark/persistence/SqliteStore.cpp): DB
-  location, schema, WAL/synchronous pragmas, individual inserts, and batch
-  inserts.
-- [`MDProducer.cpp`](../src/lib_spark/spark/md/MDProducer.cpp) and
-  [`MDProducerHandle.cpp`](../src/lib_spark/spark/md/MDProducerHandle.cpp):
-  shared-memory allocation, recorder handoff, and ordered publication.
-- [`FixedSecurity.h`](../src/lib_spark/spark/md/Shm/FixedSecurity.h) and
-  [`MDStructure.h`](../src/lib_spark/spark/md/Shm/MDStructure.h): identifier
-  widths and hot/cold storage fields.
-- [`MDP3Main.cpp`](../src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp)
-  and
-  [`MDP3Listener.h`](../src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h):
-  MDP3 construction, definition callbacks, CME ID extraction, and spread legs.
-- [`MDP3Listener.cpp`](../src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp):
-  producer/index insertion order.
-- [`OnixsGlbxSubscription.cpp`](../src/app_sparkd/sparkd/md/glbx/OnixsGlbxSubscription.cpp):
-  current dedicated MDP3 dispatcher-thread ownership.
-- [`MarketDataGateway.cpp`](../src/app_sparkd/sparkd/md/MarketDataGateway.cpp)
-  and
-  [`GeneralMarketDataListener.cpp`](../src/lib_spark/spark/md/GeneralMarketDataListener.cpp):
-  generalized source/sink topology, incoming-ID mapping, fixed-security
-  conversion, and publication.
-- [`DatabentoDataHandler.cpp`](../src/app_sparkd/sparkd/md/databento/DatabentoDataHandler.cpp),
-  [`TaifexDataProcessors.h`](../src/app_sparkd/sparkd/md/taifex/TaifexDataProcessors.h),
-  and
-  [`PyMarketDataSubscription.cpp`](../src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp):
-  representative adapter-side allocation and caching behavior.
-- [`SecurityDB.cpp`](../src/lib_spark/spark/md/SecurityDB.cpp) and
-  [`SecurityCache.cpp`](../src/lib_spark/spark/md/SecurityCache.cpp): separate
-  persistence of full security definitions.
-- [`SecurityIndex.h`](../src/lib_spark/spark/md/SecurityIndex.h),
-  [`MBOServiceServer.cpp`](../src/lib_spark/spark/md/MBOServiceServer.cpp), and
-  [`MDConsumer.cpp`](../src/lib_spark/spark/md/MDConsumer.cpp): exchange-index
-  translation followed by Spark-keyed market-data consumption and MBO access.
-- [`OrderFactory.cpp`](../src/lib_spark/spark/orderman/order_factory/OrderFactory.cpp),
-  [`FillStore.cpp`](../src/lib_spark/spark/orderman/actors/FillStore.cpp), and
-  [`PositionMap.cpp`](../src/app_sparkd/sparkd/position/PositionMap.cpp):
-  representative downstream use of Spark security keys.
-- [`MDReplayer.cpp`](../src/app_sparkd/sparkd/md/recording/MDReplayer.cpp) and
-  [`RelayClient.cpp`](../src/lib_spark/spark/md/relay/RelayClient.cpp): recorded
-  ID reuse.

docker/images/bluetradingsystems/linux_dotnet8/Dockerfile

@@ -19,10 +19,12 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
 RUN apt-get update \
   && apt-get install -y dotnet8
 
-RUN apt-get update && apt-get install -y gdb
+RUN apt-get update && apt-get install -y libpq-dev
+RUN apt-get update && apt-get install -y libsodium-dev
 RUN apt-get update && apt-get install -y fish
-RUN apt-get update && apt-get install -y vim
 RUN apt-get update && apt-get install -y less
+RUN apt-get update && apt-get install -y vim
+
 
 RUN locale-gen en_US.UTF-8
 

docker/images/bluetradingsystems/linux_dotnet8_dev/Dockerfile

@@ -98,7 +98,7 @@ RUN apt-get update && apt-get install -y libpcap0.8-dev
 RUN apt-get update && apt-get -y install libzstd-dev
 
 # Rakurai
-ARG RAKURAI_DEB_URL="https://api.bitbucket.org/2.0/repositories/bluetradesys/spark/downloads/rakurai-dex-connector_0.0.5_amd64.deb"
+ARG RAKURAI_DEB_URL="https://api.bitbucket.org/2.0/repositories/bluetradesys/spark/downloads/rakurai-dex-connector_0.0.12_amd64.deb"
 RUN --mount=type=secret,id=bitbucket_access_token \
   set -eux \
     && apt-get update \

src/DataStreamSuite/BooleanExpressions/Arguments/ArgumentComparer.cs

@@ -15,18 +15,21 @@ static class ArgumentComparer {
       }
       var obj1 = value1.GetValue(env);
       var obj2 = value2.GetValue(env);
-      if (obj1 is null) {
-         if (obj2 is null) {
+      if (isNullLike(obj1)) {
+         if (isNullLike(obj2)) {
             return 0;
          }
          return -1;
       }
-      if (obj2 is null) {
+      if (isNullLike(obj2)) {
          return 1;
       }
-      return compareObjs(obj1, obj2);
+      return compareObjs(obj1!, obj2!);
    }
 
+   static bool isNullLike(object? value) =>
+      value is null || DBNull.Value.Equals(value);
+
    static int compareObjs(object obj1, object obj2) {
       try {
          var typ1 = obj1.GetType();

src/DataStreamSuite/DataStream/BlueOrb/TimeSeriesStream.cs

@@ -15,7 +15,7 @@ namespace bts.BlueOrb {
       private int _size = 0;
 
       public TimeSeriesStream(ViewStream stream)
-         : this(stream, Int32.MaxValue) { }
+         : this(stream, 1000) { }
 
       public TimeSeriesStream(ViewStream stream, int size) {
          _stream = stream;
@@ -44,7 +44,7 @@ namespace bts.BlueOrb {
             ar.setValue("id", _id++);
             _table.add_row(ar);
          }
-         var dp_out =  BlueDataUtility.build_dp(_table, myLastValue as DataPacket);
+         var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
          myFireEvent(dp_out);
       }
    }

src/DataStreamSuite/DataStreamCore/ViewStreamDatabase.cs

@@ -117,6 +117,9 @@ namespace BCG {
                      addWorker(op.w);
                   else
                      removeWorker(op.w);
+            }
+            catch (InvalidOperationException ex) {
+               bts.Log.Gen.error($"ViewStreamDatabase worker collection was modified during enumeration: {ex}");
             } finally {
                _pending_worker_ops.Clear();
                _is_worker_call_in_progress = false;

src/DataStreamSuite/SharedStreams/BlueOrb/CrossExchangeSymbolMatcherStream.cs

@@ -1,5 +1,7 @@
 namespace bts.BlueOrb {
     using System;
+    using System.Collections.Generic;
+    using System.IO;
 
     using BCG;
 
@@ -10,7 +12,14 @@ namespace bts.BlueOrb {
 
    [LiveMarketDataStream]
     public class CrossExchangeSymbolMatcherStream : ViewStream {
-        public CrossExchangeSymbolMatcherStream([MarketDataClientParam] IMarketDataClient mdClient, IWorkScheduler scheduler) {
+        const string CanonicalSymbolColumn = "canonical_symbol";
+        readonly CrossExchangeSymbolAliasResolver symbolAliasResolver;
+
+        public CrossExchangeSymbolMatcherStream([MarketDataClientParam] IMarketDataClient mdClient, IWorkScheduler scheduler)
+            : this(mdClient, scheduler, null) { }
+
+        public CrossExchangeSymbolMatcherStream([MarketDataClientParam] IMarketDataClient mdClient, IWorkScheduler scheduler, string? symbolAliasConfigFile) {
+            symbolAliasResolver = CrossExchangeSymbolAliasResolver.Load(ResolveSymbolAliasConfigFile(symbolAliasConfigFile));
             ViewStream r;
             ViewStream okx_usdc;
             ViewStream okx_usdt;
@@ -24,6 +33,9 @@ namespace bts.BlueOrb {
             r = new InstrumentsStream(mdClient, scheduler);
             r = new DateTimeExpanstionStream(r,
                                               "expiration");
+            r = new MapStream(r,
+                              _add_canonical_symbol_schema,
+                              _add_canonical_symbol_row);
             okx_usdc = new FilterStream(r,
                                          "venue='Okx' and sec~='USDC'");
             okx_usdc = new AddColumnsStream(okx_usdc,
@@ -63,7 +75,7 @@ namespace bts.BlueOrb {
                                       r,
                                       "");
             r = new FilterStream(r,
-                                  "left_venue!=right_venue and left_symbol=right_symbol and left_cur=right_cur and left_sec_type=right_sec_type");
+                                  "left_venue!=right_venue and left_canonical_symbol=right_canonical_symbol and left_cur=right_cur and left_sec_type=right_sec_type");
             perps = new FilterStream(r,
                                       "left_expiration_year=2100 and right_expiration_year=2100");
             perps = new FilterStream(perps,
@@ -78,5 +90,127 @@ namespace bts.BlueOrb {
 
             mySetRealSource(r);
         }
+
+        static string ResolveSymbolAliasConfigFile(string? symbolAliasConfigFile) {
+            if (!string.IsNullOrWhiteSpace(symbolAliasConfigFile)) {
+                return symbolAliasConfigFile;
+            }
+
+            var home = Environment.GetEnvironmentVariable("HOME");
+            if (!string.IsNullOrWhiteSpace(home)) {
+                var sparkdConf = Path.Combine(home, ".local", "share", "bts", "sparkd", "conf", "cross_exchange_symbol_aliases.conf");
+                if (File.Exists(sparkdConf)) {
+                    return sparkdConf;
+                }
+            }
+
+            var appData = Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData);
+            if (!string.IsNullOrWhiteSpace(appData)) {
+                var appDataConf = Path.Combine(appData, "bts", "conf", "cross_exchange_symbol_aliases.conf");
+                if (File.Exists(appDataConf)) {
+                    return appDataConf;
+                }
+            }
+
+            return !string.IsNullOrWhiteSpace(home)
+                ? Path.Combine(home, ".local", "share", "bts", "sparkd", "conf", "cross_exchange_symbol_aliases.conf")
+                : "cross_exchange_symbol_aliases.conf";
+        }
+
+        IBlueTableSchema _add_canonical_symbol_schema(IBlueTableSchema schema) =>
+            BlueDataUtility.add_column(schema, CanonicalSymbolColumn, typeof(string));
+
+        IBlueRow _add_canonical_symbol_row(IBlueRow row) {
+            var rv = new MutableBlueRow(_add_canonical_symbol_schema(row.schema), row);
+            rv.setValue(CanonicalSymbolColumn, symbolAliasResolver.Canonicalize(row.get<string>("symbol")));
+            return rv;
+        }
+    }
+
+    public sealed class CrossExchangeSymbolAliasResolver {
+        static readonly System.Text.RegularExpressions.Regex TokenRegex =
+            new(@"[A-Za-z0-9]+|[^A-Za-z0-9]+", System.Text.RegularExpressions.RegexOptions.Compiled);
+
+        readonly IReadOnlyDictionary<string, string> aliases;
+
+        CrossExchangeSymbolAliasResolver(IReadOnlyDictionary<string, string> aliases) =>
+            this.aliases = aliases;
+
+        public static CrossExchangeSymbolAliasResolver Load(string configFilePath) {
+            if (!File.Exists(configFilePath)) {
+                Log.Gen.warn($"CrossExchangeSymbolMatcherStream alias config not found: {configFilePath}");
+                return new CrossExchangeSymbolAliasResolver(new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase));
+            }
+
+            var aliases = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+            foreach (var rawLine in File.ReadLines(configFilePath)) {
+                var line = StripComments(rawLine).Trim();
+                if (line.Length == 0) {
+                    continue;
+                }
+
+                if (!TryParseAlias(line, out var alias, out var canonical)) {
+                    Log.Gen.warn($"CrossExchangeSymbolMatcherStream skipping invalid alias line: '{rawLine}'");
+                    continue;
+                }
+
+                aliases[NormalizeToken(alias)] = NormalizeToken(canonical);
+            }
+
+            return new CrossExchangeSymbolAliasResolver(aliases);
+        }
+
+        public string Canonicalize(string? symbol) {
+            if (string.IsNullOrWhiteSpace(symbol)) {
+                return string.Empty;
+            }
+
+            var rv = new System.Text.StringBuilder(symbol.Length);
+            foreach (System.Text.RegularExpressions.Match match in TokenRegex.Matches(symbol)) {
+                var token = match.Value;
+                if (IsToken(token)) {
+                    var normalized = NormalizeToken(token);
+                    rv.Append(aliases.TryGetValue(normalized, out var canonical) ? canonical : normalized);
+                } else {
+                    rv.Append(token);
+                }
+            }
+            return rv.ToString();
+        }
+
+        static bool TryParseAlias(string line, out string alias, out string canonical) {
+            var match = System.Text.RegularExpressions.Regex.Match(line, @"^(?<alias>.+?)\s*(?:=|:|->)\s*(?<canonical>.+)$");
+            if (match.Success) {
+                alias = match.Groups["alias"].Value.Trim();
+                canonical = match.Groups["canonical"].Value.Trim();
+                return alias.Length > 0 && canonical.Length > 0;
+            }
+
+            var parts = line.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries);
+            if (parts.Length == 2) {
+                alias = parts[0];
+                canonical = parts[1];
+                return true;
+            }
+
+            alias = string.Empty;
+            canonical = string.Empty;
+            return false;
+        }
+
+        static string StripComments(string line) {
+            var commentIdx = line.IndexOf('#');
+            var slashIdx = line.IndexOf("//", StringComparison.Ordinal);
+            if (slashIdx >= 0 && (commentIdx < 0 || slashIdx < commentIdx)) {
+                commentIdx = slashIdx;
+            }
+            return commentIdx >= 0 ? line[..commentIdx] : line;
+        }
+
+        static bool IsToken(string token) =>
+            token.Length > 0 && char.IsLetterOrDigit(token[0]);
+
+        static string NormalizeToken(string token) =>
+            token.Trim().ToUpperInvariant();
     }
 }

src/DataStreamSuite/SharedStreams/BlueOrb/FeeAdjustedCrossExchangeArbDiscoveryStream.cs

@@ -0,0 +1,31 @@
+namespace bts.BlueOrb {
+   using BCG;
+   using Bts.DataStreamCore;
+   using Bts.PubSub;
+   using Spark.Api.MarketData;
+
+   [LiveMarketDataStream]
+   public class FeeAdjustedCrossExchangeArbDiscoveryStream : ViewStream {
+      public FeeAdjustedCrossExchangeArbDiscoveryStream(
+         [MarketDataClientParam] IMarketDataClient mdClient,
+         IWorkScheduler scheduler) {
+         ViewStream r = new CrossExchangeArbDiscoveryStream(mdClient, scheduler);
+         r = new AddColumnsStream(r, "fee:0.0002");
+         r = new AddColumnsStream(r, "capital_available:100000");
+         r = new MathStream(r, """
+            left_leg_cost = left_bid_price*left_bid_qty*left_cpp;
+            right_leg_cost = right_bid_price*right_bid_qty*right_cpp;
+            total_cost = left_leg_cost + right_leg_cost;
+            total_fees = total_cost * fee;
+            fee_adjusted_arb = arb-total_fees;
+            possible_arb = fee_adjusted_arb*(Min(capital_available,total_cost)/total_cost);
+            """);
+         r = new FilterStream(r, "arb>0");
+         r = new TimeSeriesStream(r);
+         r = new ProjectStream(r,
+            "spread_name, left_sec_name,left_venue,right_sec_name,right_venue,arb, fee_adjusted_arb, timestamp");
+
+         mySetRealSource(r);
+      }
+   }
+}

src/Directory.Build.targets

@@ -187,24 +187,32 @@
       Name="Test"
       DependsOnTargets="Build"></Target>
 
-  <Target Name="CopyNativeAssetsToOutput" AfterTargets="Build">
-    <Copy SourceFiles="@(NativeWindowsAsset)" DestinationFolder="$(OutDir)" SkipUnchangedFiles="True" Condition="Exists('%(NativeWindowsAsset.FullPath)')" />
-    <Copy SourceFiles="@(NativeLinuxAsset)" DestinationFolder="$(OutDir)" SkipUnchangedFiles="True" Condition="Exists('%(NativeLinuxAsset.FullPath)')" />
-  </Target>
+  <ItemGroup Condition="Exists('@(NativeWindowsAsset)')">
+    <Content Include="@(NativeWindowsAsset)">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      <TargetPath>%(FileName)%(Extension)</TargetPath>
+      <Pack>false</Pack>
+    </Content>
+    <Content Include="@(NativeWindowsAsset)">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      <TargetPath>runtimes/win-x64/native/%(FileName)%(Extension)</TargetPath>
+      <Pack>true</Pack>
+      <PackagePath>runtimes/win-x64/native</PackagePath>
+    </Content>
+  </ItemGroup>
 
-  <ItemGroup>
-    <Content Include="@(NativeWindowsAsset)"
-             Condition="Exists('@(NativeWindowsAsset)')"
-             Link="%(NativeWindowsAsset.FileName)%(NativeWindowsAsset.Extension)"
-             CopyToOutputDirectory="PreserveNewest"
-             Pack="true"
-             PackagePath="runtimes/win-x64/native/" />
-    <Content Include="@(NativeLinuxAsset)"
-             Condition="Exists('@(NativeLinuxAsset)')"
-             Link="%(NativeLinuxAsset.FileName)%(NativeLinuxAsset.Extension)"
-             CopyToOutputDirectory="PreserveNewest"
-             Pack="true"
-             PackagePath="runtimes/linux-x64/native/" />
+  <ItemGroup Condition="Exists('@(NativeLinuxAsset)')">
+    <Content Include="@(NativeLinuxAsset)">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      <TargetPath>%(FileName)%(Extension)</TargetPath>
+      <Pack>false</Pack>
+    </Content>
+    <Content Include="@(NativeLinuxAsset)">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      <TargetPath>runtimes/linux-x64/native/%(FileName)%(Extension)</TargetPath>
+      <Pack>true</Pack>
+      <PackagePath>runtimes/linux-x64/native</PackagePath>
+    </Content>
   </ItemGroup>
 
 </Project>

src/Spark.Admin.WebApp/src/Accounts/Types.fs

@@ -17,7 +17,9 @@ type Tab =
   | PositionManagementTab
   | MarginTab
 
-let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.BYBIT; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.GATEIO; Venue.OKX; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.TWSE; Venue.SIM; |]
+// BEGIN GENERATED ADMIN VENUES
+let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.BYBIT; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.GATEIO; Venue.GLBX; Venue.ICE; Venue.OKX; Venue.ORCA; Venue.PHOENIX; Venue.RAYDIUM; Venue.SIM; Venue.TAIFEX; Venue.TWSE; |]
+// END GENERATED ADMIN VENUES
 
 let securityTypes = [| SecurityType.SPOT; SecurityType.FUTURE; SecurityType.OPTION; SecurityType.SPREAD; |]
 
@@ -94,6 +96,16 @@ type OkxExchangeField =
   | Origin of CustomerOrFirm
   | ClearingAccount of string
 
+type OrcaExchangeField =
+  | CTI of CTICode
+  | Origin of CustomerOrFirm
+  | ClearingAccount of string
+
+type RaydiumExchangeField =
+  | CTI of CTICode
+  | Origin of CustomerOrFirm
+  | ClearingAccount of string
+
 type SIMExchangeField =
   | CTI of CTICode
   | Origin of CustomerOrFirm

src/Spark.Admin.WebApp/src/Accounts/View.fs

@@ -801,8 +801,11 @@ let venueToString (venue:bts.api.Venue) =
     | bts.api.Venue.GATEIO -> "GATEIO"
     | bts.api.Venue.FTX -> "FTX"
     | bts.api.Venue.OKX -> "OKX"
+    | bts.api.Venue.ORCA -> "ORCA"
+    | bts.api.Venue.RAYDIUM -> "RAYDIUM"
     | bts.api.Venue.TAIFEX -> "TAIFEX"
     | bts.api.Venue.TWSE -> "TWSE"
+    | value -> sprintf "%O" value
 
 let exchange_display expirationNameList =
   let s = sprintf "%O" expirationNameList.product.exchange

src/app_sparkd/CMakeLists.txt

@@ -37,6 +37,45 @@ protobuf_generate_cpp(
 )
 add_executable(sparkd ${${PROJECT_NAME}_SOURCES} ${${PROJECT_NAME}_PROTO_CXX_SOURCES})
 
+find_package(PkgConfig REQUIRED)
+pkg_check_modules(
+  emumba-deribit-connector-library
+  REQUIRED
+  IMPORTED_TARGET
+  emumba-deribit-connector
+)
+pkg_check_modules(
+  rakurai-dex-connector-library
+  REQUIRED
+  IMPORTED_TARGET
+  rakurai-dex-connector
+)
+# Rakurai's static dex SDK uses hidapi's libudev backend. Keep this explicit so
+# existing CMake build trees also pick up the dependency when the connector's
+# pkg-config metadata changes between package versions.
+find_library(RAKURAI_UDEV_LIBRARY NAMES udev libudev.so.1 REQUIRED)
+
+# The Rakurai package currently publishes a bare `udev` dependency. Minimal
+# runtime images only contain libudev.so.1, so replace that entry with the
+# resolved library while preserving its position after Rakurai's static SDK.
+get_target_property(
+  RAKURAI_LINK_LIBRARIES
+  PkgConfig::rakurai-dex-connector-library
+  INTERFACE_LINK_LIBRARIES
+)
+set(RAKURAI_RESOLVED_LINK_LIBRARIES)
+foreach(RAKURAI_LINK_LIBRARY IN LISTS RAKURAI_LINK_LIBRARIES)
+  if(RAKURAI_LINK_LIBRARY STREQUAL "udev" OR RAKURAI_LINK_LIBRARY STREQUAL "-ludev")
+    list(APPEND RAKURAI_RESOLVED_LINK_LIBRARIES ${RAKURAI_UDEV_LIBRARY})
+  else()
+    list(APPEND RAKURAI_RESOLVED_LINK_LIBRARIES ${RAKURAI_LINK_LIBRARY})
+  endif()
+endforeach()
+set_property(
+  TARGET PkgConfig::rakurai-dex-connector-library
+  PROPERTY INTERFACE_LINK_LIBRARIES ${RAKURAI_RESOLVED_LINK_LIBRARIES}
+)
+
 # include paths
 target_include_directories(
   sparkd
@@ -44,11 +83,9 @@ target_include_directories(
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
     $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>/include
     $<BUILD_INTERFACE:${emumba-deribit-connector-library_INCLUDE_DIRS}>
+    $<BUILD_INTERFACE:${rakurai-dex-connector-library_INCLUDE_DIRS}>
 )
 
-find_package(PkgConfig REQUIRED)
-pkg_check_modules(emumba-deribit-connector-library REQUIRED emumba-deribit-connector)
-
 # This dir contains rapidjsonConfig.cmake
 list(APPEND CMAKE_PREFIX_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules)
 find_package(rapidjson REQUIRED)
@@ -127,7 +164,8 @@ target_link_libraries(
   python_interop
   python_interop_module
   databento::databento
-  ${emumba-deribit-connector-library_LIBRARIES}
+  PkgConfig::emumba-deribit-connector-library
+  PkgConfig::rakurai-dex-connector-library
   tbb
   quickfix
   benchmark
@@ -140,6 +178,11 @@ target_link_libraries(
   -static-libstdc++
   -pthread)
 
+# Both vendor static connector archives currently export the same global
+# `print_books` symbol. Allow that duplicate so sparkd can link until the
+# upstream packages make the symbol internal or rename it.
+target_link_options(${PROJECT_NAME} PRIVATE -Wl,--allow-multiple-definition)
+
 # Onixs
 target_link_libraries(
   ${PROJECT_NAME}

src/app_sparkd/Dockerfile

@@ -40,6 +40,9 @@ ARG SOURCE_DIR
 ARG PYTHON_REQUIREMENTS="requirements.txt"
 ARG SPARKD_ENTRYPOINT
 
+ENV DOTNET_ENVIRONMENT=Development
+RUN dotnet dev-certs https --trust
+
 USER "root"
 RUN apt-get update && apt-get install -y liblttng-ust-dev
 RUN apt-get update && apt-get install -y libpcap0.8-dev

src/app_sparkd/configurations/all_crypto/.env

@@ -1,5 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/all_crypto
-SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/gateio_dev/requirements.txt
-SPARKD__DATA_DIR=/home/ubuntu/8000/sparkdata/data
-SPARKD__IMAGE=sparkd_symbols
-SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf

src/app_sparkd/configurations/all_crypto/gateio_overrides.json

@@ -1,11 +0,0 @@
-{
-  "currencies": [
-    "USDT"
-  ],
-  "futures": [
-    "BTC",
-    "ETH",
-    "SOL",
-    "XRP"
-  ]
-}

src/app_sparkd/configurations/all_crypto/py_bybit_md_provider.json

@@ -1,16 +0,0 @@
-{
-  "currencies": [
-    "USDT"
-  ],
-  "futures": [
-    "BTC",
-    "ETH",
-    "SOL",
-    "XRP"
-  ],
-  "use_testnet": false,
-  "logging": {
-    "console_level": "WARNING",
-    "file_level": "INFO"
-  }
-}

src/app_sparkd/configurations/all_crypto/spark.conf

@@ -1,71 +0,0 @@
-PublisherOptions {
-  value {
-    network_interface: "0.0.0.0"
-    heartbeat_ms: 5000
-    heartbeat_boot_seconds: 60
-    skip_options: true
-    hot_initial: 1024
-    hot_growth: 1024
-    cold_initial: 8192
-    cold_growth: 2048
-    channel_prefix: ""
-    disable_heartbeat: true
-  }
-}
-
-Logging {
-  value {
-    initial_queue_size: 120001
-    max_queue_growth: 4
-    file {
-      default_level: "Info"
-      subsystem {
-        name: "md"
-        level: "Diag"
-      }
-    }
-  }
-}
-
-MarketDataConfig {
-  value {
-    # python_md_providers {
-    #   venue: VENUE_GATEIO
-    #   module_path: "python_md_providers"
-    #   provider_type: "gateio_md_provider.GateIoMdProvider"
-    # }
-    python_md_providers {
-      venue: VENUE_BYBIT
-      module_path: "python_interop_plugins"
-      provider_type: "bybit_md_provider.BybitMdProvider"
-    }
-    binance_usds_futures {
-      num_depth_levels: 5
-      url_rest_base: "https://fapi.binance.com"
-      url_ws_base: "wss://fstream.binance.com"
-      security_criteria {
-        clauses {
-          security_types: SECURITY_TYPE_FUTURE
-          symbols: @SymbolLists.list[Gateio].symbol
-        }
-      }
-    }
-     okx {
-       num_depth_levels: 5
-       url_rest_base: "https://www.okx.com"
-       url_ws_base: "wss://ws.okx.com:8443"
-       security_criteria {
-         clauses {
-           security_types: SECURITY_TYPE_FUTURE
-           symbols: @SymbolLists.list[Gateio].symbol
-         }
-       }
-       is_application_ping_pong_enabled: true
-       application_ping_interval_ms: 20000
-       application_pong_timeout_ms: 10000
-       is_protocol_ping_pong_enabled: true
-       protocol_ping_interval_ms: 60000
-       protocol_pong_timeout_ms: 30000
-     }
-  }
-}

src/app_sparkd/configurations/all_crypto/symbols.conf

@@ -1,32 +0,0 @@
-  SymbolLists {
-  value {
-    list {
-      key: "Gateio"
-      symbol: "SOL"
-      symbol: "BTC"
-      symbol: "ETH"
-      symbol: "XRP"
-    }
-  }
-}
-
-Symbols {
-  value {
-    venues {
-      name: "VENUE_GATEIO"
-      spot: @SymbolLists.list[Gateio].symbol
-    }
-    venues {
-      name: "VENUE_BYBIT"
-      future: @SymbolLists.list[Gateio].symbol
-    }
-    venues {
-      name: "VENUE_BINANCE_USDS_FUTURES"
-      future: @SymbolLists.list[Gateio].symbol
-    }
-    venues {
-      name: "VENUE_OKX"
-      future: @SymbolLists.list[Gateio].symbol
-    }
-  }
-}

src/app_sparkd/configurations/cme_dev_replay_aws/.env

@@ -1,7 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/cme_dev_replay_aws
-SPARKD__DATA_DIR=/home/ubuntu/8000/sparkdata/data
-SPARKD__IMAGE=sparkd_symbols
-SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/cme_dev_replay_aws/requirements.txt
-SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf
-SPARKD__SHM_SIZE=512m
-SPARKD__MEMORY=1g

src/app_sparkd/configurations/cme_dev_replay_aws/symbols.conf

@@ -1,61 +0,0 @@
-SymbolLists {
-  value {
-    list {
-      key: "GlbxFutures"
-      symbol: "ES"
-    }
-    # list {
-    #   key: "GlbxOptions"
-    #   symbol: "E1A"
-    #   symbol: "E1B"
-    #   symbol: "E1C"
-    #   symbol: "E1D"
-    #   symbol: "E2A"
-    #   symbol: "E2B"
-    #   symbol: "E2C"
-    #   symbol: "E2D"
-    #   symbol: "E3A"
-    #   symbol: "E3B"
-    #   symbol: "E3C"
-    #   symbol: "E3D"
-    #   symbol: "E4A"
-    #   symbol: "E4B"
-    #   symbol: "E4C"
-    #   symbol: "E4D"
-    #   symbol: "E5A"
-    #   symbol: "E5B"
-    #   symbol: "E5C"
-    #   symbol: "E5D"
-    #   symbol: "EW1"
-    #   symbol: "EW2"
-    #   symbol: "EW3"
-    #   symbol: "EW4"
-    #   symbol: "ESH9"
-    #   symbol: "ESZ6"
-    #   symbol: "ESH0"
-    #   symbol: "ESM6"
-    #   symbol: "ESZ7"
-    #   symbol: "ESH6"
-    #   symbol: "ESM7"
-    #   symbol: "ESZ8"
-    #   symbol: "ESH7"
-    #   symbol: "ESH8"
-    #   symbol: "ESU6"
-    #   symbol: "ESZ5"
-    #   symbol: "ESZ9"
-    #   symbol: "E1D"
-    #   symbol: "ESZ5"
-    #   symbol: "ES"
-    # }
-  }
-}
-
-Symbols {
-  value {
-    venues {
-        name: "VENUE_GLBX"
-        future: @SymbolLists.list[GlbxFutures].symbol
-        # option: @SymbolLists.list[GlbxOptions].symbol
-    }
-  }
-}

src/app_sparkd/configurations/dex_dev/.env

@@ -0,0 +1,3 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dex_dev
+SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/dex_dev/requirements.txt
+SPARKD__SHM_SIZE=2gb

src/app_sparkd/configurations/dex_dev/README.txt

@@ -0,0 +1,2 @@
+* Connects to raydium and orca for dex
+* Execution is SIM

src/app_sparkd/configurations/dex_dev/config.json

@@ -0,0 +1,22 @@
+{
+    "app_logging": {
+        "severity": "trace",
+        "file": false,
+        "console": true
+    },
+    "dex": [
+        {
+            "id": "orca_clmm",
+            "pool_ids": ["Czfq3xZZDmsdGdUyrNLtRhGc47cXcZtLG4crryfu44zE", "4HppGTweoGQ8ZZ6UcCgwJKfi5mJD9Dqwy6htCpnbfBLW"
+            ]
+        }
+    ],
+    "cluster_mode": false,
+    "wallet_pub": "HFru3Rc3Rf6UjZtrYPsBSKvmq5uyha2DWefV77rFXU64",
+    "wallet_pvt": [168,123,234,72,218,209,180,225,42,193,240,65,11,15,105,91,95,255,231,214,188,119,90,45,119,144,177,207,203,245,250,27,241,137,138,138,103,247,64,134,70,212,107,134,75,97,87,167,53,32,169,108,75,126,254,17,213,216,74,14,245,28,231,161],
+    "md_server_uri_1": "ws://185.191.117.159:8130",
+    "node_oms_uri_1": "ws://185.191.117.159:8133",
+    "md_server_uri_2": "ws://189.1.171.199:8130",
+    "node_oms_uri_2": "ws://189.1.171.199:8133",
+    "core": 1
+}

src/app_sparkd/configurations/dex_dev/orca_1.json

@@ -0,0 +1,6 @@
+{
+  "account": "DEX_DEV",
+  "operator_id": "DEX_DEV",
+  "json_conf_file_name": "config.json",
+  "default_priority_fee": 1000
+}

src/app_sparkd/configurations/dex_dev/raydium_1.json

@@ -0,0 +1,6 @@
+{
+  "account": "DEX_DEV",
+  "operator_id": "DEX_DEV",
+  "json_conf_file_name": "config.json",
+  "default_priority_fee": 1000
+}

src/app_sparkd/configurations/dex_dev/spark.conf

@@ -0,0 +1,76 @@
+
+JetFIX {
+  value {
+    connection {
+      name: "orca_1"
+      venue: VENUE_ORCA
+      max_order_msgs_per_window: 300
+      window_duration_msecs: 1000
+    }
+    connection {
+      name: "okx_1"
+      venue: VENUE_OKX
+      max_order_msgs_per_window: 300
+      window_duration_msecs: 1000
+    }
+  }
+}
+PublisherOptions {
+  value {
+    network_interface: "0.0.0.0"
+    heartbeat_ms: 5000
+    heartbeat_boot_seconds: 60
+    skip_options: true
+    hot_initial: 1024
+    hot_growth: 1024
+    cold_initial: 8192
+    cold_growth: 2048
+    channel_prefix: ""
+    disable_heartbeat: true
+  }
+}
+MarketDataConfig {
+  value {
+    dex {
+      venue: VENUE_ORCA
+      num_depth_levels: 10
+      json_conf_file_name: "config.json"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_SPOT
+          symbols: @SymbolLists.list[Binance].symbol
+          name_regexps: "USDC|USDT$"
+        }
+      }
+    }
+    okx {
+       num_depth_levels: 5
+       url_rest_base: "https://www.okx.com"
+       url_ws_base: "wss://ws.okx.com:8443"
+       security_criteria {
+         clauses {
+           security_types: SECURITY_TYPE_SPOT
+           symbols: @SymbolLists.list[Okx].symbol
+         }
+       }
+       is_application_ping_pong_enabled: true
+       application_ping_interval_ms: 20000
+       application_pong_timeout_ms: 10000
+       is_protocol_ping_pong_enabled: true
+       protocol_ping_interval_ms: 60000
+       protocol_pong_timeout_ms: 30000
+     }
+    dex {
+      venue: VENUE_RAYDIUM
+      num_depth_levels: 10
+      json_conf_file_name: "config.json"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_SPOT
+          symbols: @SymbolLists.list[Binance].symbol
+          name_regexps: "USDC|USDT$"
+        }
+      }
+    }
+  }
+}

src/app_sparkd/configurations/dex_dev/symbols.conf

@@ -0,0 +1,84 @@
+SymbolLists {
+  value {
+    list {
+      key: "Binance"
+      symbol: "SOL"
+      symbol: "USDC"
+      #symbol: "BNB"
+      symbol: "BTC"
+      #symbol: "DOGE"
+      #symbol: "EIGEN"
+      symbol: "ETH"
+      symbol: "XRP"
+    }
+    list {
+      key: "ORCA"
+      #symbol: "AAVE"
+      #symbol: "AVAX"
+      #symbol: "BNB"
+      symbol: "BTC"
+      symbol: "USDC"
+      symbol: "SOL"
+      symbol: "ETH"
+      symbol: "XRP"
+    }
+    list {
+      key: "RAYDIUM"
+      symbol: "SOL"
+      symbol: "USDC"
+      symbol: "ETH"
+      symbol: "BTC"
+      symbol: "XRP"
+    }
+    list {
+      key: "PHOENIX"
+      symbol: "SOL"
+    }
+    list {
+      key: "BinanceUsdsFutures"
+      symbol: "SOL"
+      symbol: "BTC"
+      symbol: "USDC"
+      symbol: "ETH"
+      symbol: "XRP"
+    }
+    list {
+      key: "Okx"
+      symbol: "SOL"
+      symbol: "USDC"
+      symbol: "BTC"
+      symbol: "ETH"
+      symbol: "XRP"
+    }
+  }
+}
+
+Symbols {
+  value {
+   # venues {
+   #   name: "VENUE_BINANCE"
+   #   spot: @SymbolLists.list[Binance].symbol
+  #  }
+    #venues {
+    #  name: "VENUE_BINANCE_USDS_FUTURES"
+    #  future: @SymbolLists.list[BinanceUsdsFutures].symbol
+    #}
+    venues {
+      name: "VENUE_ORCA"
+      spot: @SymbolLists.list[ORCA].symbol
+    }
+#    venues {
+#      name: "VENUE_RAYDIUM"
+#      spot: @SymbolLists.list[RAYDIUM].symbol
+#    }
+    venues {
+      name: "VENUE_PHOENIX"
+      future: @SymbolLists.list[PHOENIX].symbol
+    }
+    venues {
+      name: "VENUE_OKX"
+      spot: @SymbolLists.list[Okx].symbol
+     # future: @SymbolLists.list[Okx].symbol
+    }
+  }
+}

src/app_sparkd/configurations/ophir_cert_live_wb/.env

@@ -1,10 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_cert_live_wb
-SPARKD__DATA_DIR=/home/wedbushprod/9000/sparkdata/data
-SPARKD__IMAGE=sparkd_symbols
-SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_cert_live_wb/requirements.txt
-SPARKD__RUNTIME_CONF_DIR=/home/wedbushprod/9000/sparkdata/conf
-SPARKD__SHM_SIZE=8g
-SPARKD__MEMORY=16g
-SPARKD__CONFIG_PORT=9000
-SPARKD__QUICKBUS_PORT=9001
-SPARKD__SIM_FIX_PORT=9002

src/app_sparkd/configurations/ophir_cert_live_wb/spark.conf

@@ -1,140 +0,0 @@
-DeploymentType {
-  value {
-    type: "dev"
-  }
-}
-PublisherOptions {
-   value {
-      network_interface_a: 'ens65f1'
-      network_interface_b: 'ens65f0'
-      #these get overriden if not entered here
-      heartbeat_ms: 5000
-      heartbeat_boot_seconds: 60
-      skip_options: false
-      hot_initial: 1024
-      hot_growth: 1024
-      cold_initial: 8192
-      cold_growth: 2048
-      channel_prefix: ""
-      disable_heartbeat: false
-   }
-}
-
-CMEOptions {
-  value {
-    feed_engine {
-      name: "mdp3feed"
-      spin_time: 0
-      channels_per_thread: 3
-      data_wait: 10
-      udp_socket_buffer_size: 268435456
-    }
-    channel {
-      id: "*"
-      feed_engine: "mdp3feed"
-      use_tcp_replay: false
-      recovery_queue_size: 640000
-      lost_packet_wait_timeout_ms: 10
-      onixs_log_mode: CME_ONIXS_LOG_MODE_UNSPECIFIED
-      #max_gap_recovery_cache_size: 4
-      use_natural_refresh: false
-      join_snapshot_on_gap: false
-    }
-  }
-}
-
-Logging {
-  value {
-    initial_queue_size: 120001
-    max_queue_growth: 4
-    console {
-      default_level: "Info"
-      subsystem {
-        name: "md"
-        level: "Info"
-      }
-      subsystem {
-        name: "OrderMan"
-        level: "Info"
-      }
-    }
-  }
-}
-
-MarketDataConfig {
-  value {
-    glbx {
-       unique_check_value: 15
-       use_databento: false
-    }
-   # recording {
-   #    venue: VENUE_GLBX
-   #    num_of_seconds_to_run_file_writer: 300
-   #    num_of_seconds_to_run_db_writer : 0
-   #    n_book_levels: 10 
-   #    batch_size: 100 #for the db writer
-   #    enable_book_recording: true
-   #    enable_trade_recording:  true
-   #    enable_bbo_recording: false
-   #    enable_tbbo_recording:  false
-   #    enable_misc_recording: true
-   #}
- }
-}
-
-JetFIX {
-  value {
-    connection {
-      name: "maw_004_64"
-      venue: VENUE_GLBX
-      socket_connect_host_ip: "69.50.112.177"
-      socket_connect_port: 44040
-      sender_compid: "MAW004"
-      target_compid: "CME"
-      target_subid: "64"
-      cme_market_segment_id: 64
-      start_time: "00:10:00"
-      end_time: "23:55:00"
-      start_day: "sunday"
-      end_day: "friday"
-      seconds_between_hbs: 15
-      sender_subid: "BTS"
-      seconds_to_pause: 2
-      max_connection_attempts: 100
-      use_ilink3: true
-    }
-  }
-}
-OrdermanCfg {
-  value {
-    executor {
-      thread_name: "oman_orders"
-      poll: true 
-      hosted_actor: "MDSource"
-      hosted_actor: "OrderManager"
-      hosted_actor: "ExecutionGateway"
-    }
-    executor {
-      thread_name: "oman_slow"
-            poll: false 
-      hosted_actor: "*"
-    }
-    fallback_cust_order_handling_inst: "Y"
-  }
-}
-NotSimExecution {
-  value {
-    entries {
-      venue: VENUE_GLBX
-    }
-  }
-}
-Ports {
-   value {
-      config_port: 9000
-      qbus_port: 9001
-      glbx_sim_fix_port: 9002
-      ice_sim_fix_port: 9003
-      replicator_port: 9080
-   }
-}

src/app_sparkd/configurations/ophir_demo/.env

@@ -0,0 +1,5 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_demo
+SPARKD__DATA_DIR=/home/ubuntu/8000/sparkdata/data
+SPARKD__IMAGE=sparkd_symbols
+SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_demo/requirements.txt
+SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf

src/app_sparkd/configurations/ophir_demo/spark.conf

@@ -0,0 +1,62 @@
+JetFIX {
+  value {
+    connection {
+      name: "glbx_1"
+      venue: VENUE_GLBX
+      max_order_msgs_per_window: 300
+      window_duration_msecs: 1000
+    }
+  }
+}
+
+PublisherOptions {
+  value {
+    network_interface: "0.0.0.0"
+    heartbeat_ms: 5000
+    heartbeat_boot_seconds: 60
+    skip_options: true
+    hot_initial: 1024
+    hot_growth: 1024
+    cold_initial: 8192
+    cold_growth: 2048
+    channel_prefix: ""
+    disable_heartbeat: true
+  }
+}
+
+Logging {
+  value {
+    initial_queue_size: 120001
+    max_queue_growth: 4
+    console {
+      default_level: "Info"
+      subsystem {
+        name: "md"
+        level: "Warn"
+      }
+      subsystem {
+        name: "OrderMan"
+        level: "Info"
+      }
+    }
+    file {
+      default_level: "Info"
+      subsystem {
+        name: "OrderMan"
+        level: "Diag"
+      }
+      subsystem {
+        name: "Http"
+        level: "Diag"
+      }
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+    replay {
+      venue: VENUE_GLBX
+    }
+  }
+}

src/app_sparkd/configurations/ophir_demo/symbols.conf

@@ -0,0 +1,8 @@
+Symbols {
+  value {
+    venues {
+      name: "VENUE_GLBX"
+      future: "ES"
+    }
+  }
+}

src/app_sparkd/configurations/ophir_prod_live_wb/spark.conf

@@ -1,152 +0,0 @@
-DeploymentType {
-  value {
-    type: "prod"
-  }
-}
-PublisherOptions {
-   value {
-      network_interface_a: 'enp193s0f0np0'
-      network_interface_b: 'enp193s0f1np1'
-      #these get overriden if not entered here
-      heartbeat_ms: 5000
-      heartbeat_boot_seconds: 60
-      skip_options: false
-      hot_initial: 1024
-      hot_growth: 1024
-      cold_initial: 8192
-      cold_growth: 2048
-      channel_prefix: ""
-      disable_heartbeat: false
-   }
-}
-
-CMEOptions {
-  value {
-    feed_engine {
-      name: "mdp3feed"
-      spin_time: 0
-      channels_per_thread: 3
-      data_wait: 10
-      udp_socket_buffer_size: 268435456
-    }
-    channel {
-      id: "*"
-      feed_engine: "mdp3feed"
-      use_tcp_replay: false
-      recovery_queue_size: 640000
-      lost_packet_wait_timeout_ms: 10
-      onixs_log_mode: CME_ONIXS_LOG_MODE_UNSPECIFIED
-      #max_gap_recovery_cache_size: 4
-      use_natural_refresh: false
-      join_snapshot_on_gap: false
-    }
-  }
-}
-
-Logging {
-  value {
-    initial_queue_size: 120001
-    max_queue_growth: 4
-    console {
-      default_level: "Info"
-      subsystem {
-        name: "md"
-        level: "Info"
-      }
-      subsystem {
-        name: "OrderMan"
-        level: "Info"
-      }
-    }
-  }
-}
-
-MarketDataConfig {
-  value {
-    glbx {
-       unique_check_value: 15
-       use_databento: false
-    }
-   # recording {
-   #    venue: VENUE_GLBX
-   #    num_of_seconds_to_run_file_writer: 300
-   #    num_of_seconds_to_run_db_writer : 0
-   #    n_book_levels: 10 
-   #    batch_size: 100 #for the db writer
-   #    enable_book_recording: true
-   #    enable_trade_recording:  true
-   #    enable_bbo_recording: false
-   #    enable_tbbo_recording:  false
-   #    enable_misc_recording: true
-   #}
- }
-}
-
-JetFIX {
-  value {
-    connection {
-      name: "u6p_7qb_54"
-      venue: VENUE_GLBX
-      socket_connect_host_ip: "205.209.201.97"
-      socket_connect_port: 10979
-      sender_compid: "U6P7QB"
-      target_compid: "CME"
-      target_subid: "54"
-      cme_market_segment_id: 54
-      start_time: "00:10:00"
-      end_time: "23:55:00"
-      start_day: "sunday"
-      end_day: "friday"
-      seconds_between_hbs: 15
-      sender_subid: "BTS"
-      seconds_to_pause: 1
-      max_order_msgs_per_window: 300
-      max_connection_attempts: 2
-      use_ilink3: true
-    }
-   connection {
-      name: "u6p_7qb_64"
-      venue: VENUE_GLBX
-      socket_connect_host_ip: "205.209.201.1"
-      socket_connect_port: 10979
-      sender_compid: "U6P7QB"
-      target_compid: "CME"
-      target_subid: "64"
-      cme_market_segment_id: 64
-      start_time: "00:10:00"
-      end_time: "23:55:00"
-      start_day: "sunday"
-      end_day: "friday"
-      seconds_between_hbs: 15
-      sender_subid: "BTS"
-      seconds_to_pause: 1
-      max_connection_attempts: 2
-      max_order_msgs_per_window: 300
-      use_ilink3: true
-    }
-  }
-}
-OrdermanCfg {
-  value {
-    executor {
-      thread_name: "oman_orders"
-      poll: true 
-      hosted_actor: "MDSource"
-      hosted_actor: "OrderManager"
-      hosted_actor: "ExecutionGateway"
-    }
-    executor {
-      thread_name: "oman_slow"
-            poll: false 
-      hosted_actor: "*"
-    }
-    fallback_cust_order_handling_inst: "Y"
-  }
-}
-NotSimExecution {
-  value {
-    entries {
-      venue: VENUE_GLBX
-    }
-  }
-}

src/app_sparkd/configurations/ophir_prod_live_wb_sim/.env

@@ -1,7 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_prod_live_wb
-SPARKD__DATA_DIR=/home/dk/8000/sparkdata/data
-SPARKD__IMAGE=sparkd_symbols
-SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_prod_live_wb/requirements.txt
-SPARKD__RUNTIME_CONF_DIR=/home/dk/8000/sparkdata/conf
-SPARKD__SHM_SIZE=8g
-SPARKD__MEMORY=16g

src/app_sparkd/configurations/ophir_prod_live_wb_sim/requirements.txt

@@ -1,5 +0,0 @@
-numpy==2.2.6
-pybit==5.11.0
-requests==2.31.0
-websocket-client==1.8.0
-websockets==15.0.1

src/app_sparkd/configurations/ophir_prod_live_wb_sim/symbols.conf

@@ -1,61 +0,0 @@
-SymbolLists {
-  value {
-    list {
-      key: "GlbxFutures"
-      symbol: "ES"
-    }
-    list {
-      key: "GlbxOptions"
-      symbol: "E1A"
-      symbol: "E1B"
-      symbol: "E1C"
-      symbol: "E1D"
-      symbol: "E2A"
-      symbol: "E2B"
-      symbol: "E2C"
-      symbol: "E2D"
-      symbol: "E3A"
-      symbol: "E3B"
-      symbol: "E3C"
-      symbol: "E3D"
-      symbol: "E4A"
-      symbol: "E4B"
-      symbol: "E4C"
-      symbol: "E4D"
-      symbol: "E5A"
-      symbol: "E5B"
-      symbol: "E5C"
-      symbol: "E5D"
-      symbol: "EW1"
-      symbol: "EW2"
-      symbol: "EW3"
-      symbol: "EW4"
-      symbol: "ESH9"
-      symbol: "ESZ6"
-      symbol: "ESH0"
-      symbol: "ESM6"
-      symbol: "ESZ7"
-      symbol: "ESH6"
-      symbol: "ESM7"
-      symbol: "ESZ8"
-      symbol: "ESH7"
-      symbol: "ESH8"
-      symbol: "ESU6"
-      symbol: "ESZ5"
-      symbol: "ESZ9"
-      symbol: "E1D"
-      symbol: "ESZ5"
-      symbol: "ES"
-    }
-  }
-}
-
-Symbols {
-  value {
-    venues {
-        name: "VENUE_GLBX"
-        future: @SymbolLists.list[GlbxFutures].symbol
-        option: @SymbolLists.list[GlbxOptions].symbol
-    }
-  }
-}

src/app_sparkd/configurations/ophir_sim/.env

@@ -0,0 +1,5 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_sim
+SPARKD__DATA_DIR=/home/tns/8000/sparkdata/data
+SPARKD__IMAGE=sparkd_symbols
+SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_sim/requirements.txt
+SPARKD__RUNTIME_CONF_DIR=/home/tns/8000/sparkdata/conf

src/app_sparkd/configurations/ophir_sim/spark.conf

@@ -1,12 +1,18 @@
-DeploymentType {
+JetFIX {
   value {
-    type: "prod"
+    connection {
+      name: "glbx_1"
+      venue: VENUE_GLBX
+      max_order_msgs_per_window: 300
+      window_duration_msecs: 1000
+    }
   }
 }
+
 PublisherOptions {
    value {
-      network_interface_a: 'ens5'
-      network_interface_b: 'ens5'
+      network_interface_a: 'ens3f0np0'
+      network_interface_b: 'ens3f1np1'
       #these get overriden if not entered here
       heartbeat_ms: 5000
       heartbeat_boot_seconds: 60
@@ -35,7 +41,7 @@ CMEOptions {
       use_tcp_replay: false
       recovery_queue_size: 640000
       lost_packet_wait_timeout_ms: 10
-      onixs_log_mode: CME_ONIXS_LOG_MODE_UNSPECIFIED
+      onixs_log_mode: CME_ONIXS_LOG_MODE_REGULAR
       #max_gap_recovery_cache_size: 4
       use_natural_refresh: false
       join_snapshot_on_gap: false
@@ -67,17 +73,32 @@ MarketDataConfig {
        unique_check_value: 15
        use_databento: false
     }
-   # recording {
-   #    venue: VENUE_GLBX
-   #    num_of_seconds_to_run_file_writer: 300
-   #    num_of_seconds_to_run_db_writer : 0
-   #    n_book_levels: 10 
-   #    batch_size: 100 #for the db writer
-   #    enable_book_recording: true
-   #    enable_trade_recording:  true
-   #    enable_bbo_recording: false
-   #    enable_tbbo_recording:  false
-   #    enable_misc_recording: true
-   #}
- }
+  }
 }
+
+# JetFIX {
+#    value {
+#     #connection: @JetFIXSimConnection.sim
+#     connection {
+#     name: "mlx_004_54"
+#     venue: VENUE_GLBX
+#     socket_connect_host_ip: "69.50.112.191"
+#     socket_connect_port: 42876
+#     sender_compid: "MLX004"
+#     target_compid: "CME"
+#     target_subid: "54"
+#     cme_market_segment_id: 54
+#     start_time: "00:10:00"
+#     end_time: "23:55:00"
+#     start_day: "sunday"
+#     end_day: "friday"
+#     seconds_between_hbs: 30
+#     sender_subid: "BTS"
+#     use_ilink3: true
+#   #   raw_data: "S4PMA"
+#   #   auto_fix_seq_num: true
+#   }
+#    }
+# }
+
+

src/app_sparkd/configurations/ophir_sim_live_wb/.env

@@ -1,7 +1,5 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_prod_live_wb
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_sim_live_wb
 SPARKD__DATA_DIR=/home/wedbushprod/8000/sparkdata/data
 SPARKD__IMAGE=sparkd_symbols
-SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_prod_live_wb/requirements.txt
+SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_sim_live_wb/requirements.txt
 SPARKD__RUNTIME_CONF_DIR=/home/wedbushprod/8000/sparkdata/conf
-SPARKD__SHM_SIZE=8g
-SPARKD__MEMORY=16g

src/app_sparkd/configurations/ophir_sim_live_wb/spark.conf

@@ -1,12 +1,18 @@
-DeploymentType {
+JetFIX {
   value {
-    type: "prod"
+    connection {
+      name: "glbx_1"
+      venue: VENUE_GLBX
+      max_order_msgs_per_window: 300
+      window_duration_msecs: 1000
+    }
   }
 }
+
 PublisherOptions {
    value {
       network_interface_a: 'enp193s0f0np0'
-      network_interface_b: 'enp193s0f1np1'
+      network_interface_b: 'enp193s0f0np0'
       #these get overriden if not entered here
       heartbeat_ms: 5000
       heartbeat_boot_seconds: 60
@@ -35,7 +41,7 @@ CMEOptions {
       use_tcp_replay: false
       recovery_queue_size: 640000
       lost_packet_wait_timeout_ms: 10
-      onixs_log_mode: CME_ONIXS_LOG_MODE_UNSPECIFIED
+      onixs_log_mode: CME_ONIXS_LOG_MODE_UNSPECIFIED:q
       #max_gap_recovery_cache_size: 4
       use_natural_refresh: false
       join_snapshot_on_gap: false
@@ -67,35 +73,44 @@ MarketDataConfig {
        unique_check_value: 15
        use_databento: false
     }
-   # recording {
-   #    venue: VENUE_GLBX
-   #    num_of_seconds_to_run_file_writer: 300
-   #    num_of_seconds_to_run_db_writer : 0
-   #    n_book_levels: 10 
-   #    batch_size: 100 #for the db writer
-   #    enable_book_recording: true
-   #    enable_trade_recording:  true
-   #    enable_bbo_recording: false
-   #    enable_tbbo_recording:  false
-   #    enable_misc_recording: true
-   #}
- }
-}
-
-OrdermanCfg {
-  value {
-    executor {
-      thread_name: "oman_orders"
-      poll: true 
-      hosted_actor: "MDSource"
-      hosted_actor: "OrderManager"
-      hosted_actor: "ExecutionGateway"
-    }
-    executor {
-      thread_name: "oman_slow"
-            poll: false 
-      hosted_actor: "*"
-    }
-    fallback_cust_order_handling_inst: "Y"
+    recording{
+       venue: VENUE_GLBX
+       num_of_seconds_to_run_file_writer: 300
+       num_of_seconds_to_run_db_writer : 0
+       n_book_levels: 10 
+       batch_size: 100 #for the db writer
+       enable_book_recording: true
+       enable_trade_recording:  true
+       enable_bbo_recording: false
+       enable_tbbo_recording:  false
+       enable_misc_recording: true
+   }
   }
 }
+
+# JetFIX {
+#    value {
+#     #connection: @JetFIXSimConnection.sim
+#     connection {
+#     name: "mlx_004_54"
+#     venue: VENUE_GLBX
+#     socket_connect_host_ip: "69.50.112.191"
+#     socket_connect_port: 42876
+#     sender_compid: "MLX004"
+#     target_compid: "CME"
+#     target_subid: "54"
+#     cme_market_segment_id: 54
+#     start_time: "00:10:00"
+#     end_time: "23:55:00"
+#     start_day: "sunday"
+#     end_day: "friday"
+#     seconds_between_hbs: 30
+#     sender_subid: "BTS"
+#     use_ilink3: true
+#   #   raw_data: "S4PMA"
+#   #   auto_fix_seq_num: true
+#   }
+#    }
+# }
+
+

src/app_sparkd/sparkd-entrypoint.sh

@@ -42,7 +42,10 @@ roll_log() {
 # Acquire sudo permission once for the script (needed for a few calls)
 echo bts | sudo -S -v
 data_root="${HOME:-/home/bts}/.local/share/bts/sparkd"
+api_server_data_root="${HOME:-/home/bts}/.local/share/bts/spark_api_server"
 sudo -n chown -R bts:bts "$data_root"
+sudo -n mkdir -p "$api_server_data_root"
+sudo -n chown -R bts:bts "$api_server_data_root"
 load_env "$data_root/conf/.env.local"
 init_timezone "$TZ"
 roll_log "$data_root/data/logging.sparkd/log"

src/app_sparkd/sparkd/cmds/md/common/VenueOption.cpp

@@ -1,74 +1,34 @@
 #include "VenueOption.h"
 #include "spark/domain_model/Security.pb.h"
 
+#include <algorithm>
+#include <cctype>
+
 namespace bts::cmds::md {
 
-#define NAME_PAIR(val)                                                         \
-   { name(val), val }
-
 EnumOptionNames<api::Venue> VenueOption::names() {
-   return {
-       NAME_PAIR(api::VENUE_BTS),
-       NAME_PAIR(api::VENUE_GLBX),
-       NAME_PAIR(api::VENUE_ICE),
-       NAME_PAIR(api::VENUE_SIM),
-       NAME_PAIR(api::VENUE_SIMICE),
-       NAME_PAIR(api::VENUE_DERIBIT),
-       NAME_PAIR(api::VENUE_FTX),
-       NAME_PAIR(api::VENUE_BINANCE),
-       NAME_PAIR(api::VENUE_BINANCE_COIN_FUTURES),
-       NAME_PAIR(api::VENUE_BINANCE_US),
-       NAME_PAIR(api::VENUE_COINBASE),
-       NAME_PAIR(api::VENUE_BINANCE_USDS_FUTURES),
-       NAME_PAIR(api::VENUE_CBOE),
-       NAME_PAIR(api::VENUE_TAIFEX),
-       NAME_PAIR(api::VENUE_TWSE),
-       NAME_PAIR(api::VENUE_BYBIT),
-       NAME_PAIR(api::VENUE_GATEIO),
-       NAME_PAIR(api::VENUE_OKX)};
+   EnumOptionNames<api::Venue> result;
+   const auto *descriptor = api::Venue_descriptor();
+   for (int i = 0; i < descriptor->value_count(); ++i) {
+      const auto venue = static_cast<api::Venue>(descriptor->value(i)->number());
+      if (venue != api::VENUE_UNSET) {
+         result.emplace(name(venue), venue);
+      }
+   }
+   return result;
 }
 
 std::string VenueOption::name(api::Venue venue) {
-   switch (venue) {
-   case api::VENUE_BTS:
-      return "bts";
-   case api::VENUE_GLBX:
-      return "glbx";
-   case api::VENUE_ICE:
-      return "ice";
-   case api::VENUE_SIM:
-      return "sim";
-   case api::VENUE_SIMICE:
-      return "simice";
-   case api::VENUE_DERIBIT:
-      return "deribit";
-   case api::VENUE_FTX:
-      return "ftx";
-   case api::VENUE_BINANCE:
-      return "binance";
-   case api::VENUE_BINANCE_COIN_FUTURES:
-      return "binance_coin_futures";
-   case api::VENUE_BINANCE_US:
-      return "binance_us";
-   case api::VENUE_COINBASE:
-      return "coinbase";
-   case api::VENUE_BINANCE_USDS_FUTURES:
-      return "binance_usds_futures";
-   case api::VENUE_CBOE:
-      return "cboe";
-   case api::VENUE_OKX:
-      return "okx";
-   case api::VENUE_TAIFEX:
-      return "taifex";
-   case api::VENUE_TWSE:
-      return "twse";
-   case api::VENUE_BYBIT:
-      return "bybit";
-   case api::VENUE_GATEIO:
-      return "gateio";
-   default:
+   if (!api::Venue_IsValid(venue)) {
       return std::to_string(static_cast<int32_t>(venue));
    }
+   std::string result = api::Venue_Name(venue);
+   constexpr auto prefix_size = sizeof("VENUE_") - 1;
+   result.erase(0, prefix_size);
+   std::transform(result.begin(), result.end(), result.begin(), [](char value) {
+      return static_cast<char>(std::tolower(static_cast<unsigned char>(value)));
+   });
+   return result;
 }
 
 } // namespace bts::cmds::md

src/app_sparkd/sparkd/execution/execution_gateway_provider.cpp

@@ -4,6 +4,7 @@
 #include "binance/spots/spots_gateway.h"
 #include "binance/usds_futures/usds_futures_gateway.h"
 #include "okx_execution/okx_execution_gateway.h"
+#include "sparkd/execution/rakurai/rakurai_execution_gateway.h"
 #include "onixs_fix/OnixFIXAdapter.h"
 #include "python_interop/plugin_manager.h"
 #include "spark/api/domain/Venue.h"
@@ -109,8 +110,7 @@ struct ExecutionGatewayProvider::PImpl {
          return make_gw<ICEExecutionGateway>(conn, conn, *l, o);
       case api::VENUE_GLBX:
          if (conn.use_ilink3()) {
-            return make_gw<ilink3::GLBXiLink3ExecutionGateway>(
-                conn, conn, *l, o);
+            make_gw<ilink3::GLBXiLink3ExecutionGateway>(conn, conn, *l, o);
          }
          return make_gw<GLBXExecutionGateway>(conn, conn, *l, o);
       case api::VENUE_DERIBIT:
@@ -120,6 +120,18 @@ struct ExecutionGatewayProvider::PImpl {
              conn, conn.venue(), std::make_unique<OnixFIXAdapter>(o), o);
       case api::VENUE_OKX:
          return make_gw<OkxExecutionGateway>(conn, o);
+      case api::VENUE_ORCA:
+      case api::VENUE_RAYDIUM: {
+         auto cfg_j = load_config_json(conn.name());
+         bts_assert_msgf(
+             cfg_j,
+             "Failed to load execution gateway config. name: {}",
+             conn.name());
+         auto cfg = cfg_j->get<rakurai::GatewayConfig>();
+         cfg.expand_env();
+         return make_gw<rakurai::RakuraiExecutionGateway>(
+             conn, o, std::move(cfg), conn);
+      }
       case api::VENUE_BINANCE: {
          auto cfg =
              parse_config_json<binance::GatewayConfig>(conn.name());

src/app_sparkd/sparkd/execution/rakurai/rakurai_execution_gateway.cpp

@@ -0,0 +1,742 @@
+#include "sparkd/execution/rakurai/rakurai_execution_gateway.h"
+
+#include "spark/config/config.h"
+#include "spark/config/protos/system_config.pb.h"
+#include "spark/domain/DomainSecurityUtils.h"
+#include "spark/domain/Price.h"
+#include "spark/domain/Quantity.h"
+#include "spark/domain_model/Security.h"
+#include "spark/logging/logging.h"
+#include "spark/md/SecurityOracle.h"
+#include "spark/orderman/CancelOrderMsg.h"
+#include "spark/orderman/CancelReplaceOrderMsg.h"
+#include "spark/orderman/CreateOrderMsg.h"
+#include "spark/orderman/RFQMsg.h"
+#include "spark/orderman/events/NewAdminDataEvent.h"
+#include "spark/orderman/events/NewSecuritiesEvent.h"
+#include "spark/orderman/orderman_utils/CancelReject.h"
+#include "spark/orderman/orderman_utils/ExecutionReport.h"
+#include "spark/orderman/orderman_utils/orderman_common.h"
+#include "spark/utils/bts_assert.h"
+#include "sparkd/rakurai/RakuraiSharedRuntime.h"
+
+#include <boost/filesystem/path.hpp>
+
+#include <cmath>
+#include <cstdint>
+#include <deque>
+#include <limits>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+namespace bts::orderman::rakurai {
+
+namespace {
+
+#define LOG_FN(level, fn, ...)                                                 \
+   LOG(OrderMan, level, "RakuraiExecutionGateway::" fn ". " __VA_ARGS__)
+#define WARN_FN(fn, ...) LOG_FN(Warn, fn, __VA_ARGS__)
+
+using ConnectorNewOrder = ::rakurai::dex_connector::order_entry::new_order;
+using ConnectorOrderState = ::rakurai::dex_connector::order_entry::order_state;
+using ConnectorSide = ::rakurai::dex_connector::order_entry::side;
+
+std::optional<std::string> extract_short_pool_id(const api::Security& security) {
+   std::string desc;
+   if (!domain::DomainSecurityUtils::get_security_desc(security, desc)) {
+      return std::nullopt;
+   }
+   const auto pos = desc.rfind('-');
+   if (pos == std::string::npos || pos + 1 >= desc.size()) {
+      return std::nullopt;
+   }
+   return desc.substr(pos + 1);
+}
+
+api::orderman::NativeOrderStatus to_order_status(ConnectorOrderState state) {
+   using Status = ::rakurai::dex_connector::data_types::txn_status_t;
+   switch (state.tx_status) {
+   case Status::accepted:
+      return api::orderman::NATIVE_ORDER_STATUS_NEW;
+   case Status::confirmed:
+      return state.is_pass ? api::orderman::NATIVE_ORDER_STATUS_FILLED
+                           : api::orderman::NATIVE_ORDER_STATUS_REJECTED;
+   case Status::failed:
+      return api::orderman::NATIVE_ORDER_STATUS_REJECTED;
+   case Status::pending:
+      return api::orderman::NATIVE_ORDER_STATUS_PENDING_NEW;
+   }
+   return api::orderman::NATIVE_ORDER_STATUS_UNDEFINED;
+}
+
+api::orderman::ExecType to_exec_type(ConnectorOrderState state) {
+   using Status = ::rakurai::dex_connector::data_types::txn_status_t;
+   switch (state.tx_status) {
+   case Status::accepted:
+      return api::orderman::EXEC_TYPE_NEW;
+   case Status::confirmed:
+      return state.is_pass ? api::orderman::EXEC_TYPE_FILL
+                           : api::orderman::EXEC_TYPE_REJECTED;
+   case Status::failed:
+      return api::orderman::EXEC_TYPE_REJECTED;
+   case Status::pending:
+      return api::orderman::EXEC_TYPE_PENDING_NEW;
+   }
+   return api::orderman::EXEC_TYPE_UNDEFINED;
+}
+
+std::string txn_status_name(
+    ::rakurai::dex_connector::data_types::txn_status_t status) {
+   using Status = ::rakurai::dex_connector::data_types::txn_status_t;
+   switch (status) {
+   case Status::accepted:
+      return "accepted";
+   case Status::confirmed:
+      return "confirmed";
+   case Status::failed:
+      return "failed";
+   case Status::pending:
+      return "pending";
+   }
+   return "unknown";
+}
+
+std::string json_for_log(const nlohmann::json& j) {
+   try {
+      return j.dump();
+   } catch (const std::exception& ex) {
+      return fmt::format("<json dump failed: {}>", ex.what());
+   }
+}
+
+} // namespace
+
+struct RakuraiExecutionGateway::PImpl {
+   struct ActiveOrder {
+      CreateOrderMsg create_msg;
+      trace::CorID cor_id;
+      std::array<std::uint8_t, 32> pool_id{};
+      double quantity_scale = 1.0;
+      std::uint64_t raw_quantity = 0;
+      std::uint64_t priority_fee = 0;
+      std::uint64_t client_id = 0;
+      std::uint64_t txn_id = 0;
+   };
+
+   struct BufferedReport {
+      std::shared_ptr<const ExecutionReport> report;
+      trace::CorID cor_id;
+   };
+
+   explicit PImpl(
+       md::SecurityOracle,
+       GatewayConfig cfg,
+       config::JetFIX_Connection jetfix_connection)
+       : config(std::move(cfg)),
+         jetfix_connection(std::make_shared<config::JetFIX_Connection>(
+             std::move(jetfix_connection))) {
+      config.expand_env();
+      venue = this->jetfix_connection->venue();
+      bts_assert(venue == api::VENUE_ORCA || venue == api::VENUE_RAYDIUM);
+      LOG_FN(
+          Info,
+          "ctor",
+          "session:{} venue:{} account:{} operator_id:{} json_conf_file_name:{} default_priority_fee:{} connector_config:{}",
+          this->jetfix_connection->name(),
+          api::Venue_Name(venue),
+          config.account,
+          config.operator_id,
+          config.json_conf_file_name,
+          config.default_priority_fee,
+          json_for_log(config.connector_config));
+   }
+
+   GatewayConfig config;
+   std::shared_ptr<config::JetFIX_Connection> jetfix_connection;
+   api::Venue venue = api::VENUE_UNSET;
+   std::mutex mutex;
+   std::unordered_map<OrderKey, ActiveOrder> active_orders;
+   std::unordered_map<std::uint64_t, OrderKey> client_id_to_order_key;
+   std::deque<OrderKey> pending_client_bindings;
+   std::deque<BufferedReport> buffered_reports;
+   std::size_t order_update_subscription_id = 0;
+   std::size_t status_subscription_id = 0;
+   bool started = false;
+   bool connected = false;
+   OnExeReport exe_report;
+   OnGwStatusChange gw_status_change;
+   OnCancelReject cancel_reject;
+   OnExchangePositionState exchange_position_state;
+   OnExchangeOpenOrdersState exchange_open_orders;
+
+   void start() {
+      {
+         std::lock_guard<std::mutex> lock(mutex);
+         if (started) {
+            LOG_FN(
+                Info, "start", "already started. venue:{}",
+                api::Venue_Name(venue));
+            return;
+         }
+         started = true;
+      }
+      LOG_FN(
+          Info,
+          "start",
+          "starting. session:{} venue:{} account:{} operator_id:{} json_conf_file_name:{} default_priority_fee:{}",
+          jetfix_connection->name(),
+          api::Venue_Name(venue),
+          config.account,
+          config.operator_id,
+          config.json_conf_file_name,
+          config.default_priority_fee);
+
+      auto& runtime = bts::rakurai::RakuraiSharedRuntime::instance();
+      order_update_subscription_id =
+          runtime.add_order_update_subscriber([this](ConnectorOrderState state) {
+             this->on_order_update(state);
+          });
+      status_subscription_id =
+          runtime.add_status_subscriber([this](bool is_connected_now) {
+             this->on_status_change(is_connected_now);
+          });
+
+      try {
+         if (!config.json_conf_file_name.empty()) {
+            const auto config_path =
+                boost::filesystem::path{
+                    config::get<config::SystemConfig>().config_path()} /
+                config.json_conf_file_name;
+            LOG_FN(
+                Info,
+                "start",
+                "ensuring shared runtime from file. path:{}",
+                config_path.string());
+            runtime.ensure_started_from_file(config_path.string());
+         } else {
+            LOG_FN(
+                Info,
+                "start",
+                "ensuring shared runtime from inline config. config:{}",
+                json_for_log(config.connector_config));
+            runtime.ensure_started_from_json(
+                config.connector_config, jetfix_connection->name());
+         }
+         LOG_FN(
+             Info,
+             "start",
+             "shared runtime start request submitted. order_update_sub_id:{} status_sub_id:{} connected_now:{}",
+             order_update_subscription_id,
+             status_subscription_id,
+             runtime.is_connected());
+      } catch (...) {
+         runtime.remove_order_update_subscriber(order_update_subscription_id);
+         runtime.remove_status_subscriber(status_subscription_id);
+         order_update_subscription_id = 0;
+         status_subscription_id = 0;
+         {
+            std::lock_guard<std::mutex> lock(mutex);
+            started = false;
+         }
+         throw;
+      }
+   }
+
+   void stop() {
+      LOG_FN(
+          Info,
+          "stop",
+          "stopping. order_update_sub_id:{} status_sub_id:{}",
+          order_update_subscription_id,
+          status_subscription_id);
+      auto& runtime = bts::rakurai::RakuraiSharedRuntime::instance();
+      if (order_update_subscription_id != 0) {
+         runtime.remove_order_update_subscriber(order_update_subscription_id);
+         order_update_subscription_id = 0;
+      }
+      if (status_subscription_id != 0) {
+         runtime.remove_status_subscriber(status_subscription_id);
+         status_subscription_id = 0;
+      }
+   }
+
+   void on_status_change(bool is_connected_now) {
+      LOG_FN(
+          Info,
+          "on_status_change",
+          "status update. venue:{} old_connected:{} new_connected:{} active_orders:{} pending_bindings:{} buffered_reports:{}",
+          api::Venue_Name(venue),
+          connected,
+          is_connected_now,
+          active_orders.size(),
+          pending_client_bindings.size(),
+          buffered_reports.size());
+      {
+         std::lock_guard<std::mutex> lock(mutex);
+         connected = is_connected_now;
+      }
+      if (gw_status_change) {
+         gw_status_change(is_connected_now);
+      }
+   }
+
+   void on_order_update(ConnectorOrderState order_state) {
+      std::lock_guard<std::mutex> lock(mutex);
+      LOG_FN(
+          Info,
+          "on_order_update",
+          "incoming order update. client_id:{} txn_id:{} txn_status:{} is_pass:{} active_orders:{} pending_bindings:{} mapped_clients:{}",
+          order_state.client_id,
+          order_state.txn_id,
+          txn_status_name(order_state.tx_status),
+          order_state.is_pass,
+          active_orders.size(),
+          pending_client_bindings.size(),
+          client_id_to_order_key.size());
+      auto key_it = client_id_to_order_key.find(order_state.client_id);
+      if (key_it == client_id_to_order_key.end()) {
+         if (pending_client_bindings.empty()) {
+            LOG_FN(
+                Warn,
+                "on_order_update",
+                "dropping order update with unknown client_id:{} because there are no pending client bindings",
+                order_state.client_id);
+            return;
+         }
+         const auto key = pending_client_bindings.front();
+         pending_client_bindings.pop_front();
+         client_id_to_order_key.emplace(order_state.client_id, key);
+         LOG_FN(
+             Info,
+             "on_order_update",
+             "bound client_id:{} to order_key:{}",
+             order_state.client_id,
+             key.value());
+         key_it = client_id_to_order_key.find(order_state.client_id);
+      }
+
+      auto order_it = active_orders.find(key_it->second);
+      if (order_it == active_orders.end()) {
+         LOG_FN(
+             Warn,
+             "on_order_update",
+             "dropping order update for client_id:{} mapped_order_key:{} because active order was not found",
+             order_state.client_id,
+             key_it->second.value());
+         return;
+      }
+
+      auto& active = order_it->second;
+      active.client_id = order_state.client_id;
+      active.txn_id = order_state.txn_id;
+      auto report = build_report_locked(active, order_state);
+      buffered_reports.push_back({std::move(report), active.cor_id});
+      LOG_FN(
+          Info,
+          "on_order_update",
+          "buffered execution report. order_key:{} client_id:{} txn_id:{} buffered_reports:{} order_status:{} exec_type:{}",
+          active.create_msg.key.value(),
+          active.client_id,
+          active.txn_id,
+          buffered_reports.size(),
+          api::orderman::NativeOrderStatus_Name(
+              buffered_reports.back().report->order_status),
+          api::orderman::ExecType_Name(buffered_reports.back().report->exec_type));
+
+      const auto order_status = buffered_reports.back().report->order_status;
+      if (order_status == api::orderman::NATIVE_ORDER_STATUS_FILLED ||
+          order_status == api::orderman::NATIVE_ORDER_STATUS_REJECTED) {
+         LOG_FN(
+             Info,
+             "on_order_update",
+             "removing terminal active order. order_key:{} client_id:{} status:{}",
+             active.create_msg.key.value(),
+             order_state.client_id,
+             api::orderman::NativeOrderStatus_Name(order_status));
+         client_id_to_order_key.erase(order_state.client_id);
+         active_orders.erase(order_it);
+      }
+   }
+
+   void send_create(const CreateOrderMsg& msg, trace::CorID cor_id) {
+      std::lock_guard<std::mutex> lock(mutex);
+      std::string security_desc = "<null>";
+      if (msg.security != nullptr) {
+         std::string desc;
+         if (domain::DomainSecurityUtils::get_security_desc(*msg.security, desc)) {
+            security_desc = std::move(desc);
+         } else {
+            security_desc = "<unresolved>";
+         }
+      }
+      LOG_FN(
+          Info,
+          "send_create",
+          "create request. order_key:{} venue:{} side:{} qty:{} price:{} tif:{} order_type:{} connected:{} security_present:{} desc:{}",
+          msg.key.value(),
+          api::Venue_Name(msg.venue()),
+          api::Side_Name(msg.side),
+          msg.quantity.full_value(),
+          msg.price,
+          api::orderman::TimeInForce_Name(msg.time_in_force),
+          api::orderman::NativeOrderType_Name(msg.order_type),
+          connected,
+          msg.security != nullptr,
+          security_desc);
+      if (msg.security == nullptr) {
+         enqueue_reject_locked(msg, cor_id, "Rakurai order missing security");
+         return;
+      }
+      if (msg.venue() != venue) {
+         enqueue_reject_locked(
+             msg,
+             cor_id,
+             fmt::format(
+                 "Rakurai gateway venue mismatch. expected:{} actual:{}",
+                 api::Venue_Name(venue),
+                 api::Venue_Name(msg.venue())));
+         return;
+      }
+      if (msg.order_type != api::orderman::NATIVE_ORDER_TYPE_MARKET) {
+         enqueue_reject_locked(
+             msg,
+             cor_id,
+             "Rakurai DEX gateway currently supports market orders only");
+         return;
+      }
+      if (!connected) {
+         enqueue_reject_locked(
+             msg, cor_id, "Rakurai execution session is not connected");
+         return;
+      }
+
+      const auto short_pool_id = extract_short_pool_id(*msg.security);
+      if (!short_pool_id.has_value()) {
+         enqueue_reject_locked(
+             msg, cor_id, "Unable to derive Rakurai pool alias from security");
+         return;
+      }
+
+      const auto metadata =
+          bts::rakurai::RakuraiSharedRuntime::instance().find_pool_metadata(
+              venue, *short_pool_id);
+      if (!metadata.has_value()) {
+         LOG_FN(
+             Warn,
+             "send_create",
+             "pool metadata missing. venue:{} short_pool_id:{}",
+             api::Venue_Name(venue),
+             *short_pool_id);
+         enqueue_reject_locked(
+             msg,
+             cor_id,
+             "Rakurai pool metadata not available yet; wait for pool definitions");
+         return;
+      }
+
+      const auto raw_qty_f =
+          msg.quantity.full_value() * metadata->quantity_scale;
+      if (!std::isfinite(raw_qty_f) || raw_qty_f <= 0.0) {
+         enqueue_reject_locked(
+             msg, cor_id, "Rakurai order quantity must be positive");
+         return;
+      }
+
+      ConnectorNewOrder order;
+      order.order_def.pool_id = metadata->security.pool_id;
+      order.order_def.dex = metadata->security.venue;
+      order.order_def.order_side =
+          msg.side == api::SIDE_BUY ? ConnectorSide::buy : ConnectorSide::sell;
+      order.qty = static_cast<std::uint64_t>(std::llround(raw_qty_f));
+      order.priority_fee = config.default_priority_fee;
+      LOG_FN(
+          Info,
+          "send_create",
+          "launching new order. order_key:{} short_pool_id:{} raw_qty:{} priority_fee:{} dex:{} quantity_scale:{}",
+          msg.key.value(),
+          *short_pool_id,
+          order.qty,
+          order.priority_fee,
+          static_cast<int>(metadata->security.venue),
+          metadata->quantity_scale);
+
+      try {
+         bts::rakurai::RakuraiSharedRuntime::instance().launch_new_order(order);
+      } catch (const std::exception& ex) {
+         WARN_FN(
+             "send_create",
+             "launch_new_order threw. order_key:{} reason:{}",
+             msg.key.value(),
+             std::string(ex.what()));
+         enqueue_reject_locked(msg, cor_id, ex.what());
+         return;
+      }
+
+      active_orders.emplace(
+          msg.key,
+          ActiveOrder{
+              msg,
+              cor_id,
+              metadata->security.pool_id,
+              metadata->quantity_scale,
+              order.qty,
+              order.priority_fee});
+      pending_client_bindings.push_back(msg.key);
+      LOG_FN(
+          Info,
+          "send_create",
+          "create request accepted locally. order_key:{} active_orders:{} pending_bindings:{}",
+          msg.key.value(),
+          active_orders.size(),
+          pending_client_bindings.size());
+   }
+
+   void send_cancel_reject(OrderKey key, const std::string& reason) {
+      WARN_FN(
+          "send_cancel_reject",
+          "cancel reject. order_key:{} reason:{}",
+          key.value(),
+          reason);
+      if (cancel_reject) {
+         cancel_reject({key, CancelRejectReason::Other, reason});
+      }
+   }
+
+   std::shared_ptr<const ExecutionReport> build_report_locked(
+       const ActiveOrder& active,
+       ConnectorOrderState state) const {
+      auto report = std::make_shared<ExecutionReport>();
+      report->account_name = account_name_for(active.create_msg);
+      report->operator_id = operator_id_for(active.create_msg);
+      report->order_key = active.create_msg.key;
+      report->msg_seq_num = 0;
+      report->security = active.create_msg.security;
+      report->leg_security = nullptr;
+      report->side = active.create_msg.side;
+      report->time_in_force = active.create_msg.time_in_force;
+      report->order_type = active.create_msg.order_type;
+      report->manual_order_indicator = active.create_msg.manual_order_indicator;
+      report->exchange = venue;
+      report->cfg = jetfix_connection.get();
+      report->exchange_order_id = ExchangeOrderID64(
+          active.client_id != 0 ? active.client_id : state.txn_id);
+      report->quantity = active.create_msg.quantity;
+      report->price = active.create_msg.price;
+      report->stop_price = active.create_msg.stop_px;
+      report->max_show = active.create_msg.display_quantity;
+      report->transact_time =
+          ExchangeTimeStamp64(static_cast<std::uint64_t>(datetime::unix_nano_now()));
+      report->exec_type = to_exec_type(state);
+      report->order_status = to_order_status(state);
+      report->filled_quantity =
+          report->order_status == api::orderman::NATIVE_ORDER_STATUS_FILLED
+              ? active.create_msg.quantity
+              : Quantity(0.0f);
+      report->last_shares = report->filled_quantity;
+      report->last_price = active.create_msg.price;
+      report->avg_price = active.create_msg.price;
+      report->msg = fmt::format(
+          "Rakurai txn_status={} txn_id={} pass={}",
+          txn_status_name(state.tx_status),
+          state.txn_id,
+          state.is_pass);
+      LOG_FN(
+          Info,
+          "build_report_locked",
+          "built report. order_key:{} client_id:{} txn_id:{} exec_type:{} order_status:{} message:{}",
+          active.create_msg.key.value(),
+          active.client_id,
+          state.txn_id,
+          api::orderman::ExecType_Name(report->exec_type),
+          api::orderman::NativeOrderStatus_Name(report->order_status),
+          report->msg);
+      return report;
+   }
+
+   void enqueue_reject_locked(
+       const CreateOrderMsg& msg,
+       trace::CorID cor_id,
+       const std::string& reason) {
+      auto report = std::make_shared<ExecutionReport>();
+      report->account_name = account_name_for(msg);
+      report->operator_id = operator_id_for(msg);
+      report->order_key = msg.key;
+      report->msg_seq_num = 0;
+      report->security = msg.security;
+      report->leg_security = nullptr;
+      report->side = msg.side;
+      report->time_in_force = msg.time_in_force;
+      report->order_type = msg.order_type;
+      report->manual_order_indicator = msg.manual_order_indicator;
+      report->exchange = venue;
+      report->cfg = jetfix_connection.get();
+      report->exchange_order_id = ExchangeOrderID64(0);
+      report->quantity = msg.quantity;
+      report->filled_quantity = Quantity(0.0f);
+      report->last_shares = Quantity(0.0f);
+      report->price = msg.price;
+      report->stop_price = msg.stop_px;
+      report->max_show = msg.display_quantity;
+      report->last_price = Price(std::numeric_limits<double>::quiet_NaN());
+      report->avg_price = Price(std::numeric_limits<double>::quiet_NaN());
+      report->transact_time =
+          ExchangeTimeStamp64(static_cast<std::uint64_t>(datetime::unix_nano_now()));
+      report->exec_type = api::orderman::EXEC_TYPE_REJECTED;
+      report->order_status = api::orderman::NATIVE_ORDER_STATUS_REJECTED;
+      report->msg = reason;
+      buffered_reports.push_back({std::move(report), cor_id});
+      WARN_FN(
+          "enqueue_reject_locked",
+          "buffered reject. order_key:{} reason:{} buffered_reports:{}",
+          msg.key.value(),
+          reason,
+          buffered_reports.size());
+   }
+
+   AccountName account_name_for(const CreateOrderMsg& msg) const {
+      const auto account = msg.routing_record != nullptr &&
+                                   !msg.routing_record->account().empty()
+                               ? msg.routing_record->account()
+                               : config.account;
+      AccountName result;
+      result->copy_from(account);
+      return result;
+   }
+
+   OperatorID operator_id_for(const CreateOrderMsg& msg) const {
+      const auto operator_id =
+          msg.routing_record != nullptr &&
+                  !msg.routing_record->operator_id().empty()
+              ? msg.routing_record->operator_id()
+              : config.operator_id;
+      OperatorID result;
+      orderman::to_array(operator_id, &result.value());
+      return result;
+   }
+};
+
+RakuraiExecutionGateway::RakuraiExecutionGateway(
+    md::SecurityOracle sec_oracle,
+    GatewayConfig config,
+    config::JetFIX_Connection jetfix_connection)
+    : me_(std::make_unique<PImpl>(
+          std::move(sec_oracle),
+          std::move(config),
+          std::move(jetfix_connection))) {}
+
+RakuraiExecutionGateway::~RakuraiExecutionGateway() { me_->stop(); }
+
+void RakuraiExecutionGateway::start() { me_->start(); }
+
+void RakuraiExecutionGateway::dispatch_incoming_fix_messages() {
+   std::deque<PImpl::BufferedReport> reports;
+   {
+      std::lock_guard<std::mutex> lock(me_->mutex);
+      reports.swap(me_->buffered_reports);
+   }
+
+   if (!reports.empty()) {
+   LOG_FN(
+        Info,
+        "dispatch_incoming_fix_messages",
+        "dispatching buffered reports. count:{} has_callback:{}",
+        reports.size(),
+        me_->exe_report != nullptr);
+   }
+   if (!me_->exe_report) {
+      return;
+   }
+
+   for (auto& buffered : reports) {
+      LOG_FN(
+          Info,
+          "dispatch_incoming_fix_messages",
+          "publishing execution report. order_key:{} exec_type:{} order_status:{} msg:{}",
+          buffered.report->order_key.value(),
+          api::orderman::ExecType_Name(buffered.report->exec_type),
+          api::orderman::NativeOrderStatus_Name(buffered.report->order_status),
+          buffered.report->msg);
+      me_->exe_report(std::move(buffered.report), buffered.cor_id);
+   }
+}
+
+void RakuraiExecutionGateway::on_securities(
+    const mercury::Update<NewSecuritiesEvent>& update) {
+   LOG_FN(
+       Info,
+       "on_securities",
+       "received securities update. type:{} has_events:{} count:{}",
+       static_cast<int>(update.event_type),
+       update.events != nullptr,
+       update.events != nullptr ? update.events->size() : 0);
+}
+
+void RakuraiExecutionGateway::on_admin_data(
+    const mercury::Update<NewAdminDataEvent>& update) {
+   LOG_FN(
+       Info,
+       "on_admin_data",
+       "received admin data update. type:{} has_events:{} count:{}",
+       static_cast<int>(update.event_type),
+       update.events != nullptr,
+       update.events != nullptr ? update.events->size() : 0);
+}
+
+void RakuraiExecutionGateway::send_fix(const jetfix::OutFIXPtr&, trace::CorID) {
+   WARN_FN("send_fix", "FIX transport is not supported for Rakurai");
+}
+
+void RakuraiExecutionGateway::send_msgs(
+    std::vector<OrderMsg> const& msgs,
+    trace::CorID cor_id) {
+   LOG_FN(
+       Info,
+       "send_msgs",
+       "received outbound message batch. count:{}",
+       msgs.size());
+   for (const auto& msg : msgs) {
+      std::visit(
+          [&](const auto& order_msg) {
+             using T = std::decay_t<decltype(order_msg)>;
+             if constexpr (std::is_same_v<T, CreateOrderMsg>) {
+                me_->send_create(order_msg, cor_id);
+             } else if constexpr (std::is_same_v<T, CancelOrderMsg>) {
+                me_->send_cancel_reject(
+                    order_msg.key,
+                    "Rakurai DEX gateway does not support order cancel");
+             } else if constexpr (std::is_same_v<T, CancelReplaceOrderMsg>) {
+                me_->send_cancel_reject(
+                    order_msg.create_fields.key,
+                    "Rakurai DEX gateway does not support order replace");
+             } else if constexpr (std::is_same_v<T, RFQMsg>) {
+                WARN_FN("send_msgs", "RFQ is not supported for Rakurai");
+             }
+          },
+          msg);
+   }
+}
+
+OnExeReport& RakuraiExecutionGateway::on_exe_report() { return me_->exe_report; }
+
+OnGwStatusChange& RakuraiExecutionGateway::on_gw_status_change() {
+   return me_->gw_status_change;
+}
+
+OnCancelReject& RakuraiExecutionGateway::on_cancel_reject() {
+   return me_->cancel_reject;
+}
+
+OnExchangePositionState& RakuraiExecutionGateway::on_exchange_position_state() {
+   return me_->exchange_position_state;
+}
+
+OnExchangeOpenOrdersState& RakuraiExecutionGateway::on_exchange_open_orders() {
+   return me_->exchange_open_orders;
+}
+
+} // namespace bts::orderman::rakurai

src/app_sparkd/sparkd/execution/rakurai/rakurai_execution_gateway.h

@@ -0,0 +1,76 @@
+#pragma once
+
+#include "spark/io/JsonReader.h"
+#include "spark/orderman/gateways/IExecutionGateway.h"
+#include "spark/os/os.h"
+
+#include <memory>
+#include <nlohmann/json.hpp>
+
+namespace bts::config {
+class JetFIX_Connection;
+}
+
+namespace bts::md {
+class SecurityOracle;
+}
+
+namespace bts::orderman::rakurai {
+
+struct GatewayConfig {
+   std::string account;
+   std::string operator_id;
+   std::string json_conf_file_name;
+   std::uint64_t default_priority_fee = 1000;
+   nlohmann::json connector_config;
+
+   void expand_env() {
+      bts::os::expand_env_in_place(account);
+      bts::os::expand_env_in_place(operator_id);
+      bts::os::expand_env_in_place(json_conf_file_name);
+   }
+};
+
+inline void from_json(const nlohmann::json& j, GatewayConfig& cfg) {
+   cfg.connector_config = j;
+
+   const bts::io::JsonReader jr{&j};
+   jr.read_or("account", cfg.account);
+   jr.read_or("operator_id", cfg.operator_id);
+   jr.read_or("json_conf_file_name", cfg.json_conf_file_name);
+   jr.read_or("default_priority_fee", cfg.default_priority_fee);
+}
+
+class RakuraiExecutionGateway : public IExecutionGateway {
+ public:
+   explicit RakuraiExecutionGateway(
+       bts::md::SecurityOracle sec_oracle,
+       GatewayConfig config,
+       bts::config::JetFIX_Connection jetfix_connection);
+   ~RakuraiExecutionGateway() override;
+
+   void start() override;
+   void dispatch_incoming_fix_messages() override;
+   void on_securities(
+       const bts::mercury::Update<bts::orderman::NewSecuritiesEvent>&)
+       override;
+   void on_admin_data(
+       const bts::mercury::Update<bts::orderman::NewAdminDataEvent>&) override;
+   void send_fix(const bts::jetfix::OutFIXPtr&, bts::trace::CorID) override;
+   void send_msgs(
+       std::vector<bts::orderman::OrderMsg> const&,
+       bts::trace::CorID) override;
+
+   bts::orderman::OnExeReport& on_exe_report() override;
+   bts::orderman::OnGwStatusChange& on_gw_status_change() override;
+   bts::orderman::OnCancelReject& on_cancel_reject() override;
+   bts::orderman::OnExchangePositionState&
+   on_exchange_position_state() override;
+   bts::orderman::OnExchangeOpenOrdersState& on_exchange_open_orders() override;
+
+ private:
+   struct PImpl;
+   std::unique_ptr<PImpl> me_;
+};
+
+} // namespace bts::orderman::rakurai

src/app_sparkd/sparkd/md/MarketDataGateway.cpp

@@ -25,6 +25,7 @@
 #include "sparkd/md/ccapi/CCApiMarketDataProvider.h"
 #include "sparkd/md/databento/DatabentoMarketDataProvider.h"
 #include "sparkd/md/emumba/EmumbaMarketDataProvider.h"
+#include "sparkd/md/rakurai/RakuraiMarketDataProvider.h"
 #include "sparkd/md/taifex/TaifexMarketDataProvider.h"
 
 #include <bits/stdint-uintn.h>
@@ -81,6 +82,8 @@ struct MarketDataGateway::PImpl {
           std::make_unique<OnixsMarketDataProvider>(this->config));
       providers.push_back(std::make_unique<DatabentoMarketDataProvider>(
           this->config, this->sec_id_store));
+      providers.push_back(std::make_unique<RakuraiMarketDataProvider>(
+          this->config, this->sec_id_store));
       providers.push_back(std::make_unique<TaifexMarketDataProvider>(
           this->config, this->sec_id_store));
       providers.push_back(std::make_unique<PyMarketDataProvider>(

src/app_sparkd/sparkd/md/glbx/OnixsGlbxSubscription.cpp

@@ -3,11 +3,9 @@
 #include "spark/core/LOCK_GUARD.h"
 #include "spark/logging/logging.h"
 #include "spark/md/IMarketDataListener.h"
-#include "spark/md/GeneralMarketDataListener.h"
 #include "spark/md/SymbolLoader.h"
 #include "spark/os/os.h"
 #include "spark/persistence/IDGenerator.h"
-#include "spark/orderman/gateways/sim/SimExecutionGatewayFactory.h"
 #include "spark/utils/Dispatcher.h"
 #include "sparkd/md/platforms/IMDPlatform.h"
 #include "sparkd/md/platforms/mdp3_onixs/MDP3Main.h"
@@ -31,10 +29,6 @@ using std::vector;
 struct OnixsGlbxSubscription::PImpl {
    GlbxMarketDataConfig config;
    shared_ptr<IDGenerator> sec_id_store;
-   // The decision is captured by start(), before the worker thread is created.
-   // A source never changes output routes while it is running.
-   shared_ptr<IMarketDataListener> simulation_listener;
-   bool simulation_enabled{false};
 
    PImpl(shared_ptr<IDGenerator> sec_id_store, GlbxMarketDataConfig config)
        : config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
@@ -72,20 +66,7 @@ struct OnixsGlbxSubscription::PImpl {
              FMT_JOIN(symbols, ", "));
          vector<shared_ptr<IMDPlatform>> platforms;
          auto main = std::make_shared<MDP3Main>();
-         if (simulation_enabled) {
-            bts_assert(simulation_listener != nullptr);
-            LOG(md, Info,
-                "GLBX MDP3 route: simulation (SimExecutionGateway is enabled)");
-            main->start(
-                {symbols.begin(), symbols.end()}, nullptr, nullptr,
-                simulation_listener);
-         } else {
-            // Keep the production route byte-for-byte on its established
-            // startup path: MDP3 owns its legacy producer and SHM channels.
-            LOG(md, Info,
-                "GLBX MDP3 route: legacy production (SimExecutionGateway is disabled)");
-            main->start({symbols.begin(), symbols.end()}, nullptr, nullptr);
-         }
+         main->start({symbols.begin(), symbols.end()}, nullptr, nullptr);
          platforms.emplace_back(std::move(main));
          dispatcher = &utils::Dispatcher::current();
          heartbeat_timer.emplace(*dispatcher, 250, [&]() {
@@ -126,24 +107,9 @@ OnixsGlbxSubscription::OnixsGlbxSubscription(
 OnixsGlbxSubscription::~OnixsGlbxSubscription() = default;
 
 void OnixsGlbxSubscription::start(
-    shared_ptr<IMarketDataListener> listener) {
+    shared_ptr<IMarketDataListener> /*listener*/) {
    std::atomic_thread_fence(std::memory_order_release);
    bts_assert(!me_->dispatcher_thread.joinable());
-   me_->simulation_enabled =
-       orderman::SimExecutionGatewayFactory::is_sim_execution_on(
-           api::VENUE_GLBX);
-   me_->simulation_listener =
-       me_->simulation_enabled ? std::move(listener) : nullptr;
-   bts_assert(!me_->simulation_enabled || me_->simulation_listener != nullptr);
-   if (me_->simulation_enabled) {
-      auto generalized_listener =
-          std::dynamic_pointer_cast<GeneralMarketDataListener>(
-              me_->simulation_listener);
-      bts_assert_msg(
-          generalized_listener != nullptr,
-          "MDP3 simulation requires a GeneralMarketDataListener");
-      generalized_listener->set_source_security_ids_are_external();
-   }
    me_->dispatcher_thread = std::thread{[&]() { me_->run(); }};
 }
 

src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp

@@ -1,58 +1,29 @@
 #include "MDP3Listener.h"
-#include "spark/md/log_messages/trade_latency_log.h"
-#include "spark/config/protos/market_data.pb.h"
-#include "spark/domain/Book.h"
-#include "spark/domain/Security.h"
-#include "spark/domain/TradeRecord.h"
+#include "spark/config/protos/market_data.pb.h"
 #include "spark/md/protos/co2.pb.h"
 #include "spark/utils/datetime.h"
 #include "spark/utils/helper_macros.h"
 #include <vector>
 
 using namespace std;
-using namespace bts::co2;
-
-namespace bts {
-namespace md {
-
-namespace {
-
-domain::AggressorSide to_domain_aggressor(onixs::AggressorSide::Enum side) {
-   if (side == onixs::AggressorSide::Buy) return domain::AggressorSide::Buy;
-   if (side == onixs::AggressorSide::Sell) return domain::AggressorSide::Sell;
-   return domain::AggressorSide::None;
-}
-
-domain::BookLevel to_domain_book_level(const PriceSize &level) {
-   if (level.is_nan() || level.size() == 0) return domain::BookLevel::empty();
-   return {domain::Price::from_double(level.price()),
-           domain::Quantity(level.size()), domain::Quantity(level.direct_size()), 0};
-}
-
-template <size_t N>
-void copy_book_side(const std::array<PriceSize, N> &source,
-                    domain::BookSide &destination) {
-   const auto count = std::min(source.size(), destination.size());
-   for (size_t i = 0; i < count; ++i) destination[i] = to_domain_book_level(source[i]);
-}
-
-} // namespace
+using namespace bts::co2;
+
+namespace bts {
+namespace md {
 
 MDP3Listener::MDP3Listener(
     CMESecurityIndex &index,
     MDProducer &producer,
-    MBOWriter *mbo,
-    const std::vector<MDProducerChannel> channels,
-    TradeSummarySignalDispatcher *trade_summary_dispatcher,
-    std::shared_ptr<IMarketDataListener> simulation_listener)
+    MBOWriter *mbo,
+    const std::vector<MDProducerChannel> channels,
+    TradeSummarySignalDispatcher *trade_summary_dispatcher)
     : _index(index), _producer(producer),
       _verbose_ticker(config::get<CMEOptions>().verbose_ticker()),
       _packet_arrived(), _latest_sequence(0),
       // _settlement_recorder(settles),
       _symbol_filter_storage(), _symbol_filter(), _channels(channels),
       _concurrency_monitor(0), _book_updates(), _mbo(mbo),
-      _trade_summary_dispatcher(trade_summary_dispatcher),
-      _simulation_listener(std::move(simulation_listener)) {
+      _trade_summary_dispatcher(trade_summary_dispatcher) {
    _book_updates.reserve(32);
 }
 
@@ -208,38 +179,16 @@ void MDP3Listener::onBookUpdate(
 // Trades
 // Called from onTradeSummary and onTradeRecovery, ends up calling
 // MDProducerHandle::add_trade
-void on_trade(
+void on_trade(
     SecurityData *data,
     const onixs::Decimal &price,
     int size,
     onixs::AggressorSide::Enum agside,
-    const onixs::Timestamp &onixs_date,
-    IMarketDataListener *simulation_listener) {
+    const onixs::Timestamp &onixs_date) {
    auto ts = utils::TimeStamp::now();
    PriceSize ps = pricesize_of_price_size(
        price, data->handle.security().display_factor, size);
-   if (simulation_listener != nullptr) {
-      // The MDP3 private handle key is the CME security id.  GMDL resolves it
-      // to the persisted Spark key before it creates its SHM security.
-      domain::TradeRecord trade;
-      trade.sec_id = domain::SecurityID(data->handle.key());
-      trade.time_stamp =
-          core::TimeStamp(
-              onixs_date.year(), static_cast<uint8_t>(onixs_date.month()),
-              onixs_date.day(), onixs_date.hour(), onixs_date.minute(),
-              onixs_date.second()) +
-          core::TimeSpan::from_millis(onixs_date.millisecond());
-      trade.agg_side = to_domain_aggressor(agside);
-      trade.trade_quantity = domain::Quantity(size);
-      trade.trade_price = domain::Price::from_double(ps.price());
-      std::vector<domain::TradeRecord const *> trades{&trade};
-      simulation_listener->on_trades(trades);
-      return;
-   }
-   auto date = ll_of_timestamp(onixs_date);
-   log_trade_latency(
-       TradeLatencyStage::MDP3_RECEIVED, data->handle.security(), date, ps,
-       map_aggressor(agside), data->electronic_volume + size, false);
+   auto date = ll_of_timestamp(onixs_date);
    data->handle.add_trade(
        ts, ps, date, map_aggressor(agside), data->electronic_volume + size);
 }
@@ -335,83 +284,24 @@ void MDP3Listener::heartbeat() {
 
 // Called by MDP3Main::prepare_listener as part of pre-load. Also called by
 // MDP3Listener::onFuture, onOption and onSpread
-void MDP3Listener::add_security(
-    FixedSecurity &security, co2::TradingStatus status) {
+void MDP3Listener::add_security(
+    const FixedSecurity &security, co2::TradingStatus status) {
    bts_assert(security.numeric_id != 0);
-   if (security.type == api::SECURITYTYPE_OPTION) {
-      const auto &option = static_cast<const FixedOptionSecurity &>(security);
-      const datetime::Date expiration(
-          option.expiration_year, option.expiration_month,
-          option.expiration_day);
-      if (!option_expiration_is_allowed(expiration)) {
-         LOG(md, Info,
-             "Skipping MDP3 option {} because it expires beyond the configured "
-             "cutoff.",
-             std::string(security.symbol()));
-         return;
-      }
-   }
-   if (_simulation_listener != nullptr) {
-      // A private producer still needs a stable key for its decoding cache.
-      // Keep that key as the CME external id; GMDL owns persisted Spark-key
-      // resolution and the only shared-memory publication.
-      security.key = security.numeric_id;
-      if (security.type == api::SECURITYTYPE_SPREAD) {
-         auto &spread = static_cast<FixedSpreadSecurityL &>(security);
-         for (size_t i = 0; i < spread.leg_count(); ++i) {
-            // The private producer validates that spread legs have keys when
-            // a security becomes visible.  At this stage the only valid
-            // identity is CME's leg SecurityID; GMDL resolves it later.
-            spread.legs[i].key = spread.legs[i].exchange_key.value();
-         }
-      }
-   }
-   auto handle = _producer.add_security(security);
+   // Set up the shared memory for this security
+   auto handle = _producer.add_security(security);
    // Add it to the index so it can receive messages
-   _index.add_security(security.numeric_id, handle);
-
-   if (_simulation_listener != nullptr) {
-      domain::Security domain_security(convert_security(security));
-      std::vector<domain::Security const *> securities{&domain_security};
-      _simulation_listener->on_securities(securities);
-      return;
-   }
+   _index.add_security(security.numeric_id, handle);
 
    handle.update_exchange_status(utils::TimeStamp::now(), status);
 }
 
 // Private functions
 
-void MDP3Listener::flush_books() {
+void MDP3Listener::flush_books() {
    // This is called at the end of the packet
    // Write all book updates to shared memory and reset for the next packet
-   for (auto &entry : _book_updates) {
-      uint32_t sequence = _latest_sequence;
-      if (_simulation_listener != nullptr) {
-         // A domain Book has direct and implied quantities per level.  Sending
-         // the merged snapshot once per packet preserves both MDP3 book kinds
-         // and gives the simulation gateway one matching/publication point.
-         domain::Book book = domain::Book::empty(
-             domain::SecurityID(entry.handle.key()));
-         copy_book_side(entry.bids.first, book.bids());
-         copy_book_side(entry.asks.first, book.asks());
-         if (entry.type_flags & book_type_implied) {
-            for (size_t i = 0; i < entry.bids.second.size() && i < book.bids().size(); ++i) {
-               const auto &implied = entry.bids.second[i];
-               if (!implied.is_nan() && implied.size() != 0) {
-                  book.bids()[i] = to_domain_book_level(implied);
-               }
-            }
-            for (size_t i = 0; i < entry.asks.second.size() && i < book.asks().size(); ++i) {
-               const auto &implied = entry.asks.second[i];
-               if (!implied.is_nan() && implied.size() != 0) {
-                  book.asks()[i] = to_domain_book_level(implied);
-               }
-            }
-         }
-         _simulation_listener->on_book(book, _packet_arrived);
-         continue;
-      }
+   for (auto &entry : _book_updates) {
+      uint32_t sequence = _latest_sequence;
       if (entry.type_flags == book_type_direct) {
          entry.handle.update_direct(
              _packet_arrived, entry.bids.first, entry.asks.first, sequence);
@@ -426,7 +316,7 @@ void MDP3Listener::flush_books() {
    }
    // Reset pending updates
    _book_updates.clear();
-}
+}
 
 void MDP3Listener::update_all_status(CoconutSecurityStatus status) {
    _index.walk_security_list(

src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h

@@ -1,12 +1,9 @@
 #pragma once
 
 #include "CMESecurityIndex.h"
-#include "spark/config/config.h"
-#include "spark/config/protos/toggles.pb.h"
 #include "spark/io/JsonDumper.h"
 #include "spark/io/json_formatter.h"
-#include "spark/md/MBOProcessor.h"
-#include "spark/md/IMarketDataListener.h"
+#include "spark/md/MBOProcessor.h"
 #include "spark/md/Shm/FixedSecurity.h"
 #include "spark/md/protos/trading_status.h"
 #include "spark/orderman/events/TradeSummarySignalDispatcher.h"
@@ -24,28 +21,6 @@
 namespace bts {
 namespace md {
 
-inline bool option_expiration_is_allowed(const datetime::Date &expiration) {
-   const auto max_days =
-       config::get<config::toggles::Mdp3OptionExpirationCutoff>()
-           .max_days_until_expiration();
-   if (expiration.year == 0) {
-      return true;
-   }
-   if (max_days == 0) {
-      return true;
-   }
-   const auto cutoff = core::TimeStamp::now().date() +
-                       core::TimeSpan::from_days(max_days);
-   const auto expiration_timestamp =
-       core::TimeStamp(expiration.year, expiration.month, expiration.day);
-   const bool allowed = expiration_timestamp <= cutoff;
-   uint32_t cutoff_year;
-   uint32_t cutoff_month;
-   uint32_t cutoff_day;
-   cutoff.to_date(cutoff_year, cutoff_month, cutoff_day);
-   return allowed;
-}
-
 inline co2::AggressorSide map_aggressor(onixs::AggressorSide::Enum agside) {
    switch (agside) {
    default:
@@ -153,10 +128,9 @@ class MDP3Listener : public onixs::HandlerListener,
    MDP3Listener(
        CMESecurityIndex &index,
        MDProducer &,
-       MBOWriter *mbo,
-       const std::vector<MDProducerChannel>,
-       TradeSummarySignalDispatcher *,
-       std::shared_ptr<IMarketDataListener> simulation_listener = nullptr);
+       MBOWriter *mbo,
+       const std::vector<MDProducerChannel>,
+       TradeSummarySignalDispatcher *);
    virtual ~MDP3Listener();
 
    //
@@ -669,7 +643,7 @@ class MDP3Listener : public onixs::HandlerListener,
    void heartbeat();
 
    // Add a new security to the index
-   void add_security(FixedSecurity &security, co2::TradingStatus);
+   void add_security(const FixedSecurity &security, co2::TradingStatus);
 
  private:
    enum SecurityFilterAction { ADD, SKIP, DELETE };
@@ -722,9 +696,8 @@ class MDP3Listener : public onixs::HandlerListener,
    };
    std::vector<BookData> _book_updates;
    MBOWriter *_mbo;
-   TradeSummarySignalDispatcher *_trade_summary_dispatcher;
-   std::shared_ptr<IMarketDataListener> _simulation_listener;
-};
+   TradeSummarySignalDispatcher *_trade_summary_dispatcher;
+};
 
 //////////////////////////////////////////
 // Template Member Function Definitions
@@ -808,11 +781,6 @@ void MDP3Listener::onOption(
           exchange_id, sec_name, Mdp3SecuritySkipReason::ALREADY_ADDED);
       return;
    }
-   if (!option_expiration_is_allowed(date_of_maturity(sec))) {
-      LOG_MDP3_ON_OPTION_SKIP(
-          exchange_id, sec_name, Mdp3SecuritySkipReason::EXPIRED);
-      return;
-   }
 
    LOG_MDP3_ON_OPTION_BEGIN(exchange_id, std::string(sec.symbol().items()));
    FixedOptionSecurity fsec;
@@ -984,13 +952,12 @@ void MDP3Listener::onBookUpdate(
    }
 }
 
-void on_trade(
+void on_trade(
     SecurityData *data,
     const onixs::Decimal &price,
     int size,
     onixs::AggressorSide::Enum agside,
-    const onixs::Timestamp &onixs_date,
-    IMarketDataListener *simulation_listener);
+    const onixs::Timestamp &onixs_date);
 
 template <typename TradeRecovery>
 void MDP3Listener::onTradeRecovery(
@@ -1005,8 +972,8 @@ void MDP3Listener::onTradeRecovery(
       onixs::Decimal price;
       arg.entry().entryPx(price);
       on_trade(
-          data, price, size, onixs::AggressorSide::NoAggressor,
-          arg.message().transactTime(), _simulation_listener.get());
+          data, price, size, onixs::AggressorSide::NoAggressor,
+          arg.message().transactTime());
    }
 }
 
@@ -1029,12 +996,10 @@ void MDP3Listener::onTradeSummary(
          for (size_t x = 0; x < sz; ++x) {
             const auto &d = details[x];
             on_trade(
-                data, price, d.lastQty(), agside, arg.message().transactTime(),
-                _simulation_listener.get());
+                data, price, d.lastQty(), agside, arg.message().transactTime());
          }
       } else {
-         on_trade(data, price, size, agside, arg.message().transactTime(),
-                  _simulation_listener.get());
+         on_trade(data, price, size, agside, arg.message().transactTime());
       }
    }
 }

src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp

@@ -36,10 +36,6 @@ struct MDP3Main::PImpl {
    CMESecurityIndex index;
    SecurityIDRecorder recorder;
    MDProducer producer;
-   // Simulation needs MDP3's per-security decoding state, but it must not
-   // become a second GLBX shared-memory writer.  This producer has no shared
-   // memory or security mapper; GMDL is the sole public producer in that mode.
-   MDProducer simulation_producer{MDProducerFlags::None};
    std::map<std::string, std::unique_ptr<onixs::FeedEngine>> engines;
    std::map<std::string, std::unique_ptr<onixs::Handler>> handlers;
    std::vector<std::unique_ptr<MDP3Listener>> listeners;
@@ -182,8 +178,7 @@ onixs::SessionSettings session_settings_for_channel(
 void MDP3Main::start(
     const std::vector<UserSymbol> &symbols,
     MBOWriter *mbo,
-    TradeSummarySignalDispatcher *tssd,
-    std::shared_ptr<IMarketDataListener> simulation_listener) {
+    TradeSummarySignalDispatcher *tssd) {
    std::ostringstream oss;
    for (const auto &us : symbols) {
       oss << us << " ";
@@ -196,8 +191,7 @@ void MDP3Main::start(
    me_->engines = create_feed_engines(channels);
    for (const auto &channel : channels) {
       unique_ptr<onixs::Handler> phandler = prepare_handler(channel.id, mbo);
-      me_->listeners.push_back(prepare_listener(
-          *phandler, channel, mbo, tssd, simulation_listener));
+      me_->listeners.push_back(prepare_listener(*phandler, channel, mbo, tssd));
       me_->handlers.insert(std::make_pair(channel.id, std::move(phandler)));
       std::stringstream symstr;
       for (const auto &sym : channel.symbols) {
@@ -392,25 +386,19 @@ unique_ptr<MDP3Listener> MDP3Main::prepare_listener(
     onixs::Handler &handler,
     const CMEChannel &channel,
     MBOWriter *mbo,
-    TradeSummarySignalDispatcher *trade_summary_dispatcher,
-    std::shared_ptr<IMarketDataListener> simulation_listener) {
+    TradeSummarySignalDispatcher *trade_summary_dispatcher) {
    LOG(md, Info, "MDP3Main::prepare_listener channel: {}", channel);
    // We need to get a handler to the producer channels so we can produce
    // heartbeats
    vector<MDProducerChannel> producer_channels;
-   MDProducer &producer = simulation_listener ? me_->simulation_producer
-                                               : me_->producer;
-   // Private simulation state does not need heartbeat SHM channels.
-   if (!simulation_listener) {
-      for (auto &symbol : channel.symbols) {
-         producer_channels.push_back(producer.get_channel(
-             symbol.symbol().c_str(), symbol.type(), symbol.exchange()));
-      }
+   for (auto &symbol : channel.symbols) {
+      producer_channels.push_back(me_->producer.get_channel(
+          symbol.symbol().c_str(), symbol.type(), symbol.exchange()));
    }
 
    unique_ptr<MDP3Listener> listener(new MDP3Listener(
-       me_->index, producer, mbo, producer_channels,
-       trade_summary_dispatcher, std::move(simulation_listener)));
+       me_->index, me_->producer, mbo, producer_channels,
+       trade_summary_dispatcher));
    for (auto &symbol : channel.symbols) {
       listener->add_filter_symbol(symbol.symbol());
    }

src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.h

@@ -2,7 +2,6 @@
 
 #include "CMESecurityIndex.h"
 #include "MDP3Listener.h"
-#include "spark/md/IMarketDataListener.h"
 #include "spark/md/coconut2.h"
 #include "spark/utils/bts_no_copy.h"
 #include "onixs.h"
@@ -38,8 +37,7 @@ class MDP3Main : public IMDPlatform {
    void start(
        const std::vector<UserSymbol> &symbols,
        MBOWriter *mbo,
-       TradeSummarySignalDispatcher *tssd,
-       std::shared_ptr<IMarketDataListener> simulation_listener = nullptr);
+       TradeSummarySignalDispatcher *tssd);
    void send_heartbeats() override;
 
    void initialize(const std::string &license_store);
@@ -49,8 +47,7 @@ class MDP3Main : public IMDPlatform {
        onixs::Handler &handler,
        const CMEChannel &channel,
        MBOWriter *,
-       TradeSummarySignalDispatcher *,
-       std::shared_ptr<IMarketDataListener> simulation_listener = nullptr);
+       TradeSummarySignalDispatcher *);
 
  private:
    struct PImpl;

src/app_sparkd/sparkd/md/rakurai/RakuraiDexDataHandler.cpp

@@ -0,0 +1,791 @@
+#include "sparkd/md/rakurai/RakuraiDexDataHandler.h"
+
+#include "spark/api/domain/SecurityType.pb.h"
+#include "spark/domain/Book.h"
+#include "spark/domain/Price.h"
+#include "spark/domain/Quantity.h"
+#include "spark/domain/Security.h"
+#include "spark/logging/logging.h"
+#include "spark/persistence/SqliteStore.h"
+#include "spark/utils/PBUtils.h"
+#include "sparkd/md/MarketDataError.h"
+#include "sparkd/rakurai/RakuraiVenueRegistry.h"
+
+#include <dex_handler.h>
+#include <dex_listener.h>
+#include <dex_utils.h>
+#include <rapidjson/document.h>
+#include <rapidjson/istreamwrapper.h>
+
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <csignal>
+#include <execinfo.h>
+#include <fstream>
+#include <iomanip>
+#include <list>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <string_view>
+#include <unordered_map>
+#include <unordered_set>
+#include <unistd.h>
+#include <vector>
+
+#include <arpa/inet.h>
+#include <netdb.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+
+namespace bts::md::rakurai {
+
+namespace {
+
+using ConnectorVenue = ::rakurai::dex_connector::data_types::venue;
+using ConnectorSecurity = ::rakurai::dex_connector::data_types::security;
+using ConnectorBookLevel = ::rakurai::dex_connector::data_types::book_level;
+using ConnectorDexConfig = ::rakurai::dex_connector::configuration::dex_config;
+using ::bts::rakurai::connector_quote_currency;
+using ::bts::rakurai::connector_security_type;
+using ::bts::rakurai::connector_venue_name;
+using ::bts::rakurai::venue_matches;
+
+struct ParsedWsUri {
+   std::string scheme;
+   std::string host;
+   std::string port;
+   std::string path;
+};
+
+struct WsProbeResult {
+   bool tcp_connected = false;
+   bool sent_request = false;
+   bool received_response = false;
+   std::string summary;
+};
+
+std::optional<ParsedWsUri> parse_ws_uri(const std::string& uri) {
+   const auto scheme_pos = uri.find("://");
+   if (scheme_pos == std::string::npos) {
+      return std::nullopt;
+   }
+
+   ParsedWsUri parsed;
+   parsed.scheme = uri.substr(0, scheme_pos);
+   auto remainder = uri.substr(scheme_pos + 3);
+
+   const auto path_pos = remainder.find('/');
+   if (path_pos == std::string::npos) {
+      parsed.path = "/";
+   } else {
+      parsed.path = remainder.substr(path_pos);
+      remainder = remainder.substr(0, path_pos);
+   }
+
+   const auto colon_pos = remainder.rfind(':');
+   if (colon_pos == std::string::npos) {
+      parsed.host = remainder;
+      parsed.port = parsed.scheme == "wss" ? "443" : "80";
+   } else {
+      parsed.host = remainder.substr(0, colon_pos);
+      parsed.port = remainder.substr(colon_pos + 1);
+   }
+
+   if (parsed.host.empty() || parsed.port.empty()) {
+      return std::nullopt;
+   }
+
+   return parsed;
+}
+
+WsProbeResult probe_ws_endpoint(const std::string& uri) {
+   WsProbeResult result;
+
+   const auto parsed = parse_ws_uri(uri);
+   if (!parsed.has_value()) {
+      result.summary = "invalid websocket URI";
+      return result;
+   }
+
+   if (parsed->scheme != "ws") {
+      result.summary = "probe only supports ws:// URIs";
+      return result;
+   }
+
+   struct addrinfo hints {};
+   hints.ai_family = AF_UNSPEC;
+   hints.ai_socktype = SOCK_STREAM;
+
+   struct addrinfo* addr_list = nullptr;
+   const int gai_rc = ::getaddrinfo(
+       parsed->host.c_str(), parsed->port.c_str(), &hints, &addr_list);
+   if (gai_rc != 0) {
+      result.summary =
+          std::string("DNS resolution failed: ") + ::gai_strerror(gai_rc);
+      return result;
+   }
+
+   int fd = -1;
+   for (auto* ai = addr_list; ai != nullptr; ai = ai->ai_next) {
+      fd = ::socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol);
+      if (fd < 0) {
+         continue;
+      }
+
+      timeval timeout {};
+      timeout.tv_sec = 3;
+      timeout.tv_usec = 0;
+      ::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout));
+      ::setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(timeout));
+
+      if (::connect(fd, ai->ai_addr, ai->ai_addrlen) == 0) {
+         result.tcp_connected = true;
+         break;
+      }
+
+      ::close(fd);
+      fd = -1;
+   }
+
+   ::freeaddrinfo(addr_list);
+
+   if (fd < 0) {
+      result.summary = "TCP connect failed";
+      return result;
+   }
+
+   const std::string request =
+       "GET " + parsed->path + " HTTP/1.1\r\n" +
+       "Host: " + parsed->host + ":" + parsed->port + "\r\n" +
+       "Upgrade: websocket\r\n"
+       "Connection: Upgrade\r\n"
+       "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n"
+       "Sec-WebSocket-Version: 13\r\n\r\n";
+
+   const ssize_t sent = ::send(fd, request.data(), request.size(), 0);
+   if (sent == static_cast<ssize_t>(request.size())) {
+      result.sent_request = true;
+   } else {
+      result.summary = "failed to send websocket probe request";
+      ::close(fd);
+      return result;
+   }
+
+   std::array<char, 1024> buffer {};
+   const ssize_t received = ::recv(fd, buffer.data(), buffer.size() - 1, 0);
+   ::close(fd);
+
+   if (received <= 0) {
+      result.summary = "no HTTP response received";
+      return result;
+   }
+
+   result.received_response = true;
+   std::string response(buffer.data(), static_cast<std::size_t>(received));
+   const auto line_end = response.find("\r\n");
+   if (line_end != std::string::npos) {
+      response.resize(line_end);
+   }
+
+   result.summary = response;
+   return result;
+}
+
+void validate_ws_endpoint_or_throw(
+    const std::string& label, const std::string& uri) {
+   const auto probe = probe_ws_endpoint(uri);
+   LOG(md, Info,
+       "Rakurai websocket probe. endpoint:{} uri:{} tcp_connected:{} sent_request:{} received_response:{} summary:{}",
+       label,
+       uri,
+       probe.tcp_connected,
+       probe.sent_request,
+       probe.received_response,
+       probe.summary);
+
+   if (!probe.tcp_connected || !probe.sent_request || !probe.received_response) {
+      throw std::runtime_error(
+          std::string("Rakurai websocket endpoint probe failed for ") + label +
+          " (" + uri + "): " + probe.summary);
+   }
+
+   if (probe.summary.rfind("HTTP/1.1 101", 0) != 0 &&
+       probe.summary.rfind("HTTP/1.0 101", 0) != 0) {
+      throw std::runtime_error(
+          std::string("Rakurai websocket endpoint returned unexpected handshake for ") +
+          label + " (" + uri + "): " + probe.summary);
+   }
+}
+
+void write_crash_text(std::string_view text) {
+   if(::write(STDERR_FILENO, text.data(), text.size()) ==-1) {
+      // If write fails, there's not much we can do, but we don't want to get
+      // stuck in a loop trying to write. So we just ignore the error.
+   };
+
+}
+
+void rakurai_crash_signal_handler(int signo, siginfo_t* info, void* /*ucontext*/) {
+   constexpr int max_frames = 64;
+   void* frames[max_frames];
+   const int frame_count = ::backtrace(frames, max_frames);
+
+   write_crash_text("\n=== Rakurai crash signal caught ===\n");
+   if (signo == SIGSEGV) {
+      write_crash_text("signal: SIGSEGV\n");
+   } else if (signo == SIGBUS) {
+      write_crash_text("signal: SIGBUS\n");
+   } else {
+      write_crash_text("signal: unknown\n");
+   }
+
+   if (info != nullptr) {
+      if (info->si_addr != nullptr) {
+         write_crash_text("fault address captured\n");
+      }
+      if (info->si_code != 0) {
+         write_crash_text("signal code captured\n");
+      }
+   }
+
+   write_crash_text("backtrace:\n");
+   ::backtrace_symbols_fd(frames, frame_count, STDERR_FILENO);
+   _Exit(128 + signo);
+}
+
+void install_rakurai_crash_signal_handlers() {
+   static std::once_flag once;
+   std::call_once(once, []() {
+      struct sigaction sa {};
+      sa.sa_flags = SA_SIGINFO | SA_RESETHAND;
+      sa.sa_sigaction = rakurai_crash_signal_handler;
+      sigemptyset(&sa.sa_mask);
+
+      sigaction(SIGSEGV, &sa, nullptr);
+      sigaction(SIGBUS, &sa, nullptr);
+   });
+}
+
+std::string bytes_to_hex(const std::array<std::uint8_t, 32>& bytes) {
+   std::ostringstream oss;
+   oss << std::hex << std::setfill('0');
+   for (std::uint8_t byte : bytes) {
+      oss << std::setw(2) << static_cast<int>(byte);
+   }
+   return oss.str();
+}
+
+std::string short_hex(const std::array<std::uint8_t, 32>& bytes) {
+   const auto hex = bytes_to_hex(bytes);
+   return hex.substr(0, std::min<std::size_t>(hex.size(), 12));
+}
+
+std::string token_display_name(const ::rakurai::dex_connector::data_types::token_info& token) {
+   return token.name.empty() ? short_hex(token.mint_address) : token.name;
+}
+
+std::string build_security_desc(
+    const ConnectorSecurity& security,
+    std::string_view short_pool_id,
+    std::string_view currency) {
+   return fmt::format(
+       "{}-{}-{}",
+       token_display_name(security.token_a),
+       currency,
+       short_pool_id);
+}
+
+std::string security_currency(const ConnectorSecurity& security) {
+   const auto configured_currency = connector_quote_currency(security.venue);
+   return configured_currency.empty()
+              ? token_display_name(security.token_b)
+              : std::string(configured_currency);
+}
+
+double quantity_scale_from_decimals(std::uint8_t decimals) {
+   double scale = 1.0;
+   for (std::uint8_t i = 0; i < decimals; ++i) {
+      scale *= 10.0;
+   }
+   return scale;
+}
+
+domain::BookLevel to_book_level(
+    const ConnectorBookLevel& source,
+    double quantity_scale) {
+   domain::BookLevel level;
+   level.price = domain::Price::from_double(source.price);
+   const auto normalized_quantity =
+       quantity_scale > 0.0
+           ? static_cast<double>(source.quantity) / quantity_scale
+           : static_cast<double>(source.quantity);
+   level.total_quantity =
+       domain::Quantity(static_cast<float>(normalized_quantity));
+   level.direct_quantity = level.total_quantity;
+   level.n_orders = 0;
+   return level;
+}
+
+std::list<ConnectorDexConfig> parse_dex_configs(
+    const rapidjson::Document& config,
+    api::domain::Venue venue_filter) {
+   std::list<ConnectorDexConfig> dex_config_list;
+   if (!config.HasMember("dex") || !config["dex"].IsArray()) {
+      return dex_config_list;
+   }
+
+   for (const auto& dex_value : config["dex"].GetArray()) {
+      if (!dex_value.IsObject() || !dex_value.HasMember("id")) {
+         continue;
+      }
+
+      const std::string dex = dex_value["id"].GetString();
+      auto dex_config =
+          bts::rakurai::make_connector_dex_config(dex, venue_filter);
+      if (!dex_config.has_value()) {
+         LOG(md, Warn, "Skipping unsupported Rakurai dex config '{}'", dex);
+         continue;
+      }
+
+      if (dex_value.HasMember("pool_ids") && dex_value["pool_ids"].IsArray()) {
+         for (const auto& pool_value : dex_value["pool_ids"].GetArray()) {
+            dex_config->interested_pools.emplace_back(
+                ::rakurai::dex_connector::solanaAddressToBytes(
+                    pool_value.GetString()));
+         }
+      }
+      if (dex_value.HasMember("symbols") && dex_value["symbols"].IsArray()) {
+         for (const auto& symbol : dex_value["symbols"].GetArray()) {
+            dex_config->interested_symbols.push_back(symbol.GetString());
+         }
+      }
+      if (dex_value.HasMember("interested_symbols") &&
+          dex_value["interested_symbols"].IsArray()) {
+         for (const auto& symbol : dex_value["interested_symbols"].GetArray()) {
+            dex_config->interested_symbols.push_back(symbol.GetString());
+         }
+      }
+
+      dex_config_list.push_back(std::move(*dex_config));
+   }
+
+   return dex_config_list;
+}
+
+class ListenerAdapter final : public ::rakurai::dex_connector::dex_listener {
+ public:
+   ListenerAdapter(
+       std::shared_ptr<IMarketDataListener> listener,
+       std::shared_ptr<persistence::IDGenerator> sec_id_store,
+       api::domain::Venue venue,
+       std::uint32_t num_depth_levels,
+       const std::vector<std::string>& excluded_securities)
+       : listener_(std::move(listener)),
+         sec_id_store_(std::move(sec_id_store)),
+         venue_(venue),
+         depth_limit_(
+             num_depth_levels == 0
+                 ? domain::Book::max_depth()
+                 : std::min<std::size_t>(
+                       num_depth_levels, domain::Book::max_depth())),
+         excluded_securities_(
+         excluded_securities.begin(), excluded_securities.end()) {}
+
+   std::string get_or_add_short_pool_id(const std::string& pool_id) {
+      constexpr std::string_view kPoolAliasNamespace = "rakurai_pool_alias";
+      const auto [exists, rowid, _] =
+          pool_alias_store_.retrieve_one(std::string(kPoolAliasNamespace), pool_id);
+      if (exists) {
+         return std::to_string(rowid);
+      }
+
+      const auto new_id =
+          pool_alias_store_.insert(std::string(kPoolAliasNamespace), pool_id, "");
+      return std::to_string(new_id);
+   }
+
+   void on_pool_definition(
+       ConnectorVenue connector_venue,
+       const std::vector<ConnectorSecurity*>& securities) override {
+      std::lock_guard<std::mutex> lock(callback_mutex_);
+      LOG(md, Info,
+          "Rakurai listener on_pool_definition start. venue:{} connector_venue:{} securities:{}",
+          int(venue_),
+          std::string(connector_venue_name(connector_venue)),
+          securities.size());
+      if (!venue_matches(venue_, connector_venue)) {
+         LOG(md, Info,
+             "Rakurai listener on_pool_definition skipped. venue:{} connector_venue:{}",
+             int(venue_),
+             std::string(connector_venue_name(connector_venue)));
+         return;
+      }
+
+      std::vector<const domain::Security*> emitted;
+      emitted.reserve(securities.size());
+
+      for (const ConnectorSecurity* security : securities) {
+         if (security == nullptr || !venue_matches(venue_, security->venue)) {
+            continue;
+         }
+
+         const auto pool_id = bytes_to_hex(security->pool_id);
+         const auto short_pool_id = get_or_add_short_pool_id(pool_id);
+         const auto symbol = token_display_name(security->token_a);
+         // const auto token_a_cpp = security->token_a.decimals;
+         const auto currency = security_currency(*security);
+         const auto desc = build_security_desc(*security, short_pool_id, currency);
+         if (excluded_securities_.contains(pool_id) ||
+             excluded_securities_.contains(short_pool_id) ||
+             excluded_securities_.contains(desc)) {
+            continue;
+         }
+
+         domain::Security emitted_security;
+         const auto security_type = connector_security_type(security->venue);
+         if (security_type == api::domain::SECURITY_TYPE_FUTURE) {
+            auto& future =
+                emitted_security.security.emplace<domain::FutureSecurity>();
+            future.expiration = core::TimeStamp::from_ymd(2100, 1, 1);
+         } else {
+            emitted_security.security = domain::SpotSecurity{};
+         }
+         emitted_security.security_common.venue = venue_;
+         emitted_security.security_common.exchange_sec_id = short_pool_id;
+         emitted_security.security_common.symbol = symbol;
+         emitted_security.security_common.currency = currency;
+         emitted_security.security_common.asset = symbol;
+         emitted_security.security_common.group =
+             connector_venue_name(security->venue);
+         emitted_security.security_common.desc = desc;
+         emitted_security.security_common.min_price_increment =
+             security->min_tick > 0.0 ? security->min_tick : 0.0;
+         emitted_security.security_common.min_quantity_increment =
+             static_cast<double>(security->min_trade_amount);
+         emitted_security.security_common.min_quantity =
+             static_cast<double>(security->min_trade_amount);
+         *emitted_security.security_common.id = sec_id_store_->get_or_add(
+             venue_, security_type, pool_id);
+
+         auto [it, is_new] = securities_.try_emplace(
+             pool_id, std::make_unique<domain::Security>(std::move(emitted_security)));
+         if (!is_new) {
+            *it->second = std::move(emitted_security);
+         }
+
+         pool_to_security_id_[pool_id] = it->second->id();
+         pool_quantity_scales_[pool_id] =
+             quantity_scale_from_decimals(security->token_a.decimals);
+         emitted.push_back(it->second.get());
+      }
+
+      if (emitted.empty()) {
+         LOG(md, Info,
+             "Rakurai listener on_pool_definition produced no securities. venue:{} connector_venue:{}",
+             int(venue_),
+             std::string(connector_venue_name(connector_venue)));
+         return;
+      }
+      LOG(md, Info,
+          "Rakurai listener on_pool_definition emitting securities. venue:{} connector_venue:{} emitted:{} tracked:{}",
+          int(venue_),
+          std::string(connector_venue_name(connector_venue)),
+          emitted.size(),
+          securities_.size());
+      listener_->on_securities(emitted);
+      LOG(md, Info,
+          "Rakurai listener on_pool_definition complete. venue:{} connector_venue:{} emitted:{}",
+          int(venue_),
+          std::string(connector_venue_name(connector_venue)),
+          emitted.size());
+   }
+
+   void on_book(
+       ConnectorVenue connector_venue,
+       const std::array<std::uint8_t, 32>& pool_id,
+       const std::vector<ConnectorBookLevel>& bids,
+       const std::vector<ConnectorBookLevel>& asks) override {
+      std::lock_guard<std::mutex> lock(callback_mutex_);
+      LOG(md, Diag,
+          "Rakurai listener on_book start. venue:{} connector_venue:{} pool:{} bids:{} asks:{}",
+          int(venue_),
+          std::string(connector_venue_name(connector_venue)),
+          short_hex(pool_id),
+          bids.size(),
+          asks.size());
+      if (!venue_matches(venue_, connector_venue)) {
+         return;
+      }
+
+      const auto pool_key = bytes_to_hex(pool_id);
+      const auto it = pool_to_security_id_.find(pool_key);
+      if (it == pool_to_security_id_.end()) {
+         LOG(md, Warn,
+             "Received Rakurai book before security definition for pool {}",
+             pool_key);
+         return;
+      }
+      const auto quantity_scale_it = pool_quantity_scales_.find(pool_key);
+      const auto quantity_scale =
+          quantity_scale_it != pool_quantity_scales_.end()
+              ? quantity_scale_it->second
+              : 1.0;
+
+      auto book = domain::Book::empty(it->second);
+      const auto bid_depth = std::min<std::size_t>(depth_limit_, bids.size());
+      const auto ask_depth = std::min<std::size_t>(depth_limit_, asks.size());
+
+      for (std::size_t i = 0; i < bid_depth; ++i) {
+         book.bids()[i] = to_book_level(bids[i], quantity_scale);
+      }
+      for (std::size_t i = 0; i < ask_depth; ++i) {
+         book.asks()[i] = to_book_level(asks[i], quantity_scale);
+      }
+
+      listener_->on_book(book);
+      LOG(md, Diag,
+          "Rakurai listener on_book complete. venue:{} connector_venue:{} pool:{}",
+          int(venue_),
+          std::string(connector_venue_name(connector_venue)),
+          short_hex(pool_id));
+   }
+
+   void on_order_update(
+       ::rakurai::dex_connector::order_entry::order_state order_state) override {
+      std::lock_guard<std::mutex> lock(callback_mutex_);
+      LOG(md, Info,
+          "Rakurai order update. client_id:{} txn_id:{} status:{}",
+          order_state.client_id,
+          order_state.txn_id,
+          static_cast<int>(order_state.tx_status));
+   }
+
+   void on_ata_verification_status(
+       const std::array<std::uint8_t, 32>& token_id,
+       const std::string& ata_address,
+       ::rakurai::dex_connector::ata_status status) override {
+      std::lock_guard<std::mutex> lock(callback_mutex_);
+      LOG(md, Info,
+          "Rakurai ATA verification. token:{} ata:{} status:{}",
+          short_hex(token_id),
+          ata_address,
+          static_cast<int>(status));
+   }
+
+ private:
+   std::shared_ptr<IMarketDataListener> listener_;
+   std::shared_ptr<persistence::IDGenerator> sec_id_store_;
+   api::domain::Venue venue_;
+   std::size_t depth_limit_;
+   std::mutex callback_mutex_;
+   std::unordered_set<std::string> excluded_securities_;
+   persistence::SqliteStore pool_alias_store_{"rakurai_pool_alias"};
+   std::unordered_map<std::string, std::unique_ptr<domain::Security>> securities_;
+   std::unordered_map<std::string, domain::SecurityID> pool_to_security_id_;
+   std::unordered_map<std::string, double> pool_quantity_scales_;
+};
+
+} // namespace
+
+void RakuraiDexDataHandler::run(
+    std::shared_ptr<IMarketDataListener> listener,
+    std::shared_ptr<persistence::IDGenerator> sec_id_store,
+    api::domain::Venue venue,
+    const std::string& config_path,
+    std::uint32_t num_depth_levels,
+   const std::vector<std::string>& excluded_securities) {
+   try {
+      install_rakurai_crash_signal_handlers();
+      LOG(md, Info,
+          "Rakurai DEX handler run start. venue:{} config:{} depth:{} excluded:{}",
+          int(venue),
+          config_path,
+          num_depth_levels,
+          excluded_securities.size());
+      std::ifstream config_file(config_path);
+      if (!config_file.is_open()) {
+         throw std::runtime_error("Unable to open Rakurai config: " + config_path);
+      }
+
+      rapidjson::IStreamWrapper isw{config_file};
+      rapidjson::Document config;
+      if (config.ParseStream(isw).HasParseError()) {
+         throw std::runtime_error(
+             "Unable to parse Rakurai config: " + config_path);
+      }
+      LOG(md, Info,
+          "Rakurai DEX handler config parsed. venue:{} config:{}",
+          int(venue),
+          config_path);
+
+      ::rakurai::dex_connector::configuration::logging_config logging_config;
+      ::rakurai::dex_connector::configuration::session_config session_config;
+      ::rakurai::dex_connector::configuration::wallet wallet_config;
+      bool cluster_mode = false;
+
+      if (config.HasMember("app_logging") && config["app_logging"].IsObject()) {
+         const auto& app_logging = config["app_logging"];
+         if (app_logging.HasMember("severity")) {
+            logging_config.severity = app_logging["severity"].GetString();
+         }
+         if (app_logging.HasMember("console")) {
+            logging_config.is_console_enabled = app_logging["console"].GetBool();
+         }
+         if (app_logging.HasMember("file")) {
+            logging_config.is_file_enabled = app_logging["file"].GetBool();
+         }
+         if (app_logging.HasMember("queuesize")) {
+            logging_config.queue_size = app_logging["queuesize"].GetUint();
+         }
+      }
+
+      if (config.HasMember("cluster_mode")) {
+         cluster_mode = config["cluster_mode"].GetBool();
+      }
+
+      auto add_uri = [&](const char* md_key, const char* oms_key) {
+         if (!config.HasMember(md_key) || !config.HasMember(oms_key)) {
+            throw std::runtime_error(
+                std::string("Incomplete Rakurai uri config in ") + config_path);
+         }
+         ::rakurai::dex_connector::configuration::uri_config uri_config;
+         uri_config.md_server_uri = config[md_key].GetString();
+         uri_config.node_oms_uri = config[oms_key].GetString();
+         session_config.uri.push_back(uri_config);
+      };
+
+      add_uri("md_server_uri_1", "node_oms_uri_1");
+      if (cluster_mode) {
+         add_uri("md_server_uri_2", "node_oms_uri_2");
+      }
+      LOG(md, Info,
+          "Rakurai DEX handler uri config prepared. venue:{} cluster_mode:{} uri_count:{}",
+          int(venue),
+          cluster_mode,
+          session_config.uri.size());
+
+      std::size_t uri_index = 1;
+      for (const auto& uri_config : session_config.uri) {
+         const auto index = std::to_string(uri_index);
+         validate_ws_endpoint_or_throw(
+             "md_server_uri_" + index, uri_config.md_server_uri);
+         validate_ws_endpoint_or_throw(
+             "node_oms_uri_" + index, uri_config.node_oms_uri);
+         ++uri_index;
+      }
+
+
+      if (config.HasMember("wallet_pub")) {
+         wallet_config.wallet_pub_key =
+             ::rakurai::dex_connector::solanaAddressToBytes(
+                 config["wallet_pub"].GetString());
+      }
+
+      if (config.HasMember("wallet_pvt") && config["wallet_pvt"].IsArray()) {
+         const auto& wallet_pvt = config["wallet_pvt"];
+         if (wallet_pvt.Size() != wallet_config.wallet_pvt_key.size()) {
+            throw std::runtime_error(
+                "Rakurai wallet_pvt must contain 64 integer entries");
+         }
+         for (rapidjson::SizeType i = 0; i < wallet_pvt.Size(); ++i) {
+            wallet_config.wallet_pvt_key[i] =
+                static_cast<std::uint8_t>(wallet_pvt[i].GetUint());
+         }
+      } else {
+         throw std::runtime_error("Missing Rakurai wallet_pvt in " + config_path);
+      }
+
+      if (config.HasMember("core")) {
+         session_config.core = config["core"].GetUint();
+      }
+      LOG(md, Info,
+          "Rakurai DEX handler session config prepared. venue:{} core:{} console:{} file:{} queue:{} severity:{}",
+          int(venue),
+          session_config.core,
+          logging_config.is_console_enabled,
+          logging_config.is_file_enabled,
+          logging_config.queue_size,
+          logging_config.severity);
+
+      auto dex_configs = parse_dex_configs(config, venue);
+      if (dex_configs.empty()) {
+         LOG(md, Warn,
+             "No Rakurai DEX subscriptions matched venue {} in {}",
+             int(venue),
+             config_path);
+         return;
+      }
+      LOG(md, Info,
+          "Rakurai DEX subscriptions prepared. venue:{} dex_configs:{}",
+          int(venue),
+          dex_configs.size());
+
+      ListenerAdapter adapter(
+          listener,
+          sec_id_store,
+          venue,
+          num_depth_levels,
+          excluded_securities);
+      LOG(md, Info,
+          "Rakurai listener adapter constructed. venue:{} listener_present:{} sec_id_store_present:{}",
+          int(venue),
+          listener != nullptr,
+          sec_id_store != nullptr);
+
+      if (config.HasMember("reconnect_retries")) {
+         session_config.reconnect_retries = config["reconnect_retries"].GetUint();
+      }
+      if (config.HasMember("reconnect_interval_sec")) {
+         session_config.reconnect_interval_sec =
+             config["reconnect_interval_sec"].GetUint();
+      }
+      if (config.HasMember("tip_transfer_account")) {
+         session_config.tip_transfer_account =
+             config["tip_transfer_account"].GetString();
+      }
+
+      ::rakurai::dex_connector::dex_handler handler(
+          dex_configs,
+          wallet_config,
+          logging_config,
+          session_config,
+          cluster_mode,
+          &adapter);
+      LOG(md, Info,
+          "Rakurai dex_handler constructed. venue:{} uri_count:{} dex_configs:{} cluster_mode:{}",
+          int(venue),
+          session_config.uri.size(),
+          dex_configs.size(),
+          cluster_mode);
+      auto trading_session =
+          std::make_unique<
+              ::rakurai::dex_connector::trading_session_interface>(
+              handler.create_new_trading_session());
+      LOG(md, Info,
+          "Rakurai trading session created. venue:{} session_present:{}",
+          int(venue),
+          trading_session != nullptr);
+      LOG(md, Info,
+          "Rakurai dex_handler about to run. venue:{}",
+          int(venue));
+      handler.run();
+      LOG(md, Info,
+          "Rakurai dex_handler run returned. venue:{}",
+          int(venue));
+   } catch (const std::exception& ex) {
+      LOG(md, Error, "Rakurai DEX handler failed: {}", std::string(ex.what()));
+      if (listener) {
+         MarketDataError error;
+         error.type = MarketDataError::DISCONNECTION;
+         error.message = ex.what();
+         listener->on_error(error);
+      }
+   }
+}
+
+} // namespace bts::md::rakurai

src/app_sparkd/sparkd/md/rakurai/RakuraiDexDataHandler.h

@@ -0,0 +1,25 @@
+#pragma once
+
+#include "spark/api/domain/Venue.pb.h"
+#include "spark/md/IMarketDataListener.h"
+#include "spark/persistence/IDGenerator.h"
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace bts::md::rakurai {
+
+class RakuraiDexDataHandler {
+ public:
+   static void run(
+       std::shared_ptr<IMarketDataListener> listener,
+       std::shared_ptr<persistence::IDGenerator> sec_id_store,
+       api::domain::Venue venue,
+       const std::string& config_path,
+       std::uint32_t num_depth_levels,
+       const std::vector<std::string>& excluded_securities);
+};
+
+} // namespace bts::md::rakurai

src/app_sparkd/sparkd/md/rakurai/RakuraiDexSubscription.cpp

@@ -0,0 +1,74 @@
+#include "sparkd/md/rakurai/RakuraiDexSubscription.h"
+
+#include "spark/config/config.h"
+#include "spark/config/protos/system_config.pb.h"
+#include "spark/logging/logging.h"
+#include "sparkd/rakurai/RakuraiSharedRuntime.h"
+
+#include <boost/filesystem/path.hpp>
+
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace bts::md {
+
+using Path = boost::filesystem::path;
+using bts::config::SystemConfig;
+
+struct RakuraiDexSubscription::PImpl {
+   api::domain::Venue venue;
+   std::string json_conf_file_name;
+   std::uint32_t num_depth_levels = 0;
+   std::vector<std::string> excluded_securities;
+   std::shared_ptr<persistence::IDGenerator> sec_id_store;
+   std::size_t subscription_id = 0;
+};
+
+RakuraiDexSubscription::RakuraiDexSubscription(
+    api::domain::Venue venue,
+    api::md::DexMarketDataConfig config,
+    std::shared_ptr<persistence::IDGenerator> sec_id_store)
+    : me_(std::make_unique<PImpl>()) {
+   me_->venue = venue;
+   me_->json_conf_file_name = config.json_conf_file_name();
+   me_->num_depth_levels = config.num_depth_levels();
+   me_->excluded_securities.assign(
+       config.excluded_securities().begin(),
+       config.excluded_securities().end());
+   me_->sec_id_store = std::move(sec_id_store);
+}
+
+RakuraiDexSubscription::~RakuraiDexSubscription() = default;
+
+void RakuraiDexSubscription::start(
+    std::shared_ptr<IMarketDataListener> listener) {
+   const auto config_path =
+       Path{config::get<SystemConfig>().config_path()} / me_->json_conf_file_name;
+
+   LOG(md, Info,
+       "Starting Rakurai DEX market data. venue:{} config:{}",
+       int(me_->venue),
+       config_path.string());
+
+   me_->subscription_id =
+       rakurai::RakuraiSharedRuntime::instance().add_market_data_subscriber(
+           std::move(listener),
+           me_->sec_id_store,
+           me_->venue,
+           me_->num_depth_levels,
+           me_->excluded_securities);
+   rakurai::RakuraiSharedRuntime::instance().ensure_started_from_file(
+       config_path.string());
+}
+
+void RakuraiDexSubscription::stop() {
+   if (me_->subscription_id != 0) {
+      rakurai::RakuraiSharedRuntime::instance().remove_market_data_subscriber(
+          me_->subscription_id);
+      me_->subscription_id = 0;
+   }
+}
+
+} // namespace bts::md

src/app_sparkd/sparkd/md/rakurai/RakuraiDexSubscription.h

@@ -0,0 +1,31 @@
+#pragma once
+
+#include "spark/api/domain/Venue.pb.h"
+#include "spark/api/md/DexMarketDataConfig.pb.h"
+#include "spark/md/IMarketDataSource.h"
+
+#include <memory>
+
+namespace bts::persistence {
+class IDGenerator;
+}
+
+namespace bts::md {
+
+class RakuraiDexSubscription : public IMarketDataSource {
+ public:
+   RakuraiDexSubscription(
+       api::domain::Venue venue,
+       api::md::DexMarketDataConfig config,
+       std::shared_ptr<persistence::IDGenerator> sec_id_store);
+   ~RakuraiDexSubscription() override;
+
+   void start(std::shared_ptr<IMarketDataListener> listener) override;
+   void stop() override;
+
+ private:
+   struct PImpl;
+   std::unique_ptr<PImpl> me_;
+};
+
+} // namespace bts::md

src/app_sparkd/sparkd/md/rakurai/RakuraiMarketDataProvider.cpp

@@ -0,0 +1,55 @@
+#include "RakuraiMarketDataProvider.h"
+#include "spark/api/domain/Venue.pb.h"
+#include "spark/api/md/MarketDataConfig.pb.h"
+#include "spark/md/IMarketDataSource.h"
+#include "spark/persistence/IDGenerator.h"
+#include "spark/logging/logging.h"
+#include "sparkd/md/rakurai/RakuraiDexSubscription.h"
+#include "sparkd/rakurai/RakuraiVenueRegistry.h"
+
+#include <memory>
+#include <vector>
+
+namespace bts::md {
+
+using bts::api::md::MarketDataConfig;
+using bts::persistence::IDGenerator;
+using std::pair;
+using std::shared_ptr;
+using std::string;
+using std::vector;
+
+struct RakuraiMarketDataProvider::PImpl {
+   MarketDataConfig config;
+   shared_ptr<IDGenerator> sec_id_store;
+
+   PImpl(MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
+       : config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
+};
+
+RakuraiMarketDataProvider::RakuraiMarketDataProvider(
+    MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
+    : me_(std::make_unique<PImpl>(std::move(config), std::move(sec_id_store))) {
+}
+
+RakuraiMarketDataProvider::~RakuraiMarketDataProvider() = default;
+
+vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>>
+RakuraiMarketDataProvider::get_sources() {
+   vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>> rv;
+   for (const auto &dex : me_->config.dex()) {
+      if (bts::rakurai::is_rakurai_venue(dex.venue())) {
+         rv.push_back(
+             {dex.venue(),
+              std::make_shared<RakuraiDexSubscription>(
+                  dex.venue(), dex, me_->sec_id_store)});
+      } else {
+         LOG(md, Warn,
+             "Skipping unsupported Rakurai dex venue:{}",
+             int(dex.venue()));
+      }
+   }
+   return rv;
+}
+
+} // namespace bts::md

src/app_sparkd/sparkd/md/rakurai/RakuraiMarketDataProvider.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#include "spark/md/IMarketDataProvider.h"
+
+#include <memory>
+
+namespace bts::persistence {
+class IDGenerator;
+}
+
+namespace bts::api::md {
+class MarketDataConfig;
+}
+
+namespace bts::md {
+
+class RakuraiMarketDataProvider : public IMarketDataProvider {
+ public:
+   explicit RakuraiMarketDataProvider(
+       api::md::MarketDataConfig config,
+       std::shared_ptr<persistence::IDGenerator> sec_id_store);
+   ~RakuraiMarketDataProvider();
+
+   std::vector<
+       std::pair<api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
+   get_sources() override;
+
+ private:
+   struct PImpl;
+   std::unique_ptr<PImpl> me_;
+};
+
+} // namespace bts::md
+

src/app_sparkd/sparkd/position/PositionServiceServer.cpp

@@ -231,16 +231,10 @@ void PositionServiceServer::create_manual_fill(
    for (auto mf : batch.manual_fill()) {
       auto id = utils::gen_id("manual_fills");
       mf.set_record_id(id);
-      // Older clients do not send a timestamp.  Retain their existing behavior,
-      // but preserve a timestamp supplied by newer clients so a manual fill can
-      // be assigned to its intended trading day.
-      if (!mf.has_timestamp() ||
-          (mf.timestamp().seconds() == 0 && mf.timestamp().nanos() == 0)) {
-         struct timeval tv;
-         gettimeofday(&tv, NULL);
-         mf.mutable_timestamp()->set_seconds(tv.tv_sec);
-         mf.mutable_timestamp()->set_nanos(tv.tv_usec * 1000);
-      }
+      struct timeval tv;
+      gettimeofday(&tv, NULL);
+      mf.mutable_timestamp()->set_seconds(tv.tv_sec);
+      mf.mutable_timestamp()->set_nanos(tv.tv_usec * 1000);
 
       me->manual_fills_store->insert(
           manual_fills_ns, std::to_string(id), utils::pb::to_string(mf));
@@ -302,16 +296,6 @@ void PositionServiceServer::update_manual_fill(
          this_resp->set_ok(false);
          this_resp->set_msg("Could not find record");
       } else {
-         // Legacy clients omit timestamps when editing.  Do not let that erase
-         // the trading date that was assigned when the fill was created.
-         if (!mf.has_timestamp() ||
-             (mf.timestamp().seconds() == 0 && mf.timestamp().nanos() == 0)) {
-            ManualFill existing;
-            if (utils::pb::from_string<ManualFill>(val, &existing) &&
-                existing.has_timestamp()) {
-               mf.mutable_timestamp()->CopyFrom(existing.timestamp());
-            }
-         }
          me->manual_fills_store->update(
              manual_fills_ns, id, utils::pb::to_string(mf));
          this_resp->set_ok(true);

src/app_sparkd/sparkd/rakurai/RakuraiSharedRuntime.cpp

@@ -0,0 +1,837 @@
+#include "sparkd/rakurai/RakuraiSharedRuntime.h"
+
+#include "spark/api/domain/SecurityType.pb.h"
+#include "spark/domain/Book.h"
+#include "spark/domain/Price.h"
+#include "spark/domain/Quantity.h"
+#include "spark/domain/Security.h"
+#include "spark/logging/logging.h"
+#include "spark/md/IMarketDataListener.h"
+#include "spark/persistence/IDGenerator.h"
+#include "spark/persistence/SqliteStore.h"
+#include "sparkd/md/MarketDataError.h"
+#include "sparkd/rakurai/RakuraiVenueRegistry.h"
+
+#include <boost/filesystem/path.hpp>
+#include <dex_handler.h>
+#include <dex_listener.h>
+#include <nlohmann/json.hpp>
+
+#include <algorithm>
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <fstream>
+#include <functional>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <stdexcept>
+#include <string>
+#include <string_view>
+#include <thread>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+namespace bts::rakurai {
+
+namespace {
+
+using ConnectorSecurity = ::rakurai::dex_connector::data_types::security;
+using ConnectorBookLevel = ::rakurai::dex_connector::data_types::book_level;
+
+std::string bytes_to_hex(const std::array<std::uint8_t, 32>& bytes) {
+   static constexpr char kHex[] = "0123456789abcdef";
+   std::string out;
+   out.reserve(bytes.size() * 2);
+   for (std::uint8_t byte : bytes) {
+      out.push_back(kHex[(byte >> 4) & 0x0f]);
+      out.push_back(kHex[byte & 0x0f]);
+   }
+   return out;
+}
+
+std::string short_hex(const std::array<std::uint8_t, 32>& bytes) {
+   const auto hex = bytes_to_hex(bytes);
+   return hex.substr(0, std::min<std::size_t>(hex.size(), 12));
+}
+
+std::string token_display_name(
+    const ::rakurai::dex_connector::data_types::token_info& token) {
+   return token.name.empty() ? short_hex(token.mint_address) : token.name;
+}
+
+std::string build_security_desc(
+    const ConnectorSecurity& security,
+    std::string_view short_pool_id,
+    std::string_view currency) {
+   return fmt::format(
+       "{}-{}-{}",
+       token_display_name(security.token_a),
+       currency,
+       short_pool_id);
+}
+
+std::string security_currency(const ConnectorSecurity& security) {
+   const auto configured_currency = connector_quote_currency(security.venue);
+   return configured_currency.empty()
+              ? token_display_name(security.token_b)
+              : std::string(configured_currency);
+}
+
+double quantity_scale_from_decimals(std::uint8_t decimals) {
+   double scale = 1.0;
+   for (std::uint8_t i = 0; i < decimals; ++i) {
+      scale *= 10.0;
+   }
+   return scale;
+}
+
+domain::BookLevel to_book_level(
+    const ConnectorBookLevel& source,
+    double quantity_scale) {
+   domain::BookLevel level;
+   level.price = domain::Price::from_double(source.price);
+   const auto normalized_quantity =
+       quantity_scale > 0.0
+           ? static_cast<double>(source.quantity) / quantity_scale
+           : static_cast<double>(source.quantity);
+   level.total_quantity =
+       domain::Quantity(static_cast<float>(normalized_quantity));
+   level.direct_quantity = level.total_quantity;
+   level.n_orders = 0;
+   return level;
+}
+
+std::list<ConnectorDexConfig> parse_dex_configs(const nlohmann::json& config) {
+   std::list<ConnectorDexConfig> dex_config_list;
+   const auto dex_it = config.find("dex");
+   if (dex_it == config.end() || !dex_it->is_array()) {
+      return dex_config_list;
+   }
+
+   for (const auto& dex_value : *dex_it) {
+      if (!dex_value.is_object()) {
+         continue;
+      }
+
+      const std::string dex = dex_value.value("id", "");
+      auto dex_config = make_connector_dex_config(dex);
+      if (!dex_config.has_value()) {
+         LOG(md, Warn, "Skipping unsupported Rakurai dex config '{}'", dex);
+         continue;
+      }
+
+      const auto pools_it = dex_value.find("pool_ids");
+      if (pools_it != dex_value.end() && pools_it->is_array()) {
+         for (const auto& pool_value : *pools_it) {
+            dex_config->interested_pools.emplace_back(
+                ::rakurai::dex_connector::solanaAddressToBytes(
+                    pool_value.get<std::string>()));
+         }
+      }
+      const auto symbols_it = dex_value.find("symbols");
+      if (symbols_it != dex_value.end() && symbols_it->is_array()) {
+         for (const auto& symbol : *symbols_it) {
+            dex_config->interested_symbols.push_back(symbol.get<std::string>());
+         }
+      }
+      const auto interested_symbols_it = dex_value.find("interested_symbols");
+      if (interested_symbols_it != dex_value.end() &&
+          interested_symbols_it->is_array()) {
+         for (const auto& symbol : *interested_symbols_it) {
+            dex_config->interested_symbols.push_back(symbol.get<std::string>());
+         }
+      }
+
+      dex_config_list.push_back(std::move(*dex_config));
+   }
+
+   return dex_config_list;
+}
+
+nlohmann::json load_json_file(const std::string& path) {
+   LOG(md, Info, "Rakurai shared runtime loading config file: {}", path);
+   std::ifstream config_file(path);
+   if (!config_file.is_open()) {
+      throw std::runtime_error("Unable to open Rakurai config: " + path);
+   }
+
+   nlohmann::json config;
+   config_file >> config;
+
+   const auto nested_config_it = config.find("json_conf_file_name");
+   if (nested_config_it != config.end() && nested_config_it->is_string() &&
+       !config.contains("dex")) {
+      const auto nested_name = nested_config_it->get<std::string>();
+      auto nested_path = boost::filesystem::path(nested_name);
+      if (nested_path.is_relative()) {
+         nested_path = boost::filesystem::path(path).parent_path() / nested_path;
+      }
+      LOG(md, Info,
+          "Rakurai shared runtime resolved wrapper config '{}' -> '{}'. wrapper:{}",
+          path,
+          nested_path.string(),
+          config.dump());
+      return load_json_file(nested_path.string());
+   }
+
+   LOG(md, Info,
+       "Rakurai shared runtime loaded connector config from '{}': {}",
+       path,
+       config.dump());
+   return config;
+}
+
+} // namespace
+
+struct RakuraiSharedRuntime::PImpl {
+   struct MarketDataSubscriber {
+      std::size_t id = 0;
+      std::shared_ptr<md::IMarketDataListener> listener;
+      std::shared_ptr<persistence::IDGenerator> sec_id_store;
+      api::domain::Venue venue = api::domain::VENUE_UNSPECIFIED;
+      std::size_t depth_limit = domain::Book::max_depth();
+      std::unordered_set<std::string> excluded_securities;
+      std::unordered_map<std::string, std::unique_ptr<domain::Security>>
+          securities;
+      std::unordered_map<std::string, domain::SecurityID> pool_to_security_id;
+      std::unordered_map<std::string, double> pool_quantity_scales;
+      std::mutex mutex;
+   };
+
+   struct ListenerAdapter final : public ::rakurai::dex_connector::dex_listener {
+      explicit ListenerAdapter(PImpl& owner) : owner(owner) {}
+
+      void on_pool_definition(
+          ConnectorVenue venue,
+          const std::vector<ConnectorSecurity*>& securities) override {
+         owner.on_pool_definition(venue, securities);
+      }
+
+      void on_book(
+          ConnectorVenue venue,
+          const std::array<std::uint8_t, 32>& pool_id,
+          const std::vector<ConnectorBookLevel>& bids,
+          const std::vector<ConnectorBookLevel>& asks) override {
+         owner.on_book(venue, pool_id, bids, asks);
+      }
+
+      void on_order_update(
+          ::rakurai::dex_connector::order_entry::order_state order_state)
+          override {
+         owner.on_order_update(order_state);
+      }
+
+      void on_ata_verification_status(
+          const std::array<std::uint8_t, 32>& token_id,
+          const std::string& ata_address,
+          ::rakurai::dex_connector::ata_status status) override {
+         LOG(md, Info,
+             "Rakurai ATA verification. token:{} ata:{} status:{}",
+             short_hex(token_id),
+             ata_address,
+             static_cast<int>(status));
+      }
+
+      PImpl& owner;
+   };
+
+   mutable std::mutex mutex;
+   std::optional<nlohmann::json> config;
+   std::string config_label;
+   std::optional<std::thread> run_thread;
+   std::unique_ptr<::rakurai::dex_connector::dex_handler> handler;
+   std::unique_ptr<::rakurai::dex_connector::trading_session_interface>
+       trading_session;
+   persistence::SqliteStore pool_alias_store{"rakurai_pool_alias"};
+   ListenerAdapter listener{*this};
+   std::unordered_map<std::size_t, std::shared_ptr<MarketDataSubscriber>>
+       market_data_subscribers;
+   std::unordered_map<std::size_t, OrderUpdateCallback> order_update_subscribers;
+   std::unordered_map<std::size_t, StatusCallback> status_subscribers;
+   std::unordered_map<std::string, PoolMetadata> pool_metadata;
+   std::unordered_map<std::string, std::string> short_pool_to_full_pool;
+   std::unordered_set<std::string> expected_pool_ids;
+   std::unordered_set<std::string> resolved_pool_ids;
+   std::size_t next_subscriber_id = 1;
+   bool started = false;
+   bool connected = false;
+
+   ~PImpl() {
+      if (run_thread.has_value() && run_thread->joinable()) {
+         run_thread->detach();
+      }
+   }
+
+   std::size_t next_id_locked() { return next_subscriber_id++; }
+
+   void ensure_started(const nlohmann::json& cfg, const std::string& label) {
+      std::lock_guard<std::mutex> lock(mutex);
+      LOG(md, Info,
+          "Rakurai shared runtime ensure_started requested. label:{} config:{}",
+          label,
+          cfg.dump());
+      if (!config.has_value()) {
+         config = cfg;
+         config_label = label;
+      } else if (*config != cfg) {
+         throw std::runtime_error(
+             "Rakurai shared runtime already configured with a different config source: " +
+             config_label + " vs " + label);
+      }
+
+      if (started) {
+         LOG(md, Info,
+             "Rakurai shared runtime already started. label:{} connected:{}",
+             config_label,
+             connected);
+         return;
+      }
+      started = true;
+      LOG(md, Info,
+          "Rakurai shared runtime launching thread. label:{}",
+          config_label);
+      run_thread.emplace([this]() { run(); });
+   }
+
+   std::size_t add_market_data_subscriber(
+       std::shared_ptr<md::IMarketDataListener> listener_in,
+       std::shared_ptr<persistence::IDGenerator> sec_id_store,
+       api::domain::Venue venue,
+       std::uint32_t num_depth_levels,
+       const std::vector<std::string>& excluded_securities) {
+      auto sub = std::make_shared<MarketDataSubscriber>();
+      sub->listener = std::move(listener_in);
+      sub->sec_id_store = std::move(sec_id_store);
+      sub->venue = venue;
+      sub->depth_limit =
+          num_depth_levels == 0
+              ? domain::Book::max_depth()
+              : std::min<std::size_t>(
+                    num_depth_levels, domain::Book::max_depth());
+      sub->excluded_securities.insert(
+          excluded_securities.begin(), excluded_securities.end());
+
+      std::vector<PoolMetadata> snapshot;
+      {
+         std::lock_guard<std::mutex> lock(mutex);
+         sub->id = next_id_locked();
+         market_data_subscribers.emplace(sub->id, sub);
+         snapshot.reserve(pool_metadata.size());
+         for (const auto& [_, metadata] : pool_metadata) {
+            if (venue_matches(venue, metadata.security.venue)) {
+               snapshot.push_back(metadata);
+            }
+         }
+      }
+
+      replay_pool_definitions(*sub, snapshot);
+      return sub->id;
+   }
+
+   void remove_market_data_subscriber(std::size_t id) {
+      std::lock_guard<std::mutex> lock(mutex);
+      market_data_subscribers.erase(id);
+   }
+
+   std::size_t add_order_update_subscriber(OrderUpdateCallback callback) {
+      std::lock_guard<std::mutex> lock(mutex);
+      const auto id = next_id_locked();
+      order_update_subscribers.emplace(id, std::move(callback));
+      return id;
+   }
+
+   void remove_order_update_subscriber(std::size_t id) {
+      std::lock_guard<std::mutex> lock(mutex);
+      order_update_subscribers.erase(id);
+   }
+
+   std::size_t add_status_subscriber(StatusCallback callback) {
+      bool current_connected = false;
+      std::size_t id = 0;
+      {
+         std::lock_guard<std::mutex> lock(mutex);
+         id = next_id_locked();
+         status_subscribers.emplace(id, callback);
+         current_connected = connected;
+      }
+      callback(current_connected);
+      return id;
+   }
+
+   void remove_status_subscriber(std::size_t id) {
+      std::lock_guard<std::mutex> lock(mutex);
+      status_subscribers.erase(id);
+   }
+
+   bool is_connected() const {
+      std::lock_guard<std::mutex> lock(mutex);
+      return connected;
+   }
+
+   void launch_new_order(const RakuraiSharedRuntime::NewOrder& order) {
+      std::lock_guard<std::mutex> lock(mutex);
+      if (!connected || trading_session == nullptr) {
+         throw std::runtime_error("Rakurai execution session is not connected");
+      }
+      // Rakurai 0.0.12 changed this API to take a mutable order reference.
+      // Keep our subscriber-facing API const and give the connector its own copy.
+      auto connector_order = order;
+      trading_session->launch_new_order(connector_order);
+   }
+
+   std::optional<PoolMetadata> find_pool_metadata(
+       api::Venue venue,
+       const std::string& short_pool_id) const {
+      std::lock_guard<std::mutex> lock(mutex);
+      const auto pool_it = short_pool_to_full_pool.find(short_pool_id);
+      if (pool_it == short_pool_to_full_pool.end()) {
+         return std::nullopt;
+      }
+      const auto metadata_it = pool_metadata.find(pool_it->second);
+      if (metadata_it == pool_metadata.end() ||
+          !venue_matches(venue, metadata_it->second.security.venue)) {
+         return std::nullopt;
+      }
+      return metadata_it->second;
+   }
+
+   void run() {
+      try {
+         nlohmann::json connector_config;
+         {
+            std::lock_guard<std::mutex> lock(mutex);
+            connector_config = *config;
+         }
+
+         auto dex_configs = parse_dex_configs(connector_config);
+         if (dex_configs.empty()) {
+            throw std::runtime_error("No Rakurai dex pools configured");
+         }
+         LOG(md, Info,
+             "Rakurai shared runtime parsed config. dex_configs:{} config:{}",
+             dex_configs.size(),
+             connector_config.dump());
+         {
+            std::lock_guard<std::mutex> lock(mutex);
+            expected_pool_ids.clear();
+            resolved_pool_ids.clear();
+            for (const auto& dex_config : dex_configs) {
+               for (const auto& pool_id : dex_config.interested_pools) {
+                  expected_pool_ids.insert(bytes_to_hex(pool_id));
+               }
+            }
+         }
+
+         ::rakurai::dex_connector::configuration::logging_config logging_config;
+         if (connector_config.contains("app_logging")) {
+            const auto& app_logging = connector_config.at("app_logging");
+            logging_config.severity =
+                app_logging.value("severity", logging_config.severity);
+            logging_config.is_console_enabled =
+                app_logging.value("console", logging_config.is_console_enabled);
+            logging_config.is_file_enabled =
+                app_logging.value("file", logging_config.is_file_enabled);
+            logging_config.queue_size =
+                app_logging.value("queuesize", logging_config.queue_size);
+         }
+
+         ::rakurai::dex_connector::configuration::wallet wallet_config;
+         wallet_config.wallet_pub_key =
+             ::rakurai::dex_connector::solanaAddressToBytes(
+                 connector_config.at("wallet_pub").get<std::string>());
+         const auto& wallet_pvt = connector_config.at("wallet_pvt");
+         if (!wallet_pvt.is_array() ||
+             wallet_pvt.size() != wallet_config.wallet_pvt_key.size()) {
+            throw std::runtime_error(
+                "Rakurai wallet_pvt must contain 64 integer entries");
+         }
+         for (std::size_t i = 0; i < wallet_config.wallet_pvt_key.size(); ++i) {
+            wallet_config.wallet_pvt_key[i] =
+                wallet_pvt.at(i).get<std::uint8_t>();
+         }
+
+         ::rakurai::dex_connector::configuration::session_config session_config;
+         auto add_uri = [&](const char* md_key, const char* oms_key) {
+            ::rakurai::dex_connector::configuration::uri_config uri;
+            uri.md_server_uri =
+                connector_config.at(md_key).get<std::string>();
+            uri.node_oms_uri =
+                connector_config.at(oms_key).get<std::string>();
+            session_config.uri.push_back(std::move(uri));
+         };
+         add_uri("md_server_uri_1", "node_oms_uri_1");
+         const bool cluster_mode =
+             connector_config.value("cluster_mode", false);
+         if (cluster_mode) {
+            add_uri("md_server_uri_2", "node_oms_uri_2");
+         }
+
+         session_config.core = connector_config.value("core", session_config.core);
+         session_config.reconnect_retries = connector_config.value(
+             "reconnect_retries", session_config.reconnect_retries);
+         session_config.reconnect_interval_sec = connector_config.value(
+             "reconnect_interval_sec", session_config.reconnect_interval_sec);
+         session_config.tip_transfer_account = connector_config.value(
+             "tip_transfer_account", session_config.tip_transfer_account);
+
+         {
+            std::lock_guard<std::mutex> lock(mutex);
+            handler = std::make_unique<::rakurai::dex_connector::dex_handler>(
+                dex_configs,
+                wallet_config,
+                logging_config,
+                session_config,
+                cluster_mode,
+                &listener);
+            trading_session = std::make_unique<
+                ::rakurai::dex_connector::trading_session_interface>(
+                handler->create_new_trading_session());
+            connected = false;
+         }
+
+         handler->run();
+      } catch (const std::exception& ex) {
+         LOG(md, Error, "Rakurai shared runtime failed: {}", std::string(ex.what()));
+         notify_md_error(ex.what());
+      }
+
+      {
+         std::lock_guard<std::mutex> lock(mutex);
+         connected = false;
+      }
+      notify_status(false);
+   }
+
+   void notify_status(bool is_connected_now) {
+      std::vector<StatusCallback> callbacks;
+      {
+         std::lock_guard<std::mutex> lock(mutex);
+         for (const auto& [_, callback] : status_subscribers) {
+            callbacks.push_back(callback);
+         }
+      }
+      for (auto& callback : callbacks) {
+         callback(is_connected_now);
+      }
+   }
+
+   void notify_md_error(const std::string& message) {
+      std::vector<std::shared_ptr<md::IMarketDataListener>> listeners;
+      {
+         std::lock_guard<std::mutex> lock(mutex);
+         for (const auto& [_, sub] : market_data_subscribers) {
+            listeners.push_back(sub->listener);
+         }
+      }
+      for (const auto& listener_ptr : listeners) {
+         if (!listener_ptr) {
+            continue;
+         }
+         md::MarketDataError error;
+         error.type = md::MarketDataError::DISCONNECTION;
+         error.message = message;
+         listener_ptr->on_error(error);
+      }
+   }
+
+   void load_aliases_locked() {
+      if (!short_pool_to_full_pool.empty()) {
+         return;
+      }
+
+      constexpr std::string_view kPoolAliasNamespace = "rakurai_pool_alias";
+      for (const auto& [ns, rows] : pool_alias_store.retrieve_all()) {
+         if (ns != kPoolAliasNamespace) {
+            continue;
+         }
+         for (const auto& [full_pool_id, alias] : rows) {
+            if (!alias.empty()) {
+               short_pool_to_full_pool.emplace(alias, full_pool_id);
+            }
+         }
+      }
+   }
+
+   std::string get_or_add_short_pool_id_locked(const std::string& pool_id) {
+      constexpr std::string_view kPoolAliasNamespace = "rakurai_pool_alias";
+      const auto [exists, rowid, _] =
+          pool_alias_store.retrieve_one(std::string(kPoolAliasNamespace), pool_id);
+      if (exists) {
+         const auto alias = std::to_string(rowid);
+         short_pool_to_full_pool.emplace(alias, pool_id);
+         return alias;
+      }
+
+      const auto new_id =
+          pool_alias_store.insert(std::string(kPoolAliasNamespace), pool_id, "");
+      const auto alias = std::to_string(new_id);
+      short_pool_to_full_pool.emplace(alias, pool_id);
+      return alias;
+   }
+
+   void on_pool_definition(
+       ConnectorVenue connector_venue,
+       const std::vector<ConnectorSecurity*>& securities) {
+      std::vector<std::shared_ptr<MarketDataSubscriber>> subscribers;
+      std::vector<PoolMetadata> touched;
+      bool should_notify_ready = false;
+      std::size_t resolved_pool_count = 0;
+      std::size_t expected_pool_count = 0;
+      {
+         std::lock_guard<std::mutex> lock(mutex);
+         load_aliases_locked();
+         for (const auto* security : securities) {
+            if (security == nullptr) {
+               continue;
+            }
+            const auto pool_id = bytes_to_hex(security->pool_id);
+            pool_metadata[pool_id] = PoolMetadata{
+                *security,
+                pool_id,
+                get_or_add_short_pool_id_locked(pool_id),
+                quantity_scale_from_decimals(security->token_a.decimals)};
+            resolved_pool_ids.insert(pool_id);
+            touched.push_back(pool_metadata.at(pool_id));
+         }
+
+         const bool has_trading_session = trading_session != nullptr;
+         const bool has_pool_definitions = !resolved_pool_ids.empty();
+         const bool all_expected_pools_resolved =
+             expected_pool_ids.empty()
+                 ? has_pool_definitions
+                 : std::all_of(
+                       expected_pool_ids.begin(),
+                       expected_pool_ids.end(),
+                       [this](const std::string& pool_id) {
+                          return resolved_pool_ids.contains(pool_id);
+                       });
+         if (!connected && has_trading_session && all_expected_pools_resolved) {
+            connected = true;
+            should_notify_ready = true;
+            resolved_pool_count = resolved_pool_ids.size();
+            expected_pool_count = expected_pool_ids.size();
+         }
+
+         for (const auto& [_, sub] : market_data_subscribers) {
+            if (venue_matches(sub->venue, connector_venue)) {
+               subscribers.push_back(sub);
+            }
+         }
+      }
+
+      if (should_notify_ready) {
+         LOG(md, Info,
+             "Rakurai shared runtime ready. resolved_pools:{} expected_pools:{}",
+             resolved_pool_count,
+             expected_pool_count);
+         notify_status(true);
+      }
+
+      for (const auto& sub : subscribers) {
+         replay_pool_definitions(*sub, touched);
+      }
+   }
+
+   void replay_pool_definitions(
+       MarketDataSubscriber& sub,
+       const std::vector<PoolMetadata>& metadata_list) {
+      std::lock_guard<std::mutex> lock(sub.mutex);
+      std::vector<const domain::Security*> emitted;
+
+      for (const auto& metadata : metadata_list) {
+         if (!venue_matches(sub.venue, metadata.security.venue)) {
+            continue;
+         }
+
+         const auto& security = metadata.security;
+         const auto& pool_id = metadata.full_pool_id;
+         const auto symbol = token_display_name(security.token_a);
+         const auto currency = security_currency(security);
+         const auto desc =
+             build_security_desc(security, metadata.short_pool_id, currency);
+         if (sub.excluded_securities.contains(pool_id) ||
+             sub.excluded_securities.contains(metadata.short_pool_id) ||
+             sub.excluded_securities.contains(desc)) {
+            continue;
+         }
+
+         domain::Security emitted_security;
+         const auto security_type = connector_security_type(security.venue);
+         if (security_type == api::domain::SECURITY_TYPE_FUTURE) {
+            auto& future =
+                emitted_security.security.emplace<domain::FutureSecurity>();
+            future.expiration = core::TimeStamp::from_ymd(2100, 1, 1);
+         } else {
+            emitted_security.security = domain::SpotSecurity{};
+         }
+         emitted_security.security_common.venue = sub.venue;
+         emitted_security.security_common.exchange_sec_id = metadata.short_pool_id;
+         emitted_security.security_common.symbol = symbol;
+         emitted_security.security_common.currency = currency;
+         emitted_security.security_common.asset = symbol;
+         emitted_security.security_common.group =
+             connector_venue_name(security.venue);
+         emitted_security.security_common.desc = desc;
+         emitted_security.security_common.min_price_increment =
+             security.min_tick > 0.0 ? security.min_tick : 0.0;
+         emitted_security.security_common.min_quantity_increment =
+             static_cast<double>(security.min_trade_amount);
+         emitted_security.security_common.min_quantity =
+             static_cast<double>(security.min_trade_amount);
+         *emitted_security.security_common.id = sub.sec_id_store->get_or_add(
+             sub.venue, security_type, pool_id);
+
+         auto [it, is_new] = sub.securities.try_emplace(
+             pool_id, std::make_unique<domain::Security>(std::move(emitted_security)));
+         if (!is_new) {
+            *it->second = std::move(emitted_security);
+         }
+
+         sub.pool_to_security_id[pool_id] = it->second->id();
+         sub.pool_quantity_scales[pool_id] = metadata.quantity_scale;
+         emitted.push_back(it->second.get());
+      }
+
+      if (!emitted.empty() && sub.listener) {
+         sub.listener->on_securities(emitted);
+      }
+   }
+
+   void on_book(
+       ConnectorVenue connector_venue,
+       const std::array<std::uint8_t, 32>& pool_id,
+       const std::vector<ConnectorBookLevel>& bids,
+       const std::vector<ConnectorBookLevel>& asks) {
+      std::vector<std::shared_ptr<MarketDataSubscriber>> subscribers;
+      {
+         std::lock_guard<std::mutex> lock(mutex);
+         for (const auto& [_, sub] : market_data_subscribers) {
+            if (venue_matches(sub->venue, connector_venue)) {
+               subscribers.push_back(sub);
+            }
+         }
+      }
+
+      const auto pool_key = bytes_to_hex(pool_id);
+      for (const auto& sub : subscribers) {
+         std::lock_guard<std::mutex> lock(sub->mutex);
+         const auto it = sub->pool_to_security_id.find(pool_key);
+         if (it == sub->pool_to_security_id.end()) {
+            continue;
+         }
+
+         const auto quantity_scale_it = sub->pool_quantity_scales.find(pool_key);
+         const auto quantity_scale =
+             quantity_scale_it != sub->pool_quantity_scales.end()
+                 ? quantity_scale_it->second
+                 : 1.0;
+
+         auto book = domain::Book::empty(it->second);
+         const auto bid_depth =
+             std::min<std::size_t>(sub->depth_limit, bids.size());
+         const auto ask_depth =
+             std::min<std::size_t>(sub->depth_limit, asks.size());
+
+         for (std::size_t i = 0; i < bid_depth; ++i) {
+            book.bids()[i] = to_book_level(bids[i], quantity_scale);
+         }
+         for (std::size_t i = 0; i < ask_depth; ++i) {
+            book.asks()[i] = to_book_level(asks[i], quantity_scale);
+         }
+
+         if (sub->listener) {
+            sub->listener->on_book(book);
+         }
+      }
+   }
+
+   void on_order_update(
+       ::rakurai::dex_connector::order_entry::order_state order_state) {
+      std::vector<OrderUpdateCallback> callbacks;
+      {
+         std::lock_guard<std::mutex> lock(mutex);
+         for (const auto& [_, callback] : order_update_subscribers) {
+            callbacks.push_back(callback);
+         }
+      }
+
+      for (auto& callback : callbacks) {
+         callback(order_state);
+      }
+   }
+};
+
+RakuraiSharedRuntime& RakuraiSharedRuntime::instance() {
+   static RakuraiSharedRuntime runtime;
+   return runtime;
+}
+
+RakuraiSharedRuntime::RakuraiSharedRuntime()
+    : me_(std::make_unique<PImpl>()) {}
+
+RakuraiSharedRuntime::~RakuraiSharedRuntime() = default;
+
+std::size_t RakuraiSharedRuntime::add_market_data_subscriber(
+    std::shared_ptr<md::IMarketDataListener> listener,
+    std::shared_ptr<persistence::IDGenerator> sec_id_store,
+    api::domain::Venue venue,
+    std::uint32_t num_depth_levels,
+    const std::vector<std::string>& excluded_securities) {
+   return me_->add_market_data_subscriber(
+       std::move(listener),
+       std::move(sec_id_store),
+       venue,
+       num_depth_levels,
+       excluded_securities);
+}
+
+void RakuraiSharedRuntime::remove_market_data_subscriber(std::size_t id) {
+   me_->remove_market_data_subscriber(id);
+}
+
+std::size_t RakuraiSharedRuntime::add_order_update_subscriber(
+    OrderUpdateCallback callback) {
+   return me_->add_order_update_subscriber(std::move(callback));
+}
+
+void RakuraiSharedRuntime::remove_order_update_subscriber(std::size_t id) {
+   me_->remove_order_update_subscriber(id);
+}
+
+std::size_t RakuraiSharedRuntime::add_status_subscriber(StatusCallback callback) {
+   return me_->add_status_subscriber(std::move(callback));
+}
+
+void RakuraiSharedRuntime::remove_status_subscriber(std::size_t id) {
+   me_->remove_status_subscriber(id);
+}
+
+void RakuraiSharedRuntime::ensure_started_from_file(
+    const std::string& config_path) {
+   me_->ensure_started(load_json_file(config_path), config_path);
+}
+
+void RakuraiSharedRuntime::ensure_started_from_json(
+    const nlohmann::json& config,
+    const std::string& config_label) {
+   me_->ensure_started(config, config_label);
+}
+
+bool RakuraiSharedRuntime::is_connected() const { return me_->is_connected(); }
+
+void RakuraiSharedRuntime::launch_new_order(const NewOrder& order) {
+   me_->launch_new_order(order);
+}
+
+std::optional<RakuraiSharedRuntime::PoolMetadata>
+RakuraiSharedRuntime::find_pool_metadata(
+    api::Venue venue,
+    const std::string& short_pool_id) const {
+   return me_->find_pool_metadata(venue, short_pool_id);
+}
+
+} // namespace bts::rakurai

src/app_sparkd/sparkd/rakurai/RakuraiSharedRuntime.h

@@ -0,0 +1,78 @@
+#pragma once
+
+#include "spark/api/domain/Venue.pb.h"
+#include "spark/domain_model/Security.pb.h"
+
+#include <dex_utils.h>
+#include <order_entry_common.h>
+
+#include <cstddef>
+#include <functional>
+#include <memory>
+#include <nlohmann/json_fwd.hpp>
+#include <optional>
+#include <string>
+#include <vector>
+
+namespace bts::md {
+class IMarketDataListener;
+}
+
+namespace bts::persistence {
+class IDGenerator;
+}
+
+namespace bts::rakurai {
+
+class RakuraiSharedRuntime {
+ public:
+   using OrderUpdate = ::rakurai::dex_connector::order_entry::order_state;
+   using NewOrder = ::rakurai::dex_connector::order_entry::new_order;
+   using OrderUpdateCallback = std::function<void(OrderUpdate)>;
+   using StatusCallback = std::function<void(bool)>;
+
+   struct PoolMetadata {
+      ::rakurai::dex_connector::data_types::security security;
+      std::string full_pool_id;
+      std::string short_pool_id;
+      double quantity_scale = 1.0;
+   };
+
+   static RakuraiSharedRuntime& instance();
+
+   std::size_t add_market_data_subscriber(
+       std::shared_ptr<md::IMarketDataListener> listener,
+       std::shared_ptr<persistence::IDGenerator> sec_id_store,
+       api::domain::Venue venue,
+       std::uint32_t num_depth_levels,
+       const std::vector<std::string>& excluded_securities);
+   void remove_market_data_subscriber(std::size_t id);
+
+   std::size_t add_order_update_subscriber(OrderUpdateCallback callback);
+   void remove_order_update_subscriber(std::size_t id);
+
+   std::size_t add_status_subscriber(StatusCallback callback);
+   void remove_status_subscriber(std::size_t id);
+
+   void ensure_started_from_file(const std::string& config_path);
+   void ensure_started_from_json(
+       const nlohmann::json& config,
+       const std::string& config_label);
+
+   bool is_connected() const;
+   void launch_new_order(const NewOrder& order);
+   std::optional<PoolMetadata> find_pool_metadata(
+       api::Venue venue,
+       const std::string& short_pool_id) const;
+
+ private:
+   RakuraiSharedRuntime();
+   ~RakuraiSharedRuntime();
+   RakuraiSharedRuntime(const RakuraiSharedRuntime&) = delete;
+   RakuraiSharedRuntime& operator=(const RakuraiSharedRuntime&) = delete;
+
+   struct PImpl;
+   std::unique_ptr<PImpl> me_;
+};
+
+} // namespace bts::rakurai

src/app_sparkd/sparkd/rakurai/RakuraiVenueRegistry.h

@@ -0,0 +1,178 @@
+// Generated by tools/generate_venues.py. Do not edit.
+#pragma once
+
+#include "spark/api/domain/SecurityType.pb.h"
+#include "spark/api/domain/Venue.pb.h"
+#include "spark/domain_model/Security.pb.h"
+
+#include <dex_utils.h>
+
+#include <optional>
+#include <string_view>
+
+namespace bts::rakurai {
+
+using ConnectorDexConfig = ::rakurai::dex_connector::configuration::dex_config;
+using ConnectorPoolType = ::rakurai::dex_connector::data_types::pool_type;
+using ConnectorVenue = ::rakurai::dex_connector::data_types::venue;
+
+inline std::string_view connector_venue_name(ConnectorVenue venue) {
+   switch (venue) {
+   case ConnectorVenue::orca_clmm:
+      return "orca_clmm";
+   case ConnectorVenue::orca_amm:
+      return "orca_amm";
+   case ConnectorVenue::raydium_amm:
+      return "raydium_amm";
+   case ConnectorVenue::raydium_cpmm:
+      return "raydium_cpmm";
+   case ConnectorVenue::raydium_clmm:
+      return "raydium_clmm";
+   case ConnectorVenue::phoenix:
+      return "phoenix";
+   default:
+      return "invalid";
+   }
+}
+
+inline bool venue_matches(
+    api::domain::Venue spark_venue, ConnectorVenue connector_venue) {
+   switch (spark_venue) {
+   case api::domain::VENUE_ORCA:
+      return connector_venue == ConnectorVenue::orca_clmm ||
+             connector_venue == ConnectorVenue::orca_amm;
+   case api::domain::VENUE_PHOENIX:
+      return connector_venue == ConnectorVenue::phoenix;
+   case api::domain::VENUE_RAYDIUM:
+      return connector_venue == ConnectorVenue::raydium_amm ||
+             connector_venue == ConnectorVenue::raydium_cpmm ||
+             connector_venue == ConnectorVenue::raydium_clmm;
+   default:
+      return false;
+   }
+}
+
+inline bool venue_matches(
+    api::Venue spark_venue, ConnectorVenue connector_venue) {
+   switch (spark_venue) {
+   case api::VENUE_ORCA:
+      return connector_venue == ConnectorVenue::orca_clmm ||
+             connector_venue == ConnectorVenue::orca_amm;
+   case api::VENUE_PHOENIX:
+      return connector_venue == ConnectorVenue::phoenix;
+   case api::VENUE_RAYDIUM:
+      return connector_venue == ConnectorVenue::raydium_amm ||
+             connector_venue == ConnectorVenue::raydium_cpmm ||
+             connector_venue == ConnectorVenue::raydium_clmm;
+   default:
+      return false;
+   }
+}
+
+inline bool is_rakurai_venue(api::domain::Venue venue) {
+   switch (venue) {
+   case api::domain::VENUE_ORCA:
+   case api::domain::VENUE_PHOENIX:
+   case api::domain::VENUE_RAYDIUM:
+      return true;
+   default:
+      return false;
+   }
+}
+
+inline api::domain::SecurityType connector_security_type(
+    ConnectorVenue venue) {
+   switch (venue) {
+   case ConnectorVenue::orca_clmm:
+      return api::domain::SECURITY_TYPE_SPOT;
+   case ConnectorVenue::orca_amm:
+      return api::domain::SECURITY_TYPE_SPOT;
+   case ConnectorVenue::raydium_amm:
+      return api::domain::SECURITY_TYPE_SPOT;
+   case ConnectorVenue::raydium_cpmm:
+      return api::domain::SECURITY_TYPE_SPOT;
+   case ConnectorVenue::raydium_clmm:
+      return api::domain::SECURITY_TYPE_SPOT;
+   case ConnectorVenue::phoenix:
+      return api::domain::SECURITY_TYPE_FUTURE;
+   default:
+      return api::domain::SECURITY_TYPE_UNSPECIFIED;
+   }
+}
+
+inline std::string_view connector_quote_currency(ConnectorVenue venue) {
+   switch (venue) {
+   case ConnectorVenue::phoenix:
+      return "USD";
+   default:
+      return {};
+   }
+}
+
+inline std::optional<ConnectorDexConfig> make_connector_dex_config(
+    std::string_view id,
+    std::optional<api::domain::Venue> venue_filter = std::nullopt) {
+   if (id == "orca_clmm") {
+      if (venue_filter.has_value() &&
+          *venue_filter != api::domain::VENUE_ORCA) {
+         return std::nullopt;
+      }
+      ConnectorDexConfig config;
+      config.dex_name = ConnectorVenue::orca_clmm;
+      config.pool_typ = ConnectorPoolType::clmm;
+      return config;
+   }
+   if (id == "orca_amm") {
+      if (venue_filter.has_value() &&
+          *venue_filter != api::domain::VENUE_ORCA) {
+         return std::nullopt;
+      }
+      ConnectorDexConfig config;
+      config.dex_name = ConnectorVenue::orca_amm;
+      config.pool_typ = ConnectorPoolType::amm;
+      return config;
+   }
+   if (id == "raydium_amm") {
+      if (venue_filter.has_value() &&
+          *venue_filter != api::domain::VENUE_RAYDIUM) {
+         return std::nullopt;
+      }
+      ConnectorDexConfig config;
+      config.dex_name = ConnectorVenue::raydium_amm;
+      config.pool_typ = ConnectorPoolType::amm;
+      return config;
+   }
+   if (id == "raydium_cpmm") {
+      if (venue_filter.has_value() &&
+          *venue_filter != api::domain::VENUE_RAYDIUM) {
+         return std::nullopt;
+      }
+      ConnectorDexConfig config;
+      config.dex_name = ConnectorVenue::raydium_cpmm;
+      config.pool_typ = ConnectorPoolType::cpmm;
+      return config;
+   }
+   if (id == "raydium_clmm") {
+      if (venue_filter.has_value() &&
+          *venue_filter != api::domain::VENUE_RAYDIUM) {
+         return std::nullopt;
+      }
+      ConnectorDexConfig config;
+      config.dex_name = ConnectorVenue::raydium_clmm;
+      config.pool_typ = ConnectorPoolType::clmm;
+      return config;
+   }
+   if (id == "phoenix") {
+      if (venue_filter.has_value() &&
+          *venue_filter != api::domain::VENUE_PHOENIX) {
+         return std::nullopt;
+      }
+      ConnectorDexConfig config;
+      config.dex_name = ConnectorVenue::phoenix;
+      config.pool_typ = ConnectorPoolType::perp;
+      return config;
+   }
+   return std::nullopt;
+}
+
+} // namespace bts::rakurai

src/client/RELEASE_AND_DEPLOYMENT.md

@@ -0,0 +1,283 @@
+# Independent release and deployment guide
+
+This guide creates and deploys the five new artifacts around an existing
+`sparkd` deployment. It deliberately does not create one shared Compose stack.
+Each owning team can build, publish, deploy, restart, and roll back its component
+without rebuilding the others.
+
+## Chosen deployment model
+
+| Component | Release artifact | Deployment unit |
+| --- | --- | --- |
+| `sparkd` | Existing Spark artifact/image | Existing Spark scripts; unchanged |
+| Spark command bridge | Self-contained Linux tarball | Independent `systemd` service |
+| BlueOrb RSS | Self-contained Linux tarball | Independent `systemd` service |
+| `blueorb-surface-python` | Python wheel and source distribution | Published library consumed by clients |
+| `spark-command-client` | Python wheel and source distribution | Published library consumed by the gateway |
+| `spark-public-websocket` | Python wheel and source distribution | Independent virtual environment and `systemd` service |
+
+The .NET tarballs are self-contained: the target host does not need the .NET
+runtime. They are directories rather than single-file executables because both
+services may carry native libraries. The Python gateway receives its two client
+libraries as wheels and has no source-level dependency on either backend.
+
+Every component uses its own semantic version. Record the compatible set for an
+environment using [`deploy/release-manifest.env.example`](deploy/release-manifest.env.example).
+Do not infer compatibility from matching version numbers.
+
+## Build prerequisites
+
+- Linux x86-64 build host for `linux-x64` artifacts.
+- .NET 8 SDK matching each repository's `global.json`.
+- Python 3.10 or newer.
+- `python -m pip install build` in the release-build environment.
+- `tar` and `sha256sum`.
+- Access to the released `Bts.Spark.Sdk` NuGet packages.
+
+Build from clean, reviewed commits. Tags should use the owning component name,
+for example `spark-command-bridge/v0.1.0` rather than one repository-wide tag.
+
+## 1. Build `blueorb-surface-python`
+
+The version comes from `/home/bts/blueorb/BlueOrb.Surface.Python/pyproject.toml`.
+Bump that value before creating a new release.
+
+```sh
+cd /home/bts/blueorb/BlueOrb.Surface.Python
+PYTHON=python3.10 ./build-release.sh 0.1.0
+```
+
+Outputs:
+
+```text
+dist/blueorb_surface_python-0.1.0-py3-none-any.whl
+dist/blueorb_surface_python-0.1.0.tar.gz
+dist/blueorb-surface-python-0.1.0.sha256
+```
+
+## 2. Build `spark-command-client`
+
+The version comes from `spark_command_client/pyproject.toml` and must be bumped
+before a new release.
+
+```sh
+cd /workspaces/spark/src/client/spark_command_client
+PYTHON=python3.10 ./build-release.sh 0.1.0
+```
+
+Outputs are written to that component's `dist/` directory with a checksum file.
+
+## 3. Build `spark-public-websocket`
+
+```sh
+cd /workspaces/spark/src/client/spark_public_websocket
+PYTHON=python3.10 ./build-release.sh 0.1.0
+```
+
+The wheel declares compatible `blueorb-surface-python` and
+`spark-command-client` ranges. A deployment must still pin exact wheel versions
+in its release manifest. The same `dist/` directory contains a versioned
+`spark-public-websocket-deploy-0.1.0.tar.gz` with its service and environment
+templates.
+
+## 4. Build the self-contained BlueOrb RSS artifact
+
+The RSS version is passed to the .NET assembly metadata and artifact name:
+
+```sh
+cd /home/bts/blueorb/RemoteStreamServer
+./build-release.sh 0.1.0 linux-x64
+```
+
+Outputs:
+
+```text
+dist/blueorb-rss-0.1.0-linux-x64.tar.gz
+dist/blueorb-rss-0.1.0-linux-x64.tar.gz.sha256
+```
+
+The extracted directory contains `RemoteStreamServer` and all managed/native
+runtime files required to start it without an installed .NET runtime.
+
+## 5. Build the self-contained Spark command bridge artifact
+
+The bridge must use the same released Spark SDK family as the target `sparkd`.
+`SPARK_NUGET_SOURCE` can be a package-feed URL or a directory containing the
+four released BTS NuGet packages.
+
+```sh
+cd /workspaces/spark/src/client/spark_command_bridge
+
+SPARK_NUGET_SOURCE=/home/bts/blueorb/third_party/nuget \
+./build-release.sh \
+  0.1.0 \
+  3.2.3-defi-2-gefcbb26de1 \
+  linux-x64
+```
+
+Outputs:
+
+```text
+dist/spark-command-bridge-0.1.0-linux-x64.tar.gz
+dist/spark-command-bridge-0.1.0-linux-x64.tar.gz.sha256
+```
+
+## Verify and publish artifacts
+
+Verify every archive before publishing it:
+
+```sh
+sha256sum -c artifact-name.sha256
+```
+
+Publish Python distributions to the team's package index. Publish .NET tarballs
+and checksum files to the binary-release store. Keep immutable artifacts for
+every deployed version; never replace an existing version in place.
+
+Before publication, install the three wheels together in a clean Python 3.10
+environment and run the package tests. Extract each .NET archive on a clean
+compatible Linux host and start it with non-production configuration.
+
+## Deploy the Spark command bridge independently
+
+The existing `sparkd` deployment must expose a configuration-server address
+reachable by the bridge. Addresses returned by that configuration service must
+also be reachable from the bridge host.
+
+Create the service identity and directories once:
+
+```sh
+sudo useradd --system --home /var/lib/spark-command-bridge \
+  --shell /usr/sbin/nologin sparkbridge
+sudo mkdir -p /opt/spark-command-bridge/releases /etc/spark-command-bridge
+```
+
+Install a version:
+
+```sh
+sudo tar -xzf spark-command-bridge-0.1.0-linux-x64.tar.gz \
+  -C /opt/spark-command-bridge/releases
+sudo ln -sfn \
+  /opt/spark-command-bridge/releases/spark-command-bridge-0.1.0-linux-x64 \
+  /opt/spark-command-bridge/current
+sudo chown -R sparkbridge:sparkbridge /opt/spark-command-bridge
+```
+
+Install configuration and the unit template:
+
+```sh
+sudo cp /opt/spark-command-bridge/current/deploy/spark-command-bridge.env.example \
+  /etc/spark-command-bridge/spark-command-bridge.env
+sudo chmod 600 /etc/spark-command-bridge/spark-command-bridge.env
+sudo cp /opt/spark-command-bridge/current/deploy/spark-command-bridge.service \
+  /etc/systemd/system/
+```
+
+Edit the environment file with the real configuration-server URL and
+credentials, then start the service:
+
+```sh
+sudo systemctl daemon-reload
+sudo systemctl enable --now spark-command-bridge
+sudo systemctl status spark-command-bridge
+journalctl -u spark-command-bridge -f
+```
+
+Only port `50051` needs to be reachable by the WebSocket service. It should not
+be exposed publicly.
+
+## Deploy BlueOrb RSS independently
+
+```sh
+sudo useradd --system --home /var/lib/blueorb-rss \
+  --shell /usr/sbin/nologin blueorb
+sudo mkdir -p /opt/blueorb-rss/releases /etc/blueorb-rss
+
+sudo tar -xzf blueorb-rss-0.1.0-linux-x64.tar.gz \
+  -C /opt/blueorb-rss/releases
+sudo ln -sfn \
+  /opt/blueorb-rss/releases/blueorb-rss-0.1.0-linux-x64 \
+  /opt/blueorb-rss/current
+sudo chown -R blueorb:blueorb /opt/blueorb-rss
+
+sudo cp /home/bts/blueorb/blueorb.config.example.json \
+  /etc/blueorb-rss/blueorb.config.json
+sudo cp /opt/blueorb-rss/current/deploy/blueorb-rss.env.example \
+  /etc/blueorb-rss/blueorb-rss.env
+sudo cp /opt/blueorb-rss/current/deploy/blueorb-rss.service \
+  /etc/systemd/system/
+
+sudo systemctl daemon-reload
+sudo systemctl enable --now blueorb-rss
+sudo systemctl status blueorb-rss
+journalctl -u blueorb-rss -f
+```
+
+Edit `/etc/blueorb-rss/blueorb.config.json` for the target environment before
+starting RSS. Port `5085` should be reachable by the WebSocket service, but not
+published to untrusted networks.
+
+## Deploy the public WebSocket independently
+
+Place the exact three approved wheels on the host or download them from the
+package index. Create one immutable virtual environment per gateway release:
+
+```sh
+sudo useradd --system --home /var/lib/spark-public-websocket \
+  --shell /usr/sbin/nologin sparkwebsocket
+sudo mkdir -p /opt/spark-public-websocket/releases /etc/spark-public-websocket
+
+sudo python3.10 -m venv /opt/spark-public-websocket/releases/0.1.0
+sudo /opt/spark-public-websocket/releases/0.1.0/bin/python -m pip install \
+  ./blueorb_surface_python-0.1.0-py3-none-any.whl \
+  ./spark_command_client-0.1.0-py3-none-any.whl \
+  ./spark_public_websocket-0.1.0-py3-none-any.whl
+
+sudo ln -sfn /opt/spark-public-websocket/releases/0.1.0 \
+  /opt/spark-public-websocket/current
+sudo chown -R sparkwebsocket:sparkwebsocket /opt/spark-public-websocket
+
+tar -xzf spark-public-websocket-deploy-0.1.0.tar.gz
+sudo cp deploy/spark-public-websocket.env.example \
+  /etc/spark-public-websocket/spark-public-websocket.env
+sudo cp deploy/spark-public-websocket.service /etc/systemd/system/
+
+sudo systemctl daemon-reload
+sudo systemctl enable --now spark-public-websocket
+sudo systemctl status spark-public-websocket
+journalctl -u spark-public-websocket -f
+```
+
+Set `BLUEORB_ADDRESS` and `SPARK_COMMAND_ADDRESS` to the independently deployed
+backend addresses. Port `8090` is the only new public listener.
+
+## Smoke test the deployed path
+
+Run the existing client from any machine allowed to reach the WebSocket:
+
+```sh
+python examples/stream_docs.py --url ws://websocket-host:8090/v1/ws
+```
+
+Then exercise `enter_order`, `update_order`, `delete_order`, and
+`create_synthetic_spread` using non-production Spark identifiers. The release is
+not functionally complete until all four commands traverse the WebSocket,
+bridge, Quickbus, and real `sparkd`.
+
+## Upgrade and rollback
+
+Never overwrite `/opt/<component>/current`. Extract or create the new version
+under `releases/`, atomically repoint `current`, and restart only that service.
+Rollback uses the same operation with the previous release directory.
+
+Python library releases do not restart RSS or the bridge. Backend releases do
+not require rebuilding the gateway unless their public contract compatibility
+changes.
+
+## Still required before production exposure
+
+- Real `sparkd` end-to-end and failure-path verification.
+- A decision on durable versus process-local order idempotency.
+- Authentication, authorization, TLS, secret management, and rate limiting.
+- Health monitoring, correlation IDs, structured timings, and alerts.
+- Independent restart, outage, rollback, load, and latency tests.

src/client/spark/AW2/LiveTheoPublisher.cs

@@ -160,7 +160,7 @@ namespace mbts.spark {
 
       private bool _compute_midmarket {
          get {
-            return false;
+            return true;
          }
       }
 

src/client/spark/Daemons/OrderTicketDaemon.cs

@@ -26,8 +26,6 @@ using LadderSettings = mbts.spark.LadderSettings;
 namespace bts.shell {
    [Export(typeof(IShellDaemon))]
    class OrderTicketDaemon : IShellDaemon {
-      static readonly bool useTicketLadder = true;
-
       readonly IBus _bus;
       readonly OrdersModel _model;
       readonly UserSettingsHub _settings_hub;
@@ -73,23 +71,13 @@ namespace bts.shell {
 
       void _show_order_ticket(SecurityMD smd) {
          var settings_src = _settings_hub.subscribe_default_settings()
-            .Select(s => s.GetLadderProfile(smd.security)?.value ?? LadderSettings.Default);
+            .Select(s => s.get_ladder_profile(smd.security)?.value ?? LadderSettings.Default);
          var accounts_src = _bus.subscribe(SparkTags.AccountsTag);
-         _show_order_window(useTicketLadder ? OrderTicketLadderWindow() : OrderTicketWindow());
-
-         Window OrderTicketWindow() =>
-            new OrderTicket {
-               DataContext = new NewOrderTicketVM(_model, smd, accounts_src, settings_src) {
-                  close_on_send = _close_on_send
-               }
-            };
-
-         Window OrderTicketLadderWindow() =>
-            new OrderTicketLadder() {
-               DataContext = new NewOrderTicketLadderVM(
-                  _model, smd, accounts_src, settings_src,
-                  ladderSecurity => _bus.post(SparkTags.ShowLadderTag, ladderSecurity))
-            };
+         _show_order_window(new OrderTicket {
+            DataContext = new NewOrderTicketVM(_model, smd, accounts_src, settings_src) {
+               close_on_send = _close_on_send
+            }
+         });
       }
 
       void _show_order_ticket(OrderTicketParams ot_params) {
@@ -126,23 +114,13 @@ namespace bts.shell {
                return;
          }
          var md = _md_map.TryGetValue(ori.order.security_key());
-         var accountsSrc = _bus.subscribe(SparkTags.AccountsTag);
-         var orderSrc = Observable.Return((md, ori.order));
-         var calcBnsSrc = Observable.Return(LadderSettings.Default.calculator_buttons);
-         _show_order_window(useTicketLadder ? OrderTicketLadderWindow() : OrderTicketWindow());
-
-         Window OrderTicketWindow() =>
-            new OrderModificationDialog {
-               DataContext = new ExistingOrderTicketVM(_model, orderSrc, accountsSrc, calcBnsSrc, md?.quantity_display)
-            };
-
-         Window OrderTicketLadderWindow() =>
-            new OrderModificationDialog {
-               DataContext = new ExistingOrderTicketLadderVM(
-                  _model, orderSrc, accountsSrc, calcBnsSrc,
-                  ladderSecurity => _bus.post(SparkTags.ShowLadderTag, ladderSecurity),
-                  md?.quantity_display)
-            };
+         _show_order_window(new OrderModificationDialog {
+            DataContext = new ExistingOrderTicketVM(_model,
+               Observable.Return((md, ori.order)),
+               _bus.subscribe(SparkTags.AccountsTag),
+               Observable.Return(LadderSettings.Default.calculator_buttons),
+               md?.quantity_display)
+         });
       }
    }
 }

src/client/spark/Model/Settings/UserSettingsExtensions.cs

@@ -14,11 +14,16 @@ namespace mbts.spark {
       /// <summary>
       /// Gets the ladder settings profile for a particular security
       /// </summary>
-      public static LadderSettingsProfile? GetLadderProfile(
+      public static LadderSettingsProfile? get_ladder_profile(
          this UserSettings settings, SecurityImmutable security) =>
          settings.ladder_profiles.FirstOrDefault(lsp =>
             lsp?.security_criteria?.includes(security) ?? false);
 
+      public static IObservable<LadderSettingsProfile?> subscribe_ladder_profile(
+         this UserSettingsHub settings_hub, SecurityImmutable security) =>
+         settings_hub.subscribe_default_settings().Select(s =>
+            s.get_ladder_profile(security));
+
       /// <summary>
       /// Gets the default account for a particular security (from the
       /// <see cref="LadderSettingsProfile"/> associated with that security)
@@ -26,7 +31,7 @@ namespace mbts.spark {
       public static IObservable<AccountTuple?> subscribe_default_account(
          this UserSettingsHub settings_hub, IBus bus, SecurityImmutable security) {
          var acc_id_src = settings_hub.subscribe_default_settings().Select(s =>
-            s.GetLadderProfile(security)?.value?.default_account);
+            s.get_ladder_profile(security)?.value?.default_account);
          return acc_id_src.CombineLatest(
             bus.subscribe(SparkTags.AccountsTag),
             (acc_id, accs) => acc_id.HasValue && accs.TryGetValue(acc_id.Value, out var acc_tup) ?

src/client/spark/Modules/Market/MarketView/MarketViewView.xaml.cs

@@ -1,10 +1,10 @@
+using bts;
 using bts.api;
 
 using mbts.spark;
 using mbts.ui.controls;
 using mbts.ui.converters;
 using mbts.ui.extensions;
-using mbts.ui.shell;
 
 using System;
 using System.Collections.Generic;
@@ -23,7 +23,6 @@ namespace mbts.ui.spark {
    /// </summary>
    partial class MarketViewView : UserControl {
       public static readonly RoutedCommand CellClickCommand = new RoutedCommand();
-      static readonly bool useTicketLadder = true;
 
       MarketViewVM _vm;
       DIYGridPopupManager _popup_man_backing;
@@ -93,7 +92,7 @@ namespace mbts.ui.spark {
 
       void _show_mini_orderbook(IReadOnlyList<SecurityImmutable> securities, Columns col) {
          var (vm, context, title, fg) = _vm.orderbook_params(securities, col);
-         _show_popup(vm, new OrderAuxiliaryPopupHeader {
+         _show_popup(vm, new AuxiliaryDataHeader {
             context = context,
             title = title,
             title_foreground = fg
@@ -105,7 +104,7 @@ namespace mbts.ui.spark {
 
       void _show_mini_fills(IEnumerable<SecurityImmutable> securities, Columns col) {
          var (vm, context, title, fg) = _vm.fills_params(securities, col);
-         _show_popup(vm, new OrderAuxiliaryPopupHeader {
+         _show_popup(vm, new AuxiliaryDataHeader {
             context = context,
             title = title,
             title_foreground = fg
@@ -115,40 +114,29 @@ namespace mbts.ui.spark {
       async void _show_order_ticket(
          SecurityMD security_md, double price = double.NaN, double size = 0) {
          var resources = new OrderTicketResources();
-         resources.MergedDictionaries.Add(new OrderAuxiliaryPopupResources());
+         resources.MergedDictionaries.Add(new AuxiliaryDataResources());
 
          var account = await _vm.get_default_account(security_md.security);
          var def = _model.factory.order(security_md.security, account);
          var calc_buttons = LadderSettings.Default.calculator_buttons;
-         var accounts = _bus.subscribe(SparkTags.AccountsTag);
-         var (vm, header) = OrderTicketFactory();
+         var ticket_vm = new NewOrderTicketVM(
+            _model,
+            security_md,
+            _bus.subscribe(SparkTags.AccountsTag),
+            new(def),
+            price,
+            size,
+            Observable.Return(calc_buttons));
 
          _show_popup(
-            vm,
-            header,
+            ticket_vm,
+            new OrderTicketHeader { bbo = ticket_vm.bbo },
             resources,
             view => Behavior.SetAutoFocusTargetProvider(view, AutoFocusTargetProvider.TypeProvider(typeof(TextBox), view)));
-
-         (ViewModelBase, object) OrderTicketFactory() {
-            if (useTicketLadder) {
-               var vm = new NewOrderTicketLadderVM(
-                  _vm.orders_model, security_md, accounts, new(def), price, size,
-                  Observable.Return(calc_buttons),
-                  ladderSecurity => _bus.post(SparkTags.ShowLadderTag, ladderSecurity));
-               var header = new OrderTicketLadderHeader() {
-                  Title = security_md.security.ToString()
-               };
-               return (vm, header);
-            } else {
-               var vm = new NewOrderTicketVM(_model, security_md, accounts, new(def), price, size, Observable.Return(calc_buttons));
-               var header = new OrderTicketHeader { bbo = vm.bbo };
-               return (vm, header);
-            }
-         }
       }
 
       void _show_popup(object content, object header) {
-         var resources = new OrderAuxiliaryPopupResources();
+         var resources = new AuxiliaryDataResources();
          resources.MergedDictionaries.Add(new AutoHideScrollbarResources());
          _show_popup(content, header, resources);
       }
@@ -160,9 +148,9 @@ namespace mbts.ui.spark {
             Header = header,
             HorizontalAlignment = HorizontalAlignment.Left,
             VerticalAlignment = VerticalAlignment.Top,
-            Resources = resources ?? new OrderAuxiliaryPopupResources()
+            Resources = resources ?? new AuxiliaryDataResources()
          };
-         view.Style = view.Resources[OrderAuxiliaryPopupResources.PopupChildStyleKey] as Style;
+         view.Style = view.Resources[AuxiliaryDataResources.PopupChildStyleKey] as Style;
          init_view?.Invoke(view);
          _popup_man.show(view);
       }

src/client/spark/Modules/Trading/Fills/FillsRowVM.cs

@@ -78,9 +78,6 @@ namespace mbts.spark {
       public Fill.Types.FillStatus fill_status => _fill.FillStatus;
       public string fill_key => _fill.FillKey;
       public string order_key => _order.Key.ToString();
-      // public bool is_aggressor => fill_tuple.is_aggressor;
-      public string display_aggressor_flag_text =>
-         display_aggressor_flag(_fill.FillAggressorFlag);
       public bool is_selected {
          get => _is_selected;
          set {
@@ -108,13 +105,6 @@ namespace mbts.spark {
       AlgoData? _root_algo => fill_tuple.root_algo;
       AlgoDefinitionMetadata? _root_algo_meta => fill_tuple.root_algo_meta;
 
-      public static string display_aggressor_flag(FillAggressorFlag flag) =>
-         flag switch {
-            FillAggressorFlag.Aggressor => "A",
-            FillAggressorFlag.Passive => "P",
-            _ => string.Empty
-         };
-
       public static FillsRowVM update(FillsRowVM r, FillTuple ft) {
          using var _ = _change_detector.detect_changes(r);
          r._badge_fill = null;

src/client/spark/Modules/Trading/Fills/FillsVM.cs

@@ -254,7 +254,7 @@ namespace mbts.spark {
          add_col(
             Columns.Price,
             DefaultColumnHeaders.Price,
-            r => _get_row(r).security.spec_or_default().format(r.fill.LastPrice),
+            r => _get_row(r).security.spec_or_default().format(r.fill.AvgPrice),
             new DoubleMinMax<FillTuple>(r => r.fill.LastPrice),
             DataGridFilterKnownTemplates.DoubleMinMax,
             fs => fs.price,
@@ -295,7 +295,7 @@ namespace mbts.spark {
          add_col(
             Columns.IsAggressor,
             DefaultColumnHeaders.IsAggressor,
-            r => FillsRowVM.display_aggressor_flag(r.fill.FillAggressorFlag),
+            r => r.is_aggressor ? SpecialCharacters.CheckMark : string.Empty,
             dv_filter(new[] { false, true }, r => r.is_aggressor),
             DataGridFilterKnownTemplates.YesNo,
             fs => fs.is_aggressor,

src/client/spark/Modules/Trading/Ladder/FairValueModel.cs

@@ -27,13 +27,13 @@ namespace Spark.Trading {
          _sec_greeks_in = sec_greeks_in;
       }
 
-      public IObservable<double> get_price_source(FairValueKey key, double? errorPrice = double.NaN) {
+      public IObservable<double> get_price_source(FairValueKey key) {
          return key.type switch {
             FairValueLine.PriceType.Bid => bid_source(),
             FairValueLine.PriceType.Ask => ask_source(),
             FairValueLine.PriceType.Mid => mid_source(),
             FairValueLine.PriceType.Theo => theo_source(),
-            _ => errorPrice.HasValue ? Observable.Return(errorPrice.Value) : throw new NotImplementedException()
+            _ => throw new NotImplementedException()
          };
 
          IObservable<double> bid_source() =>

src/client/spark/Modules/Trading/Ladder/View/LadderGridHost.cs

@@ -1,4 +1,4 @@
-using bts;
+using bts;
 using bts.api;
 using bts.api.md.auxiliary;
 using bts.collections;
@@ -15,6 +15,7 @@ using mbts.utils;
 
 using Spark.Core.Collections;
 using Spark.Core.Disposables;
+using Spark.Core.Math;
 
 using System;
 using System.Collections.Generic;
@@ -35,9 +36,8 @@ namespace mbts.spark {
    using RegionalTickFactors = user.settings.RegionalTickFactors;
 
    partial class LadderGridHost : Control {
-      const int _min_row_count = 501;
-      const int _max_row_count = 2001;
-      static readonly bool _show_scrollbar = true;
+      static readonly int _initial_row_count;
+      static readonly bool _show_scrollbar;
 
       readonly VisualHost _visual_host = new();
       readonly LadderOrdersHost _ordershost;
@@ -80,11 +80,13 @@ namespace mbts.spark {
       LadderVM _vm;
       LadderTactileAdorner _tactile_adorner;
       ulong _prices_updated;
-      ulong lastTradeTime;
       NativePriceSize[] _bids = new NativePriceSize[NativePriceHelpers.COCONUT_MAX_DEPTH], _asks = new NativePriceSize[NativePriceHelpers.COCONUT_MAX_DEPTH];
       SecurityMDExtensions.Book _depth = SecurityMDExtensions.Book.Empty;
       bool _price_dirty = false, _piq_dirty = false;
-      IList<double> _price_list;
+      LadderPriceList _price_list = new() {
+         length = _initial_row_count,
+      };
+      LadderPriceList _pending_price_list = null;
       IComparer<double> _price_comparer;
       double _last_origin_price;
       readonly MarketBreakLines _mkt_break_lines;
@@ -105,6 +107,7 @@ namespace mbts.spark {
       static LadderGridHost() {
          var knobs = config.Configuration.get<bts.config.LadderKnobs>();
          _show_scrollbar = knobs.ShowScrollbar;
+         _initial_row_count = (int)knobs.ScrollableTickWindowSize;
       }
 
       public LadderGridHost() {
@@ -129,7 +132,33 @@ namespace mbts.spark {
                _render_grid_background();
                _init_tactile_adorner();
             }
-            _scroll_to_center();
+            IDisposable scroll_viewer_hook = null;
+            loaded_hook = Disposable.Create(
+               () => {
+                  _scroll_viewer_source.value_changed -= on_scroll_viewer;
+                  Disposable.dispose(ref scroll_viewer_hook);
+               });
+            on_scroll_viewer(_scroll_viewer);
+
+            void on_scroll_viewer(ScrollViewer new_value) {
+               Disposable.dispose(ref scroll_viewer_hook);
+               if (new_value is null) {
+                  return;
+               }
+               scroll_viewer_hook = _connect_scroll_viewer(new_value);
+               delayed_init();
+               // This is a bit of a hack: the scroll viewer drifts away from the intended VerticalOffset
+               // unless it is done after a delay.
+               async void delayed_init() {
+                  await Task.Delay(100);
+                  await Dispatcher.InvokeAsync(() => {
+                     _pending_scroll_to_origin = true;
+                     _refresh_prices();
+                     _refresh_all_visuals();
+                     scroll_to_pending_target();
+                  });
+               }
+            }
          };
          recenter_command = new DelegateCommand(_scroll_to_center);
          DataContextChanged += (s, e) => _connect_vm();
@@ -253,6 +282,26 @@ namespace mbts.spark {
          set => SetValue(PIQColumnWidthProperty, value);
       }
 
+      #region TopTick
+      public static readonly DependencyProperty TopTickProperty =
+         WPFHelper.create_dp<int, LadderGridHost>(
+            nameof(TopTick),
+            FrameworkPropertyMetadataOptions.AffectsRender,
+            (o, v) => {
+               if (o._price_list is null) {
+                  return;
+               }
+               o._set_scrollbar_range(high_tick: v);
+               o._refresh_prices();
+               o._refresh_all_visuals();
+               o._ordershost.refresh_orders();
+            });
+
+      public int? TopTick {
+         get => GetValue(TopTickProperty) as int?;
+         set => SetValue(TopTickProperty, value);
+      }
+      #endregion
 
       #region CanUserSendOrders
       public static readonly DependencyProperty CanUserSendOrdersProperty =
@@ -343,7 +392,7 @@ namespace mbts.spark {
          }
       }
 
-      int _row_count = _min_row_count;
+      int _row_count => _price_list?.Count ?? 0;
 
       bts.api.domain.TickDisplaySettingsImmutable _qty_display =>
          _vm is null ?
@@ -376,19 +425,66 @@ namespace mbts.spark {
          scroll_to_price(_mkt_break_lines.origin);
 
       public bool scroll_to_price(double price) {
-         if (_scroll_viewer == null || _price_list.is_null_or_empty())
+         if (price.is_nan()) {
             return false;
-         if (_spec.tickdiff(_price_list[0], price) is long target_row) {
-            int visible_row_count = (int)(_scroll_viewer.ActualHeight / _settings.row_height);
-            int curr_row = visible_row_count + (int)(_scroll_viewer.VerticalOffset / _settings.row_height);
-            if ((target_row > curr_row && target_row < _row_count) || (target_row < (curr_row - visible_row_count) && target_row >= 0)) {
-               _scroll_to_row((int)target_row);
-               return true;
-            }
+         }
+         var tick = _spec.tickdiff(price, 0);
+         if (tick.HasValue) {
+            return scroll_to_tick(tick.Value);
          }
          return false;
       }
 
+      double _get_target_top_tick(int mid_viewport_tick) {
+         var curr_rel_off = _scroll_viewer.VerticalOffset + (_scroll_viewer.ViewportHeight / 2.0);
+         var tgt_rel_off = (_price_list.high_tick - mid_viewport_tick) * _settings.row_height;
+         var tick_diff = (curr_rel_off - tgt_rel_off) / _settings.row_height;
+         return _price_list.high_tick + tick_diff;
+      }
+
+      public bool scroll_to_tick(long tick) {
+         if (tick > int.MaxValue) {
+            _scroll_to_tick_log_throttle.Execute(() =>
+               Log.Ui.error($"Unable to scroll to tick {tick} because it exceeds int.MaxValue"));
+            return false;
+         }
+         return scroll_to_tick((int)tick);
+      }
+
+      public bool scroll_to_tick(int tick) {
+         if (_vm is null || _settings is null || _scroll_viewer is null) {
+            return false;
+         }
+         // var center_offset_ticks = (int)(_scrollViewer.ScrollableHeight / (2.0 * _settings.row_height));
+         // The scrollbar's vertical offset won't change.  Find the 'top tick' that will put the
+         // target `tick` in the center of the viewport.
+         // var center_offset_diff = _scrollViewer.VerticalOffset - (_scrollViewer.ScrollableHeight / 2.0);
+         // var tick_offset = center_offset_ticks 
+         // var tick_offset = (int)(_scrollViewer.ViewportHeight / (2.0 * _settings.row_height));
+
+         // The scrollbar's vertical offset won't change.  Find the 'top tick' that will put the
+         // target `tick` in the center of the viewport.
+         checked {
+            var top_tick = (int)_get_target_top_tick(tick);
+            if (_settings.animate_recenter) {
+               var anim = new Int32Animation {
+                  From = (int)_price_list.high_tick,
+                  To = top_tick,
+                  Duration = TimeSpan.FromSeconds(0.15)
+               };
+               var sb = new Storyboard();
+               sb.Children.Add(anim);
+               Storyboard.SetTarget(anim, this);
+               Storyboard.SetTargetProperty(anim, new PropertyPath(TopTickProperty));
+               sb.Begin();
+            } else {
+               _set_scrollbar_range(high_tick: top_tick);
+               _refresh_prices();
+            }
+         }
+         return true;
+      }
+
       public IObservable<ColumnType> subscribe_hover_column() =>
          Observable.Create<ColumnType>(o => {
             notify(Mouse.GetPosition(this));
@@ -468,6 +564,67 @@ namespace mbts.spark {
          }
       }
 
+      IDisposable _connect_scroll_viewer(ScrollViewer scroll_viewer) {
+         if (scroll_viewer is null) {
+            return null;
+         }
+         const int scroll_tick_padding = 10;
+         // const double scroll_epsilon = 1E-8;
+         var dispatcher = Dispatcher.CurrentDispatcher;
+         bool busy = false;
+         return scroll_viewer.SubscribeScrollChanged(on_scroll_changed);
+
+         async void on_scroll_changed(object sender, ScrollChangedEventArgs e) {
+            if (e.ExtentHeight == e.ViewportHeight || busy) {
+               return;
+            }
+            busy = true;
+            await dispatcher.InvokeAsync(ensure_tick_padding, DispatcherPriority.Background);
+            busy = false;
+
+            void ensure_tick_padding() {
+               if (_price_list is null || _settings is null) {
+                  return;
+               }
+               var pct = _scroll_viewer.VerticalOffset / _scroll_viewer.ScrollableHeight;
+               var new_pl = new LadderPriceList(_price_list);
+               var stp = scroll_tick_padding * _settings.row_height;
+               if (pct == 0) {
+                  if (_scroll_viewer.ExtentHeight - _scroll_viewer.ViewportHeight < stp) {
+                     new_pl.extend_up(scroll_tick_padding);
+                  } else {
+                     new_pl.shift_up(scroll_tick_padding);
+                  }
+                  _price_dirty = true;
+               }
+               if (pct == 1) {
+                  if (_scroll_viewer.ExtentHeight - _scroll_viewer.ViewportHeight < stp) {
+                     new_pl.extend_down(scroll_tick_padding);
+                  } else {
+                     new_pl.shift_down(scroll_tick_padding);
+                  }
+                  _price_dirty = true;
+               }
+               if (new_pl != _price_list) {
+                  _pending_price_list = new_pl;
+               }
+               _ordershost.refresh_orders();
+            }
+         }
+      }
+
+      void _set_scrollbar_range(int? high_tick = null, int? length = null) {
+         if (high_tick.HasValue) {
+            _price_list.high_tick = high_tick.Value;
+         }
+         if (length.HasValue) {
+            _price_list.length = length.Value;
+         }
+         _mkt_break_lines.invalidate_indices();
+         _mkt_break_lines.refresh();
+         _refresh_all_visuals();
+      }
+
       void _send_parameterized_order(Point point, Side side) {
          Disposable.dispose(ref _parameterized_order_popup);
          void on_cannot_send(string reason) {
@@ -605,7 +762,10 @@ namespace mbts.spark {
                Keyboard.IsKeyDown(Key.LeftAlt) || Keyboard.IsKeyDown(Key.RightAlt)) {
                _vm?.recenter_all_ladders();
             } else {
-               _scroll_to_center();
+               var tgt = _spec.tickdiff(_mkt_break_lines.origin, 0.0);
+               if (tgt.HasValue) {
+                  scroll_to_tick(tgt.Value);
+               }
             }
          }
       }
@@ -682,14 +842,9 @@ namespace mbts.spark {
       }
 
 
-      internal int _row_for_price(double price) {
-         if ((_price_list?.Count ?? 0) == 0 || !price.is_real()) {
-            return -1;
-         }
-         return LadderRender.index_of(_price_list, price, _price_comparer);
-      }
+      internal int _row_for_price(double price) => LadderRender.index_of(_price_list, price, _price_comparer);
 
-      // public double y_for_price(double price) => _row_for_price(price) * (_settings?.row_height ?? double.NaN);
+      public double y_for_price(double price) => _row_for_price(price) * (_settings?.row_height ?? double.NaN);
       #endregion
 
       #region Column Utility Methods
@@ -734,30 +889,11 @@ namespace mbts.spark {
                   render_market_col(_depth.Bids);
                   break;
                case ColumnType.Price:
-                  if (_settings.net_change != LadderSettings.NetChangeTerms.None) {
-                     LadderRender.render_prices(
-                        column.visual_tup.visual,
-                        column.visual_tup.glyph_context,
-                        _row_count,
-                        _settings.row_height,
-                        _price_list,
-                        _vm.md.settlement_price,
-                        _settings.net_change == LadderSettings.NetChangeTerms.Ticks,
-                        column.settings,
-                        _spec);
-                  } else {
-                     LadderRender.render_prices(
-                        column.visual_tup.visual,
-                        column.visual_tup.glyph_context,
-                        _row_count,
-                        _settings.row_height,
-                        _price_list,
-                        column.settings,
-                        _spec,
-                        _settings.price_display_format);
-                  }
+                  _refresh_prices(column);
+                  break;
+               case ColumnType.Ask:
+                  render_market_col(_depth.Asks);
                   break;
-               case ColumnType.Ask: render_market_col(_depth.Asks); break;
                case ColumnType.Volume:
                   _volume_container.mark_dirty();
                   _refresh_volume_at_price();
@@ -950,8 +1086,9 @@ namespace mbts.spark {
                Storyboard.SetTargetProperty(anim, new PropertyPath(Behavior.AnimatedScrollViewerVerticalOffsetProperty));
                sb.Begin();
             } else {
-               _scroll_to_row(_row_count / 2);
+               _scroll_to_row(_price_list.Count / 2);
             }
+            _refresh_frozen_mkt_lines();
          }
       }
 
@@ -965,23 +1102,76 @@ namespace mbts.spark {
             return Disposable.Create(() => _can_user_send_orders_changed -= o.OnNext);
          });
 
+      LadderPriceList _create_price_list() {
+         if (_settings is not null) {
+            switch (_settings.net_change) {
+               case LadderSettings.NetChangeTerms.BasisPoints:
+               case LadderSettings.NetChangeTerms.NetBasisPoints:
+                  return new() {
+                     length = _initial_row_count,
+                     regional_tick = new RegionalTickFactors(new user.settings.TickFactors(10000, 1)),
+                  };
+            }
+         }
+         return new() {
+            length = _initial_row_count,
+            regional_tick = _spec
+         };
+      }
+
       #region Timer.Tick, Refresh Methods
       void _tick() {
          if (_settings != null) {
-            bool last_trade_changed = _vm.md.get_last_trade(ref lastTradeTime, out _last_trade, out DateTime _, out ulong __);
+            ulong unused = 0;
+            bool last_trade_changed = _vm.md.get_last_trade(ref unused, out _last_trade, out DateTime _, out ulong __);
             bool depth_changed = _vm.md.get_depth(ref _prices_updated, ref _bids, ref _asks, NativePriceHelpers.COCONUT_MAX_DEPTH);
+            _depth = _vm.security_md.Depth(_bids, _asks, _vm.securities);
             _mkt_break_lines.refresh();
-            bool prices_changed =
-               _price_dirty ||
-               (double.IsNaN(_last_origin_price) && !double.IsNaN(_mkt_break_lines.origin)) ||
-               _mkt_break_lines.origin >= _price_list[0] || _mkt_break_lines.origin <= _price_list[_price_list.Count - 1];
-            if (!prices_changed && _prices_out_of_range(out double low, out double high)) {
-               prices_changed = _refresh_price_range(low, high);
+            if (_price_list is null) {
+               Log.assert(_price_dirty);
+               _price_list = _create_price_list();
+               var mid_tick = security.tickdiff(_mkt_break_lines.origin, 0) ?? 0;
+               if (mid_tick > int.MaxValue) {
+                  Log.Ui.error(
+                     $"Unable to set scrollbar range about mid_tick {mid_tick}: " +
+                     $"tick count exceeds int.MaxValue");
+               } else {
+                  _set_scrollbar_range(high_tick: (int)mid_tick + (_price_list.Count << 1));
+               }
+               _price_dirty = true;
             }
-            if (prices_changed) {
-               _refresh_price_list();
+            if (_price_dirty) {
+               if (_pending_price_list != null && !_pending_price_list.Equals(_price_list)) {
+                  var prev_price_list = _price_list;
+                  _price_list = _pending_price_list;
+                  _pending_price_list = null;
+                  if (prev_price_list.Count != _price_list.Count) {
+                     _reset_height();
+                  }
+                  // maintain previous visible price range if possible
+                  double percent_scroll_offset = _scroll_viewer.VerticalOffset / _scroll_viewer.ScrollableHeight;
+                  var n_visible_rowsd = _scroll_viewer.ViewportHeight / _settings.row_height;
+                  var n_visible_rows = (int)n_visible_rowsd;
+                  var offset = n_visible_rowsd - n_visible_rows;
+                  var n_rows_to_scroll = _price_list.Count - n_visible_rows - 1;
+                  int prev_target_index = (int)(percent_scroll_offset * n_rows_to_scroll);
+                  // int prev_target_index = (int)(percent_scroll_offset * prev_price_list.Count - 1);
+                  if (prev_price_list.TryGetAt(prev_target_index, out double prev_target_price)) {
+                     int new_target_index = LadderRender.index_of(_price_list, prev_target_price, _price_comparer);
+                     if (_price_list.within_range(new_target_index)) {
+                        // force _scrollViewer to re-calculate ScrollableHeight
+                        _scroll_viewer.UpdateLayout();
+                        double new_vertical_offset = new_target_index * _settings.row_height;
+                        // double new_vertical_offset = ((double)new_target_index / _price_list.Count) * _scrollViewer.ScrollableHeight;
+                        _scroll_viewer.ScrollToVerticalOffset(new_vertical_offset);
+                     }
+                  }
+               }
+               _refresh_prices();
+               scroll_to_pending_target();
                _refresh_all_visuals();
             } else {
+               scroll_to_pending_target();
                if (_mkt_break_lines.bid_dirty)
                   _refresh_bid_mkt_break();
                if (_mkt_break_lines.ask_dirty)
@@ -1053,75 +1243,6 @@ namespace mbts.spark {
             _settings.price_display_format);
       }
 
-      static readonly TimeSpan _price_out_of_range_render_interval = TimeSpan.FromSeconds(5);
-      DateTime _last_price_range_calc_time = DateTime.MinValue;
-
-      void _refresh_price_list() {
-         if (_settings is null || _spec is null || !_mkt_break_lines.origin.is_real()) {
-            return;
-         }
-         var prev_price_list = _price_list;
-         _price_list = (!(_settings.net_change == LadderSettings.NetChangeTerms.None)) ?
-            LadderRender.render_prices(_price_vis_tup.visual, _price_vis_tup.glyph_context, _mkt_break_lines.origin, _vm.md.settlement_price, _settings.net_change == LadderSettings.NetChangeTerms.Ticks, _row_count, _settings.row_height, _settings.price_column, _spec) :
-            LadderRender.render_prices(_price_vis_tup.visual, _price_vis_tup.glyph_context, _mkt_break_lines.origin, _row_count, _settings.row_height, _settings.price_column, _spec, _settings.price_display_format);
-         _ordershost.update_settings(_settings.row_height, _settings.orders_column);
-         if (prev_price_list != null) {
-            // maintain previous visible price range if possible
-            double percent_scroll_offset = _scroll_viewer.VerticalOffset / _scroll_viewer.ScrollableHeight;
-            int prev_target_index = (int)(percent_scroll_offset * prev_price_list.Count);
-            if (prev_price_list.TryGetAt(prev_target_index, out double prev_target_price)) {
-               int new_target_index = LadderRender.index_of(_price_list, prev_target_price, _price_comparer);
-               if (_price_list.within_range(new_target_index)) {
-                  // force _scroll_viewer to re-calculate ScrollableHeight
-                  _scroll_viewer.UpdateLayout();
-                  double new_vertical_offset = ((double)new_target_index / _price_list.Count) * _scroll_viewer.ScrollableHeight;
-                  _scroll_viewer.ScrollToVerticalOffset(new_vertical_offset);
-               }
-            }
-         }
-      }
-
-      bool _refresh_price_range(double low, double high) {
-         bool prices_changed = false;
-         // The market can be wider than the 1001 rows (ticks) we [initially] allot.
-         // Rendering prices is expensive, and will lock up the ui if done continuously.
-         // Therefore, ensure a minimum time interval has elapsed before rendering prices again.
-         var now = DateTime.Now;
-         if ((now - _last_price_range_calc_time) > _price_out_of_range_render_interval) {
-            SparkUtils.lazy_warn(!double.IsNaN(_mkt_break_lines.origin) && _mkt_break_lines.origin >= low && _mkt_break_lines.origin <= high,
-               () => fmt_log_msg($"market break ({_mkt_break_lines.origin}) is unacceptable for the market range ({low} - {high})"));
-            int prev_row_count = _row_count;
-            var n_ticks_to_low = (int)(_spec.tickdiff(_mkt_break_lines.origin, low) ?? 0).within_range(int.MinValue, int.MaxValue);
-            var n_ticks_to_high = (int)(_spec.tickdiff(high, _mkt_break_lines.origin) ?? 0).within_range(int.MinValue, int.MaxValue);
-            int desired_row_count = Math.Max(n_ticks_to_low, n_ticks_to_high) * 2 + 1;
-            desired_row_count = Math.Max(desired_row_count, _min_row_count);
-            _row_count = Math.Min(_max_row_count, desired_row_count);
-            if (_row_count != prev_row_count) {
-               if (desired_row_count > _max_row_count)
-                  Log.Ui.warn(fmt_log_msg($"ticks between market range ({low} - {high}) exceeds the maximum number of rows allowed ({_max_row_count})"));
-               else
-                  Log.Ui.info(fmt_log_msg($"increase row count to {_row_count} to accomodate market range ({low} - {high})"));
-               prices_changed = true;
-               _reset_height();
-            } else {
-               prices_changed = _mkt_break_lines.origin != _last_origin_price;
-            }
-            _last_price_range_calc_time = now;
-            //
-            string fmt_log_msg(string msg) =>
-               $"Ladder {security} (id:{security.Id}, low/high tick:{security.ContractSpec.LowTick}/{security.ContractSpec.HighTick}): {msg}.";
-         }
-         return prices_changed;
-      }
-
-      bool _prices_out_of_range(out double low, out double high) {
-         low = _bids == null ? double.NaN : TickUtils.low_tick(_bids, _asks);
-         high = _asks == null ? double.NaN : TickUtils.high_tick(_bids, _asks);
-         return
-            (!double.IsNaN(low) && _price_list[_price_list.Count - 1] > low) ||
-            (!double.IsNaN(high) && _price_list[0] < high);
-      }
-
       void _refresh_all_visuals() {
          if (_scroll_viewer is null || _settings is null) {
             return;
@@ -1175,36 +1296,50 @@ namespace mbts.spark {
       }
 
       void _refresh_session_low_line() {
-         if (_settings.session_low_line.show && _mkt_break_lines.low_render_index.HasValue)
-            _render_session_line(_session_low_vis, _mkt_break_lines.low_render_index.Value, _settings.session_low_line.color);
+         if (_settings.session_low_line.show) {
+            _render_session_line(_session_low_vis, _mkt_break_lines.low_render_index ?? -1, _settings.session_low_line.color);
+         }
       }
 
       void _refresh_session_high_line() {
-         if (_settings.session_high_line.show && _mkt_break_lines.high_render_index.HasValue)
-            _render_session_line(_session_high_vis, _mkt_break_lines.high_render_index.Value, _settings.session_high_line.color);
+         if (_settings.session_high_line.show) {
+            _render_session_line(_session_high_vis, _mkt_break_lines.high_render_index ?? -1, _settings.session_high_line.color);
+         }
       }
 
       void _refresh_last_trade() {
          var last_index = LadderRender.index_of(_price_list, _last_trade.price, _price_comparer);
          if (last_index > 0)
-            LadderRender.render_last(_last_vis_tup.visual, _last_vis_tup.glyph_context, _last_trade, last_index, _settings.row_height, _settings.last_column, _fmt_mkt_quantity);
+            LadderRender.render_last(
+               _last_vis_tup.visual,
+               _last_vis_tup.glyph_context,
+               _last_trade,
+               last_index,
+               _settings.row_height,
+               _settings.last_column,
+               _fmt_mkt_depth_item);
       }
 
       void _refresh_piq() {
          if (!_settings.piq_column.show)
             return;
          int origin_index = LadderRender.index_of(_price_list, _mkt_break_lines.origin, _price_comparer);
-         LadderRender.render_piqs(_piq_vis_tup.visual, _last_vis_tup.glyph_context, _price_list, _vm.price_vs_piq, _settings.row_height, _spec, _settings.piq_column, _fmt_mkt_quantity);
-         LadderRender.render_piqs(_piq_vis_tup.visual, _last_vis_tup.glyph_context, _price_list, _vm.price_vs_piq, _settings.row_height, _spec, _settings.piq_column, _fmt_mkt_quantity);
+         LadderRender.render_piqs(
+            _piq_vis_tup.visual,
+            _last_vis_tup.glyph_context,
+            _price_list,
+            _vm.price_vs_piq,
+            _settings.row_height,
+            _spec,
+            _settings.piq_column,
+            _fmt_mkt_quantity);
       }
 
       void _refresh_sizes() {
          Log.assert_d(_price_list != null);
          int origin_index = LadderRender.index_of(_price_list, _mkt_break_lines.origin, _price_comparer);
-         var bidItems = _bids.Select(nps => (nps.price, (double)nps.size, nps.price * nps.size));
-         var askItems = _asks.Select(nps => (nps.price, (double)nps.size, nps.price * nps.size));
-         LadderRender.render_sizes(_bid_vis_tup.visual, _bid_vis_tup.glyph_context, origin_index, _price_list, bidItems, _settings.row_height, _spec, _settings.bid_column, _fmt_price_level);
-         LadderRender.render_sizes(_ask_vis_tup.visual, _ask_vis_tup.glyph_context, origin_index, _price_list, askItems, _settings.row_height, _spec, _settings.ask_column, _fmt_price_level);
+         LadderRender.render_sizes(_bid_vis_tup.visual, _bid_vis_tup.glyph_context, origin_index, _price_list, _depth.Bids, _settings.row_height, _spec, _settings.bid_column, _fmt_price_level);
+         LadderRender.render_sizes(_ask_vis_tup.visual, _ask_vis_tup.glyph_context, origin_index, _price_list, _depth.Asks, _settings.row_height, _spec, _settings.ask_column, _fmt_price_level);
       }
 
       /// <summary>
@@ -1273,9 +1408,12 @@ namespace mbts.spark {
       void _clear_visuals() {
          if (_settings == null)
             return;
-         _price_list = new double[_row_count];
-         for (int i = 0; i < _price_list.Count; i++)
-            _price_list[i] = double.NaN;
+         if (_price_list is null) {
+            _price_list = new LadderPriceList {
+               regional_tick = _spec
+            };
+         }
+         _price_list.reset(null, null, _initial_row_count);
          Array.Clear(_bids, 0, _bids.Length);
          Array.Clear(_asks, 0, _asks.Length);
          _price_dirty = true;
@@ -1283,6 +1421,7 @@ namespace mbts.spark {
          _refresh_bid_mkt_break();
          _refresh_ask_mkt_break();
          _refresh_mid_mkt_break();
+         _refresh_fair_value_lines(true);
          _refresh_session_low_line();
          _refresh_session_high_line();
          _refresh_session_range();
@@ -1292,38 +1431,58 @@ namespace mbts.spark {
       #endregion
 
       #region Market Break, Session Low/High Utils
-      static double _get_top_price(NativePriceSize[] prices) =>
-         prices != null && prices.Length > 0 ? prices[0].price : double.NaN;
+      static double _get_top_price(NativePriceSize[] prices) {
+         if (prices == null) {
+            return double.NaN;
+         }
+         if (prices.Length == 0) {
+            return double.NaN;
+         }
+         if (prices[0].size == 0) {
+            return double.NaN;
+         }
+         return prices[0].price;
+      }
 
       double _get_bid_market_break() {
          double best_bid = _get_top_price(_bids);
          double best_ask = _get_top_price(_asks);
-         return double.IsNaN(best_bid) ?
-            double.IsNaN(best_ask) ? _vm.settlement : _spec.tickdown(best_ask) :
-            best_bid;
+         var v = best_bid
+            .if_nan(_spec.tickdown(best_ask))
+            .if_nan(_vm?.settlement ?? double.NaN);
+         return _spec.round_to_tick(v, RoundingMode.AbsDown);
       }
 
       double _normalize_price(double price) =>
          TickUtils.normalize_price(_spec, price);
 
       double _get_ask_market_break() {
+         if (_vm is null) {
+            return double.NaN;
+         }
          double best_bid = _get_top_price(_bids);
          double best_ask = _get_top_price(_asks);
-         return double.IsNaN(best_ask) ?
-            double.IsNaN(best_bid) ? _vm.settlement : best_bid :
-            _spec.tickdown(best_ask);
+         var v = _spec.tickdown(best_ask)
+            .if_nan(best_bid)
+            .if_nan(_vm?.settlement ?? double.NaN);
+         return _spec.round_to_tick(v, RoundingMode.AbsDown);
       }
 
       double _get_mid_market_break() {
          double best_bid = _get_top_price(_bids);
          double best_ask = _get_top_price(_asks);
-         return double.IsNaN(best_bid) ?
-            double.IsNaN(best_ask) ? _vm.settlement : _spec.tickdown(best_ask) :
-            double.IsNaN(best_ask) ? best_bid : TickUtils.mid_tick(_spec, best_bid, best_ask);
+         var v = TickUtils.mid_tick(_spec, best_bid, best_ask)
+            .if_nan(best_bid)
+            .if_nan(_spec?.tickdown(best_ask) ?? double.NaN)
+            .if_nan(_vm?.settlement ?? double.NaN);
+         return _spec.round_to_tick(v, RoundingMode.AbsDown);
       }
 
-      double _get_session_low() => _vm.md.session_low;
-      double _get_session_high() => _vm.md.session_high;
+      double _get_session_low() => _normalize_price(_get_session_low_raw());
+      double _get_session_high() => _normalize_price(_get_session_high_raw());
+
+      double _get_session_low_raw() => _vm?.md?.session_low ?? double.NaN;
+      double _get_session_high_raw() => _vm?.md?.session_high ?? double.NaN;
 
       void _render_mkt_break(DrawingVisual vis, double price, uint color) =>
          LadderRender.render_mkt_break(vis, _row_for_price(price), _settings.row_height, color);
@@ -1347,6 +1506,15 @@ namespace mbts.spark {
          _settings = _vm?.get_save_state() ?? s;
          _pending_scroll_to_origin |= _settings == null;
          _invalidate_spec();
+         if (_price_list != null) {
+            _pending_price_list = null;
+            if (_price_list.Count > 0) {
+               var high_price = _price_list.First();
+               var new_tickdiff = _spec.count_ticks(high_price);
+               _price_list.high_tick = (long)new_tickdiff;
+            }
+            _price_list.regional_tick = _spec;
+         }
          _bid_vis_tup.glyph_context = glyph_context(s.bid_column);
          _ask_vis_tup.glyph_context = glyph_context(s.ask_column);
          _price_vis_tup.glyph_context = glyph_context(s.price_column);
@@ -1430,22 +1598,9 @@ namespace mbts.spark {
          // if (_scroll_viewer != null) {
          //    _scroll_viewer.Background = s.last_column.background.rgb_brush();
          // }
+         scroll_to_pending_target();
          visible_columns_changed?.Invoke();
-         if (_scroll_viewer is not null && _settings is not null) {
-            _vm.md.get_last_trade(ref lastTradeTime, out _last_trade, out DateTime _, out ulong __);
-            _vm.md.get_depth(ref _prices_updated, ref _bids, ref _asks, NativePriceHelpers.COCONUT_MAX_DEPTH);
-            _mkt_break_lines.refresh();
-            double low = _bids == null ? double.NaN : TickUtils.low_tick(_bids, _asks);
-            double high = _asks == null ? double.NaN : TickUtils.high_tick(_bids, _asks);
-            _refresh_price_range(low, high);
-            _refresh_price_list();
-            _refresh_all_visuals();
-            _refresh_piq();
-            _piq_dirty = false;
-            _refresh_volume_at_price();
-            _last_origin_price = _mkt_break_lines.origin;
-            _price_dirty = false;
-         }
+         _refresh_all_visuals();
       }
 
       // This is designed to be a redundant null check for _fmt_mkt_depth_item_impl.
@@ -1476,9 +1631,6 @@ namespace mbts.spark {
          return impl is null ? value.ToString() : impl(value, price);
       }
 
-      string _fmt_mkt_quantity(NativePriceSize nps) =>
-         _fmt_mkt_quantity(nps.size, nps.price);
-
       void _init_tactile_adorner() {
          Disposable.dispose(ref _tactile_adorner);
          if (_scroll_viewer != null)
@@ -1593,7 +1745,7 @@ namespace mbts.spark {
                   return null;
                }
                if (!double.IsNaN(_session_low)) {
-                  _low_index = LadderRender.index_of(_owner._price_list, _session_low, _owner._price_comparer);
+                  _low_index = _owner._price_list.IndexOf(_session_low);
                   if (_session_low < _owner._price_list.Last()) {
                      _low_index = _owner._price_list.Count - 1;
                   }

src/client/spark/Modules/Trading/Ladder/View/LadderPriceList.cs

@@ -1,7 +1,6 @@
 using bts.api;
 using bts.utils;
 
-using Spark.Core;
 using Spark.Core.Diagnostics;
 
 using System;
@@ -31,16 +30,6 @@ namespace mbts.spark {
          regional_tick = other.regional_tick;
       }
 
-      public LadderPriceList(RegionalTickFactors regionalTick, IList<double> prices) {
-         regional_tick = regionalTick;
-         _length = prices.Count;
-         checked {
-            _high_tick = prices.Count > 0
-               ? (int)regionalTick.tickdiff(prices[0], 0).assert_value()
-               : 0;
-         }
-      }
-
       public event NotifyCollectionChangedEventHandler CollectionChanged;
 
       RegionalTickFactors _regional_tick;

src/client/spark/Modules/Trading/Ladder/View/LadderRender.cs

@@ -1,4 +1,4 @@
-using bts;
+using bts;
 using bts.api;
 using bts.api.md.auxiliary;
 using bts.collections;
@@ -101,24 +101,6 @@ namespace mbts.spark {
             settings.font_size, settings.width, row_height, HorizontalAlignment.Right,
             -4, VerticalAlignment.Center, row * row_height, true);
 
-      public static double[] gen_prices(double mid, int row_count, RegionalTickFactors spec) {
-         // require odd number
-         Log.assert(row_count % 2 == 1);
-         var result = new double[row_count];
-         int range = row_count / 2;
-         int middle = row_count / 2;
-         result[middle] = mid;
-         double up = mid, down = mid;
-         // Starting from the middle iterate out to the edges
-         for (int x = 1; x <= range; ++x) {
-            up = spec.tickup(up);
-            down = spec.tickdown(down);
-            result[middle - x] = up;
-            result[middle + x] = down;
-         }
-         return result;
-      }
-
       public static GlyphRun[] format_prices(
          IList<double> prices,
          double reference_price,
@@ -214,45 +196,6 @@ namespace mbts.spark {
          }
       }
 
-      public static double[] render_prices(DrawingVisual price, GlyphContext glyph_context, double mid, double wrt_price, bool format_in_tick_terms, int row_count, double row_height,
-         LadderColumnSettings settings, RegionalTickFactors spec) {
-         var dprices = gen_prices(mid, row_count, spec);
-         render_prices(price, glyph_context, row_count, row_height, dprices, wrt_price, format_in_tick_terms, settings, spec);
-         return dprices;
-      }
-
-      public static double[] render_prices(DrawingVisual price, GlyphContext glyph_context, double mid, int row_count, double row_height,
-         LadderColumnSettings settings, RegionalTickFactors spec, LadderSettings.PriceDisplayFormat price_display_format) {
-         var dprices = gen_prices(mid, row_count, spec);
-         render_prices(price, glyph_context, row_count, row_height, dprices, settings, spec, price_display_format);
-         return dprices;
-      }
-
-      public static void render_prices(DrawingVisual price, GlyphContext glyph_context, int row_count, double row_height, IList<double> dprices, double wrt_price, bool format_in_tick_terms, LadderColumnSettings settings, RegionalTickFactors spec) {
-         var foreground = settings.foreground.rgb_brush();
-         //var background = new SolidColorBrush(settings.background.rgb_color());
-         var terms = format_in_tick_terms ? LadderSettings.NetChangeTerms.Ticks : LadderSettings.NetChangeTerms.Dollars;
-         var fprices = format_prices(dprices, wrt_price, terms, row_height, settings, glyph_context, spec);
-         using (var dc = price.RenderOpen())
-         using (_draw_with_clip(dc, settings, row_height, row_count)) {
-            //dc.DrawRectangle(background, null, new Rect(0, 0, settings.width, row_count * row_height));
-            for (int x = 0; x < row_count; ++x)
-               dc.DrawGlyphRun(foreground, fprices[x]);
-         }
-      }
-
-      public static void render_prices(DrawingVisual price, GlyphContext glyph_context, int row_count, double row_height, IList<double> dprices, LadderColumnSettings settings, RegionalTickFactors spec, LadderSettings.PriceDisplayFormat price_display_format) {
-         var foreground = settings.foreground.rgb_brush();
-         //var background = new SolidColorBrush(settings.background.rgb_color());
-         var fprices = format_prices(dprices, double.NaN, LadderSettings.NetChangeTerms.Dollars, row_height, settings, glyph_context, spec, price_display_format);
-         using (var dc = price.RenderOpen())
-         using (_draw_with_clip(dc, settings, row_height, row_count)) {
-            //dc.DrawRectangle(background, null, new Rect(0, 0, settings.width, row_count * row_height));
-            for (int x = 0; x < row_count; ++x)
-               dc.DrawGlyphRun(foreground, fprices[x]);
-         }
-      }
-
       public static void render_grid(DrawingVisual dv, IEnumerable<LadderGridHost.IColumn> columns, int rowcount, double row_height, UInt32 line_color) {
          var height = rowcount * row_height;
          var brush = line_color.rgb_brush();

src/client/spark/Modules/Trading/ManualFills/ManualFillRow.cs

@@ -28,8 +28,7 @@ namespace bts.shell.manual_fills {
                nameof(user),
                nameof(side),
                nameof(quantity),
-               nameof(price),
-               nameof(fill_date)
+               nameof(price)
          }, typeof(ManualFillRow).GetProperty);
 
       readonly ISecurityMetadataStore _sec_meta_store;
@@ -44,7 +43,6 @@ namespace bts.shell.manual_fills {
       UserTuple? _user;
       Side _side;
       double _price;
-      DateTime _timestamp;
       bool _is_selected;
       ManualFillData? _data;
       IDisposable _committed_snapshot;
@@ -70,9 +68,6 @@ namespace bts.shell.manual_fills {
          // TickDisplaySettingsImmutable for qty, obtained via ISecurityMetadataStore.
          _quantity.conversion_data = new(1, QuantityConversionOpType.DivideActual);
          _price = data.value.manual_fill.Price;
-         _timestamp = data.value.manual_fill.HasTimestamp
-            ? data.value.manual_fill.Timestamp.to_local_time()
-            : DateTime.Now;
          if (placeholder_key.HasValue) {
             _is_dirty = backing(nameof(is_dirty), false);
             _editable_item_state = backing(nameof(editable_item_state), EditableItemState.Uncommitted);
@@ -118,22 +113,6 @@ namespace bts.shell.manual_fills {
       public TickDisplaySettingsImmutable quantity_display => _quantity_display.Value;
       public double price { get => _price; set => _set(ref _price, value); }
 
-      /// <summary>
-      /// The trading date assigned to this fill.  The time of day is retained so
-      /// that a fill entered for today continues to be classified as a day fill.
-      /// </summary>
-      public DateTime fill_date {
-         get => _timestamp.Date;
-         set {
-            var timestamp = value.Date + _timestamp.TimeOfDay;
-            if (_timestamp != timestamp) {
-               _timestamp = timestamp;
-               RaisePropertyChanged(nameof(fill_date));
-               _data = null;
-            }
-         }
-      }
-
       public DateTime? time_entered =>
          data.value.manual_fill.HasTimestamp
             ? data.value.manual_fill.Timestamp.to_local_time()
@@ -178,8 +157,7 @@ namespace bts.shell.manual_fills {
             Price = price,
             Side = side,
             Quantity = quantity,
-            SecurityKey = security?.Id ?? 0,
-            Timestamp = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(_timestamp.ToUniversalTime())
+            SecurityKey = security?.Id ?? 0
          };
 
       public ManualFillTuple to_tuple() =>

src/client/spark/Modules/Trading/ManualFills/ManualFillRowEditor.xaml

@@ -39,8 +39,6 @@
         <ColumnDefinition Width="Auto"/>
         <ColumnDefinition Width="{Binding Source={StaticResource HorizontalPush}}"/>
         <ColumnDefinition Width="Auto"/>
-        <ColumnDefinition Width="{Binding Source={StaticResource HorizontalPush}}"/>
-        <ColumnDefinition Width="Auto"/>
       </Grid.ColumnDefinitions>
       <Grid.RowDefinitions>
         <RowDefinition Height="Auto"/>
@@ -48,7 +46,7 @@
         <RowDefinition Height="*"/>
       </Grid.RowDefinitions>
       <Rectangle
-        Grid.ColumnSpan="13"
+        Grid.ColumnSpan="11"
         HorizontalAlignment="Stretch"
         VerticalAlignment="Stretch"
         Fill="{u:BTSColor header}" />
@@ -58,7 +56,6 @@
       <TextBlock Grid.Column="6" Text="Side" Style="{StaticResource HeaderText}"/>
       <TextBlock Grid.Column="8" Text="Qty" Style="{StaticResource HeaderText}"/>
       <TextBlock Grid.Column="10" Text="Price" Style="{StaticResource HeaderText}"/>
-      <TextBlock Grid.Column="12" Text="Date" Style="{StaticResource HeaderText}"/>
 
       <c:ContractPickerPopup
         Grid.Row="2"
@@ -91,7 +88,6 @@
         TickDisplaySettings="{Binding quantity_display}"
         Value="{Binding display_quantity}" />
       <usp:PriceEntryControl Grid.Row="2" Grid.Column="10" MinWidth="55" Value="{Binding price}" ContractSpec="{Binding spec}"/>
-      <DatePicker Grid.Row="2" Grid.Column="12" SelectedDate="{Binding fill_date}" />
     </Grid>
     <c:DialogSkirt Grid.Row="1" Margin="0,6,0,0">
       <c:DialogSkirt.RightItems>

src/client/spark/Modules/Trading/ManualFills/ManualFillsView.xaml

@@ -272,23 +272,6 @@
             </DataGridTemplateColumn.CellEditingTemplate>
           </DataGridTemplateColumn>
 
-          <!-- Trading Date -->
-          <DataGridTemplateColumn
-            c:DataGridEx.CellKey="{x:Static Date, MemberType={u:Nested usp:ManualFillsView.Columns}}"
-            Header="Date"
-            SortMemberPath="fill_date">
-            <DataGridTemplateColumn.CellTemplate>
-              <DataTemplate>
-                <TextBlock Text="{Binding fill_date, StringFormat={}{0:d}}" VerticalAlignment="Center" />
-              </DataTemplate>
-            </DataGridTemplateColumn.CellTemplate>
-            <DataGridTemplateColumn.CellEditingTemplate>
-              <DataTemplate>
-                <DatePicker SelectedDate="{Binding fill_date, UpdateSourceTrigger=PropertyChanged}" />
-              </DataTemplate>
-            </DataGridTemplateColumn.CellEditingTemplate>
-          </DataGridTemplateColumn>
-
           <!-- Time Entered -->
           <DataGridTextColumn
             c:DataGridEx.CellKey="{x:Static

src/client/spark/Modules/Trading/ManualFills/ManualFillsView.xaml.cs

@@ -31,7 +31,6 @@ namespace mbts.ui.spark {
          Side,
          Quantity,
          Price,
-         Date,
          TimeEntered
       }
 

src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookFilterStates.cs

@@ -26,7 +26,7 @@ namespace bts.shell.orderbook {
       [ProtoMember(5)]
       public string[] accounts;
 
-      [ProtoMember(28)]
+      [ProtoMember(5)]
       public string Tag { get; set; }
 
       [ProtoMember(6)]

src/client/spark/Modules/Trading/PNL/View/PNLView.xaml.cs

@@ -407,16 +407,16 @@ namespace bts.shell.pnl {
       void _show_auxiliary_popup<T>(T content, string context_desc, string title_desc, Color fg) {
          var view = new HeaderedContentControl {
             Content = content,
-            Header = new OrderAuxiliaryPopupHeader {
+            Header = new AuxiliaryDataHeader {
                context = context_desc,
                title = title_desc,
                title_foreground = fg
             },
             HorizontalAlignment = HorizontalAlignment.Left,
             VerticalAlignment = VerticalAlignment.Top,
-            Resources = new OrderAuxiliaryPopupResources()
+            Resources = new AuxiliaryDataResources()
          };
-         view.Style = view.Resources[OrderAuxiliaryPopupResources.PopupChildStyleKey] as Style;
+         view.Style = view.Resources[AuxiliaryDataResources.PopupChildStyleKey] as Style;
          _popup_man.show(view);
       }
 

src/client/spark/NMSheets/DataProviders/NMSheetSingleOptionMainDataSource.cs

@@ -8,15 +8,13 @@ using bts.api;
 using mbts.analytics;
 using mbts;
 
-namespace BCG
-{
+namespace BCG {
    /// For each NMSheetSymbol in an NMSheet gets all the 
    /// data together (theos, position, vol change etc) 
    /// and fires a callback.
    /// This is where all the data is gathered, the rest
    /// is all access to this
-   public class NMSheetSingleOptionMainDataSource : IDisposable
-   {
+   public class NMSheetSingleOptionMainDataSource : IDisposable {
       private ThreadedBlackboard _threaded_bb;
       private readonly ITag<NMSheetDataRequest> _in_bb_main_sheet_request_tag;
       private readonly ITag<ExchangeSecurityMap> _in_bb_secmap_tag =
@@ -26,7 +24,6 @@ namespace BCG
       private readonly ITag<OptionTheoInterpolationData> _in_bb_precomputed_sparse_theos_tag;
       private readonly ITag<IDictionary<NMSheetExpiration, double>> _in_bb_vol_shift_tag;
       private readonly ITag<NMSheetSettings> _in_bb_sheet_settings_tag;
-      private readonly ITag<IEnumerable<ITradeLogRow>> _in_bb_trade_log_tag = SparkTags.TradeLogTag;
       private readonly ITag<VolSurfaceNonVolInput> _in_exp_param_tag = SparkTags.VolSurfaceNonVolInputTag;
       private readonly ITag<bool> _in_bb_main_sheet_visibility_tag;
       private ITag<SparkTags.Nothing> _in_bb_onefive_value_tag = SparkTags.TC2Tags.VoidTag,
@@ -34,6 +31,7 @@ namespace BCG
                                        _in_bb_symbol_theo_shift_tag = SparkTags.TC2Tags.VoidTag,
                                        _in_bb_rollover_data_tag = SparkTags.TC2Tags.VoidTag,
                                        _in_bb_presaved_aw2_theos_tag = SparkTags.TC2Tags.VoidTag,
+                                       _in_bb_position_tag = SparkTags.TC2Tags.VoidTag,
                                        _in_bb_option_driver_detailed_price_tag = SparkTags.TC2Tags.VoidTag;
       private readonly NMSheetSymbol _nmsheet_symbol;
       private Action<Pair<NMSheetSymbol, NMSheetSingleOptionData>> _cb;
@@ -42,8 +40,7 @@ namespace BCG
       private string _electronic_equivelant_symbol;
       private NMSheetArchivedVolProvider _archived_vol_provider;
 
-      public NMSheetSingleOptionMainDataSource(object req_id, ThreadedBlackboard threaded_bb, NMSheetSymbol option, Action<Pair<NMSheetSymbol, NMSheetSingleOptionData>> callback)
-      {
+      public NMSheetSingleOptionMainDataSource(object req_id, ThreadedBlackboard threaded_bb, NMSheetSymbol option, Action<Pair<NMSheetSymbol, NMSheetSingleOptionData>> callback) {
          _threaded_bb = threaded_bb;
          _nmsheet_symbol = option;
          _electronic_equivelant_symbol = option.option_name;
@@ -59,8 +56,7 @@ namespace BCG
          _in_bb_sheet_settings_tag = SparkTags.TC2Tags.NMSheetSettingsTag(req_id.ToString());
          _in_bb_main_sheet_visibility_tag = SparkTags.TC2Tags.NMSheetVisibilityTag(req_id.ToString());
          _archived_vol_provider = new NMSheetArchivedVolProvider(req_id.ToString(), _threaded_bb, option,
-            () =>
-            {
+            () => {
                _rollover_data = null;
                _compute_and_fire();
             });
@@ -80,7 +76,7 @@ namespace BCG
          _threaded_bb.subscribe(_in_bb_rollover_data_tag);
          _threaded_bb.subscribe(_in_bb_presaved_aw2_theos_tag);
          _threaded_bb.subscribe(_in_exp_param_tag);
-         _threaded_bb.subscribe(_in_bb_trade_log_tag);
+         _threaded_bb.subscribe(_in_bb_position_tag);
          _threaded_bb.subscribe(_in_bb_sheet_settings_tag);
          _threaded_bb.subscribe(_in_bb_onefive_value_tag);
          _threaded_bb.subscribe(_in_bb_qs_tag);
@@ -88,17 +84,15 @@ namespace BCG
          _threaded_bb.subscribe(_in_bb_option_driver_detailed_price_tag);
       }
 
-      private void _react_to_bb(object tag, object value)
-      {
+      private void _react_to_bb(object tag, object value) {
          if (_in_bb_main_sheet_request_tag.Equals(tag) || _in_bb_secmap_tag.Equals(tag) || _in_bb_live_theos_tag.Equals(tag) || /*_in_bb_live_driver_bbo_tag.Equals(tag) ||*/ _in_bb_precomputed_sparse_theos_tag.Equals(tag) ||
             _in_bb_vol_shift_tag.Equals(tag) || _in_bb_symbol_theo_shift_tag.Equals(tag) || _in_exp_param_tag.Equals(tag) || _in_bb_presaved_aw2_theos_tag.Equals(tag) ||
-            _in_bb_trade_log_tag.Equals(tag) || _in_bb_rollover_data_tag.Equals(tag) || _in_bb_sheet_settings_tag.Equals(tag) || _in_bb_qs_tag.Equals(tag) ||
+            _in_bb_position_tag.Equals(tag) || _in_bb_rollover_data_tag.Equals(tag) || _in_bb_sheet_settings_tag.Equals(tag) || _in_bb_qs_tag.Equals(tag) ||
             _in_bb_main_sheet_visibility_tag.Equals(tag))
             _compute_and_fire();
       }
 
-      private NMSheetDriverData _get_driver(DriverMD d)
-      {
+      private NMSheetDriverData _get_driver(DriverMD d) {
          double dr_pr = d.get_driver();
          if (d.driver_security.HasValue)
             return new NMSheetDriverData(d.driver_security.symbol_or_default(),
@@ -108,10 +102,8 @@ namespace BCG
          return new NMSheetDriverData(String.Empty, Venue.Unset, DateTime.MinValue, dr_pr, dr_pr, false);
       }
 
-      private void _compute_and_fire()
-      {
-         if (_is_main_sheet_active())
-         {
+      private void _compute_and_fire() {
+         if (_is_main_sheet_active()) {
             var timer = Stopwatch.StartNew();
             NMSheetDataRequest data_request = _threaded_bb.read_state(_in_bb_main_sheet_request_tag) ?? new NMSheetDataRequest(new List<Triplet<NMSheetExpiration, bool, double>>());
             data_request = NMSheetDataRequest.restrict_for_symbol(data_request, _nmsheet_symbol);
@@ -120,8 +112,7 @@ namespace BCG
             NMSheetSettings sheet_settings = _threaded_bb.read_state(_in_bb_sheet_settings_tag) ?? new NMSheetSettings();
             ExpStrikeKeyedData<mbts.spark.TheoPoint> shared_live_theo_data_map = _threaded_bb.read_state(_in_bb_live_theos_tag);
 
-            if (data_request != null && live_driver_map != null && shared_live_theo_data_map != null)
-            {
+            if (data_request != null && live_driver_map != null && shared_live_theo_data_map != null) {
                ExpStrikeKeyedData<mbts.spark.TheoPoint> output_theo_data_map = _limit_to_requested_securities(shared_live_theo_data_map, data_request, sheet_settings);
                output_theo_data_map = _adjust_for_freezes(output_theo_data_map, data_request);
                _add_vol_shift(output_theo_data_map);
@@ -147,17 +138,14 @@ namespace BCG
          }
       }
 
-      private bool _is_main_sheet_active()
-      {
+      private bool _is_main_sheet_active() {
          return _threaded_bb.read_state(_in_bb_main_sheet_visibility_tag, true);
       }
 
-      private ExpStrikeKeyedData<mbts.spark.TheoPoint> _limit_to_requested_securities(ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_data, NMSheetDataRequest data_request, NMSheetSettings sheet_settings)
-      {
+      private ExpStrikeKeyedData<mbts.spark.TheoPoint> _limit_to_requested_securities(ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_data, NMSheetDataRequest data_request, NMSheetSettings sheet_settings) {
          HashSet<DateTime> selected_exp_map = new HashSet<DateTime>();
          HashSet<double> selected_strike_map = new HashSet<double>(sheet_settings.setting_unselected_strikes);
-         foreach (Triplet<NMSheetExpiration, bool, double> t in data_request.expiration_request_list)
-         {
+         foreach (Triplet<NMSheetExpiration, bool, double> t in data_request.expiration_request_list) {
             NMSheetExpiration nm_exp = t.first;
             if (nm_exp != null && nm_exp.option_name == _nmsheet_symbol.option_name)
                selected_exp_map.Add(nm_exp.option_expiration);
@@ -165,14 +153,12 @@ namespace BCG
          return theo_data.clone_filtered_set(selected_exp_map, true, selected_strike_map, false);
       }
 
-      private ExpStrikeKeyedData<Triplet<mbts.spark.TheoPoint, uint, uint>> _attach_sec_keys(ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_map)
-      {
+      private ExpStrikeKeyedData<Triplet<mbts.spark.TheoPoint, uint, uint>> _attach_sec_keys(ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_map) {
          theo_map = theo_map ?? new ExpStrikeKeyedData<mbts.spark.TheoPoint>();
          ExchangeSecurityMap sec_map = _threaded_bb.read_state(_in_bb_secmap_tag);
          if (sec_map != null)
             return sec_map.attach_sec_keys(theo_map, _electronic_equivelant_symbol, _nmsheet_symbol.option_exchange);
-         else
-         {
+         else {
             ExpStrikeKeyedData<Triplet<mbts.spark.TheoPoint, uint, uint>> rv = new ExpStrikeKeyedData<Triplet<mbts.spark.TheoPoint, uint, uint>>();
             foreach (Triplet<DateTime, double, mbts.spark.TheoPoint> t in theo_map.get_data_linearly())
                rv.set_data(t.first, t.second, new Triplet<mbts.spark.TheoPoint, uint, uint>(t.third, 0, 0));
@@ -180,8 +166,7 @@ namespace BCG
          }
       }
 
-      private ExpStrikeKeyedData<mbts.spark.TheoPoint> _adjust_for_freezes(ExpStrikeKeyedData<mbts.spark.TheoPoint> live_theo_map, NMSheetDataRequest data_request)
-      {
+      private ExpStrikeKeyedData<mbts.spark.TheoPoint> _adjust_for_freezes(ExpStrikeKeyedData<mbts.spark.TheoPoint> live_theo_map, NMSheetDataRequest data_request) {
          return live_theo_map;
          // TODO
          // ExpStrikeKeyedData<mbts.spark.TheoPoint> rv = new ExpStrikeKeyedData<mbts.spark.TheoPoint>();
@@ -207,11 +192,9 @@ namespace BCG
 
       private Dictionary<DateTime, NMSheetDriverData> _get_option_exp_driver_data_map(NMSheetDataRequest data_request,
                                                                                       ExpStrikeKeyedData<mbts.spark.TheoPoint> output_theo_data_map, ExpStrikeKeyedData<mbts.spark.TheoPoint> shared_live_theo_data_map,
-                                                                                      Dictionary<DateTime, Triplet<(string, Venue), DateTime, double>> live_driver_map)
-      {
+                                                                                      Dictionary<DateTime, Triplet<(string, Venue), DateTime, double>> live_driver_map) {
          Dictionary<DateTime, NMSheetDriverData> rv = new Dictionary<DateTime, NMSheetDriverData>();
-         foreach (Triplet<NMSheetExpiration, bool, double> theo_request in data_request.expiration_request_list)
-         {
+         foreach (Triplet<NMSheetExpiration, bool, double> theo_request in data_request.expiration_request_list) {
             DateTime option_expiration = theo_request.first.option_expiration;
             /// live driver
             var live_driver_info = live_driver_map.GetValueOrDefault(option_expiration)
@@ -225,11 +208,9 @@ namespace BCG
 
       private Dictionary<DateTime, NMSheetDriverData>
          _get_option_exp_driver_data_map(NMSheetDataRequest data_request,
-                                         OptionDriverPriceMap drivers)
-      {
+                                         OptionDriverPriceMap drivers) {
          var rv = new Dictionary<DateTime, NMSheetDriverData>();
-         foreach (var tr in data_request.expiration_request_list)
-         {
+         foreach (var tr in data_request.expiration_request_list) {
             NMSheetExpiration nm_exp = tr.first;
             SecurityExpiration exp = tr.first.to_sec_exp();
             drivers.exp_price_map.TryGetValue(exp, out DriverMD dr_md);
@@ -241,8 +222,7 @@ namespace BCG
          return rv;
       }
 
-      private static double _get_driverprice_for_expiration(ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_data_map, DateTime option_expiration)
-      {
+      private static double _get_driverprice_for_expiration(ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_data_map, DateTime option_expiration) {
          IDictionary<double, mbts.spark.TheoPoint> exp_data = theo_data_map.get_data_for_single_exp(option_expiration);
          if (exp_data != null)
             foreach (mbts.spark.TheoPoint tp in exp_data.Values)
@@ -250,21 +230,7 @@ namespace BCG
          return Double.NaN;
       }
 
-       private double _get_rep_driver() {
-         var map =
-            _threaded_bb.read_state(SparkTags.OptionDriverBBOTag((_nmsheet_symbol.option_name, _nmsheet_symbol.option_exchange)));
-         if (map == null)
-            return Double.NaN;
-         var price_map = map.get_price_map();
-         if (price_map == null || price_map.Count == 0)
-            return Double.NaN;
-         var md = price_map.ElementAt(0).Value;
-         double pr = md.get_driver();
-         return pr;
-      }
-
-      private IDictionary<DateTime, ATMData> _get_atm_data(ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_data)
-      {
+      private IDictionary<DateTime, ATMData> _get_atm_data(ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_data) {
          NMSheetSettings sheet_settings = _threaded_bb.read_state(_in_bb_sheet_settings_tag) ?? new NMSheetSettings();
          VolSurfaceNonVolInput vs_nvinp = _threaded_bb.read_state(_in_exp_param_tag);
          var option_exp_roll_data = vs_nvinp
@@ -274,7 +240,7 @@ namespace BCG
                exp.security_type == SecurityType.Option)
             ?.ToDictionaryCrushed(kv => kv.Key.expiration, kv => kv.Value)
             ?? new();
-         IDictionary<DateTime, ATMData> atm_data = OptionATMDataSource.compute_atm_data(_get_onefive(), theo_data, option_exp_roll_data, _get_rep_driver());
+         IDictionary<DateTime, ATMData> atm_data = OptionATMDataSource.compute_atm_data(_get_onefive(), theo_data, option_exp_roll_data);
          if (sheet_settings.setting_atm.select_atm_nearest_strike_only)
             foreach (ATMData exp_atm_data in atm_data.Values)
                exp_atm_data.reduce_to_nearest_expiration();
@@ -282,43 +248,17 @@ namespace BCG
       }
 
       OptionContractKeyedData<PNLRollOverDataUnit> _rollover_data = null;
-      private OptionContractKeyedData<PNLRollOverDataUnit> _get_rollover_data()
-      {
+      private OptionContractKeyedData<PNLRollOverDataUnit> _get_rollover_data() {
          if (_rollover_data == null)
             _rollover_data = _archived_vol_provider.get_all_data();
          return _rollover_data;
       }
 
-      private ExpStrikeKeyedData<Tuple<double, double>> _get_position_map()
-      {
-         var sheet_settings = _threaded_bb.read_state(_in_bb_sheet_settings_tag);
-         var risk_account_text = sheet_settings?.setting_layout?.risk_account ?? string.Empty;
-         var risk_accounts = risk_account_text
-            .Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)
-            .Select(a => a.Trim())
-            .Where(a => !string.IsNullOrEmpty(a))
-            .ToHashSet(StringComparer.OrdinalIgnoreCase);
-         var trades = _threaded_bb.read_state(_in_bb_trade_log_tag) ?? Enumerable.Empty<ITradeLogRow>();
-         var option_trades = trades.Where(t =>
-            t.is_active &&
-            t.security_type == SecurityType.Option &&
-            t.symbol == _nmsheet_symbol.option_name &&
-            t.venue == _nmsheet_symbol.option_exchange &&
-            (risk_accounts.Count == 0 || risk_accounts.Contains(t.account)));
-         var pos_data = SecurityPositionData.from_trades(option_trades);
-         var rv = new ExpStrikeKeyedData<Tuple<double, double>>();
-         foreach (var (sec_exp, strike_map) in pos_data.option_pos_map)
-         {
-            foreach (var (strike, strike_pos) in strike_map)
-            {
-               rv.set_data(sec_exp.expiration, strike, Tuple.Create(strike_pos.call, strike_pos.put));
-            }
-         }
-         return rv;
+      private ExpStrikeKeyedData<Tuple<double, double>> _get_position_map() {
+         return new ExpStrikeKeyedData<Tuple<double, double>>();
       }
 
-      private double _get_onefive()
-      {
+      private double _get_onefive() {
          // object dp_obj = _threaded_bb.sec_thread_retrieve_from_cache(_in_bb_onefive_value_tag);
          // if (dp_obj != null) {
          //    DataPacket dp = dp_obj as DataPacket;
@@ -330,29 +270,22 @@ namespace BCG
 
       #region Vol Shift
 
-      private void _add_vol_shift(ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_data)
-      {
+      private void _add_vol_shift(ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_data) {
          var vshift_map = _threaded_bb.read_state(_in_bb_vol_shift_tag);
          if (vshift_map != null)
             add_vol_shift(_nmsheet_symbol, theo_data, vshift_map);
       }
 
-      internal static void add_vol_shift(NMSheetSymbol apply_to_symbol, ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_data, IDictionary<NMSheetExpiration, double> exp_shift_map)
-      {
-         foreach (KeyValuePair<NMSheetExpiration, double> p in exp_shift_map)
-         {
-            if (p.Key.nmsheet_symbol.Equals(apply_to_symbol))
-            {
+      internal static void add_vol_shift(NMSheetSymbol apply_to_symbol, ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_data, IDictionary<NMSheetExpiration, double> exp_shift_map) {
+         foreach (KeyValuePair<NMSheetExpiration, double> p in exp_shift_map) {
+            if (p.Key.nmsheet_symbol.Equals(apply_to_symbol)) {
                DateTime option_expiration = p.Key.option_expiration;
                double shift = p.Value;
-               if (shift != 0)
-               {
+               if (shift != 0) {
                   IDictionary<double, mbts.spark.TheoPoint> exp_theos = theo_data.get_data_for_single_exp(option_expiration);
-                  if (exp_theos != null)
-                  {
+                  if (exp_theos != null) {
                      Dictionary<double, mbts.spark.TheoPoint> shifted_exp_theos = new Dictionary<double, mbts.spark.TheoPoint>();
-                     foreach (KeyValuePair<double, mbts.spark.TheoPoint> strike_tp in exp_theos)
-                     {
+                     foreach (KeyValuePair<double, mbts.spark.TheoPoint> strike_tp in exp_theos) {
                         mbts.spark.TheoPoint new_tp = (mbts.spark.TheoPoint)strike_tp.Value.Clone();
                         add_vol_shift(new_tp, shift);
                         shifted_exp_theos[strike_tp.Key] = new_tp;
@@ -364,8 +297,7 @@ namespace BCG
          }
       }
 
-      internal static void add_vol_shift(mbts.spark.TheoPoint v, double shift)
-      {
+      internal static void add_vol_shift(mbts.spark.TheoPoint v, double shift) {
          v.call += v.cvega * shift;
          v.put += v.pvega * shift;
       }
@@ -373,8 +305,7 @@ namespace BCG
       #endregion
 
       #region IDisposable
-      public void Dispose()
-      {
+      public void Dispose() {
          _threaded_bb.data_callback -= _react_to_bb;
       }
       #endregion

src/client/spark/NMSheets/Outside/tc2/Calculator2/MainForm/ATM/OptionATMDataSource.cs

@@ -5,7 +5,6 @@ using mbts.ui;
 using mbts;
 using BCG.Utilities;
 using mbts.spark;
-using mbts.ui.converters;
 
 namespace BCG {
    /// <summary>
@@ -35,10 +34,8 @@ namespace BCG {
       }
 
       internal static IDictionary<DateTime, ATMData> compute_atm_data(
-         double one_five, ExpStrikeKeyedData<TheoPoint> theo_data, IDictionary<DateTime, double> opt_exp_rolls,
-         double rep_driver) {
+         double one_five, ExpStrikeKeyedData<TheoPoint> theo_data, IDictionary<DateTime, double> opt_exp_rolls) {
          IDictionary<DateTime, double> exp_vs_drivers = _get_computed_drivers(theo_data);
-         exp_vs_drivers = exp_vs_drivers.ToDictionary(kv => kv.Key, kv => rep_driver);
          IDictionary<DateTime, ATMData> rv = new Dictionary<DateTime, ATMData>();
          if (one_five != 0)
             foreach (DateTime exp in new List<DateTime>(exp_vs_drivers.Keys))

src/client/spark/Orders/ExistingOrderTicketLadderVM.cs

@@ -1,236 +0,0 @@
-#nullable enable
-namespace mbts.spark;
-
-using bts;
-using bts.api;
-using bts.api.domain;
-using bts.api.orderman;
-
-using mbts.co2;
-using mbts.ui;
-using mbts.ui.shell;
-
-using Spark.Core.Disposables;
-
-using System;
-using System.Collections.Generic;
-using System.Collections.ObjectModel;
-using System.Linq;
-using System.Reactive.Linq;
-using System.Windows.Input;
-using System.Windows.Threading;
-
-class ExistingOrderTicketLadderVM : DisposableViewModelBase, IRequestViewClose {
-   const ushort numPriceLevels = 5;
-
-   readonly Property<SecurityImmutable> security;
-   readonly Property<TickDisplaySettingsImmutable> priceDisplay;
-   readonly Property<TickDisplaySettingsImmutable> quantityDisplay;
-   readonly Property<user.settings.RegionalTickFactors> priceTick;
-   readonly ObservableCollection<OrderTicketLadderPriceLevelVM> bids = [];
-   readonly ObservableCollection<OrderTicketLadderPriceLevelVM> asks = [];
-
-   readonly Property<double> price, currentPrice;
-   readonly Property<uint> currentQuantity;
-   readonly Property<AccountTuple> currentAccount;
-   readonly Property<TimeInForce> currentTimeInForce;
-   readonly Property<CustomOrderDefinition.CustomOneofCase> currentOrderType;
-   readonly Property<bool> priceDirty, quantityDirty, accountDirty, timeInForceDirty, orderTypeDirty;
-
-   Side side = Side.Buy;
-
-   public ExistingOrderTicketLadderVM(
-      OrdersModel model,
-      IObservable<(SecurityMD? SecurityMd, OrderRecordImmutable Order)> orderSource,
-      IObservable<IReadOnlyDictionary<uint, AccountTuple>> accountsSource,
-      IObservable<double[]?> calculatorButtons,
-      Action<SecurityMD> showLadder,
-      TickDisplaySettingsImmutable? quantityDisplay = null
-   ) {
-      SecurityMD? currentSecurityMd = null;
-      uint? key = null;
-      DispatcherTimer? mdPollTimer = null;
-      var nativeBids = new NativePriceSize[numPriceLevels];
-      var nativeAsks = new NativePriceSize[numPriceLevels];
-      for (int i = 0; i < numPriceLevels; i++) {
-         nativeBids[i] = NativePriceSize.NO_MARKET;
-         nativeAsks[i] = NativePriceSize.NO_MARKET;
-         Bids.Add(new(Side.Buy));
-         Asks.Add(new(Side.Sell));
-      }
-
-      var secMdSrc = orderSource.Where(ot => ot.SecurityMd.HasValue).Select(ot => ot.SecurityMd!.Value);
-      security = backing(nameof(Security), secMdSrc.Select(smd => smd.security), bts.api.Security.Defaults.SecurityImmutable);
-      priceTick = backing(nameof(PriceTick), user.settings.RegionalTickFactors.Default);
-      PropertiesVm = new OrderPropertiesEditorVM(
-         model,
-         secMdSrc,
-         accountsSource,
-         orderSource.Select(ot => new InlineOrderProperties(model, ot.Order.Def)));
-      this.quantityDisplay = backing(nameof(QuantityDisplay), quantityDisplay ?? bts.api.Security.Defaults.quantity_display_settingsi);
-      priceDisplay = backing(nameof(PriceDisplay), bts.api.Security.Defaults.price_display_settingsi);
-      SizeCalculator = new(0, calculatorButtons);
-      SetPriceCmd = new DelegateCommand<double>(SetPrice);
-      ShowLadderCmd = new DelegateCommand(() => {
-         if (currentSecurityMd.HasValue)
-            showLadder(currentSecurityMd.Value);
-      });
-
-      price = backing(nameof(Price), double.NaN);
-      currentPrice = backing(nameof(CurrentPrice), orderSource.Select(ort => ort.Order.State.Price));
-      currentQuantity = backing(nameof(CurrentQuantity), orderSource.Select(ort => ort.Order.State.Quantity));
-      currentAccount = backing(nameof(CurrentAccount), orderSource.Select(ort => new AccountTuple(ort.Order.Def.AccountId, ort.Order.Def.Account)));
-      currentTimeInForce = backing(nameof(CurrentTimeInForce), orderSource.Select(ort => ort.Order.Def.TimeInForce));
-      currentOrderType = backing(nameof(CurrentOrderType), orderSource.Select(ort => ort.Order.Def.CustomDef.CustomCase));
-      //
-      priceDirty = backing(
-         nameof(PriceDirty),
-         price.observe_current(true).CombineLatest(
-         currentPrice.observe_current(true),
-         (val, curr) => !double.IsNaN(val) && val != curr));
-      quantityDirty = backing(
-         nameof(QuantityDirty),
-         SizeCalculator.observe_actual_value()
-         .CombineLatest(currentQuantity.observe_current(true),
-         (val, curr) => val.HasValue && val.Value != curr));
-      accountDirty = backing(
-         nameof(AccountDirty),
-         PropertiesVm.observe_account(true)
-         .CombineLatest(currentAccount.observe_current(true),
-         (val, curr) => val != curr));
-      timeInForceDirty = backing(
-         nameof(TimeInForceDirty),
-         PropertiesVm.observe_time_in_force(true)
-         .CombineLatest(currentTimeInForce.observe_current(true),
-         (val, curr) => val != curr));
-      orderTypeDirty = backing(
-         nameof(OrderTypeDirty),
-         PropertiesVm.observe_order_type(true)
-         .CombineLatest(currentOrderType.observe_current(true),
-         (val, curr) => val != null && val.order_type != curr));
-      //
-      var priceDirtySrc = priceDirty.Select(vals => vals.newval).StartWith(false);
-      var quantityDirtySrc = quantityDirty.Select(vals => vals.newval).StartWith(false);
-      var sendOrderCmd = new ObservantCommand(SendOrder);
-      SendOrderCmd = sendOrderCmd;
-      var canSendOrderSrc = priceDirty.observe_current(true)
-         .CombineLatest(quantityDirty.observe_current(true), (pd, sd) => pd || sd)
-         .DistinctUntilChanged();
-      var isDirtySrc = Observable.Merge(priceDirty, quantityDirty, accountDirty, timeInForceDirty, orderTypeDirty);
-      _dispose = Disposable.Create(
-         SizeCalculator,
-         PropertiesVm,
-         secMdSrc.OnDispatcher().Subscribe(OnSecurityMd),
-         orderSource.OnDispatcher().Subscribe(ort => {
-            key = ort.Order.Key;
-            if (DoubleHelper.IsNaNOrInfinity(Price))
-               Price = ort.Order.State.Price;
-            if (!SizeCalculator.value.HasValue || !SizeCalculator.value_setter_called)
-               SizeCalculator.bypass_value_setter(ort.Order.State.Quantity, true);
-         }),
-         isDirtySrc.OnDispatcher().Subscribe(_ => RaisePropertyChanged(nameof(DirtyItemsText))),
-         canSendOrderSrc.OnDispatcher().Subscribe(sendOrderCmd));
-
-      void SetPrice(double price) {
-         Price = price;
-         RaisePropertyChanged(nameof(Price));
-      }
-
-      void SendOrder() {
-         if (!key.HasValue) {
-            return;
-         }
-         if (priceDirty.Value) {
-            if (quantityDirty.Value) {
-               model.update_order(key.Value, Price, SizeCalculator.value!.Value).ignore();
-            } else {
-               model.update_order_price(key.Value, Price).ignore();
-            }
-         } else if (quantityDirty.Value) {
-            model.update_order_size(key.Value, SizeCalculator.value!.Value).ignore();
-         } else {
-            SizeCalculator.flash_error();
-            flash_error(nameof(Price)).ignore();
-            return;
-         }
-         RequestClose?.Invoke(this, new EventArgs());
-      }
-
-      void OnSecurityMd(SecurityMD smd) {
-         currentSecurityMd = smd;
-         priceTick.set(smd.security.regional_ticks());
-         priceDisplay.set(smd.price_display);
-         this.quantityDisplay.set(smd.quantity_display);
-         SizeCalculator.display_settings = smd.quantity_display;
-         ulong lastMdPollTime = 0;
-         mdPollTimer?.Stop();
-         mdPollTimer = new DispatcherTimer(TimeSpan.FromSeconds(0.1), DispatcherPriority.DataBind, PollMd, Dispatcher.CurrentDispatcher);
-
-         void PollMd(object? sender, EventArgs e) {
-            if (smd.md.get_depth(ref lastMdPollTime, ref nativeBids, ref nativeAsks, numPriceLevels)) {
-               for (int i = 0; i < numPriceLevels; i++) {
-                  asks[numPriceLevels - i - 1].Price = nativeAsks[i].price;
-                  asks[numPriceLevels - i - 1].Quantity = nativeAsks[i].size;
-               }
-               for (int i = 0; i < numPriceLevels; i++) {
-                  bids[i].Price = nativeBids[i].price;
-                  bids[i].Quantity = nativeBids[i].size;
-               }
-            }
-         }
-      }
-   }
-
-   public event EventHandler? RequestClose;
-
-   public SecurityImmutable Security => security.Value;
-   public string Title => Security.ToString() ?? string.Empty;
-   public double Price { get; set; }
-   public user.settings.RegionalTickFactors PriceTick => priceTick.Value;
-   public TickDisplaySettingsImmutable PriceDisplay => priceDisplay.Value;
-   public TickDisplaySettingsImmutable QuantityDisplay => quantityDisplay.Value;
-   public CalculatorLogic<double> SizeCalculator { get; }
-   public Side Side {
-      get => side;
-      set => Set(ref side, value);
-   }
-
-   public OrderPropertiesEditorVM PropertiesVm { get; }
-   public ObservableCollection<OrderTicketLadderPriceLevelVM> Bids => bids;
-   public ObservableCollection<OrderTicketLadderPriceLevelVM> Asks => asks;
-
-   public double CurrentPrice => currentPrice.Value;
-   public uint CurrentQuantity => currentQuantity.Value;
-   public AccountTuple CurrentAccount => currentAccount.Value;
-   public TimeInForce CurrentTimeInForce => currentTimeInForce.Value;
-   public CustomOrderDefinition.CustomOneofCase CurrentOrderType => currentOrderType.Value;
-   public bool PriceDirty => priceDirty.Value;
-   public bool QuantityDirty => quantityDirty.Value;
-   public bool AccountDirty => accountDirty.Value;
-   public bool TimeInForceDirty => timeInForceDirty.Value;
-   public bool OrderTypeDirty => orderTypeDirty.Value;
-   public bool CanChangeAccount { get; } = false;
-   public bool CanChangeTimeInForce { get; } = false;
-   public bool CanChangeOrderType { get; } = false;
-   public string DirtyItemsText {
-      get {
-         var dirty_items = dirtyItems().ToList();
-         if (dirty_items.Count == 0) {
-            return "(no changes)";
-         }
-         return "change " + string.Join(", ", dirty_items);
-      }
-   }
-   public bool CloseOnSend { get; set; }
-   public ICommand SendOrderCmd { get; }
-   public ICommand SetPriceCmd { get; }
-   public ICommand ShowLadderCmd { get; }
-
-   IEnumerable<string> dirtyItems() {
-      if (priceDirty.Value) yield return "price";
-      if (quantityDirty.Value) yield return "quantity";
-      if (accountDirty.Value) yield return "account";
-      if (timeInForceDirty.Value) yield return "tif";
-      if (orderTypeDirty.Value) yield return "type";
-   }
-}

src/client/spark/Orders/NewOrderTicketLadderVM.cs

@@ -1,298 +0,0 @@
-#nullable enable
-namespace mbts.spark;
-
-using bts;
-using bts.api;
-using bts.api.domain;
-using bts.api.orderman;
-using bts.collections;
-using bts.utils;
-
-using mbts.co2;
-using mbts.ui;
-using mbts.ui.shell;
-using mbts.ui.spark;
-
-using Spark.Core.Disposables;
-
-using System;
-using System.Collections.Generic;
-using System.Collections.ObjectModel;
-using System.Linq;
-using System.Reactive.Linq;
-using System.Windows.Input;
-using System.Windows.Media;
-using System.Windows.Threading;
-
-class NewOrderTicketLadderVM : DisposableViewModelBase, IOrderModificationLadderVM  {
-   const ushort numPriceLevels = 5;
-   const uint numPriceRowsPerSide = 500;
-
-   readonly Property<SecurityImmutable> security;
-   readonly Property<string> title;
-   readonly Property<TickDisplaySettingsImmutable> priceDisplay;
-   readonly Property<TickDisplaySettingsImmutable> quantityDisplay;
-   readonly Property<user.settings.RegionalTickFactors> priceTick;
-   readonly ObservableCollection<OrderTicketLadderPriceLevelVM> priceLevels = [];
-   Side side = Side.Buy;
-
-   public NewOrderTicketLadderVM(
-      OrdersModel model,
-      SecurityMD securityMd,
-      IObservable<IReadOnlyDictionary<uint, AccountTuple>> accountsSource,
-      OrderDefinitionImmutable def,
-      double price,
-      double size,
-      IObservable<double[]> calculatorButtons,
-      Action<SecurityMD> showLadder)
-      : this(
-           model,
-           Observable.Return(securityMd),
-           accountsSource,
-           Observable.Return(new InlineOrderTicketProperties(model, def, price, size)),
-           calculatorButtons,
-           showLadder) { }
-
-   public NewOrderTicketLadderVM(
-      OrdersModel model,
-      SecurityMD securityMd,
-      IObservable<IReadOnlyDictionary<uint, AccountTuple>> accountsSource,
-      IObservable<LadderSettings> settings,
-      Action<SecurityMD> showLadder)
-      : this(
-           model,
-           Observable.Return(securityMd),
-           accountsSource,
-           settings.Select(s => new InlineOrderTicketProperties(model, s)),
-           settings.Select(s => s.calculator_buttons),
-           showLadder) { }
-
-   public NewOrderTicketLadderVM(
-      OrdersModel model,
-      IObservable<SecurityMD> securityMdSource,
-      IObservable<IReadOnlyDictionary<uint, AccountTuple>> accountsSource,
-      IObservable<InlineOrderTicketProperties> propsSource,
-      IObservable<double[]?> calculatorButtons,
-      Action<SecurityMD> showLadder,
-      TickDisplaySettingsImmutable? quantityDisplay = null
-   ) {
-      SecurityMD? currentSecurityMd = null;
-      DispatcherTimer? mdPollTimer = null;
-      var nativeBids = new NativePriceSize[numPriceLevels];
-      var nativeAsks = new NativePriceSize[numPriceLevels];
-      for (int i = 0; i < numPriceLevels; i++) {
-         nativeBids[i] = NativePriceSize.NO_MARKET;
-         nativeAsks[i] = NativePriceSize.NO_MARKET;
-      }
-      for (int i = 0; i < numPriceRowsPerSide; i++) {
-         priceLevels.Add(new(Side.Sell));
-      }
-      for (int i = 0; i < numPriceRowsPerSide; i++) {
-         priceLevels.Add(new(Side.Buy));
-      }
-
-      security = backing(nameof(Security), bts.api.Security.Defaults.SecurityImmutable);
-      title = backing(nameof(Title), security.observe_current(true), s => s.ToString() ?? string.Empty, string.Empty);
-      priceTick = backing(nameof(PriceTick), user.settings.RegionalTickFactors.Default);
-      PropertiesVm = new(model, securityMdSource, accountsSource, propsSource.Select(p => p.order_props));
-      this.quantityDisplay = backing(nameof(QuantityDisplay), quantityDisplay ?? bts.api.Security.Defaults.quantity_display_settingsi);
-      priceDisplay = backing(nameof(PriceDisplay), bts.api.Security.Defaults.price_display_settingsi);
-      SizeCalculator = new(0, calculatorButtons);
-      SendOrderCmd = new DelegateCommand(SendOrder);
-      SetPriceCmd = new DelegateCommand<double>(SetPrice);
-      ShowLadderCmd = new DelegateCommand(() => {
-         if (currentSecurityMd.HasValue)
-            showLadder(currentSecurityMd.Value);
-      });
-      RecenterCmd = new DelegateCommand(Recenter);
-      _dispose = Disposable.Create(
-         securityMdSource.OnDispatcher().Subscribe(OnSecurityMd),
-         SizeCalculator,
-         propsSource.OnDispatcher().Subscribe(p => OnProps(p.price, p.size)));
-
-      void OnProps(
-         InlineUserProperty<double> priceProp, InlineUserProperty<double> sizeProp) {
-         if (priceProp.has_value && (priceProp.force_override || DoubleHelper.IsNaNOrInfinity(Price)))
-            SetPrice(priceProp.value);
-         if (sizeProp.has_value && (sizeProp.force_override || SizeCalculator.value == 0))
-            SizeCalculator.value = sizeProp.value;
-      }
-
-      void SetPrice(double price) {
-         Price = price;
-         RaisePropertyChanged(nameof(Price));
-      }
-
-      void SendOrder() {
-         bool priceValid = validatePrice();
-         bool quantityValid = SizeCalculator.validate();
-         if (priceValid && quantityValid && !HasErrors && !PropertiesVm.HasErrors) {
-            model.send_order(Security, PropertiesVm.create(side), Price, SizeCalculator.value!.Value).ignore();
-         }
-      }
-
-      void Recenter() =>
-         RequestRecenter?.Invoke();
-
-      void OnSecurityMd(SecurityMD smd) {
-         currentSecurityMd = smd;
-         var spec = TickUtils.multiply_tick_size(smd.security.regional_ticks(), 1);
-         security.set(smd.security);
-         priceTick.set(smd.security.regional_ticks());
-         priceDisplay.set(smd.price_display);
-         this.quantityDisplay.set(smd.quantity_display);
-         SizeCalculator.display_settings = smd.quantity_display;
-         ulong lastMdPollTime = 0;
-         mdPollTimer?.Stop();
-         mdPollTimer = new DispatcherTimer(TimeSpan.FromSeconds(0.1), DispatcherPriority.DataBind, PollMd, Dispatcher.CurrentDispatcher);
-
-         void PollMd(object? sender, EventArgs e) {
-            if (smd.md.get_depth(ref lastMdPollTime, ref nativeBids, ref nativeAsks, numPriceLevels)) {
-               if (double.IsNaN(priceLevels[0].Price)) {
-                  // Initialize prices and recenter
-                  if (!TryGetOrigin(out var origin)) {
-                     return;
-                  }
-                  double price = spec.tickdown(origin, numPriceRowsPerSide - 1);
-                  for (int i = priceLevels.Count - 1; i >= 0; i--) {
-                     priceLevels[i].Price = price;
-                     price = spec.tickup(price);
-                  }
-                  Recenter();
-               }
-
-               var hi = priceLevels[0].Price;
-               var lo = priceLevels[^1].Price;
-               Log.assert(hi.is_real() && lo.is_real() && hi > lo);
-               var wrappedPrices = priceLevels.WrapSelector(p => p.Price, null);
-               var rPriceComp = spec.price_comparer().reverse();
-
-               // Clear old prices
-               for (int i = 0; i < numPriceLevels; i++) {
-                  // asks
-                  var px = nativeAsks[i].price;
-                  if (!double.IsNaN(px) && px.is_within_range(lo, hi)) {
-                     int idx = CollectionHelper.binary_search(wrappedPrices, px, rPriceComp);
-                     if (idx < 0)
-                        idx = ~idx;
-                     priceLevels[idx].Quantity = 0;
-                  }
-                  // bids
-                  px = nativeBids[i].price;
-                  if (!double.IsNaN(px) && px.is_within_range(lo, hi)) {
-                     int idx = CollectionHelper.binary_search(wrappedPrices, px, rPriceComp);
-                     if (idx < 0)
-                        idx = ~idx;
-                     priceLevels[idx].Quantity = 0;
-                  }
-               }
-
-               // Set new prices
-               for (int i = 0; i < numPriceLevels; i++) {
-                  // asks
-                  var px = nativeAsks[i].price;
-                  if (!double.IsNaN(px) && px.is_within_range(lo, hi)) {
-                     int idx = CollectionHelper.binary_search(wrappedPrices, px, rPriceComp);
-                     if (idx < 0)
-                        idx = ~idx;
-                     priceLevels[idx].Quantity = nativeAsks[i].size;
-                     priceLevels[idx].Side = Side.Sell;
-                  }
-                  // bids
-                  px = nativeBids[i].price;
-                  if (!double.IsNaN(px) && px.is_within_range(lo, hi)) {
-                     int idx = CollectionHelper.binary_search(wrappedPrices, px, rPriceComp);
-                     if (idx < 0)
-                        idx = ~idx;
-                     priceLevels[idx].Quantity = nativeBids[i].size;
-                     priceLevels[idx].Side = Side.Buy;
-                  }
-               }
-            }
-         }
-
-         bool TryGetOrigin(out double origin) {
-            if (!nativeBids[0].is_nan) {
-               origin = nativeBids[0].price;
-            } else if (!nativeAsks[0].is_nan) {
-               origin = nativeAsks[0].price;
-            } else {
-               origin = double.NaN;
-               return false;
-            }
-            return true;
-         }
-      }
-   }
-
-   public event Action? RequestRecenter;
-
-   public SecurityImmutable Security => security.Value;
-   public string Title => title.Value;
-   public double Price { get; set; }
-   public user.settings.RegionalTickFactors PriceTick => priceTick.Value;
-   public TickDisplaySettingsImmutable PriceDisplay => priceDisplay.Value;
-   public TickDisplaySettingsImmutable QuantityDisplay => quantityDisplay.Value;
-   public CalculatorLogic<double> SizeCalculator { get; }
-   public Side Side {
-      get => side;
-      set => Set(ref side, value);
-   }
-
-   public OrderPropertiesEditorVM PropertiesVm { get; }
-   public ObservableCollection<OrderTicketLadderPriceLevelVM> PriceLevels => priceLevels;
-   public ICommand SendOrderCmd { get; }
-   public ICommand SetPriceCmd { get; }
-   public ICommand ShowLadderCmd { get; }
-   public ICommand RecenterCmd { get; }
-
-   bool validatePrice() {
-      if (!Price.is_real()) {
-         flash_error(nameof(Price)).ignore();
-         return false;
-      }
-      return true;
-   }
-}
-
-class OrderTicketLadderPriceLevelVM : ViewModelBase {
-   Color background = Colors.Transparent;
-   Color priceForeground = BTSColors.Control.normal_foreground;
-   Side side;
-   double price = double.NaN;
-   double quantity = double.NaN;
-
-   public OrderTicketLadderPriceLevelVM(Side side) {
-      this.side = side;
-      background = BTSColors.Ladder.price_column_background;
-      priceForeground = BTSColors.Ladder.price_column_foreground;
-   }
-
-   public Side Side {
-      get => side;
-      set => Set(ref side, value, dependent_props: nameof(QuantityForeground));
-   }
-   public double Price {
-      get => price;
-      set => Set(ref price, value);
-   }
-   public double Quantity {
-      get => quantity;
-      set => Set(ref quantity, value);
-   }
-
-   public Color Background {
-      get => background;
-      set => Set(ref background, value);
-   }
-   public Color PriceForeground {
-      get => priceForeground;
-      set => Set(ref priceForeground, value);
-   }
-   public Color QuantityForeground =>
-      side switch {
-         Side.Buy => BTSColors.Ladder.bid_column_foreground,
-         Side.Sell => BTSColors.Ladder.ask_column_foreground,
-         _ => BTSColors.UI.default_foreground,
-      };
-}

src/client/spark/Orders/OrderModificationLadderControl.xaml

@@ -1,415 +0,0 @@
-<UserControl
-  x:Class="mbts.ui.spark.OrderModificationLadderControl"
-  xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
-  xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
-  xmlns:api="clr-namespace:bts.api;assembly=Bts.Spark.Api"
-  xmlns:u="clr-namespace:mbts.ui"
-  xmlns:c="clr-namespace:mbts.ui.controls"
-  xmlns:cv="clr-namespace:mbts.ui.converters"
-  xmlns:usp="clr-namespace:mbts.ui.spark"
-  xmlns:uag="clr-namespace:mbts.ui.spark.algo"
-  xmlns:sys="clr-namespace:System;assembly=mscorlib"
-  u:AttachedCommands.MiddleClick="{Binding RecenterCmd}" >
-  <UserControl.Resources>
-    <ResourceDictionary>
-      <ResourceDictionary.MergedDictionaries>
-        <ResourceDictionary Source="/view/Colors.xaml"/>
-        <ResourceDictionary Source="/view/Resources/Converters.xaml"/>
-        <ResourceDictionary Source="/view/Controls/Order/OrderDetailsPopupControl.xaml"/>
-      </ResourceDictionary.MergedDictionaries>
-
-      <!-- Consts -->
-      <sys:Double x:Key="HorizontalSpacing">8.0</sys:Double>
-      <sys:Double x:Key="VerticalSpacing">8.0</sys:Double>
-      <sys:Double x:Key="ButtonHeight">22.0</sys:Double>
-
-      <!-- Converters -->
-      <cv:NullableDoubleConverter x:Key="TickDefaultPriceConverter" DefaultValue="0"/>
-      <cv:ValueMap x:Key="SideToBrushConverter" Default="{x:Static Brushes.Transparent}">
-        <cv:KeyValue Key="{u:Enum Buy, Type=api:Side}" Value="{u:BTSBrush MarketData.bid, Opacity=0.33}" />
-        <cv:KeyValue Key="{u:Enum Sell, Type=api:Side}" Value="{u:BTSBrush MarketData.ask, Opacity=0.33}" />
-      </cv:ValueMap>
-      <cv:ValueMap x:Key="SideToIsBuyConverter" Default="{u:False}">
-        <cv:KeyValue Key="{u:Enum Buy, Type=api:Side}" Value="{u:True}" />
-      </cv:ValueMap>
-      <cv:ValueMap x:Key="SideToIsSellConverter" Default="{u:False}">
-        <cv:KeyValue Key="{u:Enum Sell, Type=api:Side}" Value="{u:True}" />
-      </cv:ValueMap>
-      <cv:ValueMap x:Key="SideToBidVisibilityConverter" Default="{u:Visibility Collapsed}">
-        <cv:KeyValue Key="{u:Enum Buy, Type=api:Side}" Value="{u:Visibility Visible}" />
-      </cv:ValueMap>
-      <cv:ValueMap x:Key="SideToAskVisibilityConverter" Default="{u:Visibility Collapsed}">
-        <cv:KeyValue Key="{u:Enum Sell, Type=api:Side}" Value="{u:Visibility Visible}" />
-      </cv:ValueMap>
-
-      <!-- Styles -->
-      <Style TargetType="Button" BasedOn="{StaticResource DefaultButton}">
-        <Setter Property="Height" Value="{StaticResource ButtonHeight}"/>
-        <Setter Property="HorizontalAlignment" Value="Stretch"/>
-      </Style>
-      <Style TargetType="ComboBox" BasedOn="{StaticResource {x:Type ComboBox}}">
-        <Setter Property="Height" Value="{StaticResource ButtonHeight}"/>
-        <Style.Triggers>
-          <Trigger Property="usp:OrderModificationLadderControl.IsDirty" Value="True">
-            <Setter Property="BorderBrush" Value="{u:BTSColor UI.input_dirty_border}"/>
-          </Trigger>
-        </Style.Triggers>
-      </Style>
-      <Style TargetType="usp:PriceEntryControl" BasedOn="{StaticResource {x:Type usp:PriceEntryControl}}">
-        <Style.Triggers>
-          <Trigger  Property="usp:OrderModificationLadderControl.IsDirty" Value="True">
-            <Setter Property="BorderBrush" Value="{u:BTSColor UI.input_dirty_border}"/>
-          </Trigger>
-        </Style.Triggers>
-      </Style>
-      <Style TargetType="c:TickUpDownNullableUInt32" BasedOn="{StaticResource {x:Type c:TickUpDownNullableUInt32}}">
-        <Style.Triggers>
-          <Trigger Property="usp:OrderModificationLadderControl.IsDirty" Value="True">
-            <Setter Property="BorderBrush" Value="{u:BTSColor UI.input_dirty_border}"/>
-          </Trigger>
-        </Style.Triggers>
-      </Style>
-
-      <DataTemplate x:Key="PriceLevelItemTemplate">
-        <Button
-          Background="{Binding Background, Converter={x:Static cv:ColorToBrushConverter.Instance}}"
-          HorizontalAlignment="Stretch"
-          HorizontalContentAlignment="Stretch"
-          MinWidth="96"
-          Padding="4,2"
-          Command="{Binding DataContext.SetPriceCmd, RelativeSource={RelativeSource AncestorType=usp:OrderModificationLadderControl}}"
-          CommandParameter="{Binding Price}">
-          <Grid>
-            <Grid.ColumnDefinitions>
-              <ColumnDefinition Width="Auto" MinWidth="40" SharedSizeGroup="Orders" />
-              <ColumnDefinition Width="*" MinWidth="40" SharedSizeGroup="BidSize" />
-              <ColumnDefinition Width="Auto" MinWidth="56" SharedSizeGroup="Price" />
-              <ColumnDefinition Width="*" MinWidth="40" SharedSizeGroup="AskSize" />
-            </Grid.ColumnDefinitions>
-            <TextBlock
-              Grid.Column="1"
-              HorizontalAlignment="Right"
-              Foreground="{Binding QuantityForeground, Converter={x:Static cv:ColorToBrushConverter.Instance}}"
-              FontSize="{x:Static u:FontSizes.Small}"
-              Visibility="{Binding Side, Converter={StaticResource SideToBidVisibilityConverter}}">
-              <TextBlock.Text>
-                <MultiBinding Converter="{x:Static cv:TickDisplayConverter.Instance}">
-                  <Binding Path="Quantity" Mode="OneWay"/>
-                  <Binding Path="DataContext.QuantityDisplay" RelativeSource="{RelativeSource AncestorType=usp:OrderModificationLadderControl}"/>
-                </MultiBinding>
-              </TextBlock.Text>
-            </TextBlock>
-            <TextBlock
-              Grid.Column="2"
-              HorizontalAlignment="Center"
-              Foreground="{Binding PriceForeground, Converter={x:Static cv:ColorToBrushConverter.Instance}}"
-              FontSize="{x:Static u:FontSizes.Small}">
-              <TextBlock.Text>
-                <MultiBinding Converter="{x:Static cv:TickDisplayConverter.Instance}">
-                  <Binding Path="Price" Mode="OneWay"/>
-                  <Binding Path="DataContext.PriceDisplay" RelativeSource="{RelativeSource AncestorType=usp:OrderModificationLadderControl}"/>
-                </MultiBinding>
-              </TextBlock.Text>
-            </TextBlock>
-            <TextBlock
-              Grid.Column="3"
-              HorizontalAlignment="Left"
-              Foreground="{Binding QuantityForeground, Converter={x:Static cv:ColorToBrushConverter.Instance}}"
-              FontSize="{x:Static u:FontSizes.Small}"
-              Visibility="{Binding Side, Converter={StaticResource SideToAskVisibilityConverter}}">
-              <TextBlock.Text>
-                <MultiBinding Converter="{x:Static cv:TickDisplayConverter.Instance}">
-                  <Binding Path="Quantity" Mode="OneWay"/>
-                  <Binding Path="DataContext.QuantityDisplay" RelativeSource="{RelativeSource AncestorType=usp:OrderModificationLadderControl}"/>
-                </MultiBinding>
-              </TextBlock.Text>
-            </TextBlock>
-          </Grid>
-        </Button>
-      </DataTemplate>
-
-      <ControlTemplate x:Key="BuyRadioButtonTemplate" TargetType="RadioButton">
-        <Grid>
-          <VisualStateManager.VisualStateGroups>
-            <VisualStateGroup x:Name="CommonStates">
-              <VisualState x:Name="Normal" />
-              <VisualState x:Name="MouseOver">
-                <Storyboard>
-                  <DoubleAnimation Storyboard.TargetName="hoverOverlay" Storyboard.TargetProperty="Opacity" To="0.3" BeginTime="0" Duration="0" />
-                </Storyboard>
-              </VisualState>
-              <VisualState x:Name="Pressed" />
-              <VisualState x:Name="Disabled">
-                <Storyboard>
-                  <ColorAnimation Storyboard.TargetName="background" Storyboard.TargetProperty="Background.Color" To="#555" Duration="0" />
-                </Storyboard>
-              </VisualState>
-            </VisualStateGroup>
-            <VisualStateGroup x:Name="CheckStates">
-              <VisualState x:Name="Checked">
-                <Storyboard>
-                  <ColorAnimation Storyboard.TargetName="background" Storyboard.TargetProperty="Background.Color" To="{u:BTSColor MarketData.bid}" BeginTime="0" Duration="0"/>
-                  <ColorAnimation Storyboard.TargetName="background" Storyboard.TargetProperty="BorderBrush.Color" To="{u:BTSColor MarketData.bid}" BeginTime="0" Duration="0"/>
-                </Storyboard>
-              </VisualState>
-              <VisualState x:Name="Unchecked" />
-              <VisualState x:Name="Indeterminate" />
-            </VisualStateGroup>
-          </VisualStateManager.VisualStateGroups>
-          <Border
-            x:Name="background"
-            BorderThickness="{TemplateBinding BorderThickness}"
-            VerticalAlignment="{TemplateBinding VerticalAlignment}"
-            HorizontalAlignment="{TemplateBinding HorizontalAlignment}"
-            Padding="{TemplateBinding Padding}"
-            CornerRadius="2">
-            <Border.Background>
-              <SolidColorBrush Color="Transparent" />
-            </Border.Background>
-            <Border.BorderBrush>
-              <SolidColorBrush Color="Black"/>
-            </Border.BorderBrush>
-          </Border>
-          <ContentPresenter
-            Name="presenter"
-            HorizontalAlignment="{TemplateBinding HorizontalContentAlignment}"
-            VerticalAlignment="{TemplateBinding HorizontalContentAlignment}" />
-          <Border Name="hoverOverlay" HorizontalAlignment="Stretch" VerticalAlignment="Stretch" Background="Black" Opacity="0" IsHitTestVisible="False" />
-        </Grid>
-      </ControlTemplate>
-
-      <ControlTemplate x:Key="SellRadioButtonTemplate" TargetType="RadioButton">
-        <Grid>
-          <VisualStateManager.VisualStateGroups>
-            <VisualStateGroup x:Name="CommonStates">
-              <VisualState x:Name="Normal" />
-              <VisualState x:Name="MouseOver">
-                <Storyboard>
-                  <DoubleAnimation Storyboard.TargetName="hoverOverlay" Storyboard.TargetProperty="Opacity" To="0.3" BeginTime="0" Duration="0" />
-                </Storyboard>
-              </VisualState>
-              <VisualState x:Name="Pressed" />
-              <VisualState x:Name="Disabled">
-                <Storyboard>
-                  <ColorAnimation Storyboard.TargetName="background" Storyboard.TargetProperty="Background.Color" To="#555" Duration="0" />
-                </Storyboard>
-              </VisualState>
-            </VisualStateGroup>
-            <VisualStateGroup x:Name="CheckStates">
-              <VisualState x:Name="Checked">
-                <Storyboard>
-                  <ColorAnimation Storyboard.TargetName="background" Storyboard.TargetProperty="Background.Color" To="{u:BTSColor MarketData.ask}" BeginTime="0" Duration="0"/>
-                  <ColorAnimation Storyboard.TargetName="background" Storyboard.TargetProperty="BorderBrush.Color" To="{u:BTSColor MarketData.ask}" BeginTime="0" Duration="0"/>
-                </Storyboard>
-              </VisualState>
-              <VisualState x:Name="Unchecked" />
-              <VisualState x:Name="Indeterminate" />
-            </VisualStateGroup>
-          </VisualStateManager.VisualStateGroups>
-          <Border
-            x:Name="background"
-            BorderThickness="{TemplateBinding BorderThickness}"
-            VerticalAlignment="{TemplateBinding VerticalAlignment}"
-            HorizontalAlignment="{TemplateBinding HorizontalAlignment}"
-            Padding="{TemplateBinding Padding}"
-            CornerRadius="2">
-            <Border.Background>
-              <SolidColorBrush Color="Transparent" />
-            </Border.Background>
-            <Border.BorderBrush>
-              <SolidColorBrush Color="Black"/>
-            </Border.BorderBrush>
-          </Border>
-          <ContentPresenter
-            Name="presenter"
-            HorizontalAlignment="{TemplateBinding HorizontalContentAlignment}"
-            VerticalAlignment="{TemplateBinding VerticalContentAlignment}" />
-          <Border Name="hoverOverlay" HorizontalAlignment="Stretch" VerticalAlignment="Stretch" Background="Black" Opacity="0" IsHitTestVisible="False" />
-        </Grid>
-      </ControlTemplate>
-
-      <Style x:Key="BuyRadioButtonStyle" TargetType="RadioButton">
-        <Setter Property="Padding" Value="2" />
-        <Setter Property="HorizontalContentAlignment" Value="Center" />
-        <Setter Property="VerticalContentAlignment" Value="Center" />
-        <Setter Property="Foreground" Value="{u:BTSBrush UI.default_text}" />
-        <Setter Property="Template" Value="{StaticResource BuyRadioButtonTemplate}" />
-      </Style>
-
-      <Style x:Key="SellRadioButtonStyle" TargetType="RadioButton">
-        <Setter Property="Padding" Value="2" />
-        <Setter Property="HorizontalContentAlignment" Value="Center" />
-        <Setter Property="VerticalContentAlignment" Value="Center" />
-        <Setter Property="Foreground" Value="{u:BTSBrush UI.default_text}" />
-        <Setter Property="Template" Value="{StaticResource SellRadioButtonTemplate}" />
-      </Style>
-
-    </ResourceDictionary>
-  </UserControl.Resources>
-
-  <Grid>
-    <Grid.ColumnDefinitions>
-      <ColumnDefinition />
-      <ColumnDefinition Width="{Binding Source={StaticResource HorizontalSpacing}}" />
-      <ColumnDefinition />
-    </Grid.ColumnDefinitions>
-
-    <!-- Ladder (market depth) -->
-    <Grid Grid.Column="2" Grid.IsSharedSizeScope="True">
-      <Grid.RowDefinitions>
-        <RowDefinition Height="Auto" />
-        <RowDefinition Height="Auto" />
-      </Grid.RowDefinitions>
-      <Grid
-        Grid.Row="0"
-        Background="{u:BTSBrush header}"
-        TextElement.Foreground="{u:BTSBrush header_text}"
-        TextElement.FontSize="{x:Static u:FontSizes.Small}"
-        TextElement.FontWeight="SemiBold">
-        <Grid.ColumnDefinitions>
-          <ColumnDefinition Width="Auto" MinWidth="40" SharedSizeGroup="Orders" />
-          <ColumnDefinition Width="*" MinWidth="40" SharedSizeGroup="BidSize" />
-          <ColumnDefinition Width="Auto" MinWidth="56" SharedSizeGroup="Price" />
-          <ColumnDefinition Width="*" MinWidth="40" SharedSizeGroup="AskSize" />
-        </Grid.ColumnDefinitions>
-        <TextBlock Grid.Column="0" HorizontalAlignment="Center" Text="Orders" />
-        <TextBlock Grid.Column="1" HorizontalAlignment="Center" Text="Bid" />
-        <TextBlock Grid.Column="2" HorizontalAlignment="Center" Text="Price" />
-        <TextBlock Grid.Column="3" HorizontalAlignment="Center" Text="Ask" />
-      </Grid>
-      <ScrollViewer Grid.Row="1" x:Name="priceLevelsScrollViewer" MaxHeight="240">
-        <ItemsControl ItemsSource="{Binding PriceLevels}" ItemTemplate="{StaticResource PriceLevelItemTemplate}" />
-      </ScrollViewer>
-    </Grid>
-
-    <!-- Order Inputs -->
-    <Grid Grid.Column="0">
-      <Grid.RowDefinitions>
-        <RowDefinition Height="Auto" />
-        <RowDefinition Height="{Binding Source={StaticResource VerticalSpacing}}" />
-        <RowDefinition Height="Auto" />
-        <RowDefinition Height="{Binding Source={StaticResource VerticalSpacing}}" />
-        <RowDefinition Height="Auto" />
-        <RowDefinition Height="{Binding Source={StaticResource VerticalSpacing}}" />
-        <RowDefinition Height="Auto" />
-        <RowDefinition Height="{Binding Source={StaticResource VerticalSpacing}}" />
-        <RowDefinition Height="Auto" />
-        <RowDefinition Height="{Binding Source={StaticResource VerticalSpacing}}" />
-        <RowDefinition Height="Auto" />
-        <RowDefinition Height="{Binding Source={StaticResource VerticalSpacing}}" />
-        <RowDefinition Height="*" />
-        <RowDefinition Height="{Binding Source={StaticResource VerticalSpacing}}" />
-        <RowDefinition Height="Auto" />
-      </Grid.RowDefinitions>
-
-      <!-- Buy/Sell toggle -->
-      <Grid HorizontalAlignment="Stretch" VerticalAlignment="Top">
-        <Grid.ColumnDefinitions>
-          <ColumnDefinition Width="1*" MinWidth="72" />
-          <ColumnDefinition Width="1*" MinWidth="72" />
-        </Grid.ColumnDefinitions>
-        <RadioButton
-          Grid.Column="0"
-          GroupName="Side"
-          IsChecked="{Binding Side, Converter={StaticResource SideToIsBuyConverter}}"
-          Style="{StaticResource BuyRadioButtonStyle}">
-          <TextBlock FontWeight="Bold" Foreground="{u:BTSBrush UI.default_text}">
-            Buy
-          </TextBlock>
-        </RadioButton>
-        <RadioButton
-          Grid.Column="1"
-          GroupName="Side"
-          IsChecked="{Binding Side, Converter={StaticResource SideToIsSellConverter}}"
-          Style="{StaticResource SellRadioButtonStyle}">
-          <TextBlock FontWeight="Bold" Foreground="{u:BTSBrush UI.default_text}">
-            Sell
-          </TextBlock>
-        </RadioButton>
-      </Grid>
-
-      <usp:PriceEntryControl
-        Grid.Row="2"
-        usp:OrderModificationControl.IsDirty="{Binding PriceDirty, FallbackValue={x:Null}}"
-        RegionalTick="{Binding PriceTick}"
-        Focusable="False"
-        TickDefaultValue="{Binding CurrentPrice, Converter={StaticResource TickDefaultPriceConverter}, FallbackValue={u:Double 0}}"
-        ToolTip="Price"
-        Value="{Binding Price}" />
-
-      <c:TickUpDownNullableDouble
-        Grid.Row="4"
-        usp:OrderModificationControl.IsDirty="{Binding QuantityDirty, FallbackValue={x:Null}}"
-        CommitOnEnter="True"
-        Focusable="False"
-        Increment="{Binding SizeCalculator.unit_quantity}"
-        Minimum="0"
-        TickDisplaySettings="{Binding SizeCalculator.display_settings}"
-        ToolTip="Quantity"
-        Value="{Binding SizeCalculator.display_value}" />
-
-      <uag:OrderDetailsPopupControl
-        Grid.Row="6"
-        DataContext="{Binding PropertiesVm}"
-        usp:OrderModificationControl.IsDirty="{Binding DataContext.order_type_dirty, RelativeSource={RelativeSource AncestorType=usp:OrderModificationControl}, FallbackValue={u:False}}"
-        HasEditableProperties="{Binding can_expand, FallbackValue={u:False}}"
-        IsAlgoTemplateToolBarReadOnly="True"
-        IsEnabled="{Binding DataContext.can_change_order_type, RelativeSource={RelativeSource AncestorType=usp:OrderModificationControl}, FallbackValue={u:True}}"
-        IsOpen="{Binding is_expanded, FallbackValue={u:False}}"
-        ItemsSource="{Binding algo_definitions_view}"
-        SelectedItem="{Binding selected_algo_definition}"
-        ToolTip="Order/Algo Type">
-        <uag:OrderDetailsPopupControl.Style>
-          <Style TargetType="uag:OrderDetailsPopupControl" BasedOn="{StaticResource {x:Type uag:OrderDetailsPopupControl}}">
-            <Style.Triggers>
-              <Trigger  Property="usp:OrderModificationControl.IsDirty" Value="True">
-                <Setter Property="BorderBrush" Value="{u:BTSColor UI.input_dirty_border}"/>
-              </Trigger>
-            </Style.Triggers>
-          </Style>
-        </uag:OrderDetailsPopupControl.Style>
-      </uag:OrderDetailsPopupControl>
-
-      <ComboBox
-        Grid.Row="8"
-        usp:OrderModificationControl.IsDirty="{Binding AccountDirty, FallbackValue={u:False}}"
-        IsEnabled="{Binding CanChangeAccount, FallbackValue={u:True}}"
-        ItemsSource="{Binding PropertiesVm.accounts}"
-        SelectedItem="{Binding PropertiesVm.selected_account}"
-        ToolTip="Account" />
-
-      <ComboBox
-        Grid.Row="10"
-        usp:OrderModificationControl.IsDirty="{Binding TimeInForceDirty, FallbackValue={u:False}}"
-        IsEnabled="{Binding CanChangeTimeInForce, FallbackValue={u:True}}"
-        ItemsSource="{Binding PropertiesVm.time_in_forces}" 
-        SelectedItem="{Binding PropertiesVm.selected_time_in_force}"
-        ToolTip="Time In Force" />
-
-      <Button
-        Grid.Row="12"
-        Height="NaN"
-        HorizontalAlignment="Stretch"
-        VerticalAlignment="Stretch"
-        HorizontalContentAlignment="Center"
-        VerticalContentAlignment="Center"
-        Background="Green"
-        Foreground="{u:BTSBrush UI.default_text}"
-        FontSize="{x:Static u:FontSizes.Largest}"
-        FontWeight="Bold"
-        Command="{Binding SendOrderCmd}"
-        ToolTip="{Binding Side, StringFormat='Send {0} Order'}">
-        Send
-      </Button>
-
-      <TextBlock
-        Grid.Row="14"
-        HorizontalAlignment="Center"
-        FontSize="{x:Static u:FontSizes.Small}">
-        <Hyperlink
-          Command="{Binding ShowLadderCmd}"
-          ToolTip="Open the original ladder for this security">Open full ladder</Hyperlink>
-      </TextBlock>
-    </Grid>
-  </Grid>
-</UserControl>

src/client/spark/Orders/OrderModificationLadderControl.xaml.cs

@@ -1,51 +0,0 @@
-#nullable enable
-
-using System;
-using System.Windows;
-using System.Windows.Controls;
-using System.Windows.Media.Animation;
-
-namespace mbts.ui.spark {
-   interface IOrderModificationLadderVM {
-      event Action? RequestRecenter;
-   }
-
-   /// <summary>
-   /// Interaction logic for OrderModificationLadderControl.xaml
-   /// </summary>
-   public partial class OrderModificationLadderControl : UserControl {
-      public static readonly DependencyProperty IsDirtyProperty = DependencyProperty.RegisterAttached(
-         "IsDirty", typeof(bool?), typeof(OrderModificationLadderControl), new PropertyMetadata(null));
-
-      public static bool? GetIsDirty(DependencyObject d) => d.GetValue(IsDirtyProperty) as bool?;
-      public static void SetIsDirty(DependencyObject d, bool? value) => d.SetValue(IsDirtyProperty, value);
-
-      public OrderModificationLadderControl() {
-         InitializeComponent();
-         DataContextChanged += onDataContextChanged;
-      }
-
-      void onDataContextChanged(object sender, DependencyPropertyChangedEventArgs e) {
-         if (e.OldValue is IOrderModificationLadderVM oldVm) {
-            oldVm.RequestRecenter -= recenter;
-         }
-         if (e.NewValue is IOrderModificationLadderVM newVm) {
-            newVm.RequestRecenter += recenter;
-         }
-      }
-
-      void recenter() {
-         var targetOffset = priceLevelsScrollViewer.ScrollableHeight / 2;
-         DoubleAnimation anim = new() {
-            From = priceLevelsScrollViewer.VerticalOffset,
-            To = targetOffset,
-            Duration = TimeSpan.FromSeconds(0.15)
-         };
-         Storyboard sb = new();
-         sb.Children.Add(anim);
-         Storyboard.SetTarget(anim, priceLevelsScrollViewer);
-         Storyboard.SetTargetProperty(anim, new PropertyPath(Behavior.AnimatedScrollViewerVerticalOffsetProperty));
-         sb.Begin();
-      }
-   }
-}

src/client/spark/Orders/OrderTicketLadder.xaml

@@ -1,120 +0,0 @@
-<Window
-  x:Class="mbts.ui.spark.OrderTicketLadder"
-  xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
-  xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
-  xmlns:msys="clr-namespace:bts.api;assembly=Bts.Spark.Api"
-  xmlns:u="clr-namespace:mbts.ui"
-  xmlns:cv="clr-namespace:mbts.ui.converters"
-  xmlns:usp="clr-namespace:mbts.ui.spark"
-  xmlns:sys="clr-namespace:System;assembly=mscorlib"
-  u:WindowStyle.SizeToContent="WidthAndHeight"
-  u:WindowStyle.TitleContent="{Binding Title, Mode=OneWay}"
-  BorderBrush="#00C000"
-  BorderThickness="1"
-  Style="{StaticResource BTSWindowTitleTemplate}">
-  <Window.Resources>
-    <ResourceDictionary>
-      <ResourceDictionary.MergedDictionaries>
-        <ResourceDictionary Source="/view/Resources/Converters.xaml"/>
-      </ResourceDictionary.MergedDictionaries>
-      <sys:Double x:Key="HorizontalSpacing">8.0</sys:Double>
-      <sys:Double x:Key="VerticalSpacing">4.0</sys:Double>
-      <sys:Double x:Key="ButtonHeight">22.0</sys:Double>
-
-      <Style x:Key="SellButton" TargetType="Button" BasedOn="{StaticResource {x:Type Button}}">
-        <Setter Property="Background" Value="{u:BTSColor MarketData.ask_dark}" />
-        <Setter Property="Foreground" Value="White" />
-        <Setter Property="MinHeight" Value="40" />
-        <Setter Property="FontWeight" Value="Bold" />
-        <Setter Property="HorizontalAlignment" Value="Stretch" />
-        <Setter Property="VerticalAlignment" Value="Stretch" />
-        <Setter Property="Content">
-          <Setter.Value>
-            <TextBlock VerticalAlignment="Center" Text="Sell" />
-          </Setter.Value>
-        </Setter>
-        <Setter Property="Template">
-          <Setter.Value>
-            <ControlTemplate TargetType="Button">
-              <Border 
-                x:Name="Border"
-                CornerRadius="0" 
-                BorderThickness="1"
-                BorderBrush="{u:BTSColor UI.Button.normal_border}"
-                Background="{u:BTSColor MarketData.ask}"
-                Padding="2"
-                Height="Auto"
-                SnapsToDevicePixels="True">
-                  <ContentPresenter 
-                    HorizontalAlignment="Center"
-                    VerticalAlignment="Center"
-                    RecognizesAccessKey="True"
-                    Name="button_content" 
-                  />
-                </Border>
-              <ControlTemplate.Triggers>
-                <Trigger Property="IsMouseOver" Value="True">
-                  <Setter TargetName="Border" Property="Background" Value="#99c00000" />
-                  <Setter TargetName="Border" Property="BorderBrush" Value="{u:BTSColor UI.Button.mouseover_border}" />
-                </Trigger>
-              </ControlTemplate.Triggers>
-            </ControlTemplate>
-          </Setter.Value>
-        </Setter>
-      </Style>
-
-      <Style x:Key="BuyButton" TargetType="Button" BasedOn="{StaticResource {x:Type Button}}">
-        <Setter Property="Background" Value="{u:BTSColor MarketData.bid_dark}" />
-        <Setter Property="Foreground" Value="White" />
-        <Setter Property="MinHeight" Value="40" />
-        <Setter Property="FontWeight" Value="Bold" />
-        <Setter Property="HorizontalAlignment" Value="Stretch" />
-        <Setter Property="VerticalAlignment" Value="Stretch" />
-        <Setter Property="Content">
-          <Setter.Value>
-            <TextBlock VerticalAlignment="Center" Text="Buy" />
-          </Setter.Value>
-        </Setter>
-        <Setter Property="Template">
-          <Setter.Value>
-            <ControlTemplate TargetType="Button">
-              <Border 
-                x:Name="Border"
-                CornerRadius="0" 
-                BorderThickness="1"
-                BorderBrush="{u:BTSColor UI.Button.normal_border}"
-                Background="{u:BTSColor MarketData.bid}"
-                Padding="2"
-                Height="Auto"
-                SnapsToDevicePixels="True">
-                <ContentPresenter 
-                  HorizontalAlignment="Center"
-                  VerticalAlignment="Center"
-                  RecognizesAccessKey="True"
-                  Name="button_content" 
-                />
-              </Border>
-              <ControlTemplate.Triggers>
-                <Trigger Property="IsMouseOver" Value="True">
-                  <Setter TargetName="Border" Property="Background" Value="#99167ed3" />
-                  <Setter TargetName="Border" Property="BorderBrush" Value="{u:BTSColor UI.Button.mouseover_border}" />
-                </Trigger>
-              </ControlTemplate.Triggers>
-            </ControlTemplate>
-          </Setter.Value>
-        </Setter>
-      </Style>
-    </ResourceDictionary>
-  </Window.Resources>
-  <Window.InputBindings>
-    <KeyBinding Key="Esc" Command="{x:Static SystemCommands.CloseWindowCommand}"/>
-  </Window.InputBindings>
-
-  <usp:OrderModificationLadderControl Margin="10,4" />
-
-  <u:WindowStyle.TitleContentTemplate>
-    <DataTemplate>
-      <TextBlock HorizontalAlignment="Center" FontWeight="Bold" Text="{Binding}" Style="{StaticResource TextBlock.Dark}" />
-    </DataTemplate>
-  </u:WindowStyle.TitleContentTemplate>
-</Window>

src/client/spark/Orders/OrderTicketLadder.xaml.cs

@@ -1,15 +0,0 @@
-using mbts.ui.utility;
-
-using System.Windows;
-
-namespace mbts.ui.spark {
-   /// <summary>
-   /// Interaction logic for OrderTicketLadder.xaml
-   /// </summary>
-   public partial class OrderTicketLadder : Window {
-      public OrderTicketLadder() {
-         InitializeComponent();
-         WindowUtilities.listen_for_close_requests(this);
-      }
-   }
-}

src/client/spark/Themes/generic.xaml

@@ -90,26 +90,9 @@
     <Setter Property="CaretBrush" Value="{u:BTSBrush UI.TextBox.normal_caret}" />
   </Style>
 
-  <Style x:Key="TextBox.Dark" TargetType="TextBox" BasedOn="{StaticResource {x:Type TextBox}}" />
-
-  <Style TargetType="DatePicker">
-    <Setter Property="Background" Value="{u:BTSBrush UI.TextBox.normal_background}" />
-    <Setter Property="Foreground" Value="{u:BTSBrush UI.TextBox.normal_foreground}" />
-    <Setter Property="BorderBrush" Value="Transparent" />
-    <Setter Property="BorderThickness" Value="0" />
-    <Setter Property="FocusVisualStyle" Value="{x:Null}" />
-  </Style>
-
-  <Style TargetType="DatePickerTextBox">
-    <Setter Property="Background" Value="{u:BTSBrush UI.TextBox.normal_background}" />
-    <Setter Property="Foreground" Value="{u:BTSBrush UI.TextBox.normal_foreground}" />
-    <Setter Property="BorderBrush" Value="Transparent" />
-    <Setter Property="BorderThickness" Value="0" />
-    <Setter Property="CaretBrush" Value="{u:BTSBrush UI.TextBox.normal_caret}" />
-    <Setter Property="FocusVisualStyle" Value="{x:Null}" />
-  </Style>
-
-  <Style x:Key="ListBoxItem.Dark" TargetType="ListBoxItem">
+  <Style x:Key="TextBox.Dark" TargetType="TextBox" BasedOn="{StaticResource {x:Type TextBox}}" />
+
+  <Style x:Key="ListBoxItem.Dark" TargetType="ListBoxItem">
     <Setter Property="Background" Value="{StaticResource ListBoxItem.Background}" />
     <Setter Property="Foreground" Value="{StaticResource ListBoxItem.Foreground}" />
     <Setter Property="BorderThickness" Value="0" />

src/client/spark/core/FontsAndColors/BTSColorExtension.cs

@@ -1,119 +1,117 @@
-#nullable enable
-namespace mbts.ui {
-   using bts.utils;
-
-   using Spark.Core.Reflection;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   using System.Reflection;
-   using System.Windows;
-   using System.Windows.Markup;
-   using System.Windows.Media;
-
-   [MarkupExtensionReturnType]
-   class BTSColorExtension : MarkupExtension {
-      static readonly Dictionary<string, Func<BTSColor>>
-         _color_map = _normalized_color_props(typeof(BTSColors), null).ToDictionary();
-
-      public BTSColorExtension() { }
-
-      public BTSColorExtension(string? name) =>
-         Name = name;
-
-      public BTSColorExtension(string? name, string? mixture) {
-         Name = name;
-         Mixture = mixture;
-      }
-
-      [ConstructorArgument("name")]
-      public string? Name { get; set; }
-
-      [ConstructorArgument("mixture")]
-      public string? Mixture { get; set; }
-
-      public ColorMixtureMode MixtureMode { get; set; } = ColorMixtureMode.Default;
-
-      public Type? Type { get; set; }
-
-      public double Opacity { get; set; } = double.NaN;
-
-      public static BTSColor get_color(string name) =>
-         _color_map.TryGetValue(name, out var color) ?
-            color() : throw new ColorNotFoundException(name);
-
-      public override object? ProvideValue(IServiceProvider service_provider) {
-         if (service_provider is null) {
-            return null;
-         }
-         if (Name is null) {
-            throw new InvalidOperationException("Name is null");
-         }
-         if (!Opacity.is_nan() && !Opacity.is_within_range(0, 1)) {
-            throw new InvalidOperationException("Opacity (pct) must between 0 and 1 (if not NaN)");
-         }
-
-         if (Name.Contains("Launcher.Button")) {
-         }
-         var pvt = service_provider.GetValueTargetProvider() ??
-            throw new InvalidOperationException("IProvideValueTarget is null");
-         var tgt_type = Type ?? _deduce_target_type(pvt);
-         var names = Name.Split(',');
-         var mixtures = Mixture?.Split(',');
-         var color = _get_color_blend(names, mixtures, MixtureMode);
-         if (!Opacity.is_nan()) {
-            color = color.with((byte)(Opacity * byte.MaxValue));
-         }
-         return tgt_type == typeof(Color) || tgt_type == typeof(Color?)
-            ? (Color)color
-            : (Brush)color;
-      }
-
-      static Type _deduce_target_type(IProvideValueTarget pvt) {
-         if (pvt.TargetProperty is null) {
-            return typeof(Color);
-         }
-         if (pvt.TargetProperty is DependencyProperty dp) {
-            return dp.PropertyType ?? throw new InvalidOperationException();
-         }
-         if (pvt.TargetObject is Setter setter) {
-            return setter.Property?.PropertyType ?? typeof(Brush);
-         }
-         return typeof(Brush);
-      }
-
-      /// <summary>
-      /// Creates a (potentially weighted) blend of colors
-      /// </summary>
-      static BTSColor _get_color_blend(string[] names, string[]? mixture, ColorMixtureMode mode) {
-         if (mixture is null || mixture.Length == 0) {
-            // create a uniform blend
-            return names.Select(get_color).blend(mode);
-         }
-         if (names.Length != mixture.Length) {
-            throw new InvalidOperationException(
-               "Names and Mixture must provide same number of elements");
-         }
-         return names.Select(get_color).Zip(mixture.Select(double.Parse)).blend();
-      }
-
-      static IEnumerable<(string, Func<BTSColor>)> _normalized_color_props(
-         Type type, string? prefix) {
-         var bts_colors = ReflectionHelper.static_property_getters<BTSColor>(type)
-            .Select(tup => (full_name(tup.name), tup.get));
-         var media_colors = ReflectionHelper.static_property_getters<Color>(type)
-            .Select(tup => (full_name(tup.name), new Func<BTSColor>(() => tup.get())));
-         var nested_types = type.GetNestedTypes(BindingFlags.Public | BindingFlags.Static);
-         var nested_colors = nested_types.SelectMany(
-            t => _normalized_color_props(t, full_name(t.Name))).ToList();
-
-         return bts_colors
-            .Concat(media_colors)
-            .Concat(nested_colors);
-
-         string full_name(string prop) =>
-            prefix is null ? prop : $"{prefix}.{prop}";
-      }
-   }
-}
+#nullable enable
+namespace mbts.ui {
+   using bts.utils;
+
+   using Spark.Core.Reflection;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   using System.Reflection;
+   using System.Windows;
+   using System.Windows.Markup;
+   using System.Windows.Media;
+
+   [MarkupExtensionReturnType]
+   class BTSColorExtension : MarkupExtension {
+      static readonly Dictionary<string, Func<BTSColor>>
+         _color_map = _normalized_color_props(typeof(BTSColors), null).ToDictionary();
+
+      public BTSColorExtension() { }
+
+      public BTSColorExtension(string? name) =>
+         Name = name;
+
+      public BTSColorExtension(string? name, string? mixture) {
+         Name = name;
+         Mixture = mixture;
+      }
+
+      [ConstructorArgument("name")]
+      public string? Name { get; set; }
+
+      [ConstructorArgument("mixture")]
+      public string? Mixture { get; set; }
+
+      public ColorMixtureMode MixtureMode { get; set; } = ColorMixtureMode.Default;
+
+      public Type? Type { get; set; }
+
+      public double Opacity { get; set; } = double.NaN;
+
+      public static BTSColor get_color(string name) =>
+         _color_map.TryGetValue(name, out var color) ?
+            color() : throw new ColorNotFoundException(name);
+
+      public override object? ProvideValue(IServiceProvider service_provider) {
+         if (service_provider is null) {
+            return null;
+         }
+         if (Name is null) {
+            throw new InvalidOperationException("Name is null");
+         }
+         if (!Opacity.is_nan() && !Opacity.is_within_range(0, 1)) {
+            throw new InvalidOperationException("Opacity (pct) must between 0 and 1 (if not NaN)");
+         }
+
+         if (Name.Contains("Launcher.Button")) {
+         }
+         var pvt = service_provider.GetValueTargetProvider() ??
+            throw new InvalidOperationException("IProvideValueTarget is null");
+         var tgt_type = Type ?? _deduce_target_type(pvt);
+         var names = Name.Split(',');
+         var mixtures = Mixture?.Split(',');
+         var color = _get_color_blend(names, mixtures, MixtureMode);
+         if (!Opacity.is_nan()) {
+            color = color.with((byte)(Opacity * byte.MaxValue));
+         }
+         return tgt_type == typeof(Color) ? (Color)color : (Brush)color;
+      }
+
+      static Type _deduce_target_type(IProvideValueTarget pvt) {
+         if (pvt.TargetProperty is null) {
+            return typeof(Color);
+         }
+         if (pvt.TargetProperty is DependencyProperty dp) {
+            return dp.PropertyType ?? throw new InvalidOperationException();
+         }
+         if (pvt.TargetObject is Setter setter) {
+            return setter.Property?.PropertyType ?? typeof(Brush);
+         }
+         return typeof(Brush);
+      }
+
+      /// <summary>
+      /// Creates a (potentially weighted) blend of colors
+      /// </summary>
+      static BTSColor _get_color_blend(string[] names, string[]? mixture, ColorMixtureMode mode) {
+         if (mixture is null || mixture.Length == 0) {
+            // create a uniform blend
+            return names.Select(get_color).blend(mode);
+         }
+         if (names.Length != mixture.Length) {
+            throw new InvalidOperationException(
+               "Names and Mixture must provide same number of elements");
+         }
+         return names.Select(get_color).Zip(mixture.Select(double.Parse)).blend();
+      }
+
+      static IEnumerable<(string, Func<BTSColor>)> _normalized_color_props(
+         Type type, string? prefix) {
+         var bts_colors = ReflectionHelper.static_property_getters<BTSColor>(type)
+            .Select(tup => (full_name(tup.name), tup.get));
+         var media_colors = ReflectionHelper.static_property_getters<Color>(type)
+            .Select(tup => (full_name(tup.name), new Func<BTSColor>(() => tup.get())));
+         var nested_types = type.GetNestedTypes(BindingFlags.Public | BindingFlags.Static);
+         var nested_colors = nested_types.SelectMany(
+            t => _normalized_color_props(t, full_name(t.Name))).ToList();
+
+         return bts_colors
+            .Concat(media_colors)
+            .Concat(nested_colors);
+
+         string full_name(string prop) =>
+            prefix is null ? prop : $"{prefix}.{prop}";
+      }
+   }
+}

src/client/spark/view/Controls/AutoHideScrollBar/AutoHideScrollBar.xaml

@@ -380,7 +380,7 @@
                 <VisualState Name="Normal"/>
                 <VisualState Name="MouseOver">
                   <Storyboard>
-                    <ColorAnimation Storyboard.TargetProperty="Background.Color" BeginTime="0" Duration="0" To="{u:BTSColor UI.ScrollBar.thumb_mouse_over}"/>
+                    <ColorAnimation Storyboard.TargetProperty="Background.Color" BeginTime="0" Duration="0" To="{StaticResource u:BTSColor UI.ScrollBar.thumb_mouse_over}"/>
                   </Storyboard>
                 </VisualState>
                 <VisualState Name="Pressed">

src/client/spark/view/Controls/Order/OrderAuxiliaryPopupResources.xaml.cs

@@ -1,49 +0,0 @@
-#nullable enable
-using mbts.spark;
-using mbts.ui.shell;
-
-using System.Windows;
-using System.Windows.Media;
-
-namespace mbts.ui.spark {
-   partial class OrderAuxiliaryPopupResources : ResourceDictionary {
-      public OrderAuxiliaryPopupResources() =>
-         InitializeComponent();
-
-      public static string PopupChildStyleKey => nameof(PopupChildStyleKey);
-   }
-
-   class OrderAuxiliaryPopupHeader : ViewModelBase {
-      string?
-         _title,
-         _context;
-      Color _title_foreground;
-      bool _keep_open;
-
-      public Color title_foreground { get => _title_foreground; set => Set(ref _title_foreground, value); }
-      public string? title { get => _title; set => Set(ref _title, value); }
-      public string? context { get => _context; set => Set(ref _context, value); }
-      public bool keep_open { get => _keep_open; set => Set(ref _keep_open, value); }
-   }
-
-   class OrderTicketHeader : ViewModelBase {
-      bool _keep_open;
-      public BBOVM? bbo { get; set; }
-      public bool keep_open { get => _keep_open; set => Set(ref _keep_open, value); }
-   }
-
-   class OrderTicketLadderHeader : ViewModelBase {
-      bool keepOpen;
-      string? title;
-
-      public string? Title {
-         get => title;
-         set => Set(ref title, value);
-      }
-
-      public bool KeepOpen {
-         get => keepOpen;
-         set => Set(ref keepOpen, value);
-      }
-   }
-}