added more detail missing security log in gmdl

The old name was unnecessarily vague

.vscode/launch.json

@@ -183,11 +183,10 @@
       "windows": {
         "program": "PubSubServer.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
-      ],
       "cwd": "build/Debug/bin/net8.0",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+      },
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -214,7 +213,7 @@
       "type": "coreclr",
       "request": "launch",
       "program": "",
-      "preLaunchTask": "Build Api Server (Debug)",
+      "preLaunchTask": "Build Spark Sdk (Debug)",
       "windows": {
         "program": "Bts.Spark.Sdk.Demo.exe",
       },
@@ -286,7 +285,7 @@
       "windows": {
         "program": "Spark.exe",
         "args": [
-          "--sparkd",
+          "--Spark:ConfigServer",
           "${input:sparkConfigServer}"
         ],
         "cwd": "build/Debug/bin/net8.0-windows",
@@ -449,14 +448,15 @@
       "default": "Local",
     },
     {
-      "id": "pubSubServerConfig",
+      "id": "pubSubServerEnvironment",
       "type": "pickString",
-      "description": "PubSub Configuration",
+      "description": "PubSubServer Configuration",
       "options": [
-        "PubSubServer",
-        "SimpleEchoClient",
+        "Development",
+        "EchoServer",
+        "Local",
       ],
-      "default": "PubSubServer",
+      "default": "Development",
     },
     {
       "id": "sdkDemoEnvironment",

compose.yaml

@@ -81,6 +81,8 @@ services:
         limits:
           memory: "${SPARKD__MEMORY:-4gb}"
   sparkd-multicast:
+    environment:
+      - TZ=America/Chicago
     volumes:
       - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
       - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
@@ -93,7 +95,7 @@ services:
         - CONFIG_DIR=${SPARKD__CONFIG_DIR}
         - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
         - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
-    shm_size: "${SPARKD__SHM_SIZE:-2048mb}"
+    shm_size: "${SPARKD__SHM_SIZE:-8192mb}"
     env_file:
       - .env
       - ${SPARKD__CONFIG_DIR}/.env
@@ -105,7 +107,38 @@ services:
         delay: 3s
       resources:
         limits:
-          memory: "${SPARKD__MEMORY:-4gb}"
+          memory: "${SPARKD__MEMORY:-16gb}"
+  
+  
+  sparkd-multicast-cert:
+    environment:
+      - TZ=America/Chicago
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
+      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
+    build:
+      dockerfile: src/app_sparkd/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
+    shm_size: "${SPARKD__SHM_SIZE:-8192mb}"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
+    network_mode: "host"
+    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
+    deploy:
+      restart_policy:
+        condition: any
+        delay: 3s
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-16gb}"
+  
   sparkd-tail:
     image: bluetradingsystems/linux_dotnet8
     volumes:

doc/sparkd-security-id-lifecycle.md

@@ -0,0 +1,942 @@
+# Security ID handling in `sparkd`
+
+This document traces how security identifiers are created, translated, persisted,
+indexed, and published by the `sparkd` market-data paths. It focuses on the
+legacy CME MDP3/OnixS pipeline and the newer generalized market-data pipeline.
+
+## Executive summary
+
+`sparkd` uses two identifiers that must not be confused:
+
+| Identifier | Meaning | Typical field | Lifetime |
+|---|---|---|---|
+| Exchange/feed ID | The venue's identifier used to route incoming definitions, books, and trades | `FixedSecurity::numeric_id`, `ExchangeKey`, or `domain::Security::security_common.exchange_sec_id` | Feed/session-specific unless the source guarantees stability |
+| Spark security key | Spark's internal non-zero identifier used in shared memory, APIs, orders, positions, and downstream services | `FixedSecurity::key`, `api::Security::id()`, `SecurityKey`; historically also called `security_id` | Intended to be stable across restarts |
+
+`invalid_security_key` is `0`; a security with key `0` is not yet publicly
+discoverable through the shared-memory consumer path.
+
+The two principal paths differ:
+
+- **MDP3/OnixS:** the definition callback sets only the CME ID in
+  `numeric_id`. `MDProducer::add_security()` allocates the shared-memory entry
+  with key `0`. `SecurityIDRecorder`, driven by a 200 ms dispatcher timer,
+  resolves or inserts a stable key in `key_ids.sqlite`; its callback finally
+  calls `MDProducerHandle::set_key()`.
+- **Generalized/newer pipeline:** `GeneralMarketDataListener::on_securities()`
+  currently uses the ID already present on the incoming domain security
+  (`use_given_sec_key == true`). Live adapters normally obtain that ID
+  synchronously from `IDGenerator` first. `_copy_security_common()` then copies
+  it to both `FixedSecurity::key` and `numeric_id`, so
+  `MDProducer::add_security()` publishes it synchronously. Only the listener's
+  own fallback `IDGenerator` call is bypassed by the current constant.
+
+## End-to-end diagrams
+
+### CME MDP3 / OnixS
+
+```mermaid
+sequenceDiagram
+    participant FE as OnixS feed-engine thread
+    participant L as MDP3Listener
+    participant P as MDProducer / shared memory
+    participant Q as SecurityIDRecorder queues
+    participant D as sparkd dispatcher thread
+    participant DB as key_ids.sqlite
+
+    FE->>L: instrument definition (CME SecurityID)
+    L->>L: set_security_common() - numeric_id = CME SecurityID, key remains 0
+    L->>P: add_security(FixedSecurity)
+    P->>P: allocate hot + cold entries with key 0
+    P->>Q: ensure_key(handle)
+    L->>L: index[CME SecurityID] = handle
+    Note over FE,L: Books can route by CME ID while the Spark key is pending
+    D->>Q: every 200 ms - work_spreads(), work_handles(), work_db()
+    Q->>DB: find (namespace, descriptor), or insert a new row
+    DB-->>Q: SQLite rowid = Spark security key
+    Q->>P: handle.set_key(rowid)
+    P->>P: hot.key first, then cold security.key, then increment security_count
+    Note over P: The security is now public to shared-memory consumers
+```
+
+### Generalized/newer pipeline (current behavior)
+
+```mermaid
+sequenceDiagram
+    participant A as Venue adapter / plugin
+    participant DB as IDGenerator / key_ids.sqlite
+    participant G as GeneralMarketDataListener
+    participant I as GeneralSecurityIndex
+    participant P as MDProducer / shared memory
+
+    A->>DB: get_or_add(venue, type, adapter key)
+    DB-->>A: SQLite rowid as domain SecurityID
+    A->>G: on_securities(domain Security list with ID)
+    G->>G: _find_sec() - use incoming security_common.id
+    G->>G: _copy_security_common() - key = id, numeric_id = id
+    G->>P: add_security(FixedSecurity with non-zero key)
+    P->>P: set_key() during add_security()
+    G->>I: index[id] = handle
+    Note over G,P: No asynchronous ID assignment in the current configuration
+```
+
+## Data-flow charts
+
+The sequence diagrams above emphasize timing and thread transitions. These
+flowcharts emphasize which identifier is carried through each representation
+and where data is retained.
+
+### MDP3 / legacy data flow
+
+```mermaid
+flowchart TD
+    CME["CME MDP3 packets<br/>CME SecurityID"] --> ONIXS["OnixS feed engine<br/>decode definition and updates"]
+
+    ONIXS --> DEF["MDP3Listener definition callback"]
+    DEF --> FIXED["FixedSecurity<br/>numeric_id = CME SecurityID<br/>key = 0"]
+
+    FIXED --> PRODUCER["MDProducer::add_security"]
+    PRODUCER --> PENDING["Hot and cold shared memory<br/>allocated but not public<br/>key = 0"]
+    PRODUCER --> QUEUE["SecurityIDRecorder<br/>pending_handle queue"]
+
+    FIXED --> CMEINDEX["CMESecurityIndex<br/>CME SecurityID to handle"]
+
+    QUEUE --> TIMER["Dispatcher timer<br/>every 200 ms"]
+    TIMER --> LEGS{"Spread legs<br/>all resolved?"}
+    LEGS -- No --> UNRESOLVED["unresolved_legs<br/>exchange IDs awaiting keys"]
+    UNRESOLVED --> CMEINDEX
+    CMEINDEX --> LEGS
+    LEGS -- Yes or outright --> LOGICAL["Logical identity<br/>venue + type + security_desc"]
+
+    LOGICAL --> SELECT["SQLite SELECT<br/>key_ids.sqlite"]
+    SELECT -- Existing row --> ROWID["SQLite rowid<br/>Spark security key"]
+    SELECT -- Missing row --> INSERT["Batched SQLite INSERT"]
+    INSERT --> ROWID
+
+    ROWID --> SETKEY["MDProducerHandle::set_key"]
+    SETKEY --> HOT["Write hot-entry key"]
+    HOT --> COLD["Write cold security key"]
+    COLD --> COUNT["Increment security_count"]
+    COUNT --> PUBLIC["Public security<br/>visible to MDConsumer"]
+
+    ONIXS --> UPDATE["Book, trade, status, or RFQ<br/>carrying CME SecurityID"]
+    UPDATE --> CMEINDEX
+    CMEINDEX --> HANDLE["MDProducerHandle"]
+    HANDLE --> LIVE["Update live shared-memory data"]
+    PUBLIC --> LIVE
+```
+
+The CME index is usable before the Spark key is assigned, so feed updates can
+reach the internal handle while the security is still hidden from consumers.
+The persistent rowid crosses the publication boundary only at `set_key()`.
+
+### Generalized/newer data flow
+
+```mermaid
+flowchart TD
+    FEED["Venue feed, Databento,<br/>Python plugin, or replay"] --> ADAPTER["Source adapter<br/>build domain::Security"]
+
+    ADAPTER --> IDTYPE{"Live definition or<br/>recorded/pre-keyed input?"}
+    IDTYPE -- Live --> IDINPUT["Logical identity<br/>venue + type + adapter-chosen name"]
+    IDINPUT --> MUTEX["IDGenerator global mutex"]
+    MUTEX --> SELECT["Synchronous SQLite SELECT<br/>key_ids.sqlite"]
+    SELECT -- Existing row --> ROWID["SQLite rowid"]
+    SELECT -- Missing row --> INSERT["Synchronous SQLite INSERT<br/>autocommit"]
+    INSERT --> ROWID
+    ROWID --> DOMAINID["domain::SecurityID<br/>used as Spark ID"]
+    IDTYPE -- Recorded or pre-keyed --> DOMAINID
+
+    DOMAINID --> DOMAINSEC["SecurityCommon::id = Spark ID"]
+    DOMAINSEC --> LISTENER["GeneralMarketDataListener::on_securities"]
+    LISTENER --> CCMAP["cc_sec_id_map<br/>incoming ID to Spark ID"]
+    LISTENER --> CONVERT["_copy_security_common<br/>FixedSecurity.key = ID<br/>numeric_id = ID"]
+
+    CONVERT --> PRODUCER["MDProducer::add_security"]
+    PRODUCER --> SETKEY["Immediate set_key<br/>no recorder timer"]
+    SETKEY --> SHM["Hot and cold shared memory<br/>immediately public"]
+    CONVERT --> GENERALINDEX["GeneralSecurityIndex<br/>ID to handle"]
+
+    FEED --> EVENT["Book, trade, or statistics<br/>with exchange/source identity"]
+    EVENT --> EVENTMAP["Adapter map<br/>source identity to domain::SecurityID"]
+    EVENTMAP --> EVENTID["Event carries domain::SecurityID"]
+    EVENT --> PYHOT["Python bridge only<br/>synchronous IDGenerator lookup"]
+    PYHOT --> EVENTID
+    PYHOT -. uses same mutex and SQLite path .-> MUTEX
+    EVENTID --> CCMAP
+    CCMAP --> GENERALINDEX
+    GENERALINDEX --> HANDLE["MDProducerHandle"]
+    HANDLE --> SHMUPDATE["Update book, trade,<br/>statistics, or status"]
+    SHM --> SHMUPDATE
+
+```
+
+In the normal newer live path, persistence occurs before the definition enters
+the generalized listener. The DB call therefore adds synchronous adapter-side
+latency, but publication needs no later timer. Most adapters use an in-memory
+map for subsequent events; the dotted Python path is the documented exception.
+
+## The publication boundary
+
+`MDProducerHandle::set_key()` is the definitive publication point for the
+legacy shared-memory representation. Its write ordering is intentional:
+
+1. Write the key into the hot book entry (`HotEntry` or `HotDepthEntry`).
+2. Write the key into the cold security entry.
+3. Increment the cold header's public `security_count`.
+
+The cold security key is written last because consumers treat a non-zero cold
+key as visibility. At that moment the corresponding hot entry must already be
+addressable. `set_key()` only permits the transition from `0` to one key; it is
+not a key-update mechanism.
+
+## Identifier representations
+
+The names are historically inconsistent. In particular, many protobuf fields
+named `security_id` contain the **Spark key**, not the exchange's ID.
+
+| Layer | Exchange/feed identifier | Spark identifier |
+|---|---|---|
+| MDP3 wire/OnixS | `securityId()` | not present |
+| Legacy fixed security | `FixedSecurity::numeric_id` (`uint64_t`) | `FixedSecurity::key` / `security_key` (`uint32_t`) |
+| Legacy API security | venue-specific `exchange_id` | top-level `Security.id()` |
+| New domain model | `SecurityCommon::exchange_sec_id` (string) is descriptive; `SecurityCommon::id` is also used to correlate source events | `domain::SecurityID` (`uint64_t`), currently copied into the Spark key |
+| Shared-memory hot book | none | `HotEntry::key` or `HotDepthEntry::key` |
+| Shared-memory cold metadata | `FixedSecurity::numeric_id` | `FixedSecurity::key` |
+| CME runtime lookup | `CMESecurityIndex` key | `SecurityData::handle.key()` |
+| General runtime lookup | incoming/canonical `domain::SecurityID` | `GeneralSecurityIndex` key and `handle.key()` (currently the same value) |
+
+The generalized model declares `domain::SecurityID` as 64-bit, but the legacy
+shared-memory `security_key` remains 32-bit. `_copy_security_common()` assigns a
+64-bit value to both `numeric_id` and the 32-bit `key`. This is safe only while
+allocated IDs fit in 32 bits; unlike the legacy lookup path, this assignment has
+no explicit checked conversion.
+
+## Which parts use exchange IDs, descriptors, or Spark security IDs?
+
+The three identifier classes serve different jobs:
+
+- **Exchange/external ID:** routes data inside a venue or vendor feed adapter.
+- **Descriptor/natural key:** recognizes a definition logically at the
+  persistence, discovery, or human-facing boundary.
+- **Spark security ID/key:** joins the instrument to books, orders, fills,
+  positions, and services after publication.
+
+The core downstream system does not repeatedly identify a security from its
+description. The description is primarily used to obtain a Spark key; after
+that boundary, most joins use the numeric Spark key.
+
+### Component-level usage matrix
+
+| Component or data path | Exchange/external ID | Descriptor or name | Spark security ID/key |
+|---|---|---|---|
+| CME MDP3 definition decoder | Reads `securityId()` and stores it in `FixedSecurity::numeric_id` | Copies definition symbol/tag 55 into `security_desc` | Leaves `FixedSecurity::key` at `0` initially |
+| `CMESecurityIndex` | Primary map key; CME ID resolves to a handle | Not used for update routing | Returns `handle.key()` when the Spark key has been assigned |
+| MDP3 books, trades, status, statistics, and RFQs | Incoming CME ID selects the handle through `CMESecurityIndex` | Not used to locate each update | The selected handle writes keyed shared-memory/trade data after publication |
+| MDP3 spread definitions | Each leg initially contains `legSecurityId()` as `exchange_key` | Parent `security_desc` identifies the spread's DB row | Each leg's Spark key and then the parent Spark key must be resolved before publication |
+| Legacy ICE OnixS | Uses ICE `marketId` as `numeric_id` and index key | ICE security description becomes the logical lookup name | Uses the same recorder-assigned Spark key pattern as MDP3 |
+| `SecurityIDRecorder` | Does not allocate from the external ID | Uses `(venue, security type, securities::descriptor())` to find/create a row | Treats the SQLite rowid as the resulting Spark key |
+| Native generalized adapters | Use exchange strings/numeric IDs to parse definitions and correlate updates | Usually pass a description or exchange instrument name to `IDGenerator` | Put the returned ID into `SecurityCommon::id` and later events |
+| Databento adapter | Uses `msg.hd.instrument_id` for feed correlation and its in-memory map | Uses the decimal instrument ID string—not the human description—as the `IDGenerator` name | Caches `instrument_id -> domain::SecurityID` and places that ID on books/trades |
+| Python bridge | Reads `exchange_security_id` from definitions, books, and trades | Uses that external-ID string as the `IDGenerator` name | Resolves it to `domain::SecurityID`; currently performs this lookup for every event |
+| `IDGenerator` | Does not interpret whether `name` came from an exchange ID | Uses exactly `(venue, type, caller-supplied name)` | Returns the SQLite rowid as a 64-bit ID |
+| `GeneralMarketDataListener` | Does not route on `exchange_sec_id` directly | Copies `sec.desc()` into fixed metadata; its disabled fallback allocator would use `sec.desc()` | Uses incoming `SecurityCommon::id`, maps events through `cc_sec_id_map`, and copies the value into both `key` and `numeric_id` |
+| `GeneralSecurityIndex` | No separate external-ID concept in the current use-given path | Not used | Maps the generalized ID/Spark key to `MDProducerHandle` |
+| `MDProducer` / shared memory | Retains `numeric_id`; in legacy feeds this is the exchange ID, but in the generalized path it is currently the Spark ID | Stores `security_desc` in the cold definition | Uses the key in hot/cold entries; non-zero key is the visibility boundary |
+| `MDConsumer` | Not normally used to find live books | Can enumerate/filter converted security definitions | Finds handles and book data by `security_key` |
+| `SecurityCache` / `securities.sqlite` | Preserves venue-specific exchange data inside the serialized security blob | Maintains descriptor/name lookup indexes and supports venue/type/name queries | Primary persisted definition key is `api::Security.id()` |
+| MBO processor and MBO service | May retain exchange **order** IDs, which are separate from security exchange IDs | Not used to find the instrument | Indexes snapshots and requests by `SecurityKey`; MBO protobuf `security_id` means Spark key |
+| Order manager and algorithms | Execution gateways eventually use venue-specific fields from the resolved security to encode orders | Descriptions appear in logs and errors, not as the primary join | Order definitions, subscriptions, book events, native orders, spread legs, and fills use `security_key` / `Security.id()` |
+| Position, PNL, and risk services | Not used as their position join | May resolve metadata for display/reporting | Positions, fills, overrides, and PNL handles are keyed by Spark security ID |
+| Relay and market-data recordings | Serialized security definitions may preserve venue exchange IDs | Preserve full descriptors in recorded definitions | Books, updates, trades, and replay correlation preserve the recorded Spark key |
+| Synthetic security service | No venue exchange ID; synthetics use `VENUE_BTS` | Synthetic name is the recorder's logical descriptor | Recorder assigns the SQLite rowid and synthetic metadata is keyed by it |
+
+### Exchange/external IDs: where they stop
+
+In MDP3 the separation is explicit:
+
+```text
+CME SecurityID
+    -> FixedSecurity.numeric_id
+    -> CMESecurityIndex lookup
+    -> MDProducerHandle
+    -> Spark security_key for downstream publication
+```
+
+The external CME ID continues to route feed messages but does not become the
+order/position key. This permits the same downstream Spark key to survive a
+restart as long as the logical descriptor mapping remains in
+`key_ids.sqlite`, even if the feed must rediscover the CME routing ID.
+
+The generalized path collapses more of these roles:
+
+```text
+exchange/vendor identity
+    -> adapter calls IDGenerator
+    -> domain::SecurityID
+    -> FixedSecurity.key and FixedSecurity.numeric_id
+    -> GeneralSecurityIndex and downstream Spark key
+```
+
+`SecurityCommon::exchange_sec_id` may still contain the original textual
+external identifier, but `GeneralMarketDataListener` does not use it to route
+events and does not put it in `numeric_id`. The adapter must translate later
+events to `domain::SecurityID` before invoking the listener.
+
+### Descriptor/name users
+
+Descriptors are used in three main places:
+
+1. **Key allocation:** `SecurityIDRecorder` uses the legacy venue-specific
+   descriptor, while `IDGenerator` uses its caller-provided `name`.
+2. **Definition lookup and discovery:** `SecurityCache` supports
+   venue/type/name lookup; symbol filters and administrative tools use security
+   metadata to select or display instruments.
+3. **Diagnostics:** logs include symbols and descriptions so humans can
+   understand a numeric key.
+
+Descriptions generally do not drive the high-rate book, trade, order, fill, or
+position paths. Those paths use numeric IDs after definition resolution.
+
+It is important that “descriptor” does not mean one universal canonical field:
+
+- Legacy CME uses Globex `security_desc` from tag 55.
+- Many generalized adapters use `sec.desc()` or an exchange instrument name.
+- Databento uses its numeric instrument ID converted to a string.
+- Python uses `exchange_security_id`.
+
+Therefore, two adapters may describe the same economic contract but request
+different logical rows and receive different Spark IDs.
+
+### Spark security-ID users
+
+Once a security is published, its Spark key is the dominant system identifier:
+
+```mermaid
+flowchart LR
+    KEY["Spark security key"] --> SHM["Hot/cold shared memory"]
+    KEY --> MDC["MDConsumer and book subscriptions"]
+    KEY --> MBO["MBO snapshots and queries"]
+    KEY --> OMS["Orders, algorithms, and spread legs"]
+    KEY --> FILLS["Fills and execution records"]
+    KEY --> POS["Positions, PNL, and risk"]
+    KEY --> API["Services and client APIs"]
+    KEY --> REC["Relay and market-data recordings"]
+    KEY --> SECDB["Serialized security definitions"]
+```
+
+Fields named `security_id`, `security_key`, or top-level `Security.id()` in
+these downstream areas almost always mean the Spark key. They should not be
+compared directly with CME `SecurityID`, Databento `instrument_id`, or another
+venue/vendor identifier without going through the relevant adapter/index
+mapping.
+
+## Persistence and allocation
+
+### The authoritative key mapping: `key_ids.sqlite`
+
+Both allocators use `SqliteStore("key_ids")`. With the normal configuration the
+file is:
+
+```text
+<SystemConfig.data_path>/databases/key_ids/key_ids.sqlite
+```
+
+The database contains one table and two indexes:
+
+```sql
+CREATE TABLE store (ns TEXT, key TEXT, value TEXT);
+CREATE UNIQUE INDEX key_index ON store(ns, key);
+CREATE INDEX ns_index ON store(ns);
+```
+
+The SQLite `rowid` is the generated Spark key. `ns` is normally
+`<VENUE>.<SECURITY_TYPE>` (for example `GLBX.FUTURE`), and `key` is the security
+descriptor chosen by the caller. `value` is normally empty. Therefore ID
+stability depends on all three of these remaining stable:
+
+- the configured `data_path` and preservation of this database;
+- the venue/type namespace;
+- the descriptor supplied by the feed adapter.
+
+The two allocator implementations share the file and convention:
+
+- `SecurityIDRecorder` (legacy): derives the namespace from legacy API enums and
+  uses `securities::descriptor()`. For CME this is the Globex
+  `security_desc`, populated from the definition's symbol/tag 55.
+- `IDGenerator` (newer): derives the namespace from new domain enums and uses
+  the caller's `name`. Most adapters pass `sec.desc()` or an exchange instrument
+  identifier; Databento deliberately passes the decimal instrument ID, and the
+  Python bridge uses the exchange security ID.
+
+### Logical-row examples
+
+The following names are illustrative. Production uses the exact string emitted
+by the selected feed/adapter, including its case, spacing, and weekly-series
+notation.
+
+#### Legacy `SecurityIDRecorder`
+
+For CME, `set_security_common()` copies the MDP3 definition's symbol/tag 55 into
+Globex `security_desc`. After converting the fixed security to
+`api::Security`, `securities::descriptor()` returns that same
+`glbx.security_desc()` string.
+
+| Instrument | Legacy enums | Example descriptor from `security_desc` | Row searched/inserted in `store` |
+|---|---|---|---|
+| Future | `VENUE_GLBX`, `SECURITYTYPE_FUTURE` | `ESZ6` | `ns='GLBX.FUTURE', key='ESZ6'` |
+| Standard option | `VENUE_GLBX`, `SECURITYTYPE_OPTION` | `ESZ6 C6500` | `ns='GLBX.OPTION', key='ESZ6 C6500'` |
+| Weekly option | `VENUE_GLBX`, `SECURITYTYPE_OPTION` | `ESZ6-W1 C6500` | `ns='GLBX.OPTION', key='ESZ6-W1 C6500'` |
+| Spot from a legacy-compatible venue path | `VENUE_COINBASE`, `SECURITYTYPE_SPOT` | `BTC-USD` | `ns='COINBASE.SPOT', key='BTC-USD'` |
+
+A weekly option does not get a special namespace. Standard and weekly options
+are both `SECURITYTYPE_OPTION`, so both live in `GLBX.OPTION`; the exact
+`security_desc` distinguishes the weekly series. For example, if the database
+assigned rowids `42001` through `42004` to the rows above, those integers—not
+the descriptor strings—would be the Spark security keys.
+
+Conceptually, legacy allocation is:
+
+```text
+future:        lookup_or_insert("GLBX.FUTURE",  "ESZ6")
+option:        lookup_or_insert("GLBX.OPTION",  "ESZ6 C6500")
+weekly option: lookup_or_insert("GLBX.OPTION",  "ESZ6-W1 C6500")
+spot:          lookup_or_insert("COINBASE.SPOT", "BTC-USD")
+```
+
+#### Newer `IDGenerator`
+
+The namespace result is similar, but the third argument is chosen by the
+adapter. Examples using a description-oriented adapter are:
+
+```cpp
+get_or_add(VENUE_GLBX, SECURITY_TYPE_FUTURE, "ESZ6");
+// store: ns="GLBX.FUTURE", key="ESZ6"
+
+get_or_add(VENUE_GLBX, SECURITY_TYPE_OPTION, "ESZ6 C6500");
+// store: ns="GLBX.OPTION", key="ESZ6 C6500"
+
+get_or_add(VENUE_GLBX, SECURITY_TYPE_OPTION, "ESZ6-W1 C6500");
+// store: ns="GLBX.OPTION", key="ESZ6-W1 C6500"
+
+get_or_add(VENUE_COINBASE, SECURITY_TYPE_SPOT, "BTC-USD");
+// store: ns="COINBASE.SPOT", key="BTC-USD"
+```
+
+Databento intentionally produces a different logical row even when the human
+description is `ESZ6`. If its instrument ID were `12345678`, it would call the
+equivalent of:
+
+```cpp
+get_or_add(VENUE_GLBX, SECURITY_TYPE_FUTURE, "12345678");
+// store: ns="GLBX.FUTURE", key="12345678"
+```
+
+Likewise, the Python bridge uses `exchange_security_id`. A spot definition with
+`exchange_security_id="BTC-USD"` produces the `COINBASE.SPOT / BTC-USD` row,
+while a provider that sends `exchange_security_id="BTCUSD"` produces a
+different row and therefore a different Spark key.
+
+These examples show why the tuple is only logical when the adapter's naming
+contract is also known. `GLBX.FUTURE / ESZ6` and `GLBX.FUTURE / 12345678` may
+describe the same contract to a human but are distinct rows to `key_ids.sqlite`.
+
+Both enum-prefix removals currently produce the same namespace spelling (for
+example, `GLBX.OPTION`), so both mechanisms allocate from one global SQLite
+rowid space. Live rows cannot share a rowid. The table does not declare
+`AUTOINCREMENT`, however, so deleting rows—especially the highest row—can allow
+SQLite to reuse a number later. Security-key rows should therefore be treated
+as append-only operational data.
+
+`SqliteStore` uses WAL mode and `PRAGMA synchronous = 0`. This favors startup
+and feed latency over maximum durability against a host/power failure. The
+unique `(ns, key)` index prevents two different rows for the same logical
+security.
+
+### Other places the key is stored
+
+| Location | Purpose | Authoritative for allocation? |
+|---|---|---|
+| Hot and cold shared-memory segments | Live publication and fast consumer lookup | No; rebuilt at process startup |
+| `CMESecurityIndex` / `GeneralSecurityIndex` and adapter maps | In-process routing from an incoming ID to a producer handle | No; in-memory only |
+| `<data_path>/databases/md/securities.sqlite` | Serialized legacy `api::Security` definitions keyed by Spark key; used by security cache and MDP3 Sunday preload | No; it stores metadata after a key exists |
+| Relay/market-data recordings | Preserve the key embedded in recorded securities, books, and trades for deterministic replay | No; replay normally reuses the recorded value |
+| Orders, fills, positions, APIs | Reference a security by the published Spark key | No |
+
+The `SecurityIDRecorder::work_db()` comment says a published security is also
+recorded in the security database, but that function does **not** write
+`securities.sqlite` in the current code. Full security definitions are persisted
+separately by `SecurityCache::flush_to_disk()` through
+`db::insert_or_replace()`.
+
+## CME MDP3 / OnixS pipeline in detail
+
+### 1. Construction and thread setup
+
+`MDP3Main::PImpl` owns one `CMESecurityIndex`, one `SecurityIDRecorder`, and one
+`MDProducer`. It constructs the recorder with:
+
+- a lookup callback that maps `ExchangeKey` through the CME index to the
+  handle's current Spark key; and
+- `Dispatcher::current()`, which drives a 200 ms recorder timer.
+
+In the current `MarketDataGateway` route, `OnixsGlbxSubscription` creates a
+dedicated `dispatcher_thread`, constructs and starts `MDP3Main` on that thread,
+then calls that thread's dispatcher `run()`. The older `md run onixs` command
+uses its command/dispatcher thread similarly.
+
+The OnixS feed engine has its own configured thread pool. Instrument and market
+data callbacks therefore originate on OnixS feed-engine threads, while recorder
+database work and final key assignment execute on the MDP3 dispatcher thread.
+
+| MDP3 stage | Executing thread | DB call? |
+|---|---|---|
+| Decode definition and set CME `numeric_id` | OnixS feed-engine callback thread | No |
+| Allocate hot/cold entry and queue handle | Same OnixS callback thread | No |
+| Add CME-ID index entry | Same OnixS callback thread | No |
+| Resolve spread legs | MDP3 dispatcher thread, on 200 ms timer | No |
+| Select/insert Spark key | MDP3 dispatcher thread, on 200 ms timer | Yes |
+| `handle.set_key()` and public visibility | MDP3 dispatcher thread, in DB callback | No additional call |
+
+### 2. Definition callback: exchange ID is set, Spark key is not
+
+For futures, options, and spreads, `MDP3Listener::set_security_common()` does:
+
+```cpp
+fsec.numeric_id = def.securityId();
+```
+
+It also copies symbol/descriptor/venue and pricing fields. A newly constructed
+`FixedSecurity` already has `key == invalid_security_key` (`0`), and this helper
+does not change it.
+
+For a spread, every leg starts with:
+
+```cpp
+leg.exchange_key = ExchangeKey(definition_leg.legSecurityId());
+leg.key = invalid_security_key;
+```
+
+Thus both the parent and legs initially carry exchange-routing identity, but no
+Spark identity.
+
+### 3. Shared-memory allocation and exchange indexing
+
+`MDP3Listener::add_security()` performs these operations in order:
+
+1. `_producer.add_security(security)` allocates a hot entry and a variable-sized
+   cold entry and copies the security into the cold segment.
+2. Because the key is `0`, `MDProducer` queues the handle through
+   `SecurityIDRecorder::ensure_key()`.
+3. `_index.add_security(security.numeric_id, handle)` immediately creates the
+   CME-ID-to-handle mapping.
+4. The initial exchange status is written through the handle.
+
+This ordering means incoming books can already be routed by CME ID while the
+Spark key is pending. Shared-memory consumers cannot discover the security yet.
+Also, `MDProducerHandle::add_trade()` only appends to the public trade queue
+when `key() != 0`, so a trade received in this short window updates the cold
+last-trade state but is not added to the keyed trade log.
+
+### 4. Recorder tick: spreads, handles, then database
+
+Every 200 ms the recorder calls, in this exact order:
+
+1. `work_spreads()` tries to fill previously unresolved spread leg keys by
+   asking `CMESecurityIndex` for each leg exchange ID.
+2. `work_handles()` drains new handles. Outrights are converted to an API
+   security and queued for DB lookup immediately. A spread without all leg keys
+   is retained in `pending_handle` and its leg pointers are registered in
+   `unresolved_legs`.
+3. `work_db()` resolves queued `(venue, type, descriptor)` requests.
+
+Because spread work precedes new-handle work, a newly seen unresolved spread
+normally waits until at least the next timer tick to resolve legs. It waits
+longer if any outright leg has not yet received its Spark key. A spread's own
+key is not requested until every leg key is non-zero.
+
+### 5. Lookup or generation
+
+`work_db()` first queries `(ns, key)`:
+
+- Existing row: use its `rowid` as the key and invoke the callback.
+- Missing row: batch it into `insert_many_resolve_ids()`. SQLite assigns a new
+  rowid inside a transaction, the row object receives that ID, and the callback
+  runs after the transaction completes.
+
+New inserts are throttled to roughly 200 per timer pass (the current `> 200`
+check can admit 201) to avoid delaying heartbeats during startup. The queues are
+TBB concurrent queues, but all SQLite access is serialized onto the recorder's
+dispatcher thread. `check_access()` asserts this single-thread rule after the
+first work pass records the dispatcher thread ID.
+
+### 6. Final assignment and visibility
+
+The queued callback captures a copy of `MDProducerHandle` and calls
+`handle.set_key(id)` on the dispatcher thread. `set_key()` writes hot, then cold,
+then increments `security_count`. From this point:
+
+- shared-memory scans report the security;
+- `CMESecurityIndex::get_key(CME_ID)` returns the Spark key;
+- trades and RFQs can be emitted with a non-zero key;
+- resolved spreads can copy this key into their leg structures.
+
+### 7. Sunday preload is a fast path
+
+For configured symbols on Sunday, `MDP3Main::prepare_listener()` loads stored
+definitions from `securities.sqlite`. It converts them with
+`set_security_key=true`, so both the security key and spread leg keys come from
+the serialized API security. `MDProducer::add_security()` calls `set_key()`
+immediately, and no recorder lookup is required for that preloaded handle. The
+CME exchange ID is still restored into `numeric_id`, allowing the normal CME
+index to be built.
+
+## Generalized/newer pipeline in detail
+
+### 1. Topology and ownership
+
+`Run::run()` starts a detached thread named `MarketDataGateway`. On that thread,
+`run_md_gwy()` obtains the process-wide `IDGenerator`, constructs
+`MarketDataGateway`, and starts the configured providers. For each `(venue,
+source)` returned by a provider, the gateway creates an `MdSink` containing:
+
+- a venue-local `GeneralSecurityIndex`;
+- a default `MDProducer` (not wired to a `SecurityIDRecorder`); and
+- a `GeneralMarketDataListener` connecting the source to that producer/index.
+
+The source decides its callback thread. Some sources call the listener while
+starting; others own network or replay threads. The generalized listener calls
+the producer inline on that source thread. The process-wide `IDGenerator`
+serializes its own calls with a mutex, but the listener's transient
+`cc_sec_id_map` is an ordinary `unordered_map`, so each source/listener pair is
+expected to serialize security/event callbacks.
+
+| Generalized stage | Executing thread | DB call? |
+|---|---|---|
+| Build/parse domain definition | Source/adaptor thread | No |
+| `IDGenerator::get_or_add()` | Same source/adaptor thread | Yes, synchronously |
+| `GeneralMarketDataListener::on_securities()` | Same source callback thread | No in current use-given mode |
+| Copy ID to `FixedSecurity::key` and `numeric_id` | Same source callback thread | No |
+| `MDProducer::add_security()` / `set_key()` | Same source callback thread | No additional call |
+| Add `GeneralSecurityIndex` entry | Same source callback thread | No |
+
+There are two important compatibility exceptions inside the provider framework:
+
+- `OnixsGlbxSubscription::start()` ignores the supplied generalized listener
+  and starts `MDP3Main`; CME/OnixS therefore still follows the asynchronous
+  legacy flow documented above.
+- `EmumbaDeribitSubscription::start()` also ignores that listener and uses
+  `EmMDListener`, which allocates via `IDGenerator` and writes through its own
+  producer/index. Its assignment is still synchronous and pre-populated, so it
+  resembles the newer ID flow even though it does not pass through
+  `GeneralMarketDataListener`.
+
+### 2. How a new ID is constructed
+
+For a normal live generalized adapter, ID creation happens **in the adapter,
+before** `GeneralMarketDataListener::on_securities()`:
+
+```text
+logical identity = (domain venue, domain security type, adapter-chosen name)
+Spark ID         = SQLite rowid for that identity in key_ids.sqlite
+```
+
+`IDGenerator::get_or_add(venue, type, name)` formats a namespace by stripping
+the enum prefixes:
+
+```text
+VENUE_TAIFEX + SECURITY_TYPE_FUTURE -> TAIFEX.FUTURE
+```
+
+It then calls `retrieve_one(ns, name)`. If the row exists, it returns that
+row's `rowid`. If absent, it inserts `(ns, name, "")`, and SQLite assigns the
+next rowid. That integer is wrapped in `domain::SecurityID` and placed in
+`SecurityCommon::id`.
+
+The third identity component is adapter-specific, and this matters when trying
+to reproduce a mapping:
+
+| Source/adaptor family | Typical `name` sent to `IDGenerator` |
+|---|---|
+| Binance, Coinbase, Deribit, OKX and similar native/CCAPI paths | Usually `sec.desc()` / exchange instrument name |
+| Taifex | product/instrument ID string |
+| Databento | `std::to_string(msg.hd.instrument_id)`, not the human-readable description |
+| Python bridge | `exchange_security_id` (and it asserts Python supplied ID is zero for definitions) |
+| Emumba Deribit | extracted Emumba security descriptor |
+| Replay | no new allocation for a recorded definition; it reuses the recorded `SecurityCommon::id` |
+
+So “security name” alone is not a universal logical key. Reproduction requires
+the exact venue enum, exact security-type enum, and the exact adapter-specific
+name used at insertion time.
+
+### 3. What `GeneralMarketDataListener` does with that ID
+
+`on_securities()` first calls `_find_sec(sec, true)`. Its current, compile-time
+behavior is controlled by:
+
+```cpp
+const bool use_given_sec_key = true;
+```
+
+Consequently, for a newly seen incoming `domain::SecurityID` it does **not**
+call the DB. It copies `security_common.id.value()` into its transient
+`cc_sec_id_map`. `_add_future()`, `_add_option()`, `_add_spread()`, or
+`_add_spot()` then calls `_copy_security_common()`, which sets:
+
+```cpp
+to.key = incoming_id;
+to.numeric_id = incoming_id;
+```
+
+This collapses the generalized source/correlation ID and Spark shared-memory
+key into the same number. The source's text `exchange_sec_id` remains on the
+domain security but is not copied into `FixedSecurity::numeric_id` by this
+path.
+
+`_commit_security()` then:
+
+1. calls `MDProducer::add_security()` with an already non-zero key;
+2. causes `MDProducerHandle::set_key()` to run immediately during that call;
+3. inserts the handle in `GeneralSecurityIndex` under `numeric_id` (the same
+   number); and
+4. returns with the security already visible in shared memory.
+
+Because the sink's producer has no recorder, supplying `id == 0` would allocate
+the shared-memory entry but never assign or publish its key. Generalized live
+sources must therefore provide a valid non-zero ID before `on_securities()`.
+
+The listener contains an alternative branch for `use_given_sec_key == false`.
+That branch would call `IDGenerator::get_or_add(me->venue, sec.type(),
+sec.desc())` itself. It is currently disabled, and `_generate_security_key()` is
+also effectively unused. Turning the flag off is not a mechanical change: an
+adapter such as Databento allocates using numeric instrument ID while the
+listener branch would use `sec.desc()`, potentially creating a second key for
+the same instrument.
+
+There is also a spread failure edge case. `_find_sec(sec, true)` records the
+parent ID in `cc_sec_id_map` before `_add_spread()` resolves its legs. If any leg
+ID is absent, `_add_spread()` logs and returns without committing the spread. A
+later repeat of the parent definition is then classified as a duplicate and
+ignored, so definition ordering is a hard requirement in the current code.
+
+### 4. How later books, trades, and statistics find the security
+
+Every later domain event carries a `domain::SecurityID`:
+
+- `Book::security_id()`;
+- `TradeRecord::sec_id`; or
+- `SecurityMarketDataStatistics::security_id`.
+
+The listener first uses `cc_sec_id_map` to translate that event ID to the value
+stored in `GeneralSecurityIndex`, then obtains the `MDProducerHandle`. With
+`use_given_sec_key == true`, this mapping is numerically identity-to-identity,
+but it still serves as the admission check that the definition was seen.
+
+Most high-throughput adapters avoid SQLite on market-data events:
+
+- Databento allocates on the definition and caches
+  `instrument_id -> domain::SecurityID` in `sec_id_map` for books/trades.
+- Taifex keeps the resolved ID in its security-details mapper.
+- Other native subscriptions maintain comparable descriptor/ID maps.
+
+The Python bridge is a notable exception: `to_bts_security_()`,
+`to_bts_trade_()`, and `to_book_()` all call `get_sec_id_()`, which calls
+`IDGenerator::get_id(..., add_if_new=true)`. As written, that performs at least
+one SQLite `SELECT` for each converted Python definition, trade, or book. It
+returns the existing ID after the first insert, but it is still a DB call and
+takes the global `IDGenerator` mutex on every event.
+
+### 5. Persistence timing and latency
+
+The newer allocator is synchronous:
+
+- Existing security: mutex acquisition plus one SQLite `SELECT` on the caller's
+  thread.
+- New security: mutex acquisition, one `SELECT`, then one SQLite `INSERT`
+  (SQLite autocommit), also on the caller's thread.
+- Listener publication: no further DB call under the current
+  `use_given_sec_key` setting.
+
+The ID therefore exists in `key_ids.sqlite` before the adapter sends the
+definition to the listener. There is no intentional 200 ms publication delay,
+but the source callback/startup path directly pays DB and mutex latency. Bulk
+definition discovery performs one lookup and possibly one insert per security;
+`IDGenerator` does not batch those operations.
+
+WAL mode helps concurrency, while `synchronous=0` reduces fsync cost. It does
+not eliminate SQL parsing/stepping, mutex contention, page-cache misses, or
+occasional file-system work. The impact is usually a startup/definition cost
+when adapters cache the result, but it becomes hot-path latency for code such as
+the current Python bridge that resolves every event through the DB.
+
+`GeneralMarketDataListener` itself does not persist the full contract definition
+to `securities.sqlite`; that remains the separate security-cache/metadata
+persistence path described earlier.
+
+## Comparison: MDP3 recorder vs generalized allocator
+
+| Property | MDP3 / `SecurityIDRecorder` | Generalized / `IDGenerator` |
+|---|---|---|
+| Logical input | venue + type + legacy `security_desc` | venue + type + adapter-chosen name |
+| Numeric output | SQLite rowid, checked/read as 32-bit `security_key` | SQLite rowid returned as 64-bit `domain::SecurityID`, then narrowed into 32-bit shared-memory key |
+| Allocation point | After definition is copied to shared memory | Normally inside adapter before listener sees definition |
+| DB execution | Asynchronous on dispatcher timer | Synchronous on caller/source thread |
+| Existing ID cost | One deferred `SELECT` | One immediate `SELECT` per call |
+| New ID cost | Deferred, batched transaction (about 200/pass) | Immediate `SELECT` + individual autocommit `INSERT` |
+| Time until public | Usually 0–200+ ms; spreads may take extra ticks | Immediate after synchronous lookup/insert completes |
+| Feed callback blocking | Queue push only for ID work | Yes, if the adapter invokes `IDGenerator` there |
+| Spread legs | Parent waits asynchronously until all leg keys resolve by exchange ID | Definition processing requires every leg's incoming ID already exist in `cc_sec_id_map`; otherwise the spread is aborted |
+| Runtime ID relationship | Exchange ID and Spark key remain distinct | Incoming/canonical ID, `numeric_id`, index ID, and Spark key are currently the same number |
+| Persistence database | `key_ids.sqlite` | The same `key_ids.sqlite` |
+| Batching | Yes for new rows | No |
+| Failure behavior | Security has an internal handle with key 0 while normal queued work is pending; DB work does not block the feed callback | DB errors/zero results occur in the synchronous source path; a zero ID can leave an unpublishable entry, with no pending-key state machine |
+
+Neither algorithm generates a UUID or a content hash. Both ultimately use an
+opaque, database-assigned SQLite rowid. It is normally increasing and looks like
+a sequence, but this schema does not request SQLite `AUTOINCREMENT`. The tuple
+used to *look up* the row is logical; the resulting number is not derivable from
+that tuple.
+
+## Can another system reconstruct the same keys?
+
+Not from `(venue, type, name)` alone. Another system can reconstruct the
+**logical lookup tuple**, but it cannot calculate the numeric Spark key because
+the key depends on the historical insertion order of every row in
+`key_ids.sqlite`.
+
+For example, two empty systems can insert the exact same `GLBX.FUTURE / ESZ6`
+row and receive different IDs if other securities were inserted before it on
+one system. They are sequential opaque IDs, not deterministic hashes and not
+UUIDs.
+
+Ways to preserve cross-system identity are:
+
+1. Copy/replicate `key_ids.sqlite` (including rowids) and keep descriptor rules
+   identical. This preserves existing numeric keys but requires coordinated
+   allocation for new ones.
+2. Export an explicit mapping of `(namespace, adapter key) -> Spark key`, then
+   import rows with their original rowids before either system allocates new
+   securities.
+3. Treat a stable external identity (venue + type + exchange instrument ID, or
+   another canonical instrument identifier) as the cross-system key and carry a
+   translation table to each system's local Spark key.
+4. Introduce a deterministic ID scheme only as a versioned migration. A hash can
+   be computed independently, but collision handling, descriptor
+   canonicalization, aliases/renames, the existing 32-bit ABI, and compatibility
+   with orders/recordings/databases all need explicit design.
+
+Copying only `securities.sqlite` is insufficient as an allocator strategy. It
+contains security blobs keyed by already-assigned IDs, but `key_ids.sqlite` is
+the mapping used to find or allocate those IDs. Conversely, copying only
+`key_ids.sqlite` preserves the numbers but not all contract metadata.
+
+## Other assignment paths in `sparkd`
+
+The two main pipelines reduce to four ways a producer handle can acquire a key:
+
+1. **Already keyed input.** `MDProducer::add_security()` sees a non-zero
+   `FixedSecurity::key` and calls `set_key()` immediately. Generalized feeds,
+   Sunday preload, relay playback, and other replay paths use this mechanism.
+2. **Recorder-backed live input.** The input key is zero and the producer owns a
+   `SecurityIDRecorder`; `ensure_key()` queues asynchronous assignment. CME
+   MDP3 and legacy OnixS ICE use this mechanism. ICE uses `marketId` as
+   `numeric_id` and otherwise follows the same recorder/index design.
+3. **Service assignment.** `SyntheticSecurityService` calls
+   `SecurityIDRecorder::assign_key()` directly on a background dispatcher. The
+   callback receives a copied `api::Security` with its top-level `id` set, then
+   publishes it to synthetic subscribers. It uses the same `key_ids.sqlite`
+   rowid allocator.
+4. **Unkeyed input with no recorder.** The entry is allocated with key zero and
+   never becomes public. This is not a valid production assignment path, but it
+   is the failure mode if a generalized source omits its ID.
+
+Relay and replay deserve care when reasoning about persistence:
+
+- The generalized `MDReplayer` forwards the recorded domain security ID and
+  emits later books/statistics with the recorded `sec_key`, so the listener's
+  current use-given mode preserves the recording's numeric identity.
+- The legacy relay producer converts a recorded `api::Security` with
+  `set_security_key=true`, so its shared-memory publication also preserves the
+  embedded key.
+- The legacy relay recorder additionally calls `bulk_insert(..., "REPLAY: ")`
+  for logical mappings. Those rows use a `REPLAY: ` namespace prefix; this is a
+  separate mapping population step, not a deterministic derivation of the key.
+
+## Operational implications
+
+- Back up `key_ids.sqlite` and `securities.sqlite` together if numeric ID and
+  definition continuity matter.
+- Do not delete/rebuild `key_ids.sqlite` on one host and assume IDs will match
+  another host, even when all instruments are identical.
+- Before moving a feed between adapters, compare the exact `name` each adapter
+  supplies to `IDGenerator`. Matching venue/type is insufficient.
+- Cache generated IDs in the source after definition processing. Avoid calling
+  `IDGenerator` from book/trade hot paths; the Python bridge currently violates
+  this guideline.
+- Keep generated rowids below the 32-bit shared-memory ceiling until the
+  `security_key` ABI is widened or explicit bounds checking is added to the
+  generalized path.
+- Ensure spread legs are announced before generalized spread parents.
+- Treat `synchronous=0` as a durability decision. A copied DB should be taken
+  with SQLite-aware backup/checkpoint handling rather than by casually copying
+  active WAL files.
+
+Useful inspection queries (run against the configured DB path) are:
+
+```sql
+SELECT rowid AS spark_key, ns, key
+FROM store
+ORDER BY rowid;
+
+SELECT rowid AS spark_key, ns, key
+FROM store
+WHERE ns = 'GLBX.FUTURE' AND key = '<exact-adapter-key>';
+```
+
+## Source map
+
+- [`SecurityIDRecorder.cpp`](../src/lib_spark/spark/md/SecurityIDRecorder.cpp):
+  legacy queues, 200 ms work ordering, logical row construction, DB callbacks,
+  and spread-leg resolution.
+- [`IDGenerator.cpp`](../src/lib_spark/spark/persistence/IDGenerator.cpp):
+  synchronous generalized lookup/allocation and process-wide mutex.
+- [`SqliteStore.cpp`](../src/lib_spark/spark/persistence/SqliteStore.cpp): DB
+  location, schema, WAL/synchronous pragmas, individual inserts, and batch
+  inserts.
+- [`MDProducer.cpp`](../src/lib_spark/spark/md/MDProducer.cpp) and
+  [`MDProducerHandle.cpp`](../src/lib_spark/spark/md/MDProducerHandle.cpp):
+  shared-memory allocation, recorder handoff, and ordered publication.
+- [`FixedSecurity.h`](../src/lib_spark/spark/md/Shm/FixedSecurity.h) and
+  [`MDStructure.h`](../src/lib_spark/spark/md/Shm/MDStructure.h): identifier
+  widths and hot/cold storage fields.
+- [`MDP3Main.cpp`](../src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp)
+  and
+  [`MDP3Listener.h`](../src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h):
+  MDP3 construction, definition callbacks, CME ID extraction, and spread legs.
+- [`MDP3Listener.cpp`](../src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp):
+  producer/index insertion order.
+- [`OnixsGlbxSubscription.cpp`](../src/app_sparkd/sparkd/md/glbx/OnixsGlbxSubscription.cpp):
+  current dedicated MDP3 dispatcher-thread ownership.
+- [`MarketDataGateway.cpp`](../src/app_sparkd/sparkd/md/MarketDataGateway.cpp)
+  and
+  [`GeneralMarketDataListener.cpp`](../src/lib_spark/spark/md/GeneralMarketDataListener.cpp):
+  generalized source/sink topology, incoming-ID mapping, fixed-security
+  conversion, and publication.
+- [`DatabentoDataHandler.cpp`](../src/app_sparkd/sparkd/md/databento/DatabentoDataHandler.cpp),
+  [`TaifexDataProcessors.h`](../src/app_sparkd/sparkd/md/taifex/TaifexDataProcessors.h),
+  and
+  [`PyMarketDataSubscription.cpp`](../src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp):
+  representative adapter-side allocation and caching behavior.
+- [`SecurityDB.cpp`](../src/lib_spark/spark/md/SecurityDB.cpp) and
+  [`SecurityCache.cpp`](../src/lib_spark/spark/md/SecurityCache.cpp): separate
+  persistence of full security definitions.
+- [`SecurityIndex.h`](../src/lib_spark/spark/md/SecurityIndex.h),
+  [`MBOServiceServer.cpp`](../src/lib_spark/spark/md/MBOServiceServer.cpp), and
+  [`MDConsumer.cpp`](../src/lib_spark/spark/md/MDConsumer.cpp): exchange-index
+  translation followed by Spark-keyed market-data consumption and MBO access.
+- [`OrderFactory.cpp`](../src/lib_spark/spark/orderman/order_factory/OrderFactory.cpp),
+  [`FillStore.cpp`](../src/lib_spark/spark/orderman/actors/FillStore.cpp), and
+  [`PositionMap.cpp`](../src/app_sparkd/sparkd/position/PositionMap.cpp):
+  representative downstream use of Spark security keys.
+- [`MDReplayer.cpp`](../src/app_sparkd/sparkd/md/recording/MDReplayer.cpp) and
+  [`RelayClient.cpp`](../src/lib_spark/spark/md/relay/RelayClient.cpp): recorded
+  ID reuse.

docker/images/bluetradingsystems/linux_dotnet8/Dockerfile

@@ -19,6 +19,11 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
 RUN apt-get update \
   && apt-get install -y dotnet8
 
+RUN apt-get update && apt-get install -y gdb
+RUN apt-get update && apt-get install -y fish
+RUN apt-get update && apt-get install -y vim
+RUN apt-get update && apt-get install -y less
+
 RUN locale-gen en_US.UTF-8
 
 ARG docker_user="bts"

src/Directory.Build.targets

@@ -187,32 +187,24 @@
       Name="Test"
       DependsOnTargets="Build"></Target>
 
-  <ItemGroup Condition="Exists('@(NativeWindowsAsset)')">
-    <Content Include="@(NativeWindowsAsset)">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-      <TargetPath>%(FileName)%(Extension)</TargetPath>
-      <Pack>false</Pack>
-    </Content>
-    <Content Include="@(NativeWindowsAsset)">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-      <TargetPath>runtimes/win-x64/native/%(FileName)%(Extension)</TargetPath>
-      <Pack>true</Pack>
-      <PackagePath>runtimes/win-x64/native</PackagePath>
-    </Content>
-  </ItemGroup>
+  <Target Name="CopyNativeAssetsToOutput" AfterTargets="Build">
+    <Copy SourceFiles="@(NativeWindowsAsset)" DestinationFolder="$(OutDir)" SkipUnchangedFiles="True" Condition="Exists('%(NativeWindowsAsset.FullPath)')" />
+    <Copy SourceFiles="@(NativeLinuxAsset)" DestinationFolder="$(OutDir)" SkipUnchangedFiles="True" Condition="Exists('%(NativeLinuxAsset.FullPath)')" />
+  </Target>
 
-  <ItemGroup Condition="Exists('@(NativeLinuxAsset)')">
-    <Content Include="@(NativeLinuxAsset)">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-      <TargetPath>%(FileName)%(Extension)</TargetPath>
-      <Pack>false</Pack>
-    </Content>
-    <Content Include="@(NativeLinuxAsset)">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-      <TargetPath>runtimes/linux-x64/native/%(FileName)%(Extension)</TargetPath>
-      <Pack>true</Pack>
-      <PackagePath>runtimes/linux-x64/native</PackagePath>
-    </Content>
+  <ItemGroup>
+    <Content Include="@(NativeWindowsAsset)"
+             Condition="Exists('@(NativeWindowsAsset)')"
+             Link="%(NativeWindowsAsset.FileName)%(NativeWindowsAsset.Extension)"
+             CopyToOutputDirectory="PreserveNewest"
+             Pack="true"
+             PackagePath="runtimes/win-x64/native/" />
+    <Content Include="@(NativeLinuxAsset)"
+             Condition="Exists('@(NativeLinuxAsset)')"
+             Link="%(NativeLinuxAsset.FileName)%(NativeLinuxAsset.Extension)"
+             CopyToOutputDirectory="PreserveNewest"
+             Pack="true"
+             PackagePath="runtimes/linux-x64/native/" />
   </ItemGroup>
 
 </Project>

src/app_sparkd/configurations/all_crypto/.env

@@ -0,0 +1,5 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/all_crypto
+SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/gateio_dev/requirements.txt
+SPARKD__DATA_DIR=/home/ubuntu/8000/sparkdata/data
+SPARKD__IMAGE=sparkd_symbols
+SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf

src/app_sparkd/configurations/all_crypto/gateio_overrides.json

@@ -0,0 +1,11 @@
+{
+  "currencies": [
+    "USDT"
+  ],
+  "futures": [
+    "BTC",
+    "ETH",
+    "SOL",
+    "XRP"
+  ]
+}

src/app_sparkd/configurations/all_crypto/py_bybit_md_provider.json

@@ -0,0 +1,16 @@
+{
+  "currencies": [
+    "USDT"
+  ],
+  "futures": [
+    "BTC",
+    "ETH",
+    "SOL",
+    "XRP"
+  ],
+  "use_testnet": false,
+  "logging": {
+    "console_level": "WARNING",
+    "file_level": "INFO"
+  }
+}

src/app_sparkd/configurations/all_crypto/spark.conf

@@ -0,0 +1,71 @@
+PublisherOptions {
+  value {
+    network_interface: "0.0.0.0"
+    heartbeat_ms: 5000
+    heartbeat_boot_seconds: 60
+    skip_options: true
+    hot_initial: 1024
+    hot_growth: 1024
+    cold_initial: 8192
+    cold_growth: 2048
+    channel_prefix: ""
+    disable_heartbeat: true
+  }
+}
+
+Logging {
+  value {
+    initial_queue_size: 120001
+    max_queue_growth: 4
+    file {
+      default_level: "Info"
+      subsystem {
+        name: "md"
+        level: "Diag"
+      }
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+    # python_md_providers {
+    #   venue: VENUE_GATEIO
+    #   module_path: "python_md_providers"
+    #   provider_type: "gateio_md_provider.GateIoMdProvider"
+    # }
+    python_md_providers {
+      venue: VENUE_BYBIT
+      module_path: "python_interop_plugins"
+      provider_type: "bybit_md_provider.BybitMdProvider"
+    }
+    binance_usds_futures {
+      num_depth_levels: 5
+      url_rest_base: "https://fapi.binance.com"
+      url_ws_base: "wss://fstream.binance.com"
+      security_criteria {
+        clauses {
+          security_types: SECURITY_TYPE_FUTURE
+          symbols: @SymbolLists.list[Gateio].symbol
+        }
+      }
+    }
+     okx {
+       num_depth_levels: 5
+       url_rest_base: "https://www.okx.com"
+       url_ws_base: "wss://ws.okx.com:8443"
+       security_criteria {
+         clauses {
+           security_types: SECURITY_TYPE_FUTURE
+           symbols: @SymbolLists.list[Gateio].symbol
+         }
+       }
+       is_application_ping_pong_enabled: true
+       application_ping_interval_ms: 20000
+       application_pong_timeout_ms: 10000
+       is_protocol_ping_pong_enabled: true
+       protocol_ping_interval_ms: 60000
+       protocol_pong_timeout_ms: 30000
+     }
+  }
+}

src/app_sparkd/configurations/all_crypto/symbols.conf

@@ -0,0 +1,32 @@
+  SymbolLists {
+  value {
+    list {
+      key: "Gateio"
+      symbol: "SOL"
+      symbol: "BTC"
+      symbol: "ETH"
+      symbol: "XRP"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+      name: "VENUE_GATEIO"
+      spot: @SymbolLists.list[Gateio].symbol
+    }
+    venues {
+      name: "VENUE_BYBIT"
+      future: @SymbolLists.list[Gateio].symbol
+    }
+    venues {
+      name: "VENUE_BINANCE_USDS_FUTURES"
+      future: @SymbolLists.list[Gateio].symbol
+    }
+    venues {
+      name: "VENUE_OKX"
+      future: @SymbolLists.list[Gateio].symbol
+    }
+  }
+}

src/app_sparkd/configurations/cme_dev_replay_aws/.env

@@ -0,0 +1,7 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/cme_dev_replay_aws
+SPARKD__DATA_DIR=/home/ubuntu/8000/sparkdata/data
+SPARKD__IMAGE=sparkd_symbols
+SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/cme_dev_replay_aws/requirements.txt
+SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf
+SPARKD__SHM_SIZE=512m
+SPARKD__MEMORY=1g

src/app_sparkd/configurations/cme_dev_replay_aws/spark.conf

@@ -1,18 +1,12 @@
-JetFIX {
+DeploymentType {
   value {
-    connection {
-      name: "glbx_1"
-      venue: VENUE_GLBX
-      max_order_msgs_per_window: 300
-      window_duration_msecs: 1000
-    }
+    type: "prod"
   }
 }
-
 PublisherOptions {
    value {
-      network_interface_a: 'ens3f0np0'
-      network_interface_b: 'ens3f1np1'
+      network_interface_a: 'ens5'
+      network_interface_b: 'ens5'
       #these get overriden if not entered here
       heartbeat_ms: 5000
       heartbeat_boot_seconds: 60
@@ -41,7 +35,7 @@ CMEOptions {
       use_tcp_replay: false
       recovery_queue_size: 640000
       lost_packet_wait_timeout_ms: 10
-      onixs_log_mode: CME_ONIXS_LOG_MODE_REGULAR
+      onixs_log_mode: CME_ONIXS_LOG_MODE_UNSPECIFIED
       #max_gap_recovery_cache_size: 4
       use_natural_refresh: false
       join_snapshot_on_gap: false
@@ -73,32 +67,17 @@ MarketDataConfig {
        unique_check_value: 15
        use_databento: false
     }
-  }
+   # recording {
+   #    venue: VENUE_GLBX
+   #    num_of_seconds_to_run_file_writer: 300
+   #    num_of_seconds_to_run_db_writer : 0
+   #    n_book_levels: 10 
+   #    batch_size: 100 #for the db writer
+   #    enable_book_recording: true
+   #    enable_trade_recording:  true
+   #    enable_bbo_recording: false
+   #    enable_tbbo_recording:  false
+   #    enable_misc_recording: true
+   #}
+ }
 }
-
-# JetFIX {
-#    value {
-#     #connection: @JetFIXSimConnection.sim
-#     connection {
-#     name: "mlx_004_54"
-#     venue: VENUE_GLBX
-#     socket_connect_host_ip: "69.50.112.191"
-#     socket_connect_port: 42876
-#     sender_compid: "MLX004"
-#     target_compid: "CME"
-#     target_subid: "54"
-#     cme_market_segment_id: 54
-#     start_time: "00:10:00"
-#     end_time: "23:55:00"
-#     start_day: "sunday"
-#     end_day: "friday"
-#     seconds_between_hbs: 30
-#     sender_subid: "BTS"
-#     use_ilink3: true
-#   #   raw_data: "S4PMA"
-#   #   auto_fix_seq_num: true
-#   }
-#    }
-# }
-
-

src/app_sparkd/configurations/cme_dev_replay_aws/symbols.conf

@@ -0,0 +1,61 @@
+SymbolLists {
+  value {
+    list {
+      key: "GlbxFutures"
+      symbol: "ES"
+    }
+    # list {
+    #   key: "GlbxOptions"
+    #   symbol: "E1A"
+    #   symbol: "E1B"
+    #   symbol: "E1C"
+    #   symbol: "E1D"
+    #   symbol: "E2A"
+    #   symbol: "E2B"
+    #   symbol: "E2C"
+    #   symbol: "E2D"
+    #   symbol: "E3A"
+    #   symbol: "E3B"
+    #   symbol: "E3C"
+    #   symbol: "E3D"
+    #   symbol: "E4A"
+    #   symbol: "E4B"
+    #   symbol: "E4C"
+    #   symbol: "E4D"
+    #   symbol: "E5A"
+    #   symbol: "E5B"
+    #   symbol: "E5C"
+    #   symbol: "E5D"
+    #   symbol: "EW1"
+    #   symbol: "EW2"
+    #   symbol: "EW3"
+    #   symbol: "EW4"
+    #   symbol: "ESH9"
+    #   symbol: "ESZ6"
+    #   symbol: "ESH0"
+    #   symbol: "ESM6"
+    #   symbol: "ESZ7"
+    #   symbol: "ESH6"
+    #   symbol: "ESM7"
+    #   symbol: "ESZ8"
+    #   symbol: "ESH7"
+    #   symbol: "ESH8"
+    #   symbol: "ESU6"
+    #   symbol: "ESZ5"
+    #   symbol: "ESZ9"
+    #   symbol: "E1D"
+    #   symbol: "ESZ5"
+    #   symbol: "ES"
+    # }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+        name: "VENUE_GLBX"
+        future: @SymbolLists.list[GlbxFutures].symbol
+        # option: @SymbolLists.list[GlbxOptions].symbol
+    }
+  }
+}

src/app_sparkd/configurations/ophir_cert_live_wb/.env

@@ -0,0 +1,10 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_cert_live_wb
+SPARKD__DATA_DIR=/home/wedbushprod/9000/sparkdata/data
+SPARKD__IMAGE=sparkd_symbols
+SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_cert_live_wb/requirements.txt
+SPARKD__RUNTIME_CONF_DIR=/home/wedbushprod/9000/sparkdata/conf
+SPARKD__SHM_SIZE=8g
+SPARKD__MEMORY=16g
+SPARKD__CONFIG_PORT=9000
+SPARKD__QUICKBUS_PORT=9001
+SPARKD__SIM_FIX_PORT=9002

src/app_sparkd/configurations/ophir_cert_live_wb/spark.conf

@@ -0,0 +1,140 @@
+DeploymentType {
+  value {
+    type: "dev"
+  }
+}
+PublisherOptions {
+   value {
+      network_interface_a: 'ens65f1'
+      network_interface_b: 'ens65f0'
+      #these get overriden if not entered here
+      heartbeat_ms: 5000
+      heartbeat_boot_seconds: 60
+      skip_options: false
+      hot_initial: 1024
+      hot_growth: 1024
+      cold_initial: 8192
+      cold_growth: 2048
+      channel_prefix: ""
+      disable_heartbeat: false
+   }
+}
+
+CMEOptions {
+  value {
+    feed_engine {
+      name: "mdp3feed"
+      spin_time: 0
+      channels_per_thread: 3
+      data_wait: 10
+      udp_socket_buffer_size: 268435456
+    }
+    channel {
+      id: "*"
+      feed_engine: "mdp3feed"
+      use_tcp_replay: false
+      recovery_queue_size: 640000
+      lost_packet_wait_timeout_ms: 10
+      onixs_log_mode: CME_ONIXS_LOG_MODE_UNSPECIFIED
+      #max_gap_recovery_cache_size: 4
+      use_natural_refresh: false
+      join_snapshot_on_gap: false
+    }
+  }
+}
+
+Logging {
+  value {
+    initial_queue_size: 120001
+    max_queue_growth: 4
+    console {
+      default_level: "Info"
+      subsystem {
+        name: "md"
+        level: "Info"
+      }
+      subsystem {
+        name: "OrderMan"
+        level: "Info"
+      }
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+    glbx {
+       unique_check_value: 15
+       use_databento: false
+    }
+   # recording {
+   #    venue: VENUE_GLBX
+   #    num_of_seconds_to_run_file_writer: 300
+   #    num_of_seconds_to_run_db_writer : 0
+   #    n_book_levels: 10 
+   #    batch_size: 100 #for the db writer
+   #    enable_book_recording: true
+   #    enable_trade_recording:  true
+   #    enable_bbo_recording: false
+   #    enable_tbbo_recording:  false
+   #    enable_misc_recording: true
+   #}
+ }
+}
+
+JetFIX {
+  value {
+    connection {
+      name: "maw_004_64"
+      venue: VENUE_GLBX
+      socket_connect_host_ip: "69.50.112.177"
+      socket_connect_port: 44040
+      sender_compid: "MAW004"
+      target_compid: "CME"
+      target_subid: "64"
+      cme_market_segment_id: 64
+      start_time: "00:10:00"
+      end_time: "23:55:00"
+      start_day: "sunday"
+      end_day: "friday"
+      seconds_between_hbs: 15
+      sender_subid: "BTS"
+      seconds_to_pause: 2
+      max_connection_attempts: 100
+      use_ilink3: true
+    }
+  }
+}
+OrdermanCfg {
+  value {
+    executor {
+      thread_name: "oman_orders"
+      poll: true 
+      hosted_actor: "MDSource"
+      hosted_actor: "OrderManager"
+      hosted_actor: "ExecutionGateway"
+    }
+    executor {
+      thread_name: "oman_slow"
+            poll: false 
+      hosted_actor: "*"
+    }
+    fallback_cust_order_handling_inst: "Y"
+  }
+}
+NotSimExecution {
+  value {
+    entries {
+      venue: VENUE_GLBX
+    }
+  }
+}
+Ports {
+   value {
+      config_port: 9000
+      qbus_port: 9001
+      glbx_sim_fix_port: 9002
+      ice_sim_fix_port: 9003
+      replicator_port: 9080
+   }
+}

src/app_sparkd/configurations/ophir_demo/.env

@@ -1,5 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_demo
-SPARKD__DATA_DIR=/home/ubuntu/8000/sparkdata/data
-SPARKD__IMAGE=sparkd_symbols
-SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_demo/requirements.txt
-SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf

src/app_sparkd/configurations/ophir_demo/spark.conf

@@ -1,62 +0,0 @@
-JetFIX {
-  value {
-    connection {
-      name: "glbx_1"
-      venue: VENUE_GLBX
-      max_order_msgs_per_window: 300
-      window_duration_msecs: 1000
-    }
-  }
-}
-
-PublisherOptions {
-  value {
-    network_interface: "0.0.0.0"
-    heartbeat_ms: 5000
-    heartbeat_boot_seconds: 60
-    skip_options: true
-    hot_initial: 1024
-    hot_growth: 1024
-    cold_initial: 8192
-    cold_growth: 2048
-    channel_prefix: ""
-    disable_heartbeat: true
-  }
-}
-
-Logging {
-  value {
-    initial_queue_size: 120001
-    max_queue_growth: 4
-    console {
-      default_level: "Info"
-      subsystem {
-        name: "md"
-        level: "Warn"
-      }
-      subsystem {
-        name: "OrderMan"
-        level: "Info"
-      }
-    }
-    file {
-      default_level: "Info"
-      subsystem {
-        name: "OrderMan"
-        level: "Diag"
-      }
-      subsystem {
-        name: "Http"
-        level: "Diag"
-      }
-    }
-  }
-}
-
-MarketDataConfig {
-  value {
-    replay {
-      venue: VENUE_GLBX
-    }
-  }
-}

src/app_sparkd/configurations/ophir_demo/symbols.conf

@@ -1,8 +0,0 @@
-Symbols {
-  value {
-    venues {
-      name: "VENUE_GLBX"
-      future: "ES"
-    }
-  }
-}

src/app_sparkd/configurations/ophir_prod_live_wb/.env

@@ -1,5 +1,7 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_sim_live_wb
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_prod_live_wb
 SPARKD__DATA_DIR=/home/wedbushprod/8000/sparkdata/data
 SPARKD__IMAGE=sparkd_symbols
-SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_sim_live_wb/requirements.txt
+SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_prod_live_wb/requirements.txt
 SPARKD__RUNTIME_CONF_DIR=/home/wedbushprod/8000/sparkdata/conf
+SPARKD__SHM_SIZE=8g
+SPARKD__MEMORY=16g

src/app_sparkd/configurations/ophir_prod_live_wb/requirements.txt

@@ -0,0 +1,5 @@
+numpy==2.2.6
+pybit==5.11.0
+requests==2.31.0
+websocket-client==1.8.0
+websockets==15.0.1

src/app_sparkd/configurations/ophir_prod_live_wb/spark.conf

@@ -0,0 +1,152 @@
+DeploymentType {
+  value {
+    type: "prod"
+  }
+}
+PublisherOptions {
+   value {
+      network_interface_a: 'enp193s0f0np0'
+      network_interface_b: 'enp193s0f1np1'
+      #these get overriden if not entered here
+      heartbeat_ms: 5000
+      heartbeat_boot_seconds: 60
+      skip_options: false
+      hot_initial: 1024
+      hot_growth: 1024
+      cold_initial: 8192
+      cold_growth: 2048
+      channel_prefix: ""
+      disable_heartbeat: false
+   }
+}
+
+CMEOptions {
+  value {
+    feed_engine {
+      name: "mdp3feed"
+      spin_time: 0
+      channels_per_thread: 3
+      data_wait: 10
+      udp_socket_buffer_size: 268435456
+    }
+    channel {
+      id: "*"
+      feed_engine: "mdp3feed"
+      use_tcp_replay: false
+      recovery_queue_size: 640000
+      lost_packet_wait_timeout_ms: 10
+      onixs_log_mode: CME_ONIXS_LOG_MODE_UNSPECIFIED
+      #max_gap_recovery_cache_size: 4
+      use_natural_refresh: false
+      join_snapshot_on_gap: false
+    }
+  }
+}
+
+Logging {
+  value {
+    initial_queue_size: 120001
+    max_queue_growth: 4
+    console {
+      default_level: "Info"
+      subsystem {
+        name: "md"
+        level: "Info"
+      }
+      subsystem {
+        name: "OrderMan"
+        level: "Info"
+      }
+    }
+  }
+}
+
+MarketDataConfig {
+  value {
+    glbx {
+       unique_check_value: 15
+       use_databento: false
+    }
+   # recording {
+   #    venue: VENUE_GLBX
+   #    num_of_seconds_to_run_file_writer: 300
+   #    num_of_seconds_to_run_db_writer : 0
+   #    n_book_levels: 10 
+   #    batch_size: 100 #for the db writer
+   #    enable_book_recording: true
+   #    enable_trade_recording:  true
+   #    enable_bbo_recording: false
+   #    enable_tbbo_recording:  false
+   #    enable_misc_recording: true
+   #}
+ }
+}
+
+JetFIX {
+  value {
+    connection {
+      name: "u6p_7qb_54"
+      venue: VENUE_GLBX
+      socket_connect_host_ip: "205.209.201.97"
+      socket_connect_port: 10979
+      sender_compid: "U6P7QB"
+      target_compid: "CME"
+      target_subid: "54"
+      cme_market_segment_id: 54
+      start_time: "00:10:00"
+      end_time: "23:55:00"
+      start_day: "sunday"
+      end_day: "friday"
+      seconds_between_hbs: 15
+      sender_subid: "BTS"
+      seconds_to_pause: 1
+      max_order_msgs_per_window: 300
+      max_connection_attempts: 2
+      use_ilink3: true
+    }
+   connection {
+      name: "u6p_7qb_64"
+      venue: VENUE_GLBX
+      socket_connect_host_ip: "205.209.201.1"
+      socket_connect_port: 10979
+      sender_compid: "U6P7QB"
+      target_compid: "CME"
+      target_subid: "64"
+      cme_market_segment_id: 64
+      start_time: "00:10:00"
+      end_time: "23:55:00"
+      start_day: "sunday"
+      end_day: "friday"
+      seconds_between_hbs: 15
+      sender_subid: "BTS"
+      seconds_to_pause: 1
+      max_connection_attempts: 2
+      max_order_msgs_per_window: 300
+      use_ilink3: true
+    }
+  }
+}
+OrdermanCfg {
+  value {
+    executor {
+      thread_name: "oman_orders"
+      poll: true 
+      hosted_actor: "MDSource"
+      hosted_actor: "OrderManager"
+      hosted_actor: "ExecutionGateway"
+    }
+    executor {
+      thread_name: "oman_slow"
+            poll: false 
+      hosted_actor: "*"
+    }
+    fallback_cust_order_handling_inst: "Y"
+  }
+}
+NotSimExecution {
+  value {
+    entries {
+      venue: VENUE_GLBX
+    }
+  }
+}

src/app_sparkd/configurations/ophir_prod_live_wb_sim/.env

@@ -0,0 +1,7 @@
+SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_prod_live_wb
+SPARKD__DATA_DIR=/home/dk/8000/sparkdata/data
+SPARKD__IMAGE=sparkd_symbols
+SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_prod_live_wb/requirements.txt
+SPARKD__RUNTIME_CONF_DIR=/home/dk/8000/sparkdata/conf
+SPARKD__SHM_SIZE=8g
+SPARKD__MEMORY=16g

src/app_sparkd/configurations/ophir_prod_live_wb_sim/requirements.txt

@@ -0,0 +1,5 @@
+numpy==2.2.6
+pybit==5.11.0
+requests==2.31.0
+websocket-client==1.8.0
+websockets==15.0.1

src/app_sparkd/configurations/ophir_prod_live_wb_sim/spark.conf

@@ -1,18 +1,12 @@
-JetFIX {
+DeploymentType {
   value {
-    connection {
-      name: "glbx_1"
-      venue: VENUE_GLBX
-      max_order_msgs_per_window: 300
-      window_duration_msecs: 1000
-    }
+    type: "prod"
   }
 }
-
 PublisherOptions {
    value {
       network_interface_a: 'enp193s0f0np0'
-      network_interface_b: 'enp193s0f0np0'
+      network_interface_b: 'enp193s0f1np1'
       #these get overriden if not entered here
       heartbeat_ms: 5000
       heartbeat_boot_seconds: 60
@@ -41,7 +35,7 @@ CMEOptions {
       use_tcp_replay: false
       recovery_queue_size: 640000
       lost_packet_wait_timeout_ms: 10
-      onixs_log_mode: CME_ONIXS_LOG_MODE_UNSPECIFIED:q
+      onixs_log_mode: CME_ONIXS_LOG_MODE_UNSPECIFIED
       #max_gap_recovery_cache_size: 4
       use_natural_refresh: false
       join_snapshot_on_gap: false
@@ -73,44 +67,35 @@ MarketDataConfig {
        unique_check_value: 15
        use_databento: false
     }
-    recording{
-       venue: VENUE_GLBX
-       num_of_seconds_to_run_file_writer: 300
-       num_of_seconds_to_run_db_writer : 0
-       n_book_levels: 10 
-       batch_size: 100 #for the db writer
-       enable_book_recording: true
-       enable_trade_recording:  true
-       enable_bbo_recording: false
-       enable_tbbo_recording:  false
-       enable_misc_recording: true
-   }
-  }
+   # recording {
+   #    venue: VENUE_GLBX
+   #    num_of_seconds_to_run_file_writer: 300
+   #    num_of_seconds_to_run_db_writer : 0
+   #    n_book_levels: 10 
+   #    batch_size: 100 #for the db writer
+   #    enable_book_recording: true
+   #    enable_trade_recording:  true
+   #    enable_bbo_recording: false
+   #    enable_tbbo_recording:  false
+   #    enable_misc_recording: true
+   #}
+ }
 }
 
-# JetFIX {
-#    value {
-#     #connection: @JetFIXSimConnection.sim
-#     connection {
-#     name: "mlx_004_54"
-#     venue: VENUE_GLBX
-#     socket_connect_host_ip: "69.50.112.191"
-#     socket_connect_port: 42876
-#     sender_compid: "MLX004"
-#     target_compid: "CME"
-#     target_subid: "54"
-#     cme_market_segment_id: 54
-#     start_time: "00:10:00"
-#     end_time: "23:55:00"
-#     start_day: "sunday"
-#     end_day: "friday"
-#     seconds_between_hbs: 30
-#     sender_subid: "BTS"
-#     use_ilink3: true
-#   #   raw_data: "S4PMA"
-#   #   auto_fix_seq_num: true
-#   }
-#    }
-# }
-
-
+OrdermanCfg {
+  value {
+    executor {
+      thread_name: "oman_orders"
+      poll: true 
+      hosted_actor: "MDSource"
+      hosted_actor: "OrderManager"
+      hosted_actor: "ExecutionGateway"
+    }
+    executor {
+      thread_name: "oman_slow"
+            poll: false 
+      hosted_actor: "*"
+    }
+    fallback_cust_order_handling_inst: "Y"
+  }
+}

src/app_sparkd/configurations/ophir_prod_live_wb_sim/symbols.conf

@@ -0,0 +1,61 @@
+SymbolLists {
+  value {
+    list {
+      key: "GlbxFutures"
+      symbol: "ES"
+    }
+    list {
+      key: "GlbxOptions"
+      symbol: "E1A"
+      symbol: "E1B"
+      symbol: "E1C"
+      symbol: "E1D"
+      symbol: "E2A"
+      symbol: "E2B"
+      symbol: "E2C"
+      symbol: "E2D"
+      symbol: "E3A"
+      symbol: "E3B"
+      symbol: "E3C"
+      symbol: "E3D"
+      symbol: "E4A"
+      symbol: "E4B"
+      symbol: "E4C"
+      symbol: "E4D"
+      symbol: "E5A"
+      symbol: "E5B"
+      symbol: "E5C"
+      symbol: "E5D"
+      symbol: "EW1"
+      symbol: "EW2"
+      symbol: "EW3"
+      symbol: "EW4"
+      symbol: "ESH9"
+      symbol: "ESZ6"
+      symbol: "ESH0"
+      symbol: "ESM6"
+      symbol: "ESZ7"
+      symbol: "ESH6"
+      symbol: "ESM7"
+      symbol: "ESZ8"
+      symbol: "ESH7"
+      symbol: "ESH8"
+      symbol: "ESU6"
+      symbol: "ESZ5"
+      symbol: "ESZ9"
+      symbol: "E1D"
+      symbol: "ESZ5"
+      symbol: "ES"
+    }
+  }
+}
+
+Symbols {
+  value {
+    venues {
+        name: "VENUE_GLBX"
+        future: @SymbolLists.list[GlbxFutures].symbol
+        option: @SymbolLists.list[GlbxOptions].symbol
+    }
+  }
+}

src/app_sparkd/configurations/ophir_sim/.env

@@ -1,5 +0,0 @@
-SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_sim
-SPARKD__DATA_DIR=/home/tns/8000/sparkdata/data
-SPARKD__IMAGE=sparkd_symbols
-SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_sim/requirements.txt
-SPARKD__RUNTIME_CONF_DIR=/home/tns/8000/sparkdata/conf

src/app_sparkd/sparkd/execution/execution_gateway_provider.cpp

@@ -109,7 +109,8 @@ struct ExecutionGatewayProvider::PImpl {
          return make_gw<ICEExecutionGateway>(conn, conn, *l, o);
       case api::VENUE_GLBX:
          if (conn.use_ilink3()) {
-            make_gw<ilink3::GLBXiLink3ExecutionGateway>(conn, conn, *l, o);
+            return make_gw<ilink3::GLBXiLink3ExecutionGateway>(
+                conn, conn, *l, o);
          }
          return make_gw<GLBXExecutionGateway>(conn, conn, *l, o);
       case api::VENUE_DERIBIT:

src/app_sparkd/sparkd/md/glbx/OnixsGlbxSubscription.cpp

@@ -3,9 +3,11 @@
 #include "spark/core/LOCK_GUARD.h"
 #include "spark/logging/logging.h"
 #include "spark/md/IMarketDataListener.h"
+#include "spark/md/GeneralMarketDataListener.h"
 #include "spark/md/SymbolLoader.h"
 #include "spark/os/os.h"
 #include "spark/persistence/IDGenerator.h"
+#include "spark/orderman/gateways/sim/SimExecutionGatewayFactory.h"
 #include "spark/utils/Dispatcher.h"
 #include "sparkd/md/platforms/IMDPlatform.h"
 #include "sparkd/md/platforms/mdp3_onixs/MDP3Main.h"
@@ -29,6 +31,10 @@ using std::vector;
 struct OnixsGlbxSubscription::PImpl {
    GlbxMarketDataConfig config;
    shared_ptr<IDGenerator> sec_id_store;
+   // The decision is captured by start(), before the worker thread is created.
+   // A source never changes output routes while it is running.
+   shared_ptr<IMarketDataListener> simulation_listener;
+   bool simulation_enabled{false};
 
    PImpl(shared_ptr<IDGenerator> sec_id_store, GlbxMarketDataConfig config)
        : config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
@@ -66,7 +72,20 @@ struct OnixsGlbxSubscription::PImpl {
              FMT_JOIN(symbols, ", "));
          vector<shared_ptr<IMDPlatform>> platforms;
          auto main = std::make_shared<MDP3Main>();
-         main->start({symbols.begin(), symbols.end()}, nullptr, nullptr);
+         if (simulation_enabled) {
+            bts_assert(simulation_listener != nullptr);
+            LOG(md, Info,
+                "GLBX MDP3 route: simulation (SimExecutionGateway is enabled)");
+            main->start(
+                {symbols.begin(), symbols.end()}, nullptr, nullptr,
+                simulation_listener);
+         } else {
+            // Keep the production route byte-for-byte on its established
+            // startup path: MDP3 owns its legacy producer and SHM channels.
+            LOG(md, Info,
+                "GLBX MDP3 route: legacy production (SimExecutionGateway is disabled)");
+            main->start({symbols.begin(), symbols.end()}, nullptr, nullptr);
+         }
          platforms.emplace_back(std::move(main));
          dispatcher = &utils::Dispatcher::current();
          heartbeat_timer.emplace(*dispatcher, 250, [&]() {
@@ -107,9 +126,24 @@ OnixsGlbxSubscription::OnixsGlbxSubscription(
 OnixsGlbxSubscription::~OnixsGlbxSubscription() = default;
 
 void OnixsGlbxSubscription::start(
-    shared_ptr<IMarketDataListener> /*listener*/) {
+    shared_ptr<IMarketDataListener> listener) {
    std::atomic_thread_fence(std::memory_order_release);
    bts_assert(!me_->dispatcher_thread.joinable());
+   me_->simulation_enabled =
+       orderman::SimExecutionGatewayFactory::is_sim_execution_on(
+           api::VENUE_GLBX);
+   me_->simulation_listener =
+       me_->simulation_enabled ? std::move(listener) : nullptr;
+   bts_assert(!me_->simulation_enabled || me_->simulation_listener != nullptr);
+   if (me_->simulation_enabled) {
+      auto generalized_listener =
+          std::dynamic_pointer_cast<GeneralMarketDataListener>(
+              me_->simulation_listener);
+      bts_assert_msg(
+          generalized_listener != nullptr,
+          "MDP3 simulation requires a GeneralMarketDataListener");
+      generalized_listener->set_source_security_ids_are_external();
+   }
    me_->dispatcher_thread = std::thread{[&]() { me_->run(); }};
 }
 

src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp

@@ -1,29 +1,58 @@
 #include "MDP3Listener.h"
-#include "spark/config/protos/market_data.pb.h"
+#include "spark/md/log_messages/trade_latency_log.h"
+#include "spark/config/protos/market_data.pb.h"
+#include "spark/domain/Book.h"
+#include "spark/domain/Security.h"
+#include "spark/domain/TradeRecord.h"
 #include "spark/md/protos/co2.pb.h"
 #include "spark/utils/datetime.h"
 #include "spark/utils/helper_macros.h"
 #include <vector>
 
 using namespace std;
-using namespace bts::co2;
-
-namespace bts {
-namespace md {
+using namespace bts::co2;
+
+namespace bts {
+namespace md {
+
+namespace {
+
+domain::AggressorSide to_domain_aggressor(onixs::AggressorSide::Enum side) {
+   if (side == onixs::AggressorSide::Buy) return domain::AggressorSide::Buy;
+   if (side == onixs::AggressorSide::Sell) return domain::AggressorSide::Sell;
+   return domain::AggressorSide::None;
+}
+
+domain::BookLevel to_domain_book_level(const PriceSize &level) {
+   if (level.is_nan() || level.size() == 0) return domain::BookLevel::empty();
+   return {domain::Price::from_double(level.price()),
+           domain::Quantity(level.size()), domain::Quantity(level.direct_size()), 0};
+}
+
+template <size_t N>
+void copy_book_side(const std::array<PriceSize, N> &source,
+                    domain::BookSide &destination) {
+   const auto count = std::min(source.size(), destination.size());
+   for (size_t i = 0; i < count; ++i) destination[i] = to_domain_book_level(source[i]);
+}
+
+} // namespace
 
 MDP3Listener::MDP3Listener(
     CMESecurityIndex &index,
     MDProducer &producer,
-    MBOWriter *mbo,
-    const std::vector<MDProducerChannel> channels,
-    TradeSummarySignalDispatcher *trade_summary_dispatcher)
+    MBOWriter *mbo,
+    const std::vector<MDProducerChannel> channels,
+    TradeSummarySignalDispatcher *trade_summary_dispatcher,
+    std::shared_ptr<IMarketDataListener> simulation_listener)
     : _index(index), _producer(producer),
       _verbose_ticker(config::get<CMEOptions>().verbose_ticker()),
       _packet_arrived(), _latest_sequence(0),
       // _settlement_recorder(settles),
       _symbol_filter_storage(), _symbol_filter(), _channels(channels),
       _concurrency_monitor(0), _book_updates(), _mbo(mbo),
-      _trade_summary_dispatcher(trade_summary_dispatcher) {
+      _trade_summary_dispatcher(trade_summary_dispatcher),
+      _simulation_listener(std::move(simulation_listener)) {
    _book_updates.reserve(32);
 }
 
@@ -179,16 +208,38 @@ void MDP3Listener::onBookUpdate(
 // Trades
 // Called from onTradeSummary and onTradeRecovery, ends up calling
 // MDProducerHandle::add_trade
-void on_trade(
+void on_trade(
     SecurityData *data,
     const onixs::Decimal &price,
     int size,
     onixs::AggressorSide::Enum agside,
-    const onixs::Timestamp &onixs_date) {
+    const onixs::Timestamp &onixs_date,
+    IMarketDataListener *simulation_listener) {
    auto ts = utils::TimeStamp::now();
    PriceSize ps = pricesize_of_price_size(
        price, data->handle.security().display_factor, size);
-   auto date = ll_of_timestamp(onixs_date);
+   if (simulation_listener != nullptr) {
+      // The MDP3 private handle key is the CME security id.  GMDL resolves it
+      // to the persisted Spark key before it creates its SHM security.
+      domain::TradeRecord trade;
+      trade.sec_id = domain::SecurityID(data->handle.key());
+      trade.time_stamp =
+          core::TimeStamp(
+              onixs_date.year(), static_cast<uint8_t>(onixs_date.month()),
+              onixs_date.day(), onixs_date.hour(), onixs_date.minute(),
+              onixs_date.second()) +
+          core::TimeSpan::from_millis(onixs_date.millisecond());
+      trade.agg_side = to_domain_aggressor(agside);
+      trade.trade_quantity = domain::Quantity(size);
+      trade.trade_price = domain::Price::from_double(ps.price());
+      std::vector<domain::TradeRecord const *> trades{&trade};
+      simulation_listener->on_trades(trades);
+      return;
+   }
+   auto date = ll_of_timestamp(onixs_date);
+   log_trade_latency(
+       TradeLatencyStage::MDP3_RECEIVED, data->handle.security(), date, ps,
+       map_aggressor(agside), data->electronic_volume + size, false);
    data->handle.add_trade(
        ts, ps, date, map_aggressor(agside), data->electronic_volume + size);
 }
@@ -284,24 +335,83 @@ void MDP3Listener::heartbeat() {
 
 // Called by MDP3Main::prepare_listener as part of pre-load. Also called by
 // MDP3Listener::onFuture, onOption and onSpread
-void MDP3Listener::add_security(
-    const FixedSecurity &security, co2::TradingStatus status) {
+void MDP3Listener::add_security(
+    FixedSecurity &security, co2::TradingStatus status) {
    bts_assert(security.numeric_id != 0);
-   // Set up the shared memory for this security
-   auto handle = _producer.add_security(security);
+   if (security.type == api::SECURITYTYPE_OPTION) {
+      const auto &option = static_cast<const FixedOptionSecurity &>(security);
+      const datetime::Date expiration(
+          option.expiration_year, option.expiration_month,
+          option.expiration_day);
+      if (!option_expiration_is_allowed(expiration)) {
+         LOG(md, Info,
+             "Skipping MDP3 option {} because it expires beyond the configured "
+             "cutoff.",
+             std::string(security.symbol()));
+         return;
+      }
+   }
+   if (_simulation_listener != nullptr) {
+      // A private producer still needs a stable key for its decoding cache.
+      // Keep that key as the CME external id; GMDL owns persisted Spark-key
+      // resolution and the only shared-memory publication.
+      security.key = security.numeric_id;
+      if (security.type == api::SECURITYTYPE_SPREAD) {
+         auto &spread = static_cast<FixedSpreadSecurityL &>(security);
+         for (size_t i = 0; i < spread.leg_count(); ++i) {
+            // The private producer validates that spread legs have keys when
+            // a security becomes visible.  At this stage the only valid
+            // identity is CME's leg SecurityID; GMDL resolves it later.
+            spread.legs[i].key = spread.legs[i].exchange_key.value();
+         }
+      }
+   }
+   auto handle = _producer.add_security(security);
    // Add it to the index so it can receive messages
-   _index.add_security(security.numeric_id, handle);
+   _index.add_security(security.numeric_id, handle);
+
+   if (_simulation_listener != nullptr) {
+      domain::Security domain_security(convert_security(security));
+      std::vector<domain::Security const *> securities{&domain_security};
+      _simulation_listener->on_securities(securities);
+      return;
+   }
 
    handle.update_exchange_status(utils::TimeStamp::now(), status);
 }
 
 // Private functions
 
-void MDP3Listener::flush_books() {
+void MDP3Listener::flush_books() {
    // This is called at the end of the packet
    // Write all book updates to shared memory and reset for the next packet
-   for (auto &entry : _book_updates) {
-      uint32_t sequence = _latest_sequence;
+   for (auto &entry : _book_updates) {
+      uint32_t sequence = _latest_sequence;
+      if (_simulation_listener != nullptr) {
+         // A domain Book has direct and implied quantities per level.  Sending
+         // the merged snapshot once per packet preserves both MDP3 book kinds
+         // and gives the simulation gateway one matching/publication point.
+         domain::Book book = domain::Book::empty(
+             domain::SecurityID(entry.handle.key()));
+         copy_book_side(entry.bids.first, book.bids());
+         copy_book_side(entry.asks.first, book.asks());
+         if (entry.type_flags & book_type_implied) {
+            for (size_t i = 0; i < entry.bids.second.size() && i < book.bids().size(); ++i) {
+               const auto &implied = entry.bids.second[i];
+               if (!implied.is_nan() && implied.size() != 0) {
+                  book.bids()[i] = to_domain_book_level(implied);
+               }
+            }
+            for (size_t i = 0; i < entry.asks.second.size() && i < book.asks().size(); ++i) {
+               const auto &implied = entry.asks.second[i];
+               if (!implied.is_nan() && implied.size() != 0) {
+                  book.asks()[i] = to_domain_book_level(implied);
+               }
+            }
+         }
+         _simulation_listener->on_book(book, _packet_arrived);
+         continue;
+      }
       if (entry.type_flags == book_type_direct) {
          entry.handle.update_direct(
              _packet_arrived, entry.bids.first, entry.asks.first, sequence);
@@ -316,7 +426,7 @@ void MDP3Listener::flush_books() {
    }
    // Reset pending updates
    _book_updates.clear();
-}
+}
 
 void MDP3Listener::update_all_status(CoconutSecurityStatus status) {
    _index.walk_security_list(

src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h

@@ -1,9 +1,12 @@
 #pragma once
 
 #include "CMESecurityIndex.h"
+#include "spark/config/config.h"
+#include "spark/config/protos/toggles.pb.h"
 #include "spark/io/JsonDumper.h"
 #include "spark/io/json_formatter.h"
-#include "spark/md/MBOProcessor.h"
+#include "spark/md/MBOProcessor.h"
+#include "spark/md/IMarketDataListener.h"
 #include "spark/md/Shm/FixedSecurity.h"
 #include "spark/md/protos/trading_status.h"
 #include "spark/orderman/events/TradeSummarySignalDispatcher.h"
@@ -21,6 +24,28 @@
 namespace bts {
 namespace md {
 
+inline bool option_expiration_is_allowed(const datetime::Date &expiration) {
+   const auto max_days =
+       config::get<config::toggles::Mdp3OptionExpirationCutoff>()
+           .max_days_until_expiration();
+   if (expiration.year == 0) {
+      return true;
+   }
+   if (max_days == 0) {
+      return true;
+   }
+   const auto cutoff = core::TimeStamp::now().date() +
+                       core::TimeSpan::from_days(max_days);
+   const auto expiration_timestamp =
+       core::TimeStamp(expiration.year, expiration.month, expiration.day);
+   const bool allowed = expiration_timestamp <= cutoff;
+   uint32_t cutoff_year;
+   uint32_t cutoff_month;
+   uint32_t cutoff_day;
+   cutoff.to_date(cutoff_year, cutoff_month, cutoff_day);
+   return allowed;
+}
+
 inline co2::AggressorSide map_aggressor(onixs::AggressorSide::Enum agside) {
    switch (agside) {
    default:
@@ -128,9 +153,10 @@ class MDP3Listener : public onixs::HandlerListener,
    MDP3Listener(
        CMESecurityIndex &index,
        MDProducer &,
-       MBOWriter *mbo,
-       const std::vector<MDProducerChannel>,
-       TradeSummarySignalDispatcher *);
+       MBOWriter *mbo,
+       const std::vector<MDProducerChannel>,
+       TradeSummarySignalDispatcher *,
+       std::shared_ptr<IMarketDataListener> simulation_listener = nullptr);
    virtual ~MDP3Listener();
 
    //
@@ -643,7 +669,7 @@ class MDP3Listener : public onixs::HandlerListener,
    void heartbeat();
 
    // Add a new security to the index
-   void add_security(const FixedSecurity &security, co2::TradingStatus);
+   void add_security(FixedSecurity &security, co2::TradingStatus);
 
  private:
    enum SecurityFilterAction { ADD, SKIP, DELETE };
@@ -696,8 +722,9 @@ class MDP3Listener : public onixs::HandlerListener,
    };
    std::vector<BookData> _book_updates;
    MBOWriter *_mbo;
-   TradeSummarySignalDispatcher *_trade_summary_dispatcher;
-};
+   TradeSummarySignalDispatcher *_trade_summary_dispatcher;
+   std::shared_ptr<IMarketDataListener> _simulation_listener;
+};
 
 //////////////////////////////////////////
 // Template Member Function Definitions
@@ -781,6 +808,11 @@ void MDP3Listener::onOption(
           exchange_id, sec_name, Mdp3SecuritySkipReason::ALREADY_ADDED);
       return;
    }
+   if (!option_expiration_is_allowed(date_of_maturity(sec))) {
+      LOG_MDP3_ON_OPTION_SKIP(
+          exchange_id, sec_name, Mdp3SecuritySkipReason::EXPIRED);
+      return;
+   }
 
    LOG_MDP3_ON_OPTION_BEGIN(exchange_id, std::string(sec.symbol().items()));
    FixedOptionSecurity fsec;
@@ -952,12 +984,13 @@ void MDP3Listener::onBookUpdate(
    }
 }
 
-void on_trade(
+void on_trade(
     SecurityData *data,
     const onixs::Decimal &price,
     int size,
     onixs::AggressorSide::Enum agside,
-    const onixs::Timestamp &onixs_date);
+    const onixs::Timestamp &onixs_date,
+    IMarketDataListener *simulation_listener);
 
 template <typename TradeRecovery>
 void MDP3Listener::onTradeRecovery(
@@ -972,8 +1005,8 @@ void MDP3Listener::onTradeRecovery(
       onixs::Decimal price;
       arg.entry().entryPx(price);
       on_trade(
-          data, price, size, onixs::AggressorSide::NoAggressor,
-          arg.message().transactTime());
+          data, price, size, onixs::AggressorSide::NoAggressor,
+          arg.message().transactTime(), _simulation_listener.get());
    }
 }
 
@@ -996,10 +1029,12 @@ void MDP3Listener::onTradeSummary(
          for (size_t x = 0; x < sz; ++x) {
             const auto &d = details[x];
             on_trade(
-                data, price, d.lastQty(), agside, arg.message().transactTime());
+                data, price, d.lastQty(), agside, arg.message().transactTime(),
+                _simulation_listener.get());
          }
       } else {
-         on_trade(data, price, size, agside, arg.message().transactTime());
+         on_trade(data, price, size, agside, arg.message().transactTime(),
+                  _simulation_listener.get());
       }
    }
 }

src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp

@@ -36,6 +36,10 @@ struct MDP3Main::PImpl {
    CMESecurityIndex index;
    SecurityIDRecorder recorder;
    MDProducer producer;
+   // Simulation needs MDP3's per-security decoding state, but it must not
+   // become a second GLBX shared-memory writer.  This producer has no shared
+   // memory or security mapper; GMDL is the sole public producer in that mode.
+   MDProducer simulation_producer{MDProducerFlags::None};
    std::map<std::string, std::unique_ptr<onixs::FeedEngine>> engines;
    std::map<std::string, std::unique_ptr<onixs::Handler>> handlers;
    std::vector<std::unique_ptr<MDP3Listener>> listeners;
@@ -178,7 +182,8 @@ onixs::SessionSettings session_settings_for_channel(
 void MDP3Main::start(
     const std::vector<UserSymbol> &symbols,
     MBOWriter *mbo,
-    TradeSummarySignalDispatcher *tssd) {
+    TradeSummarySignalDispatcher *tssd,
+    std::shared_ptr<IMarketDataListener> simulation_listener) {
    std::ostringstream oss;
    for (const auto &us : symbols) {
       oss << us << " ";
@@ -191,7 +196,8 @@ void MDP3Main::start(
    me_->engines = create_feed_engines(channels);
    for (const auto &channel : channels) {
       unique_ptr<onixs::Handler> phandler = prepare_handler(channel.id, mbo);
-      me_->listeners.push_back(prepare_listener(*phandler, channel, mbo, tssd));
+      me_->listeners.push_back(prepare_listener(
+          *phandler, channel, mbo, tssd, simulation_listener));
       me_->handlers.insert(std::make_pair(channel.id, std::move(phandler)));
       std::stringstream symstr;
       for (const auto &sym : channel.symbols) {
@@ -386,19 +392,25 @@ unique_ptr<MDP3Listener> MDP3Main::prepare_listener(
     onixs::Handler &handler,
     const CMEChannel &channel,
     MBOWriter *mbo,
-    TradeSummarySignalDispatcher *trade_summary_dispatcher) {
+    TradeSummarySignalDispatcher *trade_summary_dispatcher,
+    std::shared_ptr<IMarketDataListener> simulation_listener) {
    LOG(md, Info, "MDP3Main::prepare_listener channel: {}", channel);
    // We need to get a handler to the producer channels so we can produce
    // heartbeats
    vector<MDProducerChannel> producer_channels;
-   for (auto &symbol : channel.symbols) {
-      producer_channels.push_back(me_->producer.get_channel(
-          symbol.symbol().c_str(), symbol.type(), symbol.exchange()));
+   MDProducer &producer = simulation_listener ? me_->simulation_producer
+                                               : me_->producer;
+   // Private simulation state does not need heartbeat SHM channels.
+   if (!simulation_listener) {
+      for (auto &symbol : channel.symbols) {
+         producer_channels.push_back(producer.get_channel(
+             symbol.symbol().c_str(), symbol.type(), symbol.exchange()));
+      }
    }
 
    unique_ptr<MDP3Listener> listener(new MDP3Listener(
-       me_->index, me_->producer, mbo, producer_channels,
-       trade_summary_dispatcher));
+       me_->index, producer, mbo, producer_channels,
+       trade_summary_dispatcher, std::move(simulation_listener)));
    for (auto &symbol : channel.symbols) {
       listener->add_filter_symbol(symbol.symbol());
    }

src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.h

@@ -2,6 +2,7 @@
 
 #include "CMESecurityIndex.h"
 #include "MDP3Listener.h"
+#include "spark/md/IMarketDataListener.h"
 #include "spark/md/coconut2.h"
 #include "spark/utils/bts_no_copy.h"
 #include "onixs.h"
@@ -37,7 +38,8 @@ class MDP3Main : public IMDPlatform {
    void start(
        const std::vector<UserSymbol> &symbols,
        MBOWriter *mbo,
-       TradeSummarySignalDispatcher *tssd);
+       TradeSummarySignalDispatcher *tssd,
+       std::shared_ptr<IMarketDataListener> simulation_listener = nullptr);
    void send_heartbeats() override;
 
    void initialize(const std::string &license_store);
@@ -47,7 +49,8 @@ class MDP3Main : public IMDPlatform {
        onixs::Handler &handler,
        const CMEChannel &channel,
        MBOWriter *,
-       TradeSummarySignalDispatcher *);
+       TradeSummarySignalDispatcher *,
+       std::shared_ptr<IMarketDataListener> simulation_listener = nullptr);
 
  private:
    struct PImpl;

src/app_sparkd/sparkd/position/PositionServiceServer.cpp

@@ -231,10 +231,16 @@ void PositionServiceServer::create_manual_fill(
    for (auto mf : batch.manual_fill()) {
       auto id = utils::gen_id("manual_fills");
       mf.set_record_id(id);
-      struct timeval tv;
-      gettimeofday(&tv, NULL);
-      mf.mutable_timestamp()->set_seconds(tv.tv_sec);
-      mf.mutable_timestamp()->set_nanos(tv.tv_usec * 1000);
+      // Older clients do not send a timestamp.  Retain their existing behavior,
+      // but preserve a timestamp supplied by newer clients so a manual fill can
+      // be assigned to its intended trading day.
+      if (!mf.has_timestamp() ||
+          (mf.timestamp().seconds() == 0 && mf.timestamp().nanos() == 0)) {
+         struct timeval tv;
+         gettimeofday(&tv, NULL);
+         mf.mutable_timestamp()->set_seconds(tv.tv_sec);
+         mf.mutable_timestamp()->set_nanos(tv.tv_usec * 1000);
+      }
 
       me->manual_fills_store->insert(
           manual_fills_ns, std::to_string(id), utils::pb::to_string(mf));
@@ -296,6 +302,16 @@ void PositionServiceServer::update_manual_fill(
          this_resp->set_ok(false);
          this_resp->set_msg("Could not find record");
       } else {
+         // Legacy clients omit timestamps when editing.  Do not let that erase
+         // the trading date that was assigned when the fill was created.
+         if (!mf.has_timestamp() ||
+             (mf.timestamp().seconds() == 0 && mf.timestamp().nanos() == 0)) {
+            ManualFill existing;
+            if (utils::pb::from_string<ManualFill>(val, &existing) &&
+                existing.has_timestamp()) {
+               mf.mutable_timestamp()->CopyFrom(existing.timestamp());
+            }
+         }
          me->manual_fills_store->update(
              manual_fills_ns, id, utils::pb::to_string(mf));
          this_resp->set_ok(true);

src/client/spark/AW2/LiveTheoPublisher.cs

@@ -160,7 +160,7 @@ namespace mbts.spark {
 
       private bool _compute_midmarket {
          get {
-            return true;
+            return false;
          }
       }
 

src/client/spark/Daemons/OrderTicketDaemon.cs

@@ -26,6 +26,8 @@ using LadderSettings = mbts.spark.LadderSettings;
 namespace bts.shell {
    [Export(typeof(IShellDaemon))]
    class OrderTicketDaemon : IShellDaemon {
+      static readonly bool useTicketLadder = true;
+
       readonly IBus _bus;
       readonly OrdersModel _model;
       readonly UserSettingsHub _settings_hub;
@@ -71,13 +73,23 @@ namespace bts.shell {
 
       void _show_order_ticket(SecurityMD smd) {
          var settings_src = _settings_hub.subscribe_default_settings()
-            .Select(s => s.get_ladder_profile(smd.security)?.value ?? LadderSettings.Default);
+            .Select(s => s.GetLadderProfile(smd.security)?.value ?? LadderSettings.Default);
          var accounts_src = _bus.subscribe(SparkTags.AccountsTag);
-         _show_order_window(new OrderTicket {
-            DataContext = new NewOrderTicketVM(_model, smd, accounts_src, settings_src) {
-               close_on_send = _close_on_send
-            }
-         });
+         _show_order_window(useTicketLadder ? OrderTicketLadderWindow() : OrderTicketWindow());
+
+         Window OrderTicketWindow() =>
+            new OrderTicket {
+               DataContext = new NewOrderTicketVM(_model, smd, accounts_src, settings_src) {
+                  close_on_send = _close_on_send
+               }
+            };
+
+         Window OrderTicketLadderWindow() =>
+            new OrderTicketLadder() {
+               DataContext = new NewOrderTicketLadderVM(
+                  _model, smd, accounts_src, settings_src,
+                  ladderSecurity => _bus.post(SparkTags.ShowLadderTag, ladderSecurity))
+            };
       }
 
       void _show_order_ticket(OrderTicketParams ot_params) {
@@ -114,13 +126,23 @@ namespace bts.shell {
                return;
          }
          var md = _md_map.TryGetValue(ori.order.security_key());
-         _show_order_window(new OrderModificationDialog {
-            DataContext = new ExistingOrderTicketVM(_model,
-               Observable.Return((md, ori.order)),
-               _bus.subscribe(SparkTags.AccountsTag),
-               Observable.Return(LadderSettings.Default.calculator_buttons),
-               md?.quantity_display)
-         });
+         var accountsSrc = _bus.subscribe(SparkTags.AccountsTag);
+         var orderSrc = Observable.Return((md, ori.order));
+         var calcBnsSrc = Observable.Return(LadderSettings.Default.calculator_buttons);
+         _show_order_window(useTicketLadder ? OrderTicketLadderWindow() : OrderTicketWindow());
+
+         Window OrderTicketWindow() =>
+            new OrderModificationDialog {
+               DataContext = new ExistingOrderTicketVM(_model, orderSrc, accountsSrc, calcBnsSrc, md?.quantity_display)
+            };
+
+         Window OrderTicketLadderWindow() =>
+            new OrderModificationDialog {
+               DataContext = new ExistingOrderTicketLadderVM(
+                  _model, orderSrc, accountsSrc, calcBnsSrc,
+                  ladderSecurity => _bus.post(SparkTags.ShowLadderTag, ladderSecurity),
+                  md?.quantity_display)
+            };
       }
    }
 }

src/client/spark/Model/Settings/UserSettingsExtensions.cs

@@ -14,16 +14,11 @@ namespace mbts.spark {
       /// <summary>
       /// Gets the ladder settings profile for a particular security
       /// </summary>
-      public static LadderSettingsProfile? get_ladder_profile(
+      public static LadderSettingsProfile? GetLadderProfile(
          this UserSettings settings, SecurityImmutable security) =>
          settings.ladder_profiles.FirstOrDefault(lsp =>
             lsp?.security_criteria?.includes(security) ?? false);
 
-      public static IObservable<LadderSettingsProfile?> subscribe_ladder_profile(
-         this UserSettingsHub settings_hub, SecurityImmutable security) =>
-         settings_hub.subscribe_default_settings().Select(s =>
-            s.get_ladder_profile(security));
-
       /// <summary>
       /// Gets the default account for a particular security (from the
       /// <see cref="LadderSettingsProfile"/> associated with that security)
@@ -31,7 +26,7 @@ namespace mbts.spark {
       public static IObservable<AccountTuple?> subscribe_default_account(
          this UserSettingsHub settings_hub, IBus bus, SecurityImmutable security) {
          var acc_id_src = settings_hub.subscribe_default_settings().Select(s =>
-            s.get_ladder_profile(security)?.value?.default_account);
+            s.GetLadderProfile(security)?.value?.default_account);
          return acc_id_src.CombineLatest(
             bus.subscribe(SparkTags.AccountsTag),
             (acc_id, accs) => acc_id.HasValue && accs.TryGetValue(acc_id.Value, out var acc_tup) ?

src/client/spark/Modules/Market/MarketView/MarketViewView.xaml.cs

@@ -1,10 +1,10 @@
-using bts;
 using bts.api;
 
 using mbts.spark;
 using mbts.ui.controls;
 using mbts.ui.converters;
 using mbts.ui.extensions;
+using mbts.ui.shell;
 
 using System;
 using System.Collections.Generic;
@@ -23,6 +23,7 @@ namespace mbts.ui.spark {
    /// </summary>
    partial class MarketViewView : UserControl {
       public static readonly RoutedCommand CellClickCommand = new RoutedCommand();
+      static readonly bool useTicketLadder = true;
 
       MarketViewVM _vm;
       DIYGridPopupManager _popup_man_backing;
@@ -92,7 +93,7 @@ namespace mbts.ui.spark {
 
       void _show_mini_orderbook(IReadOnlyList<SecurityImmutable> securities, Columns col) {
          var (vm, context, title, fg) = _vm.orderbook_params(securities, col);
-         _show_popup(vm, new AuxiliaryDataHeader {
+         _show_popup(vm, new OrderAuxiliaryPopupHeader {
             context = context,
             title = title,
             title_foreground = fg
@@ -104,7 +105,7 @@ namespace mbts.ui.spark {
 
       void _show_mini_fills(IEnumerable<SecurityImmutable> securities, Columns col) {
          var (vm, context, title, fg) = _vm.fills_params(securities, col);
-         _show_popup(vm, new AuxiliaryDataHeader {
+         _show_popup(vm, new OrderAuxiliaryPopupHeader {
             context = context,
             title = title,
             title_foreground = fg
@@ -114,29 +115,40 @@ namespace mbts.ui.spark {
       async void _show_order_ticket(
          SecurityMD security_md, double price = double.NaN, double size = 0) {
          var resources = new OrderTicketResources();
-         resources.MergedDictionaries.Add(new AuxiliaryDataResources());
+         resources.MergedDictionaries.Add(new OrderAuxiliaryPopupResources());
 
          var account = await _vm.get_default_account(security_md.security);
          var def = _model.factory.order(security_md.security, account);
          var calc_buttons = LadderSettings.Default.calculator_buttons;
-         var ticket_vm = new NewOrderTicketVM(
-            _model,
-            security_md,
-            _bus.subscribe(SparkTags.AccountsTag),
-            new(def),
-            price,
-            size,
-            Observable.Return(calc_buttons));
+         var accounts = _bus.subscribe(SparkTags.AccountsTag);
+         var (vm, header) = OrderTicketFactory();
 
          _show_popup(
-            ticket_vm,
-            new OrderTicketHeader { bbo = ticket_vm.bbo },
+            vm,
+            header,
             resources,
             view => Behavior.SetAutoFocusTargetProvider(view, AutoFocusTargetProvider.TypeProvider(typeof(TextBox), view)));
+
+         (ViewModelBase, object) OrderTicketFactory() {
+            if (useTicketLadder) {
+               var vm = new NewOrderTicketLadderVM(
+                  _vm.orders_model, security_md, accounts, new(def), price, size,
+                  Observable.Return(calc_buttons),
+                  ladderSecurity => _bus.post(SparkTags.ShowLadderTag, ladderSecurity));
+               var header = new OrderTicketLadderHeader() {
+                  Title = security_md.security.ToString()
+               };
+               return (vm, header);
+            } else {
+               var vm = new NewOrderTicketVM(_model, security_md, accounts, new(def), price, size, Observable.Return(calc_buttons));
+               var header = new OrderTicketHeader { bbo = vm.bbo };
+               return (vm, header);
+            }
+         }
       }
 
       void _show_popup(object content, object header) {
-         var resources = new AuxiliaryDataResources();
+         var resources = new OrderAuxiliaryPopupResources();
          resources.MergedDictionaries.Add(new AutoHideScrollbarResources());
          _show_popup(content, header, resources);
       }
@@ -148,9 +160,9 @@ namespace mbts.ui.spark {
             Header = header,
             HorizontalAlignment = HorizontalAlignment.Left,
             VerticalAlignment = VerticalAlignment.Top,
-            Resources = resources ?? new AuxiliaryDataResources()
+            Resources = resources ?? new OrderAuxiliaryPopupResources()
          };
-         view.Style = view.Resources[AuxiliaryDataResources.PopupChildStyleKey] as Style;
+         view.Style = view.Resources[OrderAuxiliaryPopupResources.PopupChildStyleKey] as Style;
          init_view?.Invoke(view);
          _popup_man.show(view);
       }

src/client/spark/Modules/Trading/Fills/FillsRowVM.cs

@@ -78,6 +78,9 @@ namespace mbts.spark {
       public Fill.Types.FillStatus fill_status => _fill.FillStatus;
       public string fill_key => _fill.FillKey;
       public string order_key => _order.Key.ToString();
+      // public bool is_aggressor => fill_tuple.is_aggressor;
+      public string display_aggressor_flag_text =>
+         display_aggressor_flag(_fill.FillAggressorFlag);
       public bool is_selected {
          get => _is_selected;
          set {
@@ -105,6 +108,13 @@ namespace mbts.spark {
       AlgoData? _root_algo => fill_tuple.root_algo;
       AlgoDefinitionMetadata? _root_algo_meta => fill_tuple.root_algo_meta;
 
+      public static string display_aggressor_flag(FillAggressorFlag flag) =>
+         flag switch {
+            FillAggressorFlag.Aggressor => "A",
+            FillAggressorFlag.Passive => "P",
+            _ => string.Empty
+         };
+
       public static FillsRowVM update(FillsRowVM r, FillTuple ft) {
          using var _ = _change_detector.detect_changes(r);
          r._badge_fill = null;

src/client/spark/Modules/Trading/Fills/FillsVM.cs

@@ -254,7 +254,7 @@ namespace mbts.spark {
          add_col(
             Columns.Price,
             DefaultColumnHeaders.Price,
-            r => _get_row(r).security.spec_or_default().format(r.fill.AvgPrice),
+            r => _get_row(r).security.spec_or_default().format(r.fill.LastPrice),
             new DoubleMinMax<FillTuple>(r => r.fill.LastPrice),
             DataGridFilterKnownTemplates.DoubleMinMax,
             fs => fs.price,
@@ -295,7 +295,7 @@ namespace mbts.spark {
          add_col(
             Columns.IsAggressor,
             DefaultColumnHeaders.IsAggressor,
-            r => r.is_aggressor ? SpecialCharacters.CheckMark : string.Empty,
+            r => FillsRowVM.display_aggressor_flag(r.fill.FillAggressorFlag),
             dv_filter(new[] { false, true }, r => r.is_aggressor),
             DataGridFilterKnownTemplates.YesNo,
             fs => fs.is_aggressor,

src/client/spark/Modules/Trading/Ladder/FairValueModel.cs

@@ -27,13 +27,13 @@ namespace Spark.Trading {
          _sec_greeks_in = sec_greeks_in;
       }
 
-      public IObservable<double> get_price_source(FairValueKey key) {
+      public IObservable<double> get_price_source(FairValueKey key, double? errorPrice = double.NaN) {
          return key.type switch {
             FairValueLine.PriceType.Bid => bid_source(),
             FairValueLine.PriceType.Ask => ask_source(),
             FairValueLine.PriceType.Mid => mid_source(),
             FairValueLine.PriceType.Theo => theo_source(),
-            _ => throw new NotImplementedException()
+            _ => errorPrice.HasValue ? Observable.Return(errorPrice.Value) : throw new NotImplementedException()
          };
 
          IObservable<double> bid_source() =>

src/client/spark/Modules/Trading/Ladder/View/LadderGridHost.cs

@@ -1,4 +1,4 @@
-using bts;
+using bts;
 using bts.api;
 using bts.api.md.auxiliary;
 using bts.collections;
@@ -15,7 +15,6 @@ using mbts.utils;
 
 using Spark.Core.Collections;
 using Spark.Core.Disposables;
-using Spark.Core.Math;
 
 using System;
 using System.Collections.Generic;
@@ -36,8 +35,9 @@ namespace mbts.spark {
    using RegionalTickFactors = user.settings.RegionalTickFactors;
 
    partial class LadderGridHost : Control {
-      static readonly int _initial_row_count;
-      static readonly bool _show_scrollbar;
+      const int _min_row_count = 501;
+      const int _max_row_count = 2001;
+      static readonly bool _show_scrollbar = true;
 
       readonly VisualHost _visual_host = new();
       readonly LadderOrdersHost _ordershost;
@@ -80,13 +80,11 @@ namespace mbts.spark {
       LadderVM _vm;
       LadderTactileAdorner _tactile_adorner;
       ulong _prices_updated;
+      ulong lastTradeTime;
       NativePriceSize[] _bids = new NativePriceSize[NativePriceHelpers.COCONUT_MAX_DEPTH], _asks = new NativePriceSize[NativePriceHelpers.COCONUT_MAX_DEPTH];
       SecurityMDExtensions.Book _depth = SecurityMDExtensions.Book.Empty;
       bool _price_dirty = false, _piq_dirty = false;
-      LadderPriceList _price_list = new() {
-         length = _initial_row_count,
-      };
-      LadderPriceList _pending_price_list = null;
+      IList<double> _price_list;
       IComparer<double> _price_comparer;
       double _last_origin_price;
       readonly MarketBreakLines _mkt_break_lines;
@@ -107,7 +105,6 @@ namespace mbts.spark {
       static LadderGridHost() {
          var knobs = config.Configuration.get<bts.config.LadderKnobs>();
          _show_scrollbar = knobs.ShowScrollbar;
-         _initial_row_count = (int)knobs.ScrollableTickWindowSize;
       }
 
       public LadderGridHost() {
@@ -132,33 +129,7 @@ namespace mbts.spark {
                _render_grid_background();
                _init_tactile_adorner();
             }
-            IDisposable scroll_viewer_hook = null;
-            loaded_hook = Disposable.Create(
-               () => {
-                  _scroll_viewer_source.value_changed -= on_scroll_viewer;
-                  Disposable.dispose(ref scroll_viewer_hook);
-               });
-            on_scroll_viewer(_scroll_viewer);
-
-            void on_scroll_viewer(ScrollViewer new_value) {
-               Disposable.dispose(ref scroll_viewer_hook);
-               if (new_value is null) {
-                  return;
-               }
-               scroll_viewer_hook = _connect_scroll_viewer(new_value);
-               delayed_init();
-               // This is a bit of a hack: the scroll viewer drifts away from the intended VerticalOffset
-               // unless it is done after a delay.
-               async void delayed_init() {
-                  await Task.Delay(100);
-                  await Dispatcher.InvokeAsync(() => {
-                     _pending_scroll_to_origin = true;
-                     _refresh_prices();
-                     _refresh_all_visuals();
-                     scroll_to_pending_target();
-                  });
-               }
-            }
+            _scroll_to_center();
          };
          recenter_command = new DelegateCommand(_scroll_to_center);
          DataContextChanged += (s, e) => _connect_vm();
@@ -282,26 +253,6 @@ namespace mbts.spark {
          set => SetValue(PIQColumnWidthProperty, value);
       }
 
-      #region TopTick
-      public static readonly DependencyProperty TopTickProperty =
-         WPFHelper.create_dp<int, LadderGridHost>(
-            nameof(TopTick),
-            FrameworkPropertyMetadataOptions.AffectsRender,
-            (o, v) => {
-               if (o._price_list is null) {
-                  return;
-               }
-               o._set_scrollbar_range(high_tick: v);
-               o._refresh_prices();
-               o._refresh_all_visuals();
-               o._ordershost.refresh_orders();
-            });
-
-      public int? TopTick {
-         get => GetValue(TopTickProperty) as int?;
-         set => SetValue(TopTickProperty, value);
-      }
-      #endregion
 
       #region CanUserSendOrders
       public static readonly DependencyProperty CanUserSendOrdersProperty =
@@ -392,7 +343,7 @@ namespace mbts.spark {
          }
       }
 
-      int _row_count => _price_list?.Count ?? 0;
+      int _row_count = _min_row_count;
 
       bts.api.domain.TickDisplaySettingsImmutable _qty_display =>
          _vm is null ?
@@ -425,64 +376,17 @@ namespace mbts.spark {
          scroll_to_price(_mkt_break_lines.origin);
 
       public bool scroll_to_price(double price) {
-         if (price.is_nan()) {
+         if (_scroll_viewer == null || _price_list.is_null_or_empty())
             return false;
-         }
-         var tick = _spec.tickdiff(price, 0);
-         if (tick.HasValue) {
-            return scroll_to_tick(tick.Value);
-         }
-         return false;
-      }
-
-      double _get_target_top_tick(int mid_viewport_tick) {
-         var curr_rel_off = _scroll_viewer.VerticalOffset + (_scroll_viewer.ViewportHeight / 2.0);
-         var tgt_rel_off = (_price_list.high_tick - mid_viewport_tick) * _settings.row_height;
-         var tick_diff = (curr_rel_off - tgt_rel_off) / _settings.row_height;
-         return _price_list.high_tick + tick_diff;
-      }
-
-      public bool scroll_to_tick(long tick) {
-         if (tick > int.MaxValue) {
-            _scroll_to_tick_log_throttle.Execute(() =>
-               Log.Ui.error($"Unable to scroll to tick {tick} because it exceeds int.MaxValue"));
-            return false;
-         }
-         return scroll_to_tick((int)tick);
-      }
-
-      public bool scroll_to_tick(int tick) {
-         if (_vm is null || _settings is null || _scroll_viewer is null) {
-            return false;
-         }
-         // var center_offset_ticks = (int)(_scrollViewer.ScrollableHeight / (2.0 * _settings.row_height));
-         // The scrollbar's vertical offset won't change.  Find the 'top tick' that will put the
-         // target `tick` in the center of the viewport.
-         // var center_offset_diff = _scrollViewer.VerticalOffset - (_scrollViewer.ScrollableHeight / 2.0);
-         // var tick_offset = center_offset_ticks 
-         // var tick_offset = (int)(_scrollViewer.ViewportHeight / (2.0 * _settings.row_height));
-
-         // The scrollbar's vertical offset won't change.  Find the 'top tick' that will put the
-         // target `tick` in the center of the viewport.
-         checked {
-            var top_tick = (int)_get_target_top_tick(tick);
-            if (_settings.animate_recenter) {
-               var anim = new Int32Animation {
-                  From = (int)_price_list.high_tick,
-                  To = top_tick,
-                  Duration = TimeSpan.FromSeconds(0.15)
-               };
-               var sb = new Storyboard();
-               sb.Children.Add(anim);
-               Storyboard.SetTarget(anim, this);
-               Storyboard.SetTargetProperty(anim, new PropertyPath(TopTickProperty));
-               sb.Begin();
-            } else {
-               _set_scrollbar_range(high_tick: top_tick);
-               _refresh_prices();
+         if (_spec.tickdiff(_price_list[0], price) is long target_row) {
+            int visible_row_count = (int)(_scroll_viewer.ActualHeight / _settings.row_height);
+            int curr_row = visible_row_count + (int)(_scroll_viewer.VerticalOffset / _settings.row_height);
+            if ((target_row > curr_row && target_row < _row_count) || (target_row < (curr_row - visible_row_count) && target_row >= 0)) {
+               _scroll_to_row((int)target_row);
+               return true;
             }
          }
-         return true;
+         return false;
       }
 
       public IObservable<ColumnType> subscribe_hover_column() =>
@@ -564,67 +468,6 @@ namespace mbts.spark {
          }
       }
 
-      IDisposable _connect_scroll_viewer(ScrollViewer scroll_viewer) {
-         if (scroll_viewer is null) {
-            return null;
-         }
-         const int scroll_tick_padding = 10;
-         // const double scroll_epsilon = 1E-8;
-         var dispatcher = Dispatcher.CurrentDispatcher;
-         bool busy = false;
-         return scroll_viewer.SubscribeScrollChanged(on_scroll_changed);
-
-         async void on_scroll_changed(object sender, ScrollChangedEventArgs e) {
-            if (e.ExtentHeight == e.ViewportHeight || busy) {
-               return;
-            }
-            busy = true;
-            await dispatcher.InvokeAsync(ensure_tick_padding, DispatcherPriority.Background);
-            busy = false;
-
-            void ensure_tick_padding() {
-               if (_price_list is null || _settings is null) {
-                  return;
-               }
-               var pct = _scroll_viewer.VerticalOffset / _scroll_viewer.ScrollableHeight;
-               var new_pl = new LadderPriceList(_price_list);
-               var stp = scroll_tick_padding * _settings.row_height;
-               if (pct == 0) {
-                  if (_scroll_viewer.ExtentHeight - _scroll_viewer.ViewportHeight < stp) {
-                     new_pl.extend_up(scroll_tick_padding);
-                  } else {
-                     new_pl.shift_up(scroll_tick_padding);
-                  }
-                  _price_dirty = true;
-               }
-               if (pct == 1) {
-                  if (_scroll_viewer.ExtentHeight - _scroll_viewer.ViewportHeight < stp) {
-                     new_pl.extend_down(scroll_tick_padding);
-                  } else {
-                     new_pl.shift_down(scroll_tick_padding);
-                  }
-                  _price_dirty = true;
-               }
-               if (new_pl != _price_list) {
-                  _pending_price_list = new_pl;
-               }
-               _ordershost.refresh_orders();
-            }
-         }
-      }
-
-      void _set_scrollbar_range(int? high_tick = null, int? length = null) {
-         if (high_tick.HasValue) {
-            _price_list.high_tick = high_tick.Value;
-         }
-         if (length.HasValue) {
-            _price_list.length = length.Value;
-         }
-         _mkt_break_lines.invalidate_indices();
-         _mkt_break_lines.refresh();
-         _refresh_all_visuals();
-      }
-
       void _send_parameterized_order(Point point, Side side) {
          Disposable.dispose(ref _parameterized_order_popup);
          void on_cannot_send(string reason) {
@@ -762,10 +605,7 @@ namespace mbts.spark {
                Keyboard.IsKeyDown(Key.LeftAlt) || Keyboard.IsKeyDown(Key.RightAlt)) {
                _vm?.recenter_all_ladders();
             } else {
-               var tgt = _spec.tickdiff(_mkt_break_lines.origin, 0.0);
-               if (tgt.HasValue) {
-                  scroll_to_tick(tgt.Value);
-               }
+               _scroll_to_center();
             }
          }
       }
@@ -842,9 +682,14 @@ namespace mbts.spark {
       }
 
 
-      internal int _row_for_price(double price) => LadderRender.index_of(_price_list, price, _price_comparer);
+      internal int _row_for_price(double price) {
+         if ((_price_list?.Count ?? 0) == 0 || !price.is_real()) {
+            return -1;
+         }
+         return LadderRender.index_of(_price_list, price, _price_comparer);
+      }
 
-      public double y_for_price(double price) => _row_for_price(price) * (_settings?.row_height ?? double.NaN);
+      // public double y_for_price(double price) => _row_for_price(price) * (_settings?.row_height ?? double.NaN);
       #endregion
 
       #region Column Utility Methods
@@ -889,11 +734,30 @@ namespace mbts.spark {
                   render_market_col(_depth.Bids);
                   break;
                case ColumnType.Price:
-                  _refresh_prices(column);
-                  break;
-               case ColumnType.Ask:
-                  render_market_col(_depth.Asks);
+                  if (_settings.net_change != LadderSettings.NetChangeTerms.None) {
+                     LadderRender.render_prices(
+                        column.visual_tup.visual,
+                        column.visual_tup.glyph_context,
+                        _row_count,
+                        _settings.row_height,
+                        _price_list,
+                        _vm.md.settlement_price,
+                        _settings.net_change == LadderSettings.NetChangeTerms.Ticks,
+                        column.settings,
+                        _spec);
+                  } else {
+                     LadderRender.render_prices(
+                        column.visual_tup.visual,
+                        column.visual_tup.glyph_context,
+                        _row_count,
+                        _settings.row_height,
+                        _price_list,
+                        column.settings,
+                        _spec,
+                        _settings.price_display_format);
+                  }
                   break;
+               case ColumnType.Ask: render_market_col(_depth.Asks); break;
                case ColumnType.Volume:
                   _volume_container.mark_dirty();
                   _refresh_volume_at_price();
@@ -1086,9 +950,8 @@ namespace mbts.spark {
                Storyboard.SetTargetProperty(anim, new PropertyPath(Behavior.AnimatedScrollViewerVerticalOffsetProperty));
                sb.Begin();
             } else {
-               _scroll_to_row(_price_list.Count / 2);
+               _scroll_to_row(_row_count / 2);
             }
-            _refresh_frozen_mkt_lines();
          }
       }
 
@@ -1102,76 +965,23 @@ namespace mbts.spark {
             return Disposable.Create(() => _can_user_send_orders_changed -= o.OnNext);
          });
 
-      LadderPriceList _create_price_list() {
-         if (_settings is not null) {
-            switch (_settings.net_change) {
-               case LadderSettings.NetChangeTerms.BasisPoints:
-               case LadderSettings.NetChangeTerms.NetBasisPoints:
-                  return new() {
-                     length = _initial_row_count,
-                     regional_tick = new RegionalTickFactors(new user.settings.TickFactors(10000, 1)),
-                  };
-            }
-         }
-         return new() {
-            length = _initial_row_count,
-            regional_tick = _spec
-         };
-      }
-
       #region Timer.Tick, Refresh Methods
       void _tick() {
          if (_settings != null) {
-            ulong unused = 0;
-            bool last_trade_changed = _vm.md.get_last_trade(ref unused, out _last_trade, out DateTime _, out ulong __);
+            bool last_trade_changed = _vm.md.get_last_trade(ref lastTradeTime, out _last_trade, out DateTime _, out ulong __);
             bool depth_changed = _vm.md.get_depth(ref _prices_updated, ref _bids, ref _asks, NativePriceHelpers.COCONUT_MAX_DEPTH);
-            _depth = _vm.security_md.Depth(_bids, _asks, _vm.securities);
             _mkt_break_lines.refresh();
-            if (_price_list is null) {
-               Log.assert(_price_dirty);
-               _price_list = _create_price_list();
-               var mid_tick = security.tickdiff(_mkt_break_lines.origin, 0) ?? 0;
-               if (mid_tick > int.MaxValue) {
-                  Log.Ui.error(
-                     $"Unable to set scrollbar range about mid_tick {mid_tick}: " +
-                     $"tick count exceeds int.MaxValue");
-               } else {
-                  _set_scrollbar_range(high_tick: (int)mid_tick + (_price_list.Count << 1));
-               }
-               _price_dirty = true;
+            bool prices_changed =
+               _price_dirty ||
+               (double.IsNaN(_last_origin_price) && !double.IsNaN(_mkt_break_lines.origin)) ||
+               _mkt_break_lines.origin >= _price_list[0] || _mkt_break_lines.origin <= _price_list[_price_list.Count - 1];
+            if (!prices_changed && _prices_out_of_range(out double low, out double high)) {
+               prices_changed = _refresh_price_range(low, high);
             }
-            if (_price_dirty) {
-               if (_pending_price_list != null && !_pending_price_list.Equals(_price_list)) {
-                  var prev_price_list = _price_list;
-                  _price_list = _pending_price_list;
-                  _pending_price_list = null;
-                  if (prev_price_list.Count != _price_list.Count) {
-                     _reset_height();
-                  }
-                  // maintain previous visible price range if possible
-                  double percent_scroll_offset = _scroll_viewer.VerticalOffset / _scroll_viewer.ScrollableHeight;
-                  var n_visible_rowsd = _scroll_viewer.ViewportHeight / _settings.row_height;
-                  var n_visible_rows = (int)n_visible_rowsd;
-                  var offset = n_visible_rowsd - n_visible_rows;
-                  var n_rows_to_scroll = _price_list.Count - n_visible_rows - 1;
-                  int prev_target_index = (int)(percent_scroll_offset * n_rows_to_scroll);
-                  // int prev_target_index = (int)(percent_scroll_offset * prev_price_list.Count - 1);
-                  if (prev_price_list.TryGetAt(prev_target_index, out double prev_target_price)) {
-                     int new_target_index = LadderRender.index_of(_price_list, prev_target_price, _price_comparer);
-                     if (_price_list.within_range(new_target_index)) {
-                        // force _scrollViewer to re-calculate ScrollableHeight
-                        _scroll_viewer.UpdateLayout();
-                        double new_vertical_offset = new_target_index * _settings.row_height;
-                        // double new_vertical_offset = ((double)new_target_index / _price_list.Count) * _scrollViewer.ScrollableHeight;
-                        _scroll_viewer.ScrollToVerticalOffset(new_vertical_offset);
-                     }
-                  }
-               }
-               _refresh_prices();
-               scroll_to_pending_target();
+            if (prices_changed) {
+               _refresh_price_list();
                _refresh_all_visuals();
             } else {
-               scroll_to_pending_target();
                if (_mkt_break_lines.bid_dirty)
                   _refresh_bid_mkt_break();
                if (_mkt_break_lines.ask_dirty)
@@ -1243,6 +1053,75 @@ namespace mbts.spark {
             _settings.price_display_format);
       }
 
+      static readonly TimeSpan _price_out_of_range_render_interval = TimeSpan.FromSeconds(5);
+      DateTime _last_price_range_calc_time = DateTime.MinValue;
+
+      void _refresh_price_list() {
+         if (_settings is null || _spec is null || !_mkt_break_lines.origin.is_real()) {
+            return;
+         }
+         var prev_price_list = _price_list;
+         _price_list = (!(_settings.net_change == LadderSettings.NetChangeTerms.None)) ?
+            LadderRender.render_prices(_price_vis_tup.visual, _price_vis_tup.glyph_context, _mkt_break_lines.origin, _vm.md.settlement_price, _settings.net_change == LadderSettings.NetChangeTerms.Ticks, _row_count, _settings.row_height, _settings.price_column, _spec) :
+            LadderRender.render_prices(_price_vis_tup.visual, _price_vis_tup.glyph_context, _mkt_break_lines.origin, _row_count, _settings.row_height, _settings.price_column, _spec, _settings.price_display_format);
+         _ordershost.update_settings(_settings.row_height, _settings.orders_column);
+         if (prev_price_list != null) {
+            // maintain previous visible price range if possible
+            double percent_scroll_offset = _scroll_viewer.VerticalOffset / _scroll_viewer.ScrollableHeight;
+            int prev_target_index = (int)(percent_scroll_offset * prev_price_list.Count);
+            if (prev_price_list.TryGetAt(prev_target_index, out double prev_target_price)) {
+               int new_target_index = LadderRender.index_of(_price_list, prev_target_price, _price_comparer);
+               if (_price_list.within_range(new_target_index)) {
+                  // force _scroll_viewer to re-calculate ScrollableHeight
+                  _scroll_viewer.UpdateLayout();
+                  double new_vertical_offset = ((double)new_target_index / _price_list.Count) * _scroll_viewer.ScrollableHeight;
+                  _scroll_viewer.ScrollToVerticalOffset(new_vertical_offset);
+               }
+            }
+         }
+      }
+
+      bool _refresh_price_range(double low, double high) {
+         bool prices_changed = false;
+         // The market can be wider than the 1001 rows (ticks) we [initially] allot.
+         // Rendering prices is expensive, and will lock up the ui if done continuously.
+         // Therefore, ensure a minimum time interval has elapsed before rendering prices again.
+         var now = DateTime.Now;
+         if ((now - _last_price_range_calc_time) > _price_out_of_range_render_interval) {
+            SparkUtils.lazy_warn(!double.IsNaN(_mkt_break_lines.origin) && _mkt_break_lines.origin >= low && _mkt_break_lines.origin <= high,
+               () => fmt_log_msg($"market break ({_mkt_break_lines.origin}) is unacceptable for the market range ({low} - {high})"));
+            int prev_row_count = _row_count;
+            var n_ticks_to_low = (int)(_spec.tickdiff(_mkt_break_lines.origin, low) ?? 0).within_range(int.MinValue, int.MaxValue);
+            var n_ticks_to_high = (int)(_spec.tickdiff(high, _mkt_break_lines.origin) ?? 0).within_range(int.MinValue, int.MaxValue);
+            int desired_row_count = Math.Max(n_ticks_to_low, n_ticks_to_high) * 2 + 1;
+            desired_row_count = Math.Max(desired_row_count, _min_row_count);
+            _row_count = Math.Min(_max_row_count, desired_row_count);
+            if (_row_count != prev_row_count) {
+               if (desired_row_count > _max_row_count)
+                  Log.Ui.warn(fmt_log_msg($"ticks between market range ({low} - {high}) exceeds the maximum number of rows allowed ({_max_row_count})"));
+               else
+                  Log.Ui.info(fmt_log_msg($"increase row count to {_row_count} to accomodate market range ({low} - {high})"));
+               prices_changed = true;
+               _reset_height();
+            } else {
+               prices_changed = _mkt_break_lines.origin != _last_origin_price;
+            }
+            _last_price_range_calc_time = now;
+            //
+            string fmt_log_msg(string msg) =>
+               $"Ladder {security} (id:{security.Id}, low/high tick:{security.ContractSpec.LowTick}/{security.ContractSpec.HighTick}): {msg}.";
+         }
+         return prices_changed;
+      }
+
+      bool _prices_out_of_range(out double low, out double high) {
+         low = _bids == null ? double.NaN : TickUtils.low_tick(_bids, _asks);
+         high = _asks == null ? double.NaN : TickUtils.high_tick(_bids, _asks);
+         return
+            (!double.IsNaN(low) && _price_list[_price_list.Count - 1] > low) ||
+            (!double.IsNaN(high) && _price_list[0] < high);
+      }
+
       void _refresh_all_visuals() {
          if (_scroll_viewer is null || _settings is null) {
             return;
@@ -1296,50 +1175,36 @@ namespace mbts.spark {
       }
 
       void _refresh_session_low_line() {
-         if (_settings.session_low_line.show) {
-            _render_session_line(_session_low_vis, _mkt_break_lines.low_render_index ?? -1, _settings.session_low_line.color);
-         }
+         if (_settings.session_low_line.show && _mkt_break_lines.low_render_index.HasValue)
+            _render_session_line(_session_low_vis, _mkt_break_lines.low_render_index.Value, _settings.session_low_line.color);
       }
 
       void _refresh_session_high_line() {
-         if (_settings.session_high_line.show) {
-            _render_session_line(_session_high_vis, _mkt_break_lines.high_render_index ?? -1, _settings.session_high_line.color);
-         }
+         if (_settings.session_high_line.show && _mkt_break_lines.high_render_index.HasValue)
+            _render_session_line(_session_high_vis, _mkt_break_lines.high_render_index.Value, _settings.session_high_line.color);
       }
 
       void _refresh_last_trade() {
          var last_index = LadderRender.index_of(_price_list, _last_trade.price, _price_comparer);
          if (last_index > 0)
-            LadderRender.render_last(
-               _last_vis_tup.visual,
-               _last_vis_tup.glyph_context,
-               _last_trade,
-               last_index,
-               _settings.row_height,
-               _settings.last_column,
-               _fmt_mkt_depth_item);
+            LadderRender.render_last(_last_vis_tup.visual, _last_vis_tup.glyph_context, _last_trade, last_index, _settings.row_height, _settings.last_column, _fmt_mkt_quantity);
       }
 
       void _refresh_piq() {
          if (!_settings.piq_column.show)
             return;
          int origin_index = LadderRender.index_of(_price_list, _mkt_break_lines.origin, _price_comparer);
-         LadderRender.render_piqs(
-            _piq_vis_tup.visual,
-            _last_vis_tup.glyph_context,
-            _price_list,
-            _vm.price_vs_piq,
-            _settings.row_height,
-            _spec,
-            _settings.piq_column,
-            _fmt_mkt_quantity);
+         LadderRender.render_piqs(_piq_vis_tup.visual, _last_vis_tup.glyph_context, _price_list, _vm.price_vs_piq, _settings.row_height, _spec, _settings.piq_column, _fmt_mkt_quantity);
+         LadderRender.render_piqs(_piq_vis_tup.visual, _last_vis_tup.glyph_context, _price_list, _vm.price_vs_piq, _settings.row_height, _spec, _settings.piq_column, _fmt_mkt_quantity);
       }
 
       void _refresh_sizes() {
          Log.assert_d(_price_list != null);
          int origin_index = LadderRender.index_of(_price_list, _mkt_break_lines.origin, _price_comparer);
-         LadderRender.render_sizes(_bid_vis_tup.visual, _bid_vis_tup.glyph_context, origin_index, _price_list, _depth.Bids, _settings.row_height, _spec, _settings.bid_column, _fmt_price_level);
-         LadderRender.render_sizes(_ask_vis_tup.visual, _ask_vis_tup.glyph_context, origin_index, _price_list, _depth.Asks, _settings.row_height, _spec, _settings.ask_column, _fmt_price_level);
+         var bidItems = _bids.Select(nps => (nps.price, (double)nps.size, nps.price * nps.size));
+         var askItems = _asks.Select(nps => (nps.price, (double)nps.size, nps.price * nps.size));
+         LadderRender.render_sizes(_bid_vis_tup.visual, _bid_vis_tup.glyph_context, origin_index, _price_list, bidItems, _settings.row_height, _spec, _settings.bid_column, _fmt_price_level);
+         LadderRender.render_sizes(_ask_vis_tup.visual, _ask_vis_tup.glyph_context, origin_index, _price_list, askItems, _settings.row_height, _spec, _settings.ask_column, _fmt_price_level);
       }
 
       /// <summary>
@@ -1408,12 +1273,9 @@ namespace mbts.spark {
       void _clear_visuals() {
          if (_settings == null)
             return;
-         if (_price_list is null) {
-            _price_list = new LadderPriceList {
-               regional_tick = _spec
-            };
-         }
-         _price_list.reset(null, null, _initial_row_count);
+         _price_list = new double[_row_count];
+         for (int i = 0; i < _price_list.Count; i++)
+            _price_list[i] = double.NaN;
          Array.Clear(_bids, 0, _bids.Length);
          Array.Clear(_asks, 0, _asks.Length);
          _price_dirty = true;
@@ -1421,7 +1283,6 @@ namespace mbts.spark {
          _refresh_bid_mkt_break();
          _refresh_ask_mkt_break();
          _refresh_mid_mkt_break();
-         _refresh_fair_value_lines(true);
          _refresh_session_low_line();
          _refresh_session_high_line();
          _refresh_session_range();
@@ -1431,58 +1292,38 @@ namespace mbts.spark {
       #endregion
 
       #region Market Break, Session Low/High Utils
-      static double _get_top_price(NativePriceSize[] prices) {
-         if (prices == null) {
-            return double.NaN;
-         }
-         if (prices.Length == 0) {
-            return double.NaN;
-         }
-         if (prices[0].size == 0) {
-            return double.NaN;
-         }
-         return prices[0].price;
-      }
+      static double _get_top_price(NativePriceSize[] prices) =>
+         prices != null && prices.Length > 0 ? prices[0].price : double.NaN;
 
       double _get_bid_market_break() {
          double best_bid = _get_top_price(_bids);
          double best_ask = _get_top_price(_asks);
-         var v = best_bid
-            .if_nan(_spec.tickdown(best_ask))
-            .if_nan(_vm?.settlement ?? double.NaN);
-         return _spec.round_to_tick(v, RoundingMode.AbsDown);
+         return double.IsNaN(best_bid) ?
+            double.IsNaN(best_ask) ? _vm.settlement : _spec.tickdown(best_ask) :
+            best_bid;
       }
 
       double _normalize_price(double price) =>
          TickUtils.normalize_price(_spec, price);
 
       double _get_ask_market_break() {
-         if (_vm is null) {
-            return double.NaN;
-         }
          double best_bid = _get_top_price(_bids);
          double best_ask = _get_top_price(_asks);
-         var v = _spec.tickdown(best_ask)
-            .if_nan(best_bid)
-            .if_nan(_vm?.settlement ?? double.NaN);
-         return _spec.round_to_tick(v, RoundingMode.AbsDown);
+         return double.IsNaN(best_ask) ?
+            double.IsNaN(best_bid) ? _vm.settlement : best_bid :
+            _spec.tickdown(best_ask);
       }
 
       double _get_mid_market_break() {
          double best_bid = _get_top_price(_bids);
          double best_ask = _get_top_price(_asks);
-         var v = TickUtils.mid_tick(_spec, best_bid, best_ask)
-            .if_nan(best_bid)
-            .if_nan(_spec?.tickdown(best_ask) ?? double.NaN)
-            .if_nan(_vm?.settlement ?? double.NaN);
-         return _spec.round_to_tick(v, RoundingMode.AbsDown);
+         return double.IsNaN(best_bid) ?
+            double.IsNaN(best_ask) ? _vm.settlement : _spec.tickdown(best_ask) :
+            double.IsNaN(best_ask) ? best_bid : TickUtils.mid_tick(_spec, best_bid, best_ask);
       }
 
-      double _get_session_low() => _normalize_price(_get_session_low_raw());
-      double _get_session_high() => _normalize_price(_get_session_high_raw());
-
-      double _get_session_low_raw() => _vm?.md?.session_low ?? double.NaN;
-      double _get_session_high_raw() => _vm?.md?.session_high ?? double.NaN;
+      double _get_session_low() => _vm.md.session_low;
+      double _get_session_high() => _vm.md.session_high;
 
       void _render_mkt_break(DrawingVisual vis, double price, uint color) =>
          LadderRender.render_mkt_break(vis, _row_for_price(price), _settings.row_height, color);
@@ -1506,15 +1347,6 @@ namespace mbts.spark {
          _settings = _vm?.get_save_state() ?? s;
          _pending_scroll_to_origin |= _settings == null;
          _invalidate_spec();
-         if (_price_list != null) {
-            _pending_price_list = null;
-            if (_price_list.Count > 0) {
-               var high_price = _price_list.First();
-               var new_tickdiff = _spec.count_ticks(high_price);
-               _price_list.high_tick = (long)new_tickdiff;
-            }
-            _price_list.regional_tick = _spec;
-         }
          _bid_vis_tup.glyph_context = glyph_context(s.bid_column);
          _ask_vis_tup.glyph_context = glyph_context(s.ask_column);
          _price_vis_tup.glyph_context = glyph_context(s.price_column);
@@ -1598,9 +1430,22 @@ namespace mbts.spark {
          // if (_scroll_viewer != null) {
          //    _scroll_viewer.Background = s.last_column.background.rgb_brush();
          // }
-         scroll_to_pending_target();
          visible_columns_changed?.Invoke();
-         _refresh_all_visuals();
+         if (_scroll_viewer is not null && _settings is not null) {
+            _vm.md.get_last_trade(ref lastTradeTime, out _last_trade, out DateTime _, out ulong __);
+            _vm.md.get_depth(ref _prices_updated, ref _bids, ref _asks, NativePriceHelpers.COCONUT_MAX_DEPTH);
+            _mkt_break_lines.refresh();
+            double low = _bids == null ? double.NaN : TickUtils.low_tick(_bids, _asks);
+            double high = _asks == null ? double.NaN : TickUtils.high_tick(_bids, _asks);
+            _refresh_price_range(low, high);
+            _refresh_price_list();
+            _refresh_all_visuals();
+            _refresh_piq();
+            _piq_dirty = false;
+            _refresh_volume_at_price();
+            _last_origin_price = _mkt_break_lines.origin;
+            _price_dirty = false;
+         }
       }
 
       // This is designed to be a redundant null check for _fmt_mkt_depth_item_impl.
@@ -1631,6 +1476,9 @@ namespace mbts.spark {
          return impl is null ? value.ToString() : impl(value, price);
       }
 
+      string _fmt_mkt_quantity(NativePriceSize nps) =>
+         _fmt_mkt_quantity(nps.size, nps.price);
+
       void _init_tactile_adorner() {
          Disposable.dispose(ref _tactile_adorner);
          if (_scroll_viewer != null)
@@ -1745,7 +1593,7 @@ namespace mbts.spark {
                   return null;
                }
                if (!double.IsNaN(_session_low)) {
-                  _low_index = _owner._price_list.IndexOf(_session_low);
+                  _low_index = LadderRender.index_of(_owner._price_list, _session_low, _owner._price_comparer);
                   if (_session_low < _owner._price_list.Last()) {
                      _low_index = _owner._price_list.Count - 1;
                   }

src/client/spark/Modules/Trading/Ladder/View/LadderPriceList.cs

@@ -1,6 +1,7 @@
 using bts.api;
 using bts.utils;
 
+using Spark.Core;
 using Spark.Core.Diagnostics;
 
 using System;
@@ -30,6 +31,16 @@ namespace mbts.spark {
          regional_tick = other.regional_tick;
       }
 
+      public LadderPriceList(RegionalTickFactors regionalTick, IList<double> prices) {
+         regional_tick = regionalTick;
+         _length = prices.Count;
+         checked {
+            _high_tick = prices.Count > 0
+               ? (int)regionalTick.tickdiff(prices[0], 0).assert_value()
+               : 0;
+         }
+      }
+
       public event NotifyCollectionChangedEventHandler CollectionChanged;
 
       RegionalTickFactors _regional_tick;

src/client/spark/Modules/Trading/Ladder/View/LadderRender.cs

@@ -1,4 +1,4 @@
-using bts;
+using bts;
 using bts.api;
 using bts.api.md.auxiliary;
 using bts.collections;
@@ -101,6 +101,24 @@ namespace mbts.spark {
             settings.font_size, settings.width, row_height, HorizontalAlignment.Right,
             -4, VerticalAlignment.Center, row * row_height, true);
 
+      public static double[] gen_prices(double mid, int row_count, RegionalTickFactors spec) {
+         // require odd number
+         Log.assert(row_count % 2 == 1);
+         var result = new double[row_count];
+         int range = row_count / 2;
+         int middle = row_count / 2;
+         result[middle] = mid;
+         double up = mid, down = mid;
+         // Starting from the middle iterate out to the edges
+         for (int x = 1; x <= range; ++x) {
+            up = spec.tickup(up);
+            down = spec.tickdown(down);
+            result[middle - x] = up;
+            result[middle + x] = down;
+         }
+         return result;
+      }
+
       public static GlyphRun[] format_prices(
          IList<double> prices,
          double reference_price,
@@ -196,6 +214,45 @@ namespace mbts.spark {
          }
       }
 
+      public static double[] render_prices(DrawingVisual price, GlyphContext glyph_context, double mid, double wrt_price, bool format_in_tick_terms, int row_count, double row_height,
+         LadderColumnSettings settings, RegionalTickFactors spec) {
+         var dprices = gen_prices(mid, row_count, spec);
+         render_prices(price, glyph_context, row_count, row_height, dprices, wrt_price, format_in_tick_terms, settings, spec);
+         return dprices;
+      }
+
+      public static double[] render_prices(DrawingVisual price, GlyphContext glyph_context, double mid, int row_count, double row_height,
+         LadderColumnSettings settings, RegionalTickFactors spec, LadderSettings.PriceDisplayFormat price_display_format) {
+         var dprices = gen_prices(mid, row_count, spec);
+         render_prices(price, glyph_context, row_count, row_height, dprices, settings, spec, price_display_format);
+         return dprices;
+      }
+
+      public static void render_prices(DrawingVisual price, GlyphContext glyph_context, int row_count, double row_height, IList<double> dprices, double wrt_price, bool format_in_tick_terms, LadderColumnSettings settings, RegionalTickFactors spec) {
+         var foreground = settings.foreground.rgb_brush();
+         //var background = new SolidColorBrush(settings.background.rgb_color());
+         var terms = format_in_tick_terms ? LadderSettings.NetChangeTerms.Ticks : LadderSettings.NetChangeTerms.Dollars;
+         var fprices = format_prices(dprices, wrt_price, terms, row_height, settings, glyph_context, spec);
+         using (var dc = price.RenderOpen())
+         using (_draw_with_clip(dc, settings, row_height, row_count)) {
+            //dc.DrawRectangle(background, null, new Rect(0, 0, settings.width, row_count * row_height));
+            for (int x = 0; x < row_count; ++x)
+               dc.DrawGlyphRun(foreground, fprices[x]);
+         }
+      }
+
+      public static void render_prices(DrawingVisual price, GlyphContext glyph_context, int row_count, double row_height, IList<double> dprices, LadderColumnSettings settings, RegionalTickFactors spec, LadderSettings.PriceDisplayFormat price_display_format) {
+         var foreground = settings.foreground.rgb_brush();
+         //var background = new SolidColorBrush(settings.background.rgb_color());
+         var fprices = format_prices(dprices, double.NaN, LadderSettings.NetChangeTerms.Dollars, row_height, settings, glyph_context, spec, price_display_format);
+         using (var dc = price.RenderOpen())
+         using (_draw_with_clip(dc, settings, row_height, row_count)) {
+            //dc.DrawRectangle(background, null, new Rect(0, 0, settings.width, row_count * row_height));
+            for (int x = 0; x < row_count; ++x)
+               dc.DrawGlyphRun(foreground, fprices[x]);
+         }
+      }
+
       public static void render_grid(DrawingVisual dv, IEnumerable<LadderGridHost.IColumn> columns, int rowcount, double row_height, UInt32 line_color) {
          var height = rowcount * row_height;
          var brush = line_color.rgb_brush();

src/client/spark/Modules/Trading/ManualFills/ManualFillRow.cs

@@ -28,7 +28,8 @@ namespace bts.shell.manual_fills {
                nameof(user),
                nameof(side),
                nameof(quantity),
-               nameof(price)
+               nameof(price),
+               nameof(fill_date)
          }, typeof(ManualFillRow).GetProperty);
 
       readonly ISecurityMetadataStore _sec_meta_store;
@@ -43,6 +44,7 @@ namespace bts.shell.manual_fills {
       UserTuple? _user;
       Side _side;
       double _price;
+      DateTime _timestamp;
       bool _is_selected;
       ManualFillData? _data;
       IDisposable _committed_snapshot;
@@ -68,6 +70,9 @@ namespace bts.shell.manual_fills {
          // TickDisplaySettingsImmutable for qty, obtained via ISecurityMetadataStore.
          _quantity.conversion_data = new(1, QuantityConversionOpType.DivideActual);
          _price = data.value.manual_fill.Price;
+         _timestamp = data.value.manual_fill.HasTimestamp
+            ? data.value.manual_fill.Timestamp.to_local_time()
+            : DateTime.Now;
          if (placeholder_key.HasValue) {
             _is_dirty = backing(nameof(is_dirty), false);
             _editable_item_state = backing(nameof(editable_item_state), EditableItemState.Uncommitted);
@@ -113,6 +118,22 @@ namespace bts.shell.manual_fills {
       public TickDisplaySettingsImmutable quantity_display => _quantity_display.Value;
       public double price { get => _price; set => _set(ref _price, value); }
 
+      /// <summary>
+      /// The trading date assigned to this fill.  The time of day is retained so
+      /// that a fill entered for today continues to be classified as a day fill.
+      /// </summary>
+      public DateTime fill_date {
+         get => _timestamp.Date;
+         set {
+            var timestamp = value.Date + _timestamp.TimeOfDay;
+            if (_timestamp != timestamp) {
+               _timestamp = timestamp;
+               RaisePropertyChanged(nameof(fill_date));
+               _data = null;
+            }
+         }
+      }
+
       public DateTime? time_entered =>
          data.value.manual_fill.HasTimestamp
             ? data.value.manual_fill.Timestamp.to_local_time()
@@ -157,7 +178,8 @@ namespace bts.shell.manual_fills {
             Price = price,
             Side = side,
             Quantity = quantity,
-            SecurityKey = security?.Id ?? 0
+            SecurityKey = security?.Id ?? 0,
+            Timestamp = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(_timestamp.ToUniversalTime())
          };
 
       public ManualFillTuple to_tuple() =>

src/client/spark/Modules/Trading/ManualFills/ManualFillRowEditor.xaml

@@ -39,6 +39,8 @@
         <ColumnDefinition Width="Auto"/>
         <ColumnDefinition Width="{Binding Source={StaticResource HorizontalPush}}"/>
         <ColumnDefinition Width="Auto"/>
+        <ColumnDefinition Width="{Binding Source={StaticResource HorizontalPush}}"/>
+        <ColumnDefinition Width="Auto"/>
       </Grid.ColumnDefinitions>
       <Grid.RowDefinitions>
         <RowDefinition Height="Auto"/>
@@ -46,7 +48,7 @@
         <RowDefinition Height="*"/>
       </Grid.RowDefinitions>
       <Rectangle
-        Grid.ColumnSpan="11"
+        Grid.ColumnSpan="13"
         HorizontalAlignment="Stretch"
         VerticalAlignment="Stretch"
         Fill="{u:BTSColor header}" />
@@ -56,6 +58,7 @@
       <TextBlock Grid.Column="6" Text="Side" Style="{StaticResource HeaderText}"/>
       <TextBlock Grid.Column="8" Text="Qty" Style="{StaticResource HeaderText}"/>
       <TextBlock Grid.Column="10" Text="Price" Style="{StaticResource HeaderText}"/>
+      <TextBlock Grid.Column="12" Text="Date" Style="{StaticResource HeaderText}"/>
 
       <c:ContractPickerPopup
         Grid.Row="2"
@@ -88,6 +91,7 @@
         TickDisplaySettings="{Binding quantity_display}"
         Value="{Binding display_quantity}" />
       <usp:PriceEntryControl Grid.Row="2" Grid.Column="10" MinWidth="55" Value="{Binding price}" ContractSpec="{Binding spec}"/>
+      <DatePicker Grid.Row="2" Grid.Column="12" SelectedDate="{Binding fill_date}" />
     </Grid>
     <c:DialogSkirt Grid.Row="1" Margin="0,6,0,0">
       <c:DialogSkirt.RightItems>

src/client/spark/Modules/Trading/ManualFills/ManualFillsView.xaml

@@ -272,6 +272,23 @@
             </DataGridTemplateColumn.CellEditingTemplate>
           </DataGridTemplateColumn>
 
+          <!-- Trading Date -->
+          <DataGridTemplateColumn
+            c:DataGridEx.CellKey="{x:Static Date, MemberType={u:Nested usp:ManualFillsView.Columns}}"
+            Header="Date"
+            SortMemberPath="fill_date">
+            <DataGridTemplateColumn.CellTemplate>
+              <DataTemplate>
+                <TextBlock Text="{Binding fill_date, StringFormat={}{0:d}}" VerticalAlignment="Center" />
+              </DataTemplate>
+            </DataGridTemplateColumn.CellTemplate>
+            <DataGridTemplateColumn.CellEditingTemplate>
+              <DataTemplate>
+                <DatePicker SelectedDate="{Binding fill_date, UpdateSourceTrigger=PropertyChanged}" />
+              </DataTemplate>
+            </DataGridTemplateColumn.CellEditingTemplate>
+          </DataGridTemplateColumn>
+
           <!-- Time Entered -->
           <DataGridTextColumn
             c:DataGridEx.CellKey="{x:Static

src/client/spark/Modules/Trading/ManualFills/ManualFillsView.xaml.cs

@@ -31,6 +31,7 @@ namespace mbts.ui.spark {
          Side,
          Quantity,
          Price,
+         Date,
          TimeEntered
       }
 

src/client/spark/Modules/Trading/OrderBook/Settings/OrderBookFilterStates.cs

@@ -26,7 +26,7 @@ namespace bts.shell.orderbook {
       [ProtoMember(5)]
       public string[] accounts;
 
-      [ProtoMember(5)]
+      [ProtoMember(28)]
       public string Tag { get; set; }
 
       [ProtoMember(6)]

src/client/spark/Modules/Trading/PNL/View/PNLView.xaml.cs

@@ -407,16 +407,16 @@ namespace bts.shell.pnl {
       void _show_auxiliary_popup<T>(T content, string context_desc, string title_desc, Color fg) {
          var view = new HeaderedContentControl {
             Content = content,
-            Header = new AuxiliaryDataHeader {
+            Header = new OrderAuxiliaryPopupHeader {
                context = context_desc,
                title = title_desc,
                title_foreground = fg
             },
             HorizontalAlignment = HorizontalAlignment.Left,
             VerticalAlignment = VerticalAlignment.Top,
-            Resources = new AuxiliaryDataResources()
+            Resources = new OrderAuxiliaryPopupResources()
          };
-         view.Style = view.Resources[AuxiliaryDataResources.PopupChildStyleKey] as Style;
+         view.Style = view.Resources[OrderAuxiliaryPopupResources.PopupChildStyleKey] as Style;
          _popup_man.show(view);
       }
 

src/client/spark/NMSheets/DataProviders/NMSheetSingleOptionMainDataSource.cs

@@ -8,13 +8,15 @@ using bts.api;
 using mbts.analytics;
 using mbts;
 
-namespace BCG {
+namespace BCG
+{
    /// For each NMSheetSymbol in an NMSheet gets all the 
    /// data together (theos, position, vol change etc) 
    /// and fires a callback.
    /// This is where all the data is gathered, the rest
    /// is all access to this
-   public class NMSheetSingleOptionMainDataSource : IDisposable {
+   public class NMSheetSingleOptionMainDataSource : IDisposable
+   {
       private ThreadedBlackboard _threaded_bb;
       private readonly ITag<NMSheetDataRequest> _in_bb_main_sheet_request_tag;
       private readonly ITag<ExchangeSecurityMap> _in_bb_secmap_tag =
@@ -24,6 +26,7 @@ namespace BCG {
       private readonly ITag<OptionTheoInterpolationData> _in_bb_precomputed_sparse_theos_tag;
       private readonly ITag<IDictionary<NMSheetExpiration, double>> _in_bb_vol_shift_tag;
       private readonly ITag<NMSheetSettings> _in_bb_sheet_settings_tag;
+      private readonly ITag<IEnumerable<ITradeLogRow>> _in_bb_trade_log_tag = SparkTags.TradeLogTag;
       private readonly ITag<VolSurfaceNonVolInput> _in_exp_param_tag = SparkTags.VolSurfaceNonVolInputTag;
       private readonly ITag<bool> _in_bb_main_sheet_visibility_tag;
       private ITag<SparkTags.Nothing> _in_bb_onefive_value_tag = SparkTags.TC2Tags.VoidTag,
@@ -31,7 +34,6 @@ namespace BCG {
                                        _in_bb_symbol_theo_shift_tag = SparkTags.TC2Tags.VoidTag,
                                        _in_bb_rollover_data_tag = SparkTags.TC2Tags.VoidTag,
                                        _in_bb_presaved_aw2_theos_tag = SparkTags.TC2Tags.VoidTag,
-                                       _in_bb_position_tag = SparkTags.TC2Tags.VoidTag,
                                        _in_bb_option_driver_detailed_price_tag = SparkTags.TC2Tags.VoidTag;
       private readonly NMSheetSymbol _nmsheet_symbol;
       private Action<Pair<NMSheetSymbol, NMSheetSingleOptionData>> _cb;
@@ -40,7 +42,8 @@ namespace BCG {
       private string _electronic_equivelant_symbol;
       private NMSheetArchivedVolProvider _archived_vol_provider;
 
-      public NMSheetSingleOptionMainDataSource(object req_id, ThreadedBlackboard threaded_bb, NMSheetSymbol option, Action<Pair<NMSheetSymbol, NMSheetSingleOptionData>> callback) {
+      public NMSheetSingleOptionMainDataSource(object req_id, ThreadedBlackboard threaded_bb, NMSheetSymbol option, Action<Pair<NMSheetSymbol, NMSheetSingleOptionData>> callback)
+      {
          _threaded_bb = threaded_bb;
          _nmsheet_symbol = option;
          _electronic_equivelant_symbol = option.option_name;
@@ -56,7 +59,8 @@ namespace BCG {
          _in_bb_sheet_settings_tag = SparkTags.TC2Tags.NMSheetSettingsTag(req_id.ToString());
          _in_bb_main_sheet_visibility_tag = SparkTags.TC2Tags.NMSheetVisibilityTag(req_id.ToString());
          _archived_vol_provider = new NMSheetArchivedVolProvider(req_id.ToString(), _threaded_bb, option,
-            () => {
+            () =>
+            {
                _rollover_data = null;
                _compute_and_fire();
             });
@@ -76,7 +80,7 @@ namespace BCG {
          _threaded_bb.subscribe(_in_bb_rollover_data_tag);
          _threaded_bb.subscribe(_in_bb_presaved_aw2_theos_tag);
          _threaded_bb.subscribe(_in_exp_param_tag);
-         _threaded_bb.subscribe(_in_bb_position_tag);
+         _threaded_bb.subscribe(_in_bb_trade_log_tag);
          _threaded_bb.subscribe(_in_bb_sheet_settings_tag);
          _threaded_bb.subscribe(_in_bb_onefive_value_tag);
          _threaded_bb.subscribe(_in_bb_qs_tag);
@@ -84,15 +88,17 @@ namespace BCG {
          _threaded_bb.subscribe(_in_bb_option_driver_detailed_price_tag);
       }
 
-      private void _react_to_bb(object tag, object value) {
+      private void _react_to_bb(object tag, object value)
+      {
          if (_in_bb_main_sheet_request_tag.Equals(tag) || _in_bb_secmap_tag.Equals(tag) || _in_bb_live_theos_tag.Equals(tag) || /*_in_bb_live_driver_bbo_tag.Equals(tag) ||*/ _in_bb_precomputed_sparse_theos_tag.Equals(tag) ||
             _in_bb_vol_shift_tag.Equals(tag) || _in_bb_symbol_theo_shift_tag.Equals(tag) || _in_exp_param_tag.Equals(tag) || _in_bb_presaved_aw2_theos_tag.Equals(tag) ||
-            _in_bb_position_tag.Equals(tag) || _in_bb_rollover_data_tag.Equals(tag) || _in_bb_sheet_settings_tag.Equals(tag) || _in_bb_qs_tag.Equals(tag) ||
+            _in_bb_trade_log_tag.Equals(tag) || _in_bb_rollover_data_tag.Equals(tag) || _in_bb_sheet_settings_tag.Equals(tag) || _in_bb_qs_tag.Equals(tag) ||
             _in_bb_main_sheet_visibility_tag.Equals(tag))
             _compute_and_fire();
       }
 
-      private NMSheetDriverData _get_driver(DriverMD d) {
+      private NMSheetDriverData _get_driver(DriverMD d)
+      {
          double dr_pr = d.get_driver();
          if (d.driver_security.HasValue)
             return new NMSheetDriverData(d.driver_security.symbol_or_default(),
@@ -102,8 +108,10 @@ namespace BCG {
          return new NMSheetDriverData(String.Empty, Venue.Unset, DateTime.MinValue, dr_pr, dr_pr, false);
       }
 
-      private void _compute_and_fire() {
-         if (_is_main_sheet_active()) {
+      private void _compute_and_fire()
+      {
+         if (_is_main_sheet_active())
+         {
             var timer = Stopwatch.StartNew();
             NMSheetDataRequest data_request = _threaded_bb.read_state(_in_bb_main_sheet_request_tag) ?? new NMSheetDataRequest(new List<Triplet<NMSheetExpiration, bool, double>>());
             data_request = NMSheetDataRequest.restrict_for_symbol(data_request, _nmsheet_symbol);
@@ -112,7 +120,8 @@ namespace BCG {
             NMSheetSettings sheet_settings = _threaded_bb.read_state(_in_bb_sheet_settings_tag) ?? new NMSheetSettings();
             ExpStrikeKeyedData<mbts.spark.TheoPoint> shared_live_theo_data_map = _threaded_bb.read_state(_in_bb_live_theos_tag);
 
-            if (data_request != null && live_driver_map != null && shared_live_theo_data_map != null) {
+            if (data_request != null && live_driver_map != null && shared_live_theo_data_map != null)
+            {
                ExpStrikeKeyedData<mbts.spark.TheoPoint> output_theo_data_map = _limit_to_requested_securities(shared_live_theo_data_map, data_request, sheet_settings);
                output_theo_data_map = _adjust_for_freezes(output_theo_data_map, data_request);
                _add_vol_shift(output_theo_data_map);
@@ -138,14 +147,17 @@ namespace BCG {
          }
       }
 
-      private bool _is_main_sheet_active() {
+      private bool _is_main_sheet_active()
+      {
          return _threaded_bb.read_state(_in_bb_main_sheet_visibility_tag, true);
       }
 
-      private ExpStrikeKeyedData<mbts.spark.TheoPoint> _limit_to_requested_securities(ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_data, NMSheetDataRequest data_request, NMSheetSettings sheet_settings) {
+      private ExpStrikeKeyedData<mbts.spark.TheoPoint> _limit_to_requested_securities(ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_data, NMSheetDataRequest data_request, NMSheetSettings sheet_settings)
+      {
          HashSet<DateTime> selected_exp_map = new HashSet<DateTime>();
          HashSet<double> selected_strike_map = new HashSet<double>(sheet_settings.setting_unselected_strikes);
-         foreach (Triplet<NMSheetExpiration, bool, double> t in data_request.expiration_request_list) {
+         foreach (Triplet<NMSheetExpiration, bool, double> t in data_request.expiration_request_list)
+         {
             NMSheetExpiration nm_exp = t.first;
             if (nm_exp != null && nm_exp.option_name == _nmsheet_symbol.option_name)
                selected_exp_map.Add(nm_exp.option_expiration);
@@ -153,12 +165,14 @@ namespace BCG {
          return theo_data.clone_filtered_set(selected_exp_map, true, selected_strike_map, false);
       }
 
-      private ExpStrikeKeyedData<Triplet<mbts.spark.TheoPoint, uint, uint>> _attach_sec_keys(ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_map) {
+      private ExpStrikeKeyedData<Triplet<mbts.spark.TheoPoint, uint, uint>> _attach_sec_keys(ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_map)
+      {
          theo_map = theo_map ?? new ExpStrikeKeyedData<mbts.spark.TheoPoint>();
          ExchangeSecurityMap sec_map = _threaded_bb.read_state(_in_bb_secmap_tag);
          if (sec_map != null)
             return sec_map.attach_sec_keys(theo_map, _electronic_equivelant_symbol, _nmsheet_symbol.option_exchange);
-         else {
+         else
+         {
             ExpStrikeKeyedData<Triplet<mbts.spark.TheoPoint, uint, uint>> rv = new ExpStrikeKeyedData<Triplet<mbts.spark.TheoPoint, uint, uint>>();
             foreach (Triplet<DateTime, double, mbts.spark.TheoPoint> t in theo_map.get_data_linearly())
                rv.set_data(t.first, t.second, new Triplet<mbts.spark.TheoPoint, uint, uint>(t.third, 0, 0));
@@ -166,7 +180,8 @@ namespace BCG {
          }
       }
 
-      private ExpStrikeKeyedData<mbts.spark.TheoPoint> _adjust_for_freezes(ExpStrikeKeyedData<mbts.spark.TheoPoint> live_theo_map, NMSheetDataRequest data_request) {
+      private ExpStrikeKeyedData<mbts.spark.TheoPoint> _adjust_for_freezes(ExpStrikeKeyedData<mbts.spark.TheoPoint> live_theo_map, NMSheetDataRequest data_request)
+      {
          return live_theo_map;
          // TODO
          // ExpStrikeKeyedData<mbts.spark.TheoPoint> rv = new ExpStrikeKeyedData<mbts.spark.TheoPoint>();
@@ -192,9 +207,11 @@ namespace BCG {
 
       private Dictionary<DateTime, NMSheetDriverData> _get_option_exp_driver_data_map(NMSheetDataRequest data_request,
                                                                                       ExpStrikeKeyedData<mbts.spark.TheoPoint> output_theo_data_map, ExpStrikeKeyedData<mbts.spark.TheoPoint> shared_live_theo_data_map,
-                                                                                      Dictionary<DateTime, Triplet<(string, Venue), DateTime, double>> live_driver_map) {
+                                                                                      Dictionary<DateTime, Triplet<(string, Venue), DateTime, double>> live_driver_map)
+      {
          Dictionary<DateTime, NMSheetDriverData> rv = new Dictionary<DateTime, NMSheetDriverData>();
-         foreach (Triplet<NMSheetExpiration, bool, double> theo_request in data_request.expiration_request_list) {
+         foreach (Triplet<NMSheetExpiration, bool, double> theo_request in data_request.expiration_request_list)
+         {
             DateTime option_expiration = theo_request.first.option_expiration;
             /// live driver
             var live_driver_info = live_driver_map.GetValueOrDefault(option_expiration)
@@ -208,9 +225,11 @@ namespace BCG {
 
       private Dictionary<DateTime, NMSheetDriverData>
          _get_option_exp_driver_data_map(NMSheetDataRequest data_request,
-                                         OptionDriverPriceMap drivers) {
+                                         OptionDriverPriceMap drivers)
+      {
          var rv = new Dictionary<DateTime, NMSheetDriverData>();
-         foreach (var tr in data_request.expiration_request_list) {
+         foreach (var tr in data_request.expiration_request_list)
+         {
             NMSheetExpiration nm_exp = tr.first;
             SecurityExpiration exp = tr.first.to_sec_exp();
             drivers.exp_price_map.TryGetValue(exp, out DriverMD dr_md);
@@ -222,7 +241,8 @@ namespace BCG {
          return rv;
       }
 
-      private static double _get_driverprice_for_expiration(ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_data_map, DateTime option_expiration) {
+      private static double _get_driverprice_for_expiration(ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_data_map, DateTime option_expiration)
+      {
          IDictionary<double, mbts.spark.TheoPoint> exp_data = theo_data_map.get_data_for_single_exp(option_expiration);
          if (exp_data != null)
             foreach (mbts.spark.TheoPoint tp in exp_data.Values)
@@ -230,7 +250,21 @@ namespace BCG {
          return Double.NaN;
       }
 
-      private IDictionary<DateTime, ATMData> _get_atm_data(ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_data) {
+       private double _get_rep_driver() {
+         var map =
+            _threaded_bb.read_state(SparkTags.OptionDriverBBOTag((_nmsheet_symbol.option_name, _nmsheet_symbol.option_exchange)));
+         if (map == null)
+            return Double.NaN;
+         var price_map = map.get_price_map();
+         if (price_map == null || price_map.Count == 0)
+            return Double.NaN;
+         var md = price_map.ElementAt(0).Value;
+         double pr = md.get_driver();
+         return pr;
+      }
+
+      private IDictionary<DateTime, ATMData> _get_atm_data(ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_data)
+      {
          NMSheetSettings sheet_settings = _threaded_bb.read_state(_in_bb_sheet_settings_tag) ?? new NMSheetSettings();
          VolSurfaceNonVolInput vs_nvinp = _threaded_bb.read_state(_in_exp_param_tag);
          var option_exp_roll_data = vs_nvinp
@@ -240,7 +274,7 @@ namespace BCG {
                exp.security_type == SecurityType.Option)
             ?.ToDictionaryCrushed(kv => kv.Key.expiration, kv => kv.Value)
             ?? new();
-         IDictionary<DateTime, ATMData> atm_data = OptionATMDataSource.compute_atm_data(_get_onefive(), theo_data, option_exp_roll_data);
+         IDictionary<DateTime, ATMData> atm_data = OptionATMDataSource.compute_atm_data(_get_onefive(), theo_data, option_exp_roll_data, _get_rep_driver());
          if (sheet_settings.setting_atm.select_atm_nearest_strike_only)
             foreach (ATMData exp_atm_data in atm_data.Values)
                exp_atm_data.reduce_to_nearest_expiration();
@@ -248,17 +282,43 @@ namespace BCG {
       }
 
       OptionContractKeyedData<PNLRollOverDataUnit> _rollover_data = null;
-      private OptionContractKeyedData<PNLRollOverDataUnit> _get_rollover_data() {
+      private OptionContractKeyedData<PNLRollOverDataUnit> _get_rollover_data()
+      {
          if (_rollover_data == null)
             _rollover_data = _archived_vol_provider.get_all_data();
          return _rollover_data;
       }
 
-      private ExpStrikeKeyedData<Tuple<double, double>> _get_position_map() {
-         return new ExpStrikeKeyedData<Tuple<double, double>>();
+      private ExpStrikeKeyedData<Tuple<double, double>> _get_position_map()
+      {
+         var sheet_settings = _threaded_bb.read_state(_in_bb_sheet_settings_tag);
+         var risk_account_text = sheet_settings?.setting_layout?.risk_account ?? string.Empty;
+         var risk_accounts = risk_account_text
+            .Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)
+            .Select(a => a.Trim())
+            .Where(a => !string.IsNullOrEmpty(a))
+            .ToHashSet(StringComparer.OrdinalIgnoreCase);
+         var trades = _threaded_bb.read_state(_in_bb_trade_log_tag) ?? Enumerable.Empty<ITradeLogRow>();
+         var option_trades = trades.Where(t =>
+            t.is_active &&
+            t.security_type == SecurityType.Option &&
+            t.symbol == _nmsheet_symbol.option_name &&
+            t.venue == _nmsheet_symbol.option_exchange &&
+            (risk_accounts.Count == 0 || risk_accounts.Contains(t.account)));
+         var pos_data = SecurityPositionData.from_trades(option_trades);
+         var rv = new ExpStrikeKeyedData<Tuple<double, double>>();
+         foreach (var (sec_exp, strike_map) in pos_data.option_pos_map)
+         {
+            foreach (var (strike, strike_pos) in strike_map)
+            {
+               rv.set_data(sec_exp.expiration, strike, Tuple.Create(strike_pos.call, strike_pos.put));
+            }
+         }
+         return rv;
       }
 
-      private double _get_onefive() {
+      private double _get_onefive()
+      {
          // object dp_obj = _threaded_bb.sec_thread_retrieve_from_cache(_in_bb_onefive_value_tag);
          // if (dp_obj != null) {
          //    DataPacket dp = dp_obj as DataPacket;
@@ -270,22 +330,29 @@ namespace BCG {
 
       #region Vol Shift
 
-      private void _add_vol_shift(ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_data) {
+      private void _add_vol_shift(ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_data)
+      {
          var vshift_map = _threaded_bb.read_state(_in_bb_vol_shift_tag);
          if (vshift_map != null)
             add_vol_shift(_nmsheet_symbol, theo_data, vshift_map);
       }
 
-      internal static void add_vol_shift(NMSheetSymbol apply_to_symbol, ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_data, IDictionary<NMSheetExpiration, double> exp_shift_map) {
-         foreach (KeyValuePair<NMSheetExpiration, double> p in exp_shift_map) {
-            if (p.Key.nmsheet_symbol.Equals(apply_to_symbol)) {
+      internal static void add_vol_shift(NMSheetSymbol apply_to_symbol, ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_data, IDictionary<NMSheetExpiration, double> exp_shift_map)
+      {
+         foreach (KeyValuePair<NMSheetExpiration, double> p in exp_shift_map)
+         {
+            if (p.Key.nmsheet_symbol.Equals(apply_to_symbol))
+            {
                DateTime option_expiration = p.Key.option_expiration;
                double shift = p.Value;
-               if (shift != 0) {
+               if (shift != 0)
+               {
                   IDictionary<double, mbts.spark.TheoPoint> exp_theos = theo_data.get_data_for_single_exp(option_expiration);
-                  if (exp_theos != null) {
+                  if (exp_theos != null)
+                  {
                      Dictionary<double, mbts.spark.TheoPoint> shifted_exp_theos = new Dictionary<double, mbts.spark.TheoPoint>();
-                     foreach (KeyValuePair<double, mbts.spark.TheoPoint> strike_tp in exp_theos) {
+                     foreach (KeyValuePair<double, mbts.spark.TheoPoint> strike_tp in exp_theos)
+                     {
                         mbts.spark.TheoPoint new_tp = (mbts.spark.TheoPoint)strike_tp.Value.Clone();
                         add_vol_shift(new_tp, shift);
                         shifted_exp_theos[strike_tp.Key] = new_tp;
@@ -297,7 +364,8 @@ namespace BCG {
          }
       }
 
-      internal static void add_vol_shift(mbts.spark.TheoPoint v, double shift) {
+      internal static void add_vol_shift(mbts.spark.TheoPoint v, double shift)
+      {
          v.call += v.cvega * shift;
          v.put += v.pvega * shift;
       }
@@ -305,7 +373,8 @@ namespace BCG {
       #endregion
 
       #region IDisposable
-      public void Dispose() {
+      public void Dispose()
+      {
          _threaded_bb.data_callback -= _react_to_bb;
       }
       #endregion

src/client/spark/NMSheets/Outside/tc2/Calculator2/MainForm/ATM/OptionATMDataSource.cs

@@ -5,6 +5,7 @@ using mbts.ui;
 using mbts;
 using BCG.Utilities;
 using mbts.spark;
+using mbts.ui.converters;
 
 namespace BCG {
    /// <summary>
@@ -34,8 +35,10 @@ namespace BCG {
       }
 
       internal static IDictionary<DateTime, ATMData> compute_atm_data(
-         double one_five, ExpStrikeKeyedData<TheoPoint> theo_data, IDictionary<DateTime, double> opt_exp_rolls) {
+         double one_five, ExpStrikeKeyedData<TheoPoint> theo_data, IDictionary<DateTime, double> opt_exp_rolls,
+         double rep_driver) {
          IDictionary<DateTime, double> exp_vs_drivers = _get_computed_drivers(theo_data);
+         exp_vs_drivers = exp_vs_drivers.ToDictionary(kv => kv.Key, kv => rep_driver);
          IDictionary<DateTime, ATMData> rv = new Dictionary<DateTime, ATMData>();
          if (one_five != 0)
             foreach (DateTime exp in new List<DateTime>(exp_vs_drivers.Keys))

src/client/spark/Orders/ExistingOrderTicketLadderVM.cs

@@ -0,0 +1,236 @@
+#nullable enable
+namespace mbts.spark;
+
+using bts;
+using bts.api;
+using bts.api.domain;
+using bts.api.orderman;
+
+using mbts.co2;
+using mbts.ui;
+using mbts.ui.shell;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Collections.Generic;
+using System.Collections.ObjectModel;
+using System.Linq;
+using System.Reactive.Linq;
+using System.Windows.Input;
+using System.Windows.Threading;
+
+class ExistingOrderTicketLadderVM : DisposableViewModelBase, IRequestViewClose {
+   const ushort numPriceLevels = 5;
+
+   readonly Property<SecurityImmutable> security;
+   readonly Property<TickDisplaySettingsImmutable> priceDisplay;
+   readonly Property<TickDisplaySettingsImmutable> quantityDisplay;
+   readonly Property<user.settings.RegionalTickFactors> priceTick;
+   readonly ObservableCollection<OrderTicketLadderPriceLevelVM> bids = [];
+   readonly ObservableCollection<OrderTicketLadderPriceLevelVM> asks = [];
+
+   readonly Property<double> price, currentPrice;
+   readonly Property<uint> currentQuantity;
+   readonly Property<AccountTuple> currentAccount;
+   readonly Property<TimeInForce> currentTimeInForce;
+   readonly Property<CustomOrderDefinition.CustomOneofCase> currentOrderType;
+   readonly Property<bool> priceDirty, quantityDirty, accountDirty, timeInForceDirty, orderTypeDirty;
+
+   Side side = Side.Buy;
+
+   public ExistingOrderTicketLadderVM(
+      OrdersModel model,
+      IObservable<(SecurityMD? SecurityMd, OrderRecordImmutable Order)> orderSource,
+      IObservable<IReadOnlyDictionary<uint, AccountTuple>> accountsSource,
+      IObservable<double[]?> calculatorButtons,
+      Action<SecurityMD> showLadder,
+      TickDisplaySettingsImmutable? quantityDisplay = null
+   ) {
+      SecurityMD? currentSecurityMd = null;
+      uint? key = null;
+      DispatcherTimer? mdPollTimer = null;
+      var nativeBids = new NativePriceSize[numPriceLevels];
+      var nativeAsks = new NativePriceSize[numPriceLevels];
+      for (int i = 0; i < numPriceLevels; i++) {
+         nativeBids[i] = NativePriceSize.NO_MARKET;
+         nativeAsks[i] = NativePriceSize.NO_MARKET;
+         Bids.Add(new(Side.Buy));
+         Asks.Add(new(Side.Sell));
+      }
+
+      var secMdSrc = orderSource.Where(ot => ot.SecurityMd.HasValue).Select(ot => ot.SecurityMd!.Value);
+      security = backing(nameof(Security), secMdSrc.Select(smd => smd.security), bts.api.Security.Defaults.SecurityImmutable);
+      priceTick = backing(nameof(PriceTick), user.settings.RegionalTickFactors.Default);
+      PropertiesVm = new OrderPropertiesEditorVM(
+         model,
+         secMdSrc,
+         accountsSource,
+         orderSource.Select(ot => new InlineOrderProperties(model, ot.Order.Def)));
+      this.quantityDisplay = backing(nameof(QuantityDisplay), quantityDisplay ?? bts.api.Security.Defaults.quantity_display_settingsi);
+      priceDisplay = backing(nameof(PriceDisplay), bts.api.Security.Defaults.price_display_settingsi);
+      SizeCalculator = new(0, calculatorButtons);
+      SetPriceCmd = new DelegateCommand<double>(SetPrice);
+      ShowLadderCmd = new DelegateCommand(() => {
+         if (currentSecurityMd.HasValue)
+            showLadder(currentSecurityMd.Value);
+      });
+
+      price = backing(nameof(Price), double.NaN);
+      currentPrice = backing(nameof(CurrentPrice), orderSource.Select(ort => ort.Order.State.Price));
+      currentQuantity = backing(nameof(CurrentQuantity), orderSource.Select(ort => ort.Order.State.Quantity));
+      currentAccount = backing(nameof(CurrentAccount), orderSource.Select(ort => new AccountTuple(ort.Order.Def.AccountId, ort.Order.Def.Account)));
+      currentTimeInForce = backing(nameof(CurrentTimeInForce), orderSource.Select(ort => ort.Order.Def.TimeInForce));
+      currentOrderType = backing(nameof(CurrentOrderType), orderSource.Select(ort => ort.Order.Def.CustomDef.CustomCase));
+      //
+      priceDirty = backing(
+         nameof(PriceDirty),
+         price.observe_current(true).CombineLatest(
+         currentPrice.observe_current(true),
+         (val, curr) => !double.IsNaN(val) && val != curr));
+      quantityDirty = backing(
+         nameof(QuantityDirty),
+         SizeCalculator.observe_actual_value()
+         .CombineLatest(currentQuantity.observe_current(true),
+         (val, curr) => val.HasValue && val.Value != curr));
+      accountDirty = backing(
+         nameof(AccountDirty),
+         PropertiesVm.observe_account(true)
+         .CombineLatest(currentAccount.observe_current(true),
+         (val, curr) => val != curr));
+      timeInForceDirty = backing(
+         nameof(TimeInForceDirty),
+         PropertiesVm.observe_time_in_force(true)
+         .CombineLatest(currentTimeInForce.observe_current(true),
+         (val, curr) => val != curr));
+      orderTypeDirty = backing(
+         nameof(OrderTypeDirty),
+         PropertiesVm.observe_order_type(true)
+         .CombineLatest(currentOrderType.observe_current(true),
+         (val, curr) => val != null && val.order_type != curr));
+      //
+      var priceDirtySrc = priceDirty.Select(vals => vals.newval).StartWith(false);
+      var quantityDirtySrc = quantityDirty.Select(vals => vals.newval).StartWith(false);
+      var sendOrderCmd = new ObservantCommand(SendOrder);
+      SendOrderCmd = sendOrderCmd;
+      var canSendOrderSrc = priceDirty.observe_current(true)
+         .CombineLatest(quantityDirty.observe_current(true), (pd, sd) => pd || sd)
+         .DistinctUntilChanged();
+      var isDirtySrc = Observable.Merge(priceDirty, quantityDirty, accountDirty, timeInForceDirty, orderTypeDirty);
+      _dispose = Disposable.Create(
+         SizeCalculator,
+         PropertiesVm,
+         secMdSrc.OnDispatcher().Subscribe(OnSecurityMd),
+         orderSource.OnDispatcher().Subscribe(ort => {
+            key = ort.Order.Key;
+            if (DoubleHelper.IsNaNOrInfinity(Price))
+               Price = ort.Order.State.Price;
+            if (!SizeCalculator.value.HasValue || !SizeCalculator.value_setter_called)
+               SizeCalculator.bypass_value_setter(ort.Order.State.Quantity, true);
+         }),
+         isDirtySrc.OnDispatcher().Subscribe(_ => RaisePropertyChanged(nameof(DirtyItemsText))),
+         canSendOrderSrc.OnDispatcher().Subscribe(sendOrderCmd));
+
+      void SetPrice(double price) {
+         Price = price;
+         RaisePropertyChanged(nameof(Price));
+      }
+
+      void SendOrder() {
+         if (!key.HasValue) {
+            return;
+         }
+         if (priceDirty.Value) {
+            if (quantityDirty.Value) {
+               model.update_order(key.Value, Price, SizeCalculator.value!.Value).ignore();
+            } else {
+               model.update_order_price(key.Value, Price).ignore();
+            }
+         } else if (quantityDirty.Value) {
+            model.update_order_size(key.Value, SizeCalculator.value!.Value).ignore();
+         } else {
+            SizeCalculator.flash_error();
+            flash_error(nameof(Price)).ignore();
+            return;
+         }
+         RequestClose?.Invoke(this, new EventArgs());
+      }
+
+      void OnSecurityMd(SecurityMD smd) {
+         currentSecurityMd = smd;
+         priceTick.set(smd.security.regional_ticks());
+         priceDisplay.set(smd.price_display);
+         this.quantityDisplay.set(smd.quantity_display);
+         SizeCalculator.display_settings = smd.quantity_display;
+         ulong lastMdPollTime = 0;
+         mdPollTimer?.Stop();
+         mdPollTimer = new DispatcherTimer(TimeSpan.FromSeconds(0.1), DispatcherPriority.DataBind, PollMd, Dispatcher.CurrentDispatcher);
+
+         void PollMd(object? sender, EventArgs e) {
+            if (smd.md.get_depth(ref lastMdPollTime, ref nativeBids, ref nativeAsks, numPriceLevels)) {
+               for (int i = 0; i < numPriceLevels; i++) {
+                  asks[numPriceLevels - i - 1].Price = nativeAsks[i].price;
+                  asks[numPriceLevels - i - 1].Quantity = nativeAsks[i].size;
+               }
+               for (int i = 0; i < numPriceLevels; i++) {
+                  bids[i].Price = nativeBids[i].price;
+                  bids[i].Quantity = nativeBids[i].size;
+               }
+            }
+         }
+      }
+   }
+
+   public event EventHandler? RequestClose;
+
+   public SecurityImmutable Security => security.Value;
+   public string Title => Security.ToString() ?? string.Empty;
+   public double Price { get; set; }
+   public user.settings.RegionalTickFactors PriceTick => priceTick.Value;
+   public TickDisplaySettingsImmutable PriceDisplay => priceDisplay.Value;
+   public TickDisplaySettingsImmutable QuantityDisplay => quantityDisplay.Value;
+   public CalculatorLogic<double> SizeCalculator { get; }
+   public Side Side {
+      get => side;
+      set => Set(ref side, value);
+   }
+
+   public OrderPropertiesEditorVM PropertiesVm { get; }
+   public ObservableCollection<OrderTicketLadderPriceLevelVM> Bids => bids;
+   public ObservableCollection<OrderTicketLadderPriceLevelVM> Asks => asks;
+
+   public double CurrentPrice => currentPrice.Value;
+   public uint CurrentQuantity => currentQuantity.Value;
+   public AccountTuple CurrentAccount => currentAccount.Value;
+   public TimeInForce CurrentTimeInForce => currentTimeInForce.Value;
+   public CustomOrderDefinition.CustomOneofCase CurrentOrderType => currentOrderType.Value;
+   public bool PriceDirty => priceDirty.Value;
+   public bool QuantityDirty => quantityDirty.Value;
+   public bool AccountDirty => accountDirty.Value;
+   public bool TimeInForceDirty => timeInForceDirty.Value;
+   public bool OrderTypeDirty => orderTypeDirty.Value;
+   public bool CanChangeAccount { get; } = false;
+   public bool CanChangeTimeInForce { get; } = false;
+   public bool CanChangeOrderType { get; } = false;
+   public string DirtyItemsText {
+      get {
+         var dirty_items = dirtyItems().ToList();
+         if (dirty_items.Count == 0) {
+            return "(no changes)";
+         }
+         return "change " + string.Join(", ", dirty_items);
+      }
+   }
+   public bool CloseOnSend { get; set; }
+   public ICommand SendOrderCmd { get; }
+   public ICommand SetPriceCmd { get; }
+   public ICommand ShowLadderCmd { get; }
+
+   IEnumerable<string> dirtyItems() {
+      if (priceDirty.Value) yield return "price";
+      if (quantityDirty.Value) yield return "quantity";
+      if (accountDirty.Value) yield return "account";
+      if (timeInForceDirty.Value) yield return "tif";
+      if (orderTypeDirty.Value) yield return "type";
+   }
+}

src/client/spark/Orders/NewOrderTicketLadderVM.cs

@@ -0,0 +1,298 @@
+#nullable enable
+namespace mbts.spark;
+
+using bts;
+using bts.api;
+using bts.api.domain;
+using bts.api.orderman;
+using bts.collections;
+using bts.utils;
+
+using mbts.co2;
+using mbts.ui;
+using mbts.ui.shell;
+using mbts.ui.spark;
+
+using Spark.Core.Disposables;
+
+using System;
+using System.Collections.Generic;
+using System.Collections.ObjectModel;
+using System.Linq;
+using System.Reactive.Linq;
+using System.Windows.Input;
+using System.Windows.Media;
+using System.Windows.Threading;
+
+class NewOrderTicketLadderVM : DisposableViewModelBase, IOrderModificationLadderVM  {
+   const ushort numPriceLevels = 5;
+   const uint numPriceRowsPerSide = 500;
+
+   readonly Property<SecurityImmutable> security;
+   readonly Property<string> title;
+   readonly Property<TickDisplaySettingsImmutable> priceDisplay;
+   readonly Property<TickDisplaySettingsImmutable> quantityDisplay;
+   readonly Property<user.settings.RegionalTickFactors> priceTick;
+   readonly ObservableCollection<OrderTicketLadderPriceLevelVM> priceLevels = [];
+   Side side = Side.Buy;
+
+   public NewOrderTicketLadderVM(
+      OrdersModel model,
+      SecurityMD securityMd,
+      IObservable<IReadOnlyDictionary<uint, AccountTuple>> accountsSource,
+      OrderDefinitionImmutable def,
+      double price,
+      double size,
+      IObservable<double[]> calculatorButtons,
+      Action<SecurityMD> showLadder)
+      : this(
+           model,
+           Observable.Return(securityMd),
+           accountsSource,
+           Observable.Return(new InlineOrderTicketProperties(model, def, price, size)),
+           calculatorButtons,
+           showLadder) { }
+
+   public NewOrderTicketLadderVM(
+      OrdersModel model,
+      SecurityMD securityMd,
+      IObservable<IReadOnlyDictionary<uint, AccountTuple>> accountsSource,
+      IObservable<LadderSettings> settings,
+      Action<SecurityMD> showLadder)
+      : this(
+           model,
+           Observable.Return(securityMd),
+           accountsSource,
+           settings.Select(s => new InlineOrderTicketProperties(model, s)),
+           settings.Select(s => s.calculator_buttons),
+           showLadder) { }
+
+   public NewOrderTicketLadderVM(
+      OrdersModel model,
+      IObservable<SecurityMD> securityMdSource,
+      IObservable<IReadOnlyDictionary<uint, AccountTuple>> accountsSource,
+      IObservable<InlineOrderTicketProperties> propsSource,
+      IObservable<double[]?> calculatorButtons,
+      Action<SecurityMD> showLadder,
+      TickDisplaySettingsImmutable? quantityDisplay = null
+   ) {
+      SecurityMD? currentSecurityMd = null;
+      DispatcherTimer? mdPollTimer = null;
+      var nativeBids = new NativePriceSize[numPriceLevels];
+      var nativeAsks = new NativePriceSize[numPriceLevels];
+      for (int i = 0; i < numPriceLevels; i++) {
+         nativeBids[i] = NativePriceSize.NO_MARKET;
+         nativeAsks[i] = NativePriceSize.NO_MARKET;
+      }
+      for (int i = 0; i < numPriceRowsPerSide; i++) {
+         priceLevels.Add(new(Side.Sell));
+      }
+      for (int i = 0; i < numPriceRowsPerSide; i++) {
+         priceLevels.Add(new(Side.Buy));
+      }
+
+      security = backing(nameof(Security), bts.api.Security.Defaults.SecurityImmutable);
+      title = backing(nameof(Title), security.observe_current(true), s => s.ToString() ?? string.Empty, string.Empty);
+      priceTick = backing(nameof(PriceTick), user.settings.RegionalTickFactors.Default);
+      PropertiesVm = new(model, securityMdSource, accountsSource, propsSource.Select(p => p.order_props));
+      this.quantityDisplay = backing(nameof(QuantityDisplay), quantityDisplay ?? bts.api.Security.Defaults.quantity_display_settingsi);
+      priceDisplay = backing(nameof(PriceDisplay), bts.api.Security.Defaults.price_display_settingsi);
+      SizeCalculator = new(0, calculatorButtons);
+      SendOrderCmd = new DelegateCommand(SendOrder);
+      SetPriceCmd = new DelegateCommand<double>(SetPrice);
+      ShowLadderCmd = new DelegateCommand(() => {
+         if (currentSecurityMd.HasValue)
+            showLadder(currentSecurityMd.Value);
+      });
+      RecenterCmd = new DelegateCommand(Recenter);
+      _dispose = Disposable.Create(
+         securityMdSource.OnDispatcher().Subscribe(OnSecurityMd),
+         SizeCalculator,
+         propsSource.OnDispatcher().Subscribe(p => OnProps(p.price, p.size)));
+
+      void OnProps(
+         InlineUserProperty<double> priceProp, InlineUserProperty<double> sizeProp) {
+         if (priceProp.has_value && (priceProp.force_override || DoubleHelper.IsNaNOrInfinity(Price)))
+            SetPrice(priceProp.value);
+         if (sizeProp.has_value && (sizeProp.force_override || SizeCalculator.value == 0))
+            SizeCalculator.value = sizeProp.value;
+      }
+
+      void SetPrice(double price) {
+         Price = price;
+         RaisePropertyChanged(nameof(Price));
+      }
+
+      void SendOrder() {
+         bool priceValid = validatePrice();
+         bool quantityValid = SizeCalculator.validate();
+         if (priceValid && quantityValid && !HasErrors && !PropertiesVm.HasErrors) {
+            model.send_order(Security, PropertiesVm.create(side), Price, SizeCalculator.value!.Value).ignore();
+         }
+      }
+
+      void Recenter() =>
+         RequestRecenter?.Invoke();
+
+      void OnSecurityMd(SecurityMD smd) {
+         currentSecurityMd = smd;
+         var spec = TickUtils.multiply_tick_size(smd.security.regional_ticks(), 1);
+         security.set(smd.security);
+         priceTick.set(smd.security.regional_ticks());
+         priceDisplay.set(smd.price_display);
+         this.quantityDisplay.set(smd.quantity_display);
+         SizeCalculator.display_settings = smd.quantity_display;
+         ulong lastMdPollTime = 0;
+         mdPollTimer?.Stop();
+         mdPollTimer = new DispatcherTimer(TimeSpan.FromSeconds(0.1), DispatcherPriority.DataBind, PollMd, Dispatcher.CurrentDispatcher);
+
+         void PollMd(object? sender, EventArgs e) {
+            if (smd.md.get_depth(ref lastMdPollTime, ref nativeBids, ref nativeAsks, numPriceLevels)) {
+               if (double.IsNaN(priceLevels[0].Price)) {
+                  // Initialize prices and recenter
+                  if (!TryGetOrigin(out var origin)) {
+                     return;
+                  }
+                  double price = spec.tickdown(origin, numPriceRowsPerSide - 1);
+                  for (int i = priceLevels.Count - 1; i >= 0; i--) {
+                     priceLevels[i].Price = price;
+                     price = spec.tickup(price);
+                  }
+                  Recenter();
+               }
+
+               var hi = priceLevels[0].Price;
+               var lo = priceLevels[^1].Price;
+               Log.assert(hi.is_real() && lo.is_real() && hi > lo);
+               var wrappedPrices = priceLevels.WrapSelector(p => p.Price, null);
+               var rPriceComp = spec.price_comparer().reverse();
+
+               // Clear old prices
+               for (int i = 0; i < numPriceLevels; i++) {
+                  // asks
+                  var px = nativeAsks[i].price;
+                  if (!double.IsNaN(px) && px.is_within_range(lo, hi)) {
+                     int idx = CollectionHelper.binary_search(wrappedPrices, px, rPriceComp);
+                     if (idx < 0)
+                        idx = ~idx;
+                     priceLevels[idx].Quantity = 0;
+                  }
+                  // bids
+                  px = nativeBids[i].price;
+                  if (!double.IsNaN(px) && px.is_within_range(lo, hi)) {
+                     int idx = CollectionHelper.binary_search(wrappedPrices, px, rPriceComp);
+                     if (idx < 0)
+                        idx = ~idx;
+                     priceLevels[idx].Quantity = 0;
+                  }
+               }
+
+               // Set new prices
+               for (int i = 0; i < numPriceLevels; i++) {
+                  // asks
+                  var px = nativeAsks[i].price;
+                  if (!double.IsNaN(px) && px.is_within_range(lo, hi)) {
+                     int idx = CollectionHelper.binary_search(wrappedPrices, px, rPriceComp);
+                     if (idx < 0)
+                        idx = ~idx;
+                     priceLevels[idx].Quantity = nativeAsks[i].size;
+                     priceLevels[idx].Side = Side.Sell;
+                  }
+                  // bids
+                  px = nativeBids[i].price;
+                  if (!double.IsNaN(px) && px.is_within_range(lo, hi)) {
+                     int idx = CollectionHelper.binary_search(wrappedPrices, px, rPriceComp);
+                     if (idx < 0)
+                        idx = ~idx;
+                     priceLevels[idx].Quantity = nativeBids[i].size;
+                     priceLevels[idx].Side = Side.Buy;
+                  }
+               }
+            }
+         }
+
+         bool TryGetOrigin(out double origin) {
+            if (!nativeBids[0].is_nan) {
+               origin = nativeBids[0].price;
+            } else if (!nativeAsks[0].is_nan) {
+               origin = nativeAsks[0].price;
+            } else {
+               origin = double.NaN;
+               return false;
+            }
+            return true;
+         }
+      }
+   }
+
+   public event Action? RequestRecenter;
+
+   public SecurityImmutable Security => security.Value;
+   public string Title => title.Value;
+   public double Price { get; set; }
+   public user.settings.RegionalTickFactors PriceTick => priceTick.Value;
+   public TickDisplaySettingsImmutable PriceDisplay => priceDisplay.Value;
+   public TickDisplaySettingsImmutable QuantityDisplay => quantityDisplay.Value;
+   public CalculatorLogic<double> SizeCalculator { get; }
+   public Side Side {
+      get => side;
+      set => Set(ref side, value);
+   }
+
+   public OrderPropertiesEditorVM PropertiesVm { get; }
+   public ObservableCollection<OrderTicketLadderPriceLevelVM> PriceLevels => priceLevels;
+   public ICommand SendOrderCmd { get; }
+   public ICommand SetPriceCmd { get; }
+   public ICommand ShowLadderCmd { get; }
+   public ICommand RecenterCmd { get; }
+
+   bool validatePrice() {
+      if (!Price.is_real()) {
+         flash_error(nameof(Price)).ignore();
+         return false;
+      }
+      return true;
+   }
+}
+
+class OrderTicketLadderPriceLevelVM : ViewModelBase {
+   Color background = Colors.Transparent;
+   Color priceForeground = BTSColors.Control.normal_foreground;
+   Side side;
+   double price = double.NaN;
+   double quantity = double.NaN;
+
+   public OrderTicketLadderPriceLevelVM(Side side) {
+      this.side = side;
+      background = BTSColors.Ladder.price_column_background;
+      priceForeground = BTSColors.Ladder.price_column_foreground;
+   }
+
+   public Side Side {
+      get => side;
+      set => Set(ref side, value, dependent_props: nameof(QuantityForeground));
+   }
+   public double Price {
+      get => price;
+      set => Set(ref price, value);
+   }
+   public double Quantity {
+      get => quantity;
+      set => Set(ref quantity, value);
+   }
+
+   public Color Background {
+      get => background;
+      set => Set(ref background, value);
+   }
+   public Color PriceForeground {
+      get => priceForeground;
+      set => Set(ref priceForeground, value);
+   }
+   public Color QuantityForeground =>
+      side switch {
+         Side.Buy => BTSColors.Ladder.bid_column_foreground,
+         Side.Sell => BTSColors.Ladder.ask_column_foreground,
+         _ => BTSColors.UI.default_foreground,
+      };
+}

src/client/spark/Orders/OrderModificationLadderControl.xaml

@@ -0,0 +1,415 @@
+<UserControl
+  x:Class="mbts.ui.spark.OrderModificationLadderControl"
+  xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
+  xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+  xmlns:api="clr-namespace:bts.api;assembly=Bts.Spark.Api"
+  xmlns:u="clr-namespace:mbts.ui"
+  xmlns:c="clr-namespace:mbts.ui.controls"
+  xmlns:cv="clr-namespace:mbts.ui.converters"
+  xmlns:usp="clr-namespace:mbts.ui.spark"
+  xmlns:uag="clr-namespace:mbts.ui.spark.algo"
+  xmlns:sys="clr-namespace:System;assembly=mscorlib"
+  u:AttachedCommands.MiddleClick="{Binding RecenterCmd}" >
+  <UserControl.Resources>
+    <ResourceDictionary>
+      <ResourceDictionary.MergedDictionaries>
+        <ResourceDictionary Source="/view/Colors.xaml"/>
+        <ResourceDictionary Source="/view/Resources/Converters.xaml"/>
+        <ResourceDictionary Source="/view/Controls/Order/OrderDetailsPopupControl.xaml"/>
+      </ResourceDictionary.MergedDictionaries>
+
+      <!-- Consts -->
+      <sys:Double x:Key="HorizontalSpacing">8.0</sys:Double>
+      <sys:Double x:Key="VerticalSpacing">8.0</sys:Double>
+      <sys:Double x:Key="ButtonHeight">22.0</sys:Double>
+
+      <!-- Converters -->
+      <cv:NullableDoubleConverter x:Key="TickDefaultPriceConverter" DefaultValue="0"/>
+      <cv:ValueMap x:Key="SideToBrushConverter" Default="{x:Static Brushes.Transparent}">
+        <cv:KeyValue Key="{u:Enum Buy, Type=api:Side}" Value="{u:BTSBrush MarketData.bid, Opacity=0.33}" />
+        <cv:KeyValue Key="{u:Enum Sell, Type=api:Side}" Value="{u:BTSBrush MarketData.ask, Opacity=0.33}" />
+      </cv:ValueMap>
+      <cv:ValueMap x:Key="SideToIsBuyConverter" Default="{u:False}">
+        <cv:KeyValue Key="{u:Enum Buy, Type=api:Side}" Value="{u:True}" />
+      </cv:ValueMap>
+      <cv:ValueMap x:Key="SideToIsSellConverter" Default="{u:False}">
+        <cv:KeyValue Key="{u:Enum Sell, Type=api:Side}" Value="{u:True}" />
+      </cv:ValueMap>
+      <cv:ValueMap x:Key="SideToBidVisibilityConverter" Default="{u:Visibility Collapsed}">
+        <cv:KeyValue Key="{u:Enum Buy, Type=api:Side}" Value="{u:Visibility Visible}" />
+      </cv:ValueMap>
+      <cv:ValueMap x:Key="SideToAskVisibilityConverter" Default="{u:Visibility Collapsed}">
+        <cv:KeyValue Key="{u:Enum Sell, Type=api:Side}" Value="{u:Visibility Visible}" />
+      </cv:ValueMap>
+
+      <!-- Styles -->
+      <Style TargetType="Button" BasedOn="{StaticResource DefaultButton}">
+        <Setter Property="Height" Value="{StaticResource ButtonHeight}"/>
+        <Setter Property="HorizontalAlignment" Value="Stretch"/>
+      </Style>
+      <Style TargetType="ComboBox" BasedOn="{StaticResource {x:Type ComboBox}}">
+        <Setter Property="Height" Value="{StaticResource ButtonHeight}"/>
+        <Style.Triggers>
+          <Trigger Property="usp:OrderModificationLadderControl.IsDirty" Value="True">
+            <Setter Property="BorderBrush" Value="{u:BTSColor UI.input_dirty_border}"/>
+          </Trigger>
+        </Style.Triggers>
+      </Style>
+      <Style TargetType="usp:PriceEntryControl" BasedOn="{StaticResource {x:Type usp:PriceEntryControl}}">
+        <Style.Triggers>
+          <Trigger  Property="usp:OrderModificationLadderControl.IsDirty" Value="True">
+            <Setter Property="BorderBrush" Value="{u:BTSColor UI.input_dirty_border}"/>
+          </Trigger>
+        </Style.Triggers>
+      </Style>
+      <Style TargetType="c:TickUpDownNullableUInt32" BasedOn="{StaticResource {x:Type c:TickUpDownNullableUInt32}}">
+        <Style.Triggers>
+          <Trigger Property="usp:OrderModificationLadderControl.IsDirty" Value="True">
+            <Setter Property="BorderBrush" Value="{u:BTSColor UI.input_dirty_border}"/>
+          </Trigger>
+        </Style.Triggers>
+      </Style>
+
+      <DataTemplate x:Key="PriceLevelItemTemplate">
+        <Button
+          Background="{Binding Background, Converter={x:Static cv:ColorToBrushConverter.Instance}}"
+          HorizontalAlignment="Stretch"
+          HorizontalContentAlignment="Stretch"
+          MinWidth="96"
+          Padding="4,2"
+          Command="{Binding DataContext.SetPriceCmd, RelativeSource={RelativeSource AncestorType=usp:OrderModificationLadderControl}}"
+          CommandParameter="{Binding Price}">
+          <Grid>
+            <Grid.ColumnDefinitions>
+              <ColumnDefinition Width="Auto" MinWidth="40" SharedSizeGroup="Orders" />
+              <ColumnDefinition Width="*" MinWidth="40" SharedSizeGroup="BidSize" />
+              <ColumnDefinition Width="Auto" MinWidth="56" SharedSizeGroup="Price" />
+              <ColumnDefinition Width="*" MinWidth="40" SharedSizeGroup="AskSize" />
+            </Grid.ColumnDefinitions>
+            <TextBlock
+              Grid.Column="1"
+              HorizontalAlignment="Right"
+              Foreground="{Binding QuantityForeground, Converter={x:Static cv:ColorToBrushConverter.Instance}}"
+              FontSize="{x:Static u:FontSizes.Small}"
+              Visibility="{Binding Side, Converter={StaticResource SideToBidVisibilityConverter}}">
+              <TextBlock.Text>
+                <MultiBinding Converter="{x:Static cv:TickDisplayConverter.Instance}">
+                  <Binding Path="Quantity" Mode="OneWay"/>
+                  <Binding Path="DataContext.QuantityDisplay" RelativeSource="{RelativeSource AncestorType=usp:OrderModificationLadderControl}"/>
+                </MultiBinding>
+              </TextBlock.Text>
+            </TextBlock>
+            <TextBlock
+              Grid.Column="2"
+              HorizontalAlignment="Center"
+              Foreground="{Binding PriceForeground, Converter={x:Static cv:ColorToBrushConverter.Instance}}"
+              FontSize="{x:Static u:FontSizes.Small}">
+              <TextBlock.Text>
+                <MultiBinding Converter="{x:Static cv:TickDisplayConverter.Instance}">
+                  <Binding Path="Price" Mode="OneWay"/>
+                  <Binding Path="DataContext.PriceDisplay" RelativeSource="{RelativeSource AncestorType=usp:OrderModificationLadderControl}"/>
+                </MultiBinding>
+              </TextBlock.Text>
+            </TextBlock>
+            <TextBlock
+              Grid.Column="3"
+              HorizontalAlignment="Left"
+              Foreground="{Binding QuantityForeground, Converter={x:Static cv:ColorToBrushConverter.Instance}}"
+              FontSize="{x:Static u:FontSizes.Small}"
+              Visibility="{Binding Side, Converter={StaticResource SideToAskVisibilityConverter}}">
+              <TextBlock.Text>
+                <MultiBinding Converter="{x:Static cv:TickDisplayConverter.Instance}">
+                  <Binding Path="Quantity" Mode="OneWay"/>
+                  <Binding Path="DataContext.QuantityDisplay" RelativeSource="{RelativeSource AncestorType=usp:OrderModificationLadderControl}"/>
+                </MultiBinding>
+              </TextBlock.Text>
+            </TextBlock>
+          </Grid>
+        </Button>
+      </DataTemplate>
+
+      <ControlTemplate x:Key="BuyRadioButtonTemplate" TargetType="RadioButton">
+        <Grid>
+          <VisualStateManager.VisualStateGroups>
+            <VisualStateGroup x:Name="CommonStates">
+              <VisualState x:Name="Normal" />
+              <VisualState x:Name="MouseOver">
+                <Storyboard>
+                  <DoubleAnimation Storyboard.TargetName="hoverOverlay" Storyboard.TargetProperty="Opacity" To="0.3" BeginTime="0" Duration="0" />
+                </Storyboard>
+              </VisualState>
+              <VisualState x:Name="Pressed" />
+              <VisualState x:Name="Disabled">
+                <Storyboard>
+                  <ColorAnimation Storyboard.TargetName="background" Storyboard.TargetProperty="Background.Color" To="#555" Duration="0" />
+                </Storyboard>
+              </VisualState>
+            </VisualStateGroup>
+            <VisualStateGroup x:Name="CheckStates">
+              <VisualState x:Name="Checked">
+                <Storyboard>
+                  <ColorAnimation Storyboard.TargetName="background" Storyboard.TargetProperty="Background.Color" To="{u:BTSColor MarketData.bid}" BeginTime="0" Duration="0"/>
+                  <ColorAnimation Storyboard.TargetName="background" Storyboard.TargetProperty="BorderBrush.Color" To="{u:BTSColor MarketData.bid}" BeginTime="0" Duration="0"/>
+                </Storyboard>
+              </VisualState>
+              <VisualState x:Name="Unchecked" />
+              <VisualState x:Name="Indeterminate" />
+            </VisualStateGroup>
+          </VisualStateManager.VisualStateGroups>
+          <Border
+            x:Name="background"
+            BorderThickness="{TemplateBinding BorderThickness}"
+            VerticalAlignment="{TemplateBinding VerticalAlignment}"
+            HorizontalAlignment="{TemplateBinding HorizontalAlignment}"
+            Padding="{TemplateBinding Padding}"
+            CornerRadius="2">
+            <Border.Background>
+              <SolidColorBrush Color="Transparent" />
+            </Border.Background>
+            <Border.BorderBrush>
+              <SolidColorBrush Color="Black"/>
+            </Border.BorderBrush>
+          </Border>
+          <ContentPresenter
+            Name="presenter"
+            HorizontalAlignment="{TemplateBinding HorizontalContentAlignment}"
+            VerticalAlignment="{TemplateBinding HorizontalContentAlignment}" />
+          <Border Name="hoverOverlay" HorizontalAlignment="Stretch" VerticalAlignment="Stretch" Background="Black" Opacity="0" IsHitTestVisible="False" />
+        </Grid>
+      </ControlTemplate>
+
+      <ControlTemplate x:Key="SellRadioButtonTemplate" TargetType="RadioButton">
+        <Grid>
+          <VisualStateManager.VisualStateGroups>
+            <VisualStateGroup x:Name="CommonStates">
+              <VisualState x:Name="Normal" />
+              <VisualState x:Name="MouseOver">
+                <Storyboard>
+                  <DoubleAnimation Storyboard.TargetName="hoverOverlay" Storyboard.TargetProperty="Opacity" To="0.3" BeginTime="0" Duration="0" />
+                </Storyboard>
+              </VisualState>
+              <VisualState x:Name="Pressed" />
+              <VisualState x:Name="Disabled">
+                <Storyboard>
+                  <ColorAnimation Storyboard.TargetName="background" Storyboard.TargetProperty="Background.Color" To="#555" Duration="0" />
+                </Storyboard>
+              </VisualState>
+            </VisualStateGroup>
+            <VisualStateGroup x:Name="CheckStates">
+              <VisualState x:Name="Checked">
+                <Storyboard>
+                  <ColorAnimation Storyboard.TargetName="background" Storyboard.TargetProperty="Background.Color" To="{u:BTSColor MarketData.ask}" BeginTime="0" Duration="0"/>
+                  <ColorAnimation Storyboard.TargetName="background" Storyboard.TargetProperty="BorderBrush.Color" To="{u:BTSColor MarketData.ask}" BeginTime="0" Duration="0"/>
+                </Storyboard>
+              </VisualState>
+              <VisualState x:Name="Unchecked" />
+              <VisualState x:Name="Indeterminate" />
+            </VisualStateGroup>
+          </VisualStateManager.VisualStateGroups>
+          <Border
+            x:Name="background"
+            BorderThickness="{TemplateBinding BorderThickness}"
+            VerticalAlignment="{TemplateBinding VerticalAlignment}"
+            HorizontalAlignment="{TemplateBinding HorizontalAlignment}"
+            Padding="{TemplateBinding Padding}"
+            CornerRadius="2">
+            <Border.Background>
+              <SolidColorBrush Color="Transparent" />
+            </Border.Background>
+            <Border.BorderBrush>
+              <SolidColorBrush Color="Black"/>
+            </Border.BorderBrush>
+          </Border>
+          <ContentPresenter
+            Name="presenter"
+            HorizontalAlignment="{TemplateBinding HorizontalContentAlignment}"
+            VerticalAlignment="{TemplateBinding VerticalContentAlignment}" />
+          <Border Name="hoverOverlay" HorizontalAlignment="Stretch" VerticalAlignment="Stretch" Background="Black" Opacity="0" IsHitTestVisible="False" />
+        </Grid>
+      </ControlTemplate>
+
+      <Style x:Key="BuyRadioButtonStyle" TargetType="RadioButton">
+        <Setter Property="Padding" Value="2" />
+        <Setter Property="HorizontalContentAlignment" Value="Center" />
+        <Setter Property="VerticalContentAlignment" Value="Center" />
+        <Setter Property="Foreground" Value="{u:BTSBrush UI.default_text}" />
+        <Setter Property="Template" Value="{StaticResource BuyRadioButtonTemplate}" />
+      </Style>
+
+      <Style x:Key="SellRadioButtonStyle" TargetType="RadioButton">
+        <Setter Property="Padding" Value="2" />
+        <Setter Property="HorizontalContentAlignment" Value="Center" />
+        <Setter Property="VerticalContentAlignment" Value="Center" />
+        <Setter Property="Foreground" Value="{u:BTSBrush UI.default_text}" />
+        <Setter Property="Template" Value="{StaticResource SellRadioButtonTemplate}" />
+      </Style>
+
+    </ResourceDictionary>
+  </UserControl.Resources>
+
+  <Grid>
+    <Grid.ColumnDefinitions>
+      <ColumnDefinition />
+      <ColumnDefinition Width="{Binding Source={StaticResource HorizontalSpacing}}" />
+      <ColumnDefinition />
+    </Grid.ColumnDefinitions>
+
+    <!-- Ladder (market depth) -->
+    <Grid Grid.Column="2" Grid.IsSharedSizeScope="True">
+      <Grid.RowDefinitions>
+        <RowDefinition Height="Auto" />
+        <RowDefinition Height="Auto" />
+      </Grid.RowDefinitions>
+      <Grid
+        Grid.Row="0"
+        Background="{u:BTSBrush header}"
+        TextElement.Foreground="{u:BTSBrush header_text}"
+        TextElement.FontSize="{x:Static u:FontSizes.Small}"
+        TextElement.FontWeight="SemiBold">
+        <Grid.ColumnDefinitions>
+          <ColumnDefinition Width="Auto" MinWidth="40" SharedSizeGroup="Orders" />
+          <ColumnDefinition Width="*" MinWidth="40" SharedSizeGroup="BidSize" />
+          <ColumnDefinition Width="Auto" MinWidth="56" SharedSizeGroup="Price" />
+          <ColumnDefinition Width="*" MinWidth="40" SharedSizeGroup="AskSize" />
+        </Grid.ColumnDefinitions>
+        <TextBlock Grid.Column="0" HorizontalAlignment="Center" Text="Orders" />
+        <TextBlock Grid.Column="1" HorizontalAlignment="Center" Text="Bid" />
+        <TextBlock Grid.Column="2" HorizontalAlignment="Center" Text="Price" />
+        <TextBlock Grid.Column="3" HorizontalAlignment="Center" Text="Ask" />
+      </Grid>
+      <ScrollViewer Grid.Row="1" x:Name="priceLevelsScrollViewer" MaxHeight="240">
+        <ItemsControl ItemsSource="{Binding PriceLevels}" ItemTemplate="{StaticResource PriceLevelItemTemplate}" />
+      </ScrollViewer>
+    </Grid>
+
+    <!-- Order Inputs -->
+    <Grid Grid.Column="0">
+      <Grid.RowDefinitions>
+        <RowDefinition Height="Auto" />
+        <RowDefinition Height="{Binding Source={StaticResource VerticalSpacing}}" />
+        <RowDefinition Height="Auto" />
+        <RowDefinition Height="{Binding Source={StaticResource VerticalSpacing}}" />
+        <RowDefinition Height="Auto" />
+        <RowDefinition Height="{Binding Source={StaticResource VerticalSpacing}}" />
+        <RowDefinition Height="Auto" />
+        <RowDefinition Height="{Binding Source={StaticResource VerticalSpacing}}" />
+        <RowDefinition Height="Auto" />
+        <RowDefinition Height="{Binding Source={StaticResource VerticalSpacing}}" />
+        <RowDefinition Height="Auto" />
+        <RowDefinition Height="{Binding Source={StaticResource VerticalSpacing}}" />
+        <RowDefinition Height="*" />
+        <RowDefinition Height="{Binding Source={StaticResource VerticalSpacing}}" />
+        <RowDefinition Height="Auto" />
+      </Grid.RowDefinitions>
+
+      <!-- Buy/Sell toggle -->
+      <Grid HorizontalAlignment="Stretch" VerticalAlignment="Top">
+        <Grid.ColumnDefinitions>
+          <ColumnDefinition Width="1*" MinWidth="72" />
+          <ColumnDefinition Width="1*" MinWidth="72" />
+        </Grid.ColumnDefinitions>
+        <RadioButton
+          Grid.Column="0"
+          GroupName="Side"
+          IsChecked="{Binding Side, Converter={StaticResource SideToIsBuyConverter}}"
+          Style="{StaticResource BuyRadioButtonStyle}">
+          <TextBlock FontWeight="Bold" Foreground="{u:BTSBrush UI.default_text}">
+            Buy
+          </TextBlock>
+        </RadioButton>
+        <RadioButton
+          Grid.Column="1"
+          GroupName="Side"
+          IsChecked="{Binding Side, Converter={StaticResource SideToIsSellConverter}}"
+          Style="{StaticResource SellRadioButtonStyle}">
+          <TextBlock FontWeight="Bold" Foreground="{u:BTSBrush UI.default_text}">
+            Sell
+          </TextBlock>
+        </RadioButton>
+      </Grid>
+
+      <usp:PriceEntryControl
+        Grid.Row="2"
+        usp:OrderModificationControl.IsDirty="{Binding PriceDirty, FallbackValue={x:Null}}"
+        RegionalTick="{Binding PriceTick}"
+        Focusable="False"
+        TickDefaultValue="{Binding CurrentPrice, Converter={StaticResource TickDefaultPriceConverter}, FallbackValue={u:Double 0}}"
+        ToolTip="Price"
+        Value="{Binding Price}" />
+
+      <c:TickUpDownNullableDouble
+        Grid.Row="4"
+        usp:OrderModificationControl.IsDirty="{Binding QuantityDirty, FallbackValue={x:Null}}"
+        CommitOnEnter="True"
+        Focusable="False"
+        Increment="{Binding SizeCalculator.unit_quantity}"
+        Minimum="0"
+        TickDisplaySettings="{Binding SizeCalculator.display_settings}"
+        ToolTip="Quantity"
+        Value="{Binding SizeCalculator.display_value}" />
+
+      <uag:OrderDetailsPopupControl
+        Grid.Row="6"
+        DataContext="{Binding PropertiesVm}"
+        usp:OrderModificationControl.IsDirty="{Binding DataContext.order_type_dirty, RelativeSource={RelativeSource AncestorType=usp:OrderModificationControl}, FallbackValue={u:False}}"
+        HasEditableProperties="{Binding can_expand, FallbackValue={u:False}}"
+        IsAlgoTemplateToolBarReadOnly="True"
+        IsEnabled="{Binding DataContext.can_change_order_type, RelativeSource={RelativeSource AncestorType=usp:OrderModificationControl}, FallbackValue={u:True}}"
+        IsOpen="{Binding is_expanded, FallbackValue={u:False}}"
+        ItemsSource="{Binding algo_definitions_view}"
+        SelectedItem="{Binding selected_algo_definition}"
+        ToolTip="Order/Algo Type">
+        <uag:OrderDetailsPopupControl.Style>
+          <Style TargetType="uag:OrderDetailsPopupControl" BasedOn="{StaticResource {x:Type uag:OrderDetailsPopupControl}}">
+            <Style.Triggers>
+              <Trigger  Property="usp:OrderModificationControl.IsDirty" Value="True">
+                <Setter Property="BorderBrush" Value="{u:BTSColor UI.input_dirty_border}"/>
+              </Trigger>
+            </Style.Triggers>
+          </Style>
+        </uag:OrderDetailsPopupControl.Style>
+      </uag:OrderDetailsPopupControl>
+
+      <ComboBox
+        Grid.Row="8"
+        usp:OrderModificationControl.IsDirty="{Binding AccountDirty, FallbackValue={u:False}}"
+        IsEnabled="{Binding CanChangeAccount, FallbackValue={u:True}}"
+        ItemsSource="{Binding PropertiesVm.accounts}"
+        SelectedItem="{Binding PropertiesVm.selected_account}"
+        ToolTip="Account" />
+
+      <ComboBox
+        Grid.Row="10"
+        usp:OrderModificationControl.IsDirty="{Binding TimeInForceDirty, FallbackValue={u:False}}"
+        IsEnabled="{Binding CanChangeTimeInForce, FallbackValue={u:True}}"
+        ItemsSource="{Binding PropertiesVm.time_in_forces}" 
+        SelectedItem="{Binding PropertiesVm.selected_time_in_force}"
+        ToolTip="Time In Force" />
+
+      <Button
+        Grid.Row="12"
+        Height="NaN"
+        HorizontalAlignment="Stretch"
+        VerticalAlignment="Stretch"
+        HorizontalContentAlignment="Center"
+        VerticalContentAlignment="Center"
+        Background="Green"
+        Foreground="{u:BTSBrush UI.default_text}"
+        FontSize="{x:Static u:FontSizes.Largest}"
+        FontWeight="Bold"
+        Command="{Binding SendOrderCmd}"
+        ToolTip="{Binding Side, StringFormat='Send {0} Order'}">
+        Send
+      </Button>
+
+      <TextBlock
+        Grid.Row="14"
+        HorizontalAlignment="Center"
+        FontSize="{x:Static u:FontSizes.Small}">
+        <Hyperlink
+          Command="{Binding ShowLadderCmd}"
+          ToolTip="Open the original ladder for this security">Open full ladder</Hyperlink>
+      </TextBlock>
+    </Grid>
+  </Grid>
+</UserControl>

src/client/spark/Orders/OrderModificationLadderControl.xaml.cs

@@ -0,0 +1,51 @@
+#nullable enable
+
+using System;
+using System.Windows;
+using System.Windows.Controls;
+using System.Windows.Media.Animation;
+
+namespace mbts.ui.spark {
+   interface IOrderModificationLadderVM {
+      event Action? RequestRecenter;
+   }
+
+   /// <summary>
+   /// Interaction logic for OrderModificationLadderControl.xaml
+   /// </summary>
+   public partial class OrderModificationLadderControl : UserControl {
+      public static readonly DependencyProperty IsDirtyProperty = DependencyProperty.RegisterAttached(
+         "IsDirty", typeof(bool?), typeof(OrderModificationLadderControl), new PropertyMetadata(null));
+
+      public static bool? GetIsDirty(DependencyObject d) => d.GetValue(IsDirtyProperty) as bool?;
+      public static void SetIsDirty(DependencyObject d, bool? value) => d.SetValue(IsDirtyProperty, value);
+
+      public OrderModificationLadderControl() {
+         InitializeComponent();
+         DataContextChanged += onDataContextChanged;
+      }
+
+      void onDataContextChanged(object sender, DependencyPropertyChangedEventArgs e) {
+         if (e.OldValue is IOrderModificationLadderVM oldVm) {
+            oldVm.RequestRecenter -= recenter;
+         }
+         if (e.NewValue is IOrderModificationLadderVM newVm) {
+            newVm.RequestRecenter += recenter;
+         }
+      }
+
+      void recenter() {
+         var targetOffset = priceLevelsScrollViewer.ScrollableHeight / 2;
+         DoubleAnimation anim = new() {
+            From = priceLevelsScrollViewer.VerticalOffset,
+            To = targetOffset,
+            Duration = TimeSpan.FromSeconds(0.15)
+         };
+         Storyboard sb = new();
+         sb.Children.Add(anim);
+         Storyboard.SetTarget(anim, priceLevelsScrollViewer);
+         Storyboard.SetTargetProperty(anim, new PropertyPath(Behavior.AnimatedScrollViewerVerticalOffsetProperty));
+         sb.Begin();
+      }
+   }
+}

src/client/spark/Orders/OrderTicketLadder.xaml

@@ -0,0 +1,120 @@
+<Window
+  x:Class="mbts.ui.spark.OrderTicketLadder"
+  xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
+  xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+  xmlns:msys="clr-namespace:bts.api;assembly=Bts.Spark.Api"
+  xmlns:u="clr-namespace:mbts.ui"
+  xmlns:cv="clr-namespace:mbts.ui.converters"
+  xmlns:usp="clr-namespace:mbts.ui.spark"
+  xmlns:sys="clr-namespace:System;assembly=mscorlib"
+  u:WindowStyle.SizeToContent="WidthAndHeight"
+  u:WindowStyle.TitleContent="{Binding Title, Mode=OneWay}"
+  BorderBrush="#00C000"
+  BorderThickness="1"
+  Style="{StaticResource BTSWindowTitleTemplate}">
+  <Window.Resources>
+    <ResourceDictionary>
+      <ResourceDictionary.MergedDictionaries>
+        <ResourceDictionary Source="/view/Resources/Converters.xaml"/>
+      </ResourceDictionary.MergedDictionaries>
+      <sys:Double x:Key="HorizontalSpacing">8.0</sys:Double>
+      <sys:Double x:Key="VerticalSpacing">4.0</sys:Double>
+      <sys:Double x:Key="ButtonHeight">22.0</sys:Double>
+
+      <Style x:Key="SellButton" TargetType="Button" BasedOn="{StaticResource {x:Type Button}}">
+        <Setter Property="Background" Value="{u:BTSColor MarketData.ask_dark}" />
+        <Setter Property="Foreground" Value="White" />
+        <Setter Property="MinHeight" Value="40" />
+        <Setter Property="FontWeight" Value="Bold" />
+        <Setter Property="HorizontalAlignment" Value="Stretch" />
+        <Setter Property="VerticalAlignment" Value="Stretch" />
+        <Setter Property="Content">
+          <Setter.Value>
+            <TextBlock VerticalAlignment="Center" Text="Sell" />
+          </Setter.Value>
+        </Setter>
+        <Setter Property="Template">
+          <Setter.Value>
+            <ControlTemplate TargetType="Button">
+              <Border 
+                x:Name="Border"
+                CornerRadius="0" 
+                BorderThickness="1"
+                BorderBrush="{u:BTSColor UI.Button.normal_border}"
+                Background="{u:BTSColor MarketData.ask}"
+                Padding="2"
+                Height="Auto"
+                SnapsToDevicePixels="True">
+                  <ContentPresenter 
+                    HorizontalAlignment="Center"
+                    VerticalAlignment="Center"
+                    RecognizesAccessKey="True"
+                    Name="button_content" 
+                  />
+                </Border>
+              <ControlTemplate.Triggers>
+                <Trigger Property="IsMouseOver" Value="True">
+                  <Setter TargetName="Border" Property="Background" Value="#99c00000" />
+                  <Setter TargetName="Border" Property="BorderBrush" Value="{u:BTSColor UI.Button.mouseover_border}" />
+                </Trigger>
+              </ControlTemplate.Triggers>
+            </ControlTemplate>
+          </Setter.Value>
+        </Setter>
+      </Style>
+
+      <Style x:Key="BuyButton" TargetType="Button" BasedOn="{StaticResource {x:Type Button}}">
+        <Setter Property="Background" Value="{u:BTSColor MarketData.bid_dark}" />
+        <Setter Property="Foreground" Value="White" />
+        <Setter Property="MinHeight" Value="40" />
+        <Setter Property="FontWeight" Value="Bold" />
+        <Setter Property="HorizontalAlignment" Value="Stretch" />
+        <Setter Property="VerticalAlignment" Value="Stretch" />
+        <Setter Property="Content">
+          <Setter.Value>
+            <TextBlock VerticalAlignment="Center" Text="Buy" />
+          </Setter.Value>
+        </Setter>
+        <Setter Property="Template">
+          <Setter.Value>
+            <ControlTemplate TargetType="Button">
+              <Border 
+                x:Name="Border"
+                CornerRadius="0" 
+                BorderThickness="1"
+                BorderBrush="{u:BTSColor UI.Button.normal_border}"
+                Background="{u:BTSColor MarketData.bid}"
+                Padding="2"
+                Height="Auto"
+                SnapsToDevicePixels="True">
+                <ContentPresenter 
+                  HorizontalAlignment="Center"
+                  VerticalAlignment="Center"
+                  RecognizesAccessKey="True"
+                  Name="button_content" 
+                />
+              </Border>
+              <ControlTemplate.Triggers>
+                <Trigger Property="IsMouseOver" Value="True">
+                  <Setter TargetName="Border" Property="Background" Value="#99167ed3" />
+                  <Setter TargetName="Border" Property="BorderBrush" Value="{u:BTSColor UI.Button.mouseover_border}" />
+                </Trigger>
+              </ControlTemplate.Triggers>
+            </ControlTemplate>
+          </Setter.Value>
+        </Setter>
+      </Style>
+    </ResourceDictionary>
+  </Window.Resources>
+  <Window.InputBindings>
+    <KeyBinding Key="Esc" Command="{x:Static SystemCommands.CloseWindowCommand}"/>
+  </Window.InputBindings>
+
+  <usp:OrderModificationLadderControl Margin="10,4" />
+
+  <u:WindowStyle.TitleContentTemplate>
+    <DataTemplate>
+      <TextBlock HorizontalAlignment="Center" FontWeight="Bold" Text="{Binding}" Style="{StaticResource TextBlock.Dark}" />
+    </DataTemplate>
+  </u:WindowStyle.TitleContentTemplate>
+</Window>

src/client/spark/Orders/OrderTicketLadder.xaml.cs

@@ -0,0 +1,15 @@
+using mbts.ui.utility;
+
+using System.Windows;
+
+namespace mbts.ui.spark {
+   /// <summary>
+   /// Interaction logic for OrderTicketLadder.xaml
+   /// </summary>
+   public partial class OrderTicketLadder : Window {
+      public OrderTicketLadder() {
+         InitializeComponent();
+         WindowUtilities.listen_for_close_requests(this);
+      }
+   }
+}

src/client/spark/Themes/generic.xaml

@@ -90,9 +90,26 @@
     <Setter Property="CaretBrush" Value="{u:BTSBrush UI.TextBox.normal_caret}" />
   </Style>
 
-  <Style x:Key="TextBox.Dark" TargetType="TextBox" BasedOn="{StaticResource {x:Type TextBox}}" />
-
-  <Style x:Key="ListBoxItem.Dark" TargetType="ListBoxItem">
+  <Style x:Key="TextBox.Dark" TargetType="TextBox" BasedOn="{StaticResource {x:Type TextBox}}" />
+
+  <Style TargetType="DatePicker">
+    <Setter Property="Background" Value="{u:BTSBrush UI.TextBox.normal_background}" />
+    <Setter Property="Foreground" Value="{u:BTSBrush UI.TextBox.normal_foreground}" />
+    <Setter Property="BorderBrush" Value="Transparent" />
+    <Setter Property="BorderThickness" Value="0" />
+    <Setter Property="FocusVisualStyle" Value="{x:Null}" />
+  </Style>
+
+  <Style TargetType="DatePickerTextBox">
+    <Setter Property="Background" Value="{u:BTSBrush UI.TextBox.normal_background}" />
+    <Setter Property="Foreground" Value="{u:BTSBrush UI.TextBox.normal_foreground}" />
+    <Setter Property="BorderBrush" Value="Transparent" />
+    <Setter Property="BorderThickness" Value="0" />
+    <Setter Property="CaretBrush" Value="{u:BTSBrush UI.TextBox.normal_caret}" />
+    <Setter Property="FocusVisualStyle" Value="{x:Null}" />
+  </Style>
+
+  <Style x:Key="ListBoxItem.Dark" TargetType="ListBoxItem">
     <Setter Property="Background" Value="{StaticResource ListBoxItem.Background}" />
     <Setter Property="Foreground" Value="{StaticResource ListBoxItem.Foreground}" />
     <Setter Property="BorderThickness" Value="0" />

src/client/spark/core/FontsAndColors/BTSColorExtension.cs

@@ -1,117 +1,119 @@
-#nullable enable
-namespace mbts.ui {
-   using bts.utils;
-
-   using Spark.Core.Reflection;
-
-   using System;
-   using System.Collections.Generic;
-   using System.Linq;
-   using System.Reflection;
-   using System.Windows;
-   using System.Windows.Markup;
-   using System.Windows.Media;
-
-   [MarkupExtensionReturnType]
-   class BTSColorExtension : MarkupExtension {
-      static readonly Dictionary<string, Func<BTSColor>>
-         _color_map = _normalized_color_props(typeof(BTSColors), null).ToDictionary();
-
-      public BTSColorExtension() { }
-
-      public BTSColorExtension(string? name) =>
-         Name = name;
-
-      public BTSColorExtension(string? name, string? mixture) {
-         Name = name;
-         Mixture = mixture;
-      }
-
-      [ConstructorArgument("name")]
-      public string? Name { get; set; }
-
-      [ConstructorArgument("mixture")]
-      public string? Mixture { get; set; }
-
-      public ColorMixtureMode MixtureMode { get; set; } = ColorMixtureMode.Default;
-
-      public Type? Type { get; set; }
-
-      public double Opacity { get; set; } = double.NaN;
-
-      public static BTSColor get_color(string name) =>
-         _color_map.TryGetValue(name, out var color) ?
-            color() : throw new ColorNotFoundException(name);
-
-      public override object? ProvideValue(IServiceProvider service_provider) {
-         if (service_provider is null) {
-            return null;
-         }
-         if (Name is null) {
-            throw new InvalidOperationException("Name is null");
-         }
-         if (!Opacity.is_nan() && !Opacity.is_within_range(0, 1)) {
-            throw new InvalidOperationException("Opacity (pct) must between 0 and 1 (if not NaN)");
-         }
-
-         if (Name.Contains("Launcher.Button")) {
-         }
-         var pvt = service_provider.GetValueTargetProvider() ??
-            throw new InvalidOperationException("IProvideValueTarget is null");
-         var tgt_type = Type ?? _deduce_target_type(pvt);
-         var names = Name.Split(',');
-         var mixtures = Mixture?.Split(',');
-         var color = _get_color_blend(names, mixtures, MixtureMode);
-         if (!Opacity.is_nan()) {
-            color = color.with((byte)(Opacity * byte.MaxValue));
-         }
-         return tgt_type == typeof(Color) ? (Color)color : (Brush)color;
-      }
-
-      static Type _deduce_target_type(IProvideValueTarget pvt) {
-         if (pvt.TargetProperty is null) {
-            return typeof(Color);
-         }
-         if (pvt.TargetProperty is DependencyProperty dp) {
-            return dp.PropertyType ?? throw new InvalidOperationException();
-         }
-         if (pvt.TargetObject is Setter setter) {
-            return setter.Property?.PropertyType ?? typeof(Brush);
-         }
-         return typeof(Brush);
-      }
-
-      /// <summary>
-      /// Creates a (potentially weighted) blend of colors
-      /// </summary>
-      static BTSColor _get_color_blend(string[] names, string[]? mixture, ColorMixtureMode mode) {
-         if (mixture is null || mixture.Length == 0) {
-            // create a uniform blend
-            return names.Select(get_color).blend(mode);
-         }
-         if (names.Length != mixture.Length) {
-            throw new InvalidOperationException(
-               "Names and Mixture must provide same number of elements");
-         }
-         return names.Select(get_color).Zip(mixture.Select(double.Parse)).blend();
-      }
-
-      static IEnumerable<(string, Func<BTSColor>)> _normalized_color_props(
-         Type type, string? prefix) {
-         var bts_colors = ReflectionHelper.static_property_getters<BTSColor>(type)
-            .Select(tup => (full_name(tup.name), tup.get));
-         var media_colors = ReflectionHelper.static_property_getters<Color>(type)
-            .Select(tup => (full_name(tup.name), new Func<BTSColor>(() => tup.get())));
-         var nested_types = type.GetNestedTypes(BindingFlags.Public | BindingFlags.Static);
-         var nested_colors = nested_types.SelectMany(
-            t => _normalized_color_props(t, full_name(t.Name))).ToList();
-
-         return bts_colors
-            .Concat(media_colors)
-            .Concat(nested_colors);
-
-         string full_name(string prop) =>
-            prefix is null ? prop : $"{prefix}.{prop}";
-      }
-   }
-}
+#nullable enable
+namespace mbts.ui {
+   using bts.utils;
+
+   using Spark.Core.Reflection;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   using System.Reflection;
+   using System.Windows;
+   using System.Windows.Markup;
+   using System.Windows.Media;
+
+   [MarkupExtensionReturnType]
+   class BTSColorExtension : MarkupExtension {
+      static readonly Dictionary<string, Func<BTSColor>>
+         _color_map = _normalized_color_props(typeof(BTSColors), null).ToDictionary();
+
+      public BTSColorExtension() { }
+
+      public BTSColorExtension(string? name) =>
+         Name = name;
+
+      public BTSColorExtension(string? name, string? mixture) {
+         Name = name;
+         Mixture = mixture;
+      }
+
+      [ConstructorArgument("name")]
+      public string? Name { get; set; }
+
+      [ConstructorArgument("mixture")]
+      public string? Mixture { get; set; }
+
+      public ColorMixtureMode MixtureMode { get; set; } = ColorMixtureMode.Default;
+
+      public Type? Type { get; set; }
+
+      public double Opacity { get; set; } = double.NaN;
+
+      public static BTSColor get_color(string name) =>
+         _color_map.TryGetValue(name, out var color) ?
+            color() : throw new ColorNotFoundException(name);
+
+      public override object? ProvideValue(IServiceProvider service_provider) {
+         if (service_provider is null) {
+            return null;
+         }
+         if (Name is null) {
+            throw new InvalidOperationException("Name is null");
+         }
+         if (!Opacity.is_nan() && !Opacity.is_within_range(0, 1)) {
+            throw new InvalidOperationException("Opacity (pct) must between 0 and 1 (if not NaN)");
+         }
+
+         if (Name.Contains("Launcher.Button")) {
+         }
+         var pvt = service_provider.GetValueTargetProvider() ??
+            throw new InvalidOperationException("IProvideValueTarget is null");
+         var tgt_type = Type ?? _deduce_target_type(pvt);
+         var names = Name.Split(',');
+         var mixtures = Mixture?.Split(',');
+         var color = _get_color_blend(names, mixtures, MixtureMode);
+         if (!Opacity.is_nan()) {
+            color = color.with((byte)(Opacity * byte.MaxValue));
+         }
+         return tgt_type == typeof(Color) || tgt_type == typeof(Color?)
+            ? (Color)color
+            : (Brush)color;
+      }
+
+      static Type _deduce_target_type(IProvideValueTarget pvt) {
+         if (pvt.TargetProperty is null) {
+            return typeof(Color);
+         }
+         if (pvt.TargetProperty is DependencyProperty dp) {
+            return dp.PropertyType ?? throw new InvalidOperationException();
+         }
+         if (pvt.TargetObject is Setter setter) {
+            return setter.Property?.PropertyType ?? typeof(Brush);
+         }
+         return typeof(Brush);
+      }
+
+      /// <summary>
+      /// Creates a (potentially weighted) blend of colors
+      /// </summary>
+      static BTSColor _get_color_blend(string[] names, string[]? mixture, ColorMixtureMode mode) {
+         if (mixture is null || mixture.Length == 0) {
+            // create a uniform blend
+            return names.Select(get_color).blend(mode);
+         }
+         if (names.Length != mixture.Length) {
+            throw new InvalidOperationException(
+               "Names and Mixture must provide same number of elements");
+         }
+         return names.Select(get_color).Zip(mixture.Select(double.Parse)).blend();
+      }
+
+      static IEnumerable<(string, Func<BTSColor>)> _normalized_color_props(
+         Type type, string? prefix) {
+         var bts_colors = ReflectionHelper.static_property_getters<BTSColor>(type)
+            .Select(tup => (full_name(tup.name), tup.get));
+         var media_colors = ReflectionHelper.static_property_getters<Color>(type)
+            .Select(tup => (full_name(tup.name), new Func<BTSColor>(() => tup.get())));
+         var nested_types = type.GetNestedTypes(BindingFlags.Public | BindingFlags.Static);
+         var nested_colors = nested_types.SelectMany(
+            t => _normalized_color_props(t, full_name(t.Name))).ToList();
+
+         return bts_colors
+            .Concat(media_colors)
+            .Concat(nested_colors);
+
+         string full_name(string prop) =>
+            prefix is null ? prop : $"{prefix}.{prop}";
+      }
+   }
+}

src/client/spark/view/Controls/AutoHideScrollBar/AutoHideScrollBar.xaml

@@ -380,7 +380,7 @@
                 <VisualState Name="Normal"/>
                 <VisualState Name="MouseOver">
                   <Storyboard>
-                    <ColorAnimation Storyboard.TargetProperty="Background.Color" BeginTime="0" Duration="0" To="{StaticResource u:BTSColor UI.ScrollBar.thumb_mouse_over}"/>
+                    <ColorAnimation Storyboard.TargetProperty="Background.Color" BeginTime="0" Duration="0" To="{u:BTSColor UI.ScrollBar.thumb_mouse_over}"/>
                   </Storyboard>
                 </VisualState>
                 <VisualState Name="Pressed">

src/client/spark/view/Controls/Order/OrderAuxiliaryPopupResources.xaml

@@ -1,5 +1,5 @@
 <ResourceDictionary
-  x:Class="mbts.ui.spark.AuxiliaryDataResources"
+  x:Class="mbts.ui.spark.OrderAuxiliaryPopupResources"
   x:ClassModifier="internal"
   xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
   xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
@@ -17,7 +17,7 @@
     <sp:MiniOrderBookView/>
   </DataTemplate>
   <!--Header (Title Bar)-->
-  <DataTemplate DataType="{x:Type usp:AuxiliaryDataHeader}">
+  <DataTemplate DataType="{x:Type usp:OrderAuxiliaryPopupHeader}">
     <Grid MaxWidth="{Binding ActualWidth, RelativeSource={RelativeSource AncestorType=ContentPresenter}}">
       <Grid.ColumnDefinitions>
         <ColumnDefinition Width="Auto"/>
@@ -101,6 +101,20 @@
     </Grid>
   </DataTemplate>
 
+  <DataTemplate DataType="{x:Type usp:OrderTicketLadderHeader}">
+    <Grid>
+      <Grid.ColumnDefinitions>
+        <ColumnDefinition Width="Auto"/>
+        <ColumnDefinition Width="*"/>
+        <ColumnDefinition Width="Auto"/>
+        <ColumnDefinition Width="Auto"/>
+      </Grid.ColumnDefinitions>
+      <TextBlock FontWeight="Bold" Text="{Binding Title}" Style="{StaticResource TextBlock.Dark}" />
+      <CheckBox Grid.Column="2" VerticalAlignment="Center" Margin="4,0,0,0" Focusable="False" IsChecked="{Binding StaysOpen, RelativeSource={RelativeSource AncestorType=Popup}}">Keep Open</CheckBox>
+      <Button Grid.Column="3" HorizontalAlignment="Right" Margin="4" Command="{x:Static ApplicationCommands.Close}" CommandTarget="{Binding RelativeSource={RelativeSource AncestorType=HeaderedContentControl}}" Style="{StaticResource RoundCloseButton}"/>
+    </Grid>
+  </DataTemplate>
+
   <ControlTemplate x:Key="PopupChildTemplate" TargetType="HeaderedContentControl">
     <Border
       Background="{TemplateBinding Background}"

src/client/spark/view/Controls/Order/OrderAuxiliaryPopupResources.xaml.cs

@@ -0,0 +1,49 @@
+#nullable enable
+using mbts.spark;
+using mbts.ui.shell;
+
+using System.Windows;
+using System.Windows.Media;
+
+namespace mbts.ui.spark {
+   partial class OrderAuxiliaryPopupResources : ResourceDictionary {
+      public OrderAuxiliaryPopupResources() =>
+         InitializeComponent();
+
+      public static string PopupChildStyleKey => nameof(PopupChildStyleKey);
+   }
+
+   class OrderAuxiliaryPopupHeader : ViewModelBase {
+      string?
+         _title,
+         _context;
+      Color _title_foreground;
+      bool _keep_open;
+
+      public Color title_foreground { get => _title_foreground; set => Set(ref _title_foreground, value); }
+      public string? title { get => _title; set => Set(ref _title, value); }
+      public string? context { get => _context; set => Set(ref _context, value); }
+      public bool keep_open { get => _keep_open; set => Set(ref _keep_open, value); }
+   }
+
+   class OrderTicketHeader : ViewModelBase {
+      bool _keep_open;
+      public BBOVM? bbo { get; set; }
+      public bool keep_open { get => _keep_open; set => Set(ref _keep_open, value); }
+   }
+
+   class OrderTicketLadderHeader : ViewModelBase {
+      bool keepOpen;
+      string? title;
+
+      public string? Title {
+         get => title;
+         set => Set(ref title, value);
+      }
+
+      public bool KeepOpen {
+         get => keepOpen;
+         set => Set(ref keepOpen, value);
+      }
+   }
+}

src/client/spark/view/Controls/Order/OrderTicketResources.xaml

@@ -102,6 +102,10 @@
     </Grid>
   </DataTemplate>
 
+  <DataTemplate DataType="{x:Type sp:NewOrderTicketLadderVM}">
+    <usp:OrderModificationLadderControl />
+  </DataTemplate>
+
   <DataTemplate DataType="{x:Type sp:ExistingOrderTicketVM}">
     <Grid Margin="10">
       <Grid.RowDefinitions>

src/client/spark/view/Controls/Popup/AuxiliaryDataView.xaml.cs

@@ -1,36 +0,0 @@
-using mbts.spark;
-using mbts.ui.shell;
-using System.Windows;
-using System.Windows.Controls;
-using System.Windows.Media;
-
-namespace mbts.ui.spark {
-   partial class AuxiliaryDataResources : ResourceDictionary {
-      public AuxiliaryDataResources() =>
-         InitializeComponent();
-
-      public static string PopupChildStyleKey => nameof(PopupChildStyleKey);
-   }
-
-   class AuxiliaryDataHeader : ViewModelBase {
-      string
-         _title,
-         _context;
-      Color _title_foreground;
-      bool _keep_open;
-
-      public Color title_foreground { get => _title_foreground; set => Set(ref _title_foreground, value); }
-      public string title { get => _title; set => Set(ref _title, value); }
-      public string context { get => _context; set => Set(ref _context, value); }
-      public bool keep_open { get => _keep_open; set => Set(ref _keep_open, value); }
-   }
-
-   class OrderTicketHeader : ViewModelBase {
-      bool _keep_open;
-      public BBOVM bbo { get; set; }
-      public bool keep_open { get => _keep_open; set => Set(ref _keep_open, value); }
-   }
-
-   class ConstrainedTextBlock : TextBlock {
-   }
-}

src/client/spark/view/Extensions/Command/AttachedCommands.cs

@@ -26,6 +26,39 @@ namespace mbts.ui {
       public static void SetDoubleClickParameter(DependencyObject target, object value) { target.SetValue(DoubleClickParameterProperty, value); }
       #endregion
 
+      #region MiddleClick
+      public static readonly DependencyProperty MiddleClickProperty =
+         DependencyProperty.RegisterAttached("MiddleClick", typeof(ICommand), typeof(AttachedCommands), new PropertyMetadata(onMiddleClickChanged));
+
+      static void onMiddleClickChanged(DependencyObject d, DependencyPropertyChangedEventArgs e) {
+         var el = (UIElement)d;
+         el.ClearMouseBindings(MouseAction.MiddleClick);
+         if (e.NewValue is ICommand cmd) {
+            var mb = new MouseBinding(cmd, new MouseGesture(MouseAction.MiddleClick));
+            var bd = new Binding { Path = new PropertyPath(MiddleClickParameterProperty), Source = d };
+            BindingOperations.SetBinding(mb, InputBinding.CommandParameterProperty, bd);
+            el.InputBindings.Add(mb);
+         }
+      }
+
+      public static ICommand GetMiddleClick(DependencyObject target) =>
+         target.GetValue(MiddleClickProperty) as ICommand;
+
+      public static void SetMiddleClick(DependencyObject target, ICommand value) =>
+         target.SetValue(MiddleClickProperty, value);
+      #endregion
+
+      #region MiddleClickParameter
+      public static readonly DependencyProperty MiddleClickParameterProperty =
+         DependencyProperty.RegisterAttached("MiddleClickParameter", typeof(object), typeof(AttachedCommands));
+
+      public static object GetMiddleClickParameter(DependencyObject target) =>
+         target.GetValue(MiddleClickParameterProperty);
+
+      public static void SetMiddleClickParameter(DependencyObject target, object value) =>
+         target.SetValue(MiddleClickParameterProperty, value);
+      #endregion
+
       #region Enter
       public static readonly DependencyProperty EnterProperty = DependencyProperty.RegisterAttached("Enter", typeof(ICommand), typeof(AttachedCommands), new PropertyMetadata(OnEnterChanged));
       static void OnEnterChanged(DependencyObject d, DependencyPropertyChangedEventArgs e) {

src/cs/Bts.Spark.Api/Orders/OrdersClient.cs

@@ -11,6 +11,7 @@ namespace Spark.Api.Orders {
 
    using Spark.Api.Admin;
    using Spark.Api.MarketData;
+   using Spark.Api.Position;
    using Spark.Api.Reactive;
    using Spark.Core.Disposables;
    using Spark.Core.Reactive;
@@ -40,10 +41,12 @@ namespace Spark.Api.Orders {
       ) {
          _order_service = order_service;
          var scheduler = ThreadPoolScheduler.Instance;
-         interval ??= TimeSpan.FromSeconds(0.1);
+         if(interval is not null)
+            Log.Api.info($"originally OrdersClient polling interval {interval}");
+         interval = TimeSpan.FromMilliseconds(1);
+         Log.Api.info($"OrdersClient polling interval {interval}");
          var bootstraps = new DisposableBuilder();
 
-         positions = position_service.subscribei_position_data().Table().Cache();
          _orders = bootstraps.emplace(
             admin_client
                .permission_filter(
@@ -73,6 +76,11 @@ namespace Spark.Api.Orders {
                .Cache()
                .ToObservable());
 
+         var raw_fills = admin_client.permission_filter(
+            order_service.subscribei_fills().Naggle(interval.Value, scheduler),
+            f => f.Account)
+            .Table();
+
          var fill_order_tuples_join = fills.LeftJoin(
             _orders,
             f => f.val.FillKey,
@@ -94,6 +102,12 @@ namespace Spark.Api.Orders {
                .Cache()
                .ToObservable());
 
+         var raw_manual_fills = admin_client
+            .permission_filter(
+               position_service.subscribei_manual_fills().Naggle(interval.Value, scheduler),
+               f => f.AccountId)
+            .Table();
+
          var manual_fill_tuples_account_join = manual_fills_securities.LeftJoin(
             admin_client.Accounts,
             mf => mf.val.RecordId,
@@ -117,6 +131,11 @@ namespace Spark.Api.Orders {
                .Cache()
                .ToObservable());
 
+         positions = bootstraps.emplace(
+            ComputedPositionsDataSource.create(raw_fills, raw_manual_fills)
+               .Cache()
+               .ToObservable());
+
          _dispose = bootstraps;
       }
       public async Task<Response> EnterOrderAsync(

src/cs/Bts.Spark.Api/Position/ComputedPositionsDataSource.cs

@@ -0,0 +1,194 @@
+#nullable enable
+namespace Spark.Api.Position {
+   using bts;
+   using bts.api;
+   using bts.api.core;
+   using bts.api.orderman;
+   using bts.api.position;
+
+   using System;
+   using System.Collections.Generic;
+   using System.Linq;
+   using System.Reactive.Linq;
+
+   public static class ComputedPositionsDataSource {
+      public static IObservable<TableUpdate<PositionUnitImmutable, int>> create(
+         IObservable<TableUpdate<FillImmutable, string>> fills,
+         IObservable<TableUpdate<ManualFillImmutable, int>> manual_fills) {
+         var record_ids = new Dictionary<AccountSecurityKey, int>();
+         int next_record_id = 1;
+
+         return Observable.CombineLatest(
+            fills,
+            manual_fills,
+            (fill_update, manual_fill_update) => build_snapshot(
+               fill_update.Index.Values,
+               manual_fill_update.Index.Values)
+               .ToDictionary(PositionUnitImmutable.key_fn))
+            .Table(PositionUnitImmutable.key_fn);
+
+         IEnumerable<PositionUnitImmutable> build_snapshot(
+            IEnumerable<FillImmutable> electronic_fills,
+            IEnumerable<ManualFillImmutable> manual_fill_rows) {
+            var positions = new SortedDictionary<AccountSecurityKey, PositionAccumulator>();
+
+            foreach (var fill in electronic_fills) {
+               if (fill.FillStatus == Fill.Types.FillStatus.Busted ||
+                  fill.SecurityKey == 0 ||
+                  fill.InternalAccountId == 0) {
+                  continue;
+               }
+               var key = new AccountSecurityKey(fill.InternalAccountId, fill.SecurityKey);
+               var acc = get_or_create(positions, key, fill.Account);
+               acc.add_electronic_fill(fill);
+               positions[key] = acc;
+            }
+
+            foreach (var fill in manual_fill_rows) {
+               if (fill.SecurityKey == 0 || fill.AccountId == 0) {
+                  continue;
+               }
+               var key = new AccountSecurityKey(fill.AccountId, fill.SecurityKey);
+               var acc = get_or_create(positions, key, fill.AccountName);
+               acc.add_manual_fill(fill);
+               positions[key] = acc;
+            }
+
+            foreach (var (key, acc) in positions) {
+               if (!record_ids.TryGetValue(key, out var record_id)) {
+                  record_id = next_record_id++;
+                  record_ids[key] = record_id;
+               }
+               yield return acc.to_position(record_id);
+            }
+         }
+
+         static PositionAccumulator get_or_create(
+            IDictionary<AccountSecurityKey, PositionAccumulator> positions,
+            AccountSecurityKey key,
+            string account_name) {
+            if (!positions.TryGetValue(key, out var acc)) {
+               return new PositionAccumulator(key.account_id, key.security_id, account_name);
+            } else if (string.IsNullOrEmpty(acc.account) && !string.IsNullOrEmpty(account_name)) {
+               acc.account = account_name;
+            }
+            return acc;
+         }
+      }
+
+      struct PositionAccumulator {
+         public PositionAccumulator(uint account_id, uint security_id, string account) {
+            this.account_id = account_id;
+            this.security_id = security_id;
+            this.account = account;
+            day = default;
+            sod = default;
+            local = default;
+            admin_day = default;
+            admin_sod = default;
+            raw_day = default;
+         }
+
+         public readonly uint account_id;
+         public readonly uint security_id;
+         public string account;
+         PositionEntryAccumulator day;
+         PositionEntryAccumulator sod;
+         PositionEntryAccumulator local;
+         PositionEntryAccumulator admin_day;
+         PositionEntryAccumulator admin_sod;
+         PositionEntryAccumulator raw_day;
+
+         public void add_electronic_fill(FillImmutable fill) {
+            var qty = fill.Side == Side.Sell ? -fill.LastShares : fill.LastShares;
+            if (is_sod(fill.FillKind)) {
+               sod.add(qty, fill.LastPrice);
+               return;
+            }
+            day.add(qty, fill.LastPrice);
+            raw_day.add(qty, fill.LastPrice);
+         }
+
+         public void add_manual_fill(ManualFillImmutable fill) {
+            var qty = fill.Side == Side.Sell ? -fill.Quantity : fill.Quantity;
+            switch (fill.FillType) {
+               case ManualFill.Types.FillType.AdminSod:
+                  sod.add(qty, fill.Price);
+                  admin_sod.add(qty, fill.Price);
+                  break;
+               case ManualFill.Types.FillType.AdminDay:
+                  day.add(qty, fill.Price);
+                  raw_day.add(qty, fill.Price);
+                  admin_day.add(qty, fill.Price);
+                  break;
+               default:
+                  day.add(qty, fill.Price);
+                  raw_day.add(qty, fill.Price);
+                  local.add(qty, fill.Price);
+                  break;
+            }
+         }
+
+         public PositionUnitImmutable to_position(int record_id) {
+            var sod_entry = sod.to_entry();
+            var day_entry = day.to_entry();
+            var raw_day_entry = raw_day.to_entry();
+            var local_entry = local.to_entry();
+            var admin_sod_entry = admin_sod.to_entry();
+            var admin_day_entry = admin_day.to_entry();
+
+            return new PositionUnitImmutable(new PositionUnit {
+               RecordId = record_id,
+               Account = account,
+               AccountId = account_id,
+               SecurityId = security_id,
+               SodLongPos = sod_entry.LongPos,
+               SodLongAvgPrice = sod_entry.LongAvgPrice,
+               SodShortPos = sod_entry.ShortPos,
+               SodShortAvgPrice = sod_entry.ShortAvgPrice,
+               DayLongPos = day_entry.LongPos,
+               DayLongAvgPrice = day_entry.LongAvgPrice,
+               DayShortPos = day_entry.ShortPos,
+               DayShortAvgPrice = day_entry.ShortAvgPrice,
+               Sod = sod_entry,
+               Day = day_entry,
+               RawDay = raw_day_entry,
+               Local = local_entry,
+               AdminSod = admin_sod_entry,
+               AdminDay = admin_day_entry
+            });
+         }
+
+         static bool is_sod(Fill.Types.FillKind kind) =>
+            kind == Fill.Types.FillKind.Sod ||
+            kind == Fill.Types.FillKind.SodSpreadLeg ||
+            kind == Fill.Types.FillKind.SodStripLeg;
+      }
+
+      struct PositionEntryAccumulator {
+         double long_qty;
+         double long_cost;
+         double short_qty;
+         double short_cost;
+
+         public void add(double signed_qty, double price) {
+            if (signed_qty > 0) {
+               long_qty += signed_qty;
+               long_cost += signed_qty * price;
+            } else if (signed_qty < 0) {
+               var abs_qty = Math.Abs(signed_qty);
+               short_qty += abs_qty;
+               short_cost += abs_qty * price;
+            }
+         }
+
+         public PositionEntry to_entry() =>
+            new() {
+               LongPos = long_qty,
+               LongAvgPrice = long_qty == 0 ? 0 : long_cost / long_qty,
+               ShortPos = short_qty,
+               ShortAvgPrice = short_qty == 0 ? 0 : short_cost / short_qty
+            };
+      }
+   }
+}

src/cs/Bts.Spark.Api/Position/PositionModel.cs

@@ -15,6 +15,7 @@ namespace Spark.Trading {
    using Spark.Api.Data;
    using Spark.Api.MarketData;
    using Spark.Api.Orders;
+   using Spark.Api.Position;
    using Spark.Api.Reactive;
    using Spark.Core;
    using Spark.Core.Data;
@@ -60,13 +61,21 @@ namespace Spark.Trading {
                .Cache()
                .ToObservable());
 
+         var raw_fills = admin_model
+            .permission_filter(
+               order_service.subscribei_fills().Naggle(interval, scheduler),
+               f => f.Account)
+            .Table();
+
+         var raw_manual_fills = admin_model
+            .permission_filter(
+               _position_service.subscribei_manual_fills().Naggle(interval, scheduler),
+               f => f.AccountId)
+            .Table();
+
          on_positions = bootstraps.emplace(
-            admin_model
-               .permission_filter(
-                  _position_service.subscribei_position_data().Naggle(interval, scheduler),
-                  p => p.AccountId)
-               .join(sec_resolver, pos => pos.SecurityId)
-               .Table()
+            ComputedPositionsDataSource.create(raw_fills, raw_manual_fills)
+               .join_table(sec_resolver, pos => pos.SecurityId)
                .Cache()
                .ToObservable());
          var users = admin_model.Users

src/cs/Bts.Spark.Api/Services/SparkApiServices.cs

@@ -8,6 +8,7 @@ using bts.api.core;
 using bts.api.orderman;
 using bts.api.position;
 using bts.co2;
+using bts.metrics;
 using bts.md;
 
 using Microsoft.Extensions.Configuration;
@@ -86,6 +87,7 @@ public static class SparkApiServices {
          services.AddSparkdService<OrderService>();
          services.AddSingleton<IOrdersClient, OrdersClient>();
       }
+      services.AddSparkdService<ShellLatencyService>();
       if (configurationSection.GetValue("EnablePosition", true)) {
          services.TryAddSingleton<ISecurityResolver>(ActivatorUtilities.GetServiceOrCreateInstance<PassiveSecurityResolver>);
          services.AddSparkdService<PositionService>()
@@ -176,6 +178,7 @@ public static class SparkApiServices {
          services.AddSparkdService<OrderService>();
          services.AddSingleton<IOrdersClient, OrdersClient>();
       }
+      services.AddSparkdService<ShellLatencyService>();
       if (isPositionsEnabled) {
          services.AddSparkdService<PositionService>();
       }

src/cs/Bts.Spark.Api/data/SecurityCriteriaExtensions.cs

@@ -190,6 +190,10 @@ namespace mbts.spark {
                result = SingleSecurityCriteria.Empty;
                return false;
          }
+         if (id is null && (name is null || venue is null)) {
+            result = SingleSecurityCriteria.Empty;
+            return false;
+         }
          result = new(venue, type, asset, name, id);
          return true;
       }

src/cs/Bts.Spark.Api/data/ticks/TickUtils.cs

@@ -3,6 +3,7 @@ using bts.api;
 using bts.data.ticks;
 using bts.utils;
 
+using mbts.co2;
 using mbts.user.settings;
 
 using Spark.Core;
@@ -113,6 +114,24 @@ namespace mbts.spark {
          return rv;
       }
 
+      public static double low_tick(NativePriceSize[] bids, NativePriceSize[] asks) {
+         double low = bids[bids.Length - 1].price;
+         if (double.IsNaN(low))
+            low = bids.LastOrDefault(nps => nps.price, p => !double.IsNaN(p), double.NaN);
+         if (double.IsNaN(low))
+            low = asks[0].price;
+         return low;
+      }
+
+      public static double high_tick(NativePriceSize[] bids, NativePriceSize[] asks) {
+         double high = asks[asks.Length - 1].price;
+         if (double.IsNaN(high))
+            high = asks.LastOrDefault(nps => nps.price, p => !double.IsNaN(p), double.NaN);
+         if (double.IsNaN(high))
+            high = bids[0].price;
+         return high;
+      }
+
       public static long? tickdiff(this ContractSpecificationImmutable spec, double x, double y) =>
          get_regional_ticks(spec).tickdiff(x, y);
 

src/cs/Bts.Spark.Sdk.Demo/TopOfBookQuoter.cs

@@ -41,8 +41,8 @@ class TopOfBookQuoter : IHostedService {
          throw new ArgumentException("Failed to parse venue", nameof(config));
       }
       var quantity = config.Quantity;
-      if (!(quantity > 0)) {
-         throw new ArgumentException("Quantity is not positive", nameof(config));
+      if (quantity == 0 || !double.IsRealNumber(quantity)) {
+         throw new ArgumentException("Quantity must be real and non-zero", nameof(config));
       }
       this.config = new(venue, contract, account, config.OffsetTicks, quantity);
    }
@@ -86,11 +86,12 @@ class TopOfBookQuoter : IHostedService {
       if (!bk.Security.HasValue) {
          return;
       }
-      var bidPrice = bk[Side.Buy, 0].Price;
-      var orderPrice = bidPrice;
       var orderQty = config.Quantity;
+      var bidPrice = bk[orderQty < 0 ? Side.Sell : Side.Buy, 0].Price;
+      orderQty = Math.Abs(orderQty);
+      var orderPrice = bidPrice;
       if (config.OffsetTicks != 0) {
-         orderPrice = bk.Security.Value.tick_factors().tickdown(orderPrice, config.OffsetTicks);
+         orderPrice = bk.Security.Value.tick_factors().tick(orderPrice, config.OffsetTicks);
       }
       if (securityToOrderId.TryGetValue(bk.SecurityId, out var orderId)
          && orders.TryGetValue(orderId, out var orderTree)

src/cs/Bts.Spark.Sdk.Demo/VenueParser.cs

@@ -4,7 +4,7 @@ using bts.api;
 
 public static class VenueParser {
    public static bool TryParse(string? venueStr, out Venue venue) {
-      if (venueStr is null || (!Enum.TryParse(venueStr, false, out venue) && !Enum.TryParse("VENUE_" + venueStr, false, out venue))) {
+      if (venueStr is null || (!Enum.TryParse(venueStr, true, out venue) && !Enum.TryParse("VENUE_" + venueStr, true, out venue))) {
          venue = Venue.Unset;
          return false;
       }

src/cs/Bts.Spark.Sdk/AuthenticatedSimpleSparkClient.cs

@@ -3,6 +3,7 @@ namespace Bts.Spark.Sdk;
 using bts.api;
 using bts.api.core;
 using bts.api.orderman;
+using bts.metrics;
 
 using global::Spark.Api.Admin;
 using global::Spark.Api.MarketData;
@@ -28,6 +29,7 @@ public class AuthenticatedSimpleSparkClient : ISimpleSparkClient, IHostedService
    readonly IOrdersClient ordersClient;
    readonly IAdminClient adminClient;
    readonly ISecurityResolver securityResolver;
+   readonly ShellLatencyService shellLatencyService;
    readonly ISparkLoginService loginService;
    readonly SimpleSparkClientConfig config;
    readonly ILogger<AuthenticatedSimpleSparkClient> logger;
@@ -39,6 +41,7 @@ public class AuthenticatedSimpleSparkClient : ISimpleSparkClient, IHostedService
       IOrdersClient ordersClient,
       IAdminClient adminClient,
       ISecurityResolver securityResolver,
+      ShellLatencyService shellLatencyService,
       ISparkLoginService loginService,
       IOptions<SimpleSparkClientConfig> config,
       ILogger<AuthenticatedSimpleSparkClient> logger) {
@@ -46,6 +49,7 @@ public class AuthenticatedSimpleSparkClient : ISimpleSparkClient, IHostedService
       this.ordersClient = ordersClient;
       this.adminClient = adminClient;
       this.securityResolver = securityResolver;
+      this.shellLatencyService = shellLatencyService;
       this.loginService = loginService;
       this.config = config.Value;
       this.logger = logger;
@@ -61,7 +65,7 @@ public class AuthenticatedSimpleSparkClient : ISimpleSparkClient, IHostedService
       }
       cancellationToken.ThrowIfCancellationRequested();
       var user = loginRes.user.Value;
-      simpleClient = new(user.name, user.id, mdClient, ordersClient, adminClient, securityResolver, config.OrderPoolCapacity, config.OrderPoolReplenishInterval);
+      simpleClient = new(user.name, user.id, mdClient, ordersClient, shellLatencyService, adminClient, securityResolver, config.OrderPoolCapacity, config.OrderPoolReplenishInterval);
       simpleClientSubject.OnNext(simpleClient);
       logger.LogInformation("SimpleSparkClient started");
    }
@@ -97,6 +101,9 @@ public class AuthenticatedSimpleSparkClient : ISimpleSparkClient, IHostedService
    public IObservable<TableUpdate<PriceTicker, uint>> PriceTicker(SecurityCriteria? securityCriteria) =>
       simpleClient?.PriceTicker(securityCriteria) ?? simpleClientSubject.Take(1).Select(c => c.PriceTicker(securityCriteria)).FlattenSeries();
 
+   public Task<bool> PingAsync() =>
+      simpleClient?.PingAsync() ?? pingBeforeStartAsync();
+
    public Task<Response> CreateOrder(OrderDefinition orderDef, double quantity, double price) =>
       orderOp(cli => cli.CreateOrder(orderDef, quantity, price));
 
@@ -134,4 +141,15 @@ public class AuthenticatedSimpleSparkClient : ISimpleSparkClient, IHostedService
       async Task<T> LateImpl() =>
          await op(await simpleClientSubject.FirstAsync());
    }
+
+   async Task<bool> pingBeforeStartAsync() {
+      var request = new BufferMessage() {
+         Sequence = (ulong)DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
+         Data = "spark-sdk-ping"
+      };
+      var result = await shellLatencyService.ping_async(request);
+      return result.success &&
+         result.value.Sequence == request.Sequence &&
+         result.value.Data == request.Data;
+   }
 }

src/cs/Bts.Spark.Sdk/ExecutionFacadeSimpleSparkClient.cs

@@ -48,6 +48,9 @@ public class ExecutionFacadeSimpleSparkClient : ISimpleSparkClient {
    public IObservable<TableUpdate<PriceTicker, uint>> PriceTicker(SecurityCriteria? securityCriteria) =>
       sparkClient.PriceTicker(securityCriteria);
 
+   public Task<bool> PingAsync() =>
+      sparkClient.PingAsync();
+
    public Task<Response> CreateOrder(OrderDefinition orderDef, double quantity, double price) {
       logger.LogInformation("CreateOrder {account} {tag} {securityId} {side} {quantity} @ {price}", orderDef.Account, orderDef.Tag, orderDef.SecurityKey, orderDef.Side, quantity, price);
       return Task.FromResult(facadeResponse());

src/cs/Bts.Spark.Sdk/ISimpleSparkClient.cs

@@ -12,6 +12,8 @@ using mbts.spark;
 using System;
 
 public interface ISimpleSparkClient {
+   Task<bool> PingAsync();
+
    // Market Data
    IObservable<TableUpdate<SecurityImmutable, uint>> Securities(SecurityCriteria? securityCriteria);
    IObservable<TableUpdate<BookRecord, uint>> Books(SecurityCriteria? securityCriteria);

src/cs/Bts.Spark.Sdk/SimpleSparkClient.cs

@@ -16,12 +16,14 @@ using global::Spark.Core.Disposables;
 using global::Spark.Core.Reactive;
 
 using mbts.spark;
+using bts.metrics;
 
 using System.Reactive.Linq;
 
 public class SimpleSparkClient : ISimpleSparkClient, IDisposable {
    readonly IMarketDataClient mdClient;
    readonly IOrdersClient ordersClient;
+   readonly ShellLatencyService shellLatencyService;
    readonly OrderPool limitOrderPool;
    readonly OrderPool marketOrderPool;
    readonly TableUpdateDemuxer<OrderRecordTupleWithRoot, uint, uint> orderSecurityDmx;
@@ -46,6 +48,7 @@ public class SimpleSparkClient : ISimpleSparkClient, IDisposable {
       uint userId,
       IMarketDataClient mdClient,
       IOrdersClient ordersClient,
+      ShellLatencyService shellLatencyService,
       IAdminClient adminClient,
       ISecurityResolver securityResolver,
       int orderPoolCapacity,
@@ -53,6 +56,7 @@ public class SimpleSparkClient : ISimpleSparkClient, IDisposable {
    ) {
       this.mdClient = mdClient;
       this.ordersClient = ordersClient;
+      this.shellLatencyService = shellLatencyService;
       limitOrderPool = new(orderPoolCapacity, () => new() { User = userName, UserId = userId, TimeInForce = TimeInForce.Gtc, CustomDef = new() { Limit = new() } });
       marketOrderPool = new(orderPoolCapacity, () => new() { User = userName, UserId = userId, TimeInForce = TimeInForce.Gtc, CustomDef = new() { Market = new() } });
       orderSecurityDmx = new TableUpdateDemuxer<OrderRecordTupleWithRoot, uint, uint>(o => o.order.order.Key, o => o.security_id);
@@ -140,6 +144,17 @@ public class SimpleSparkClient : ISimpleSparkClient, IDisposable {
       }
    }
 
+   public async Task<bool> PingAsync() {
+      var request = new BufferMessage() {
+         Sequence = (ulong)DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
+         Data = "spark-sdk-ping"
+      };
+      var result = await shellLatencyService.ping_async(request);
+      return result.success &&
+         result.value.Sequence == request.Sequence &&
+         result.value.Data == request.Data;
+   }
+
    public IObservable<TableUpdate<BookRecord, uint>> Books(SecurityCriteria? securityCriteria) {
       var booksSource = this.booksSource;
       var secFilter = securityCriteria?.ToPredicate();
@@ -271,28 +286,32 @@ public class SimpleSparkClient : ISimpleSparkClient, IDisposable {
    public Task<Response> CreateLimitOrder(string account, Venue venue, string contract, Side side, double quantity, double price, string? tag) {
       var def = buildOrder(account, venue, contract, limitOrderPool);
       def.Side = side;
-      def.Tag = tag;
+      if (tag is not null)
+         def.Tag = tag;
       return ordersClient.TryEnterOrderAsync(def, price, quantity);
    }
 
    public Task<Response> CreateLimitOrder(string account, uint securityId, Side side, double quantity, double price, string? tag) {
       var def = buildOrder(account, securityId, limitOrderPool);
       def.Side = side;
-      def.Tag = tag;
+      if (tag is not null)
+         def.Tag = tag;
       return ordersClient.TryEnterOrderAsync(def, price, quantity);
    }
 
    public Task<Response> CreateMarketOrder(string account, Venue venue, string contract, Side side, double quantity, string? tag) {
       var def = buildOrder(account, venue, contract, marketOrderPool);
       def.Side = side;
-      def.Tag = tag;
+      if (tag is not null)
+         def.Tag = tag;
       return ordersClient.TryEnterOrderAsync(def, double.NaN, quantity);
    }
 
    public Task<Response> CreateMarketOrder(string account, uint securityId, Side side, double quantity, string? tag) {
       var def = buildOrder(account, securityId, marketOrderPool);
       def.Side = side;
-      def.Tag = tag;
+      if (tag is not null)
+         def.Tag = tag;
       return ordersClient.TryEnterOrderAsync(def, double.NaN, quantity);
    }
 

src/lib_spark/spark/config/profiles/default/cme_reference_data.conf

@@ -3,9 +3,9 @@
 # source: tools/cpp_helper/generate-cpps.ps1
 #
 
-CppConfig
-{
-	value
+CppConfig
+{
+	value
 	{
 		cpp_entry { key: "0A" value: 100 }
 		cpp_entry { key: "0B" value: 100 }
@@ -537,6 +537,31 @@ CppConfig
 		cpp_entry { key: "EQ1" value: 50 }
 		cpp_entry { key: "ERL" value: 5 }
 		cpp_entry { key: "ES" value: 50 }
+		cpp_entry { key: "E1A" value: 50}
+		cpp_entry { key: "E1B" value: 50}
+		cpp_entry { key: "E1C" value: 50}
+		cpp_entry { key: "E1D" value: 50}
+		cpp_entry { key: "E2A" value: 50}
+		cpp_entry { key: "E2B" value: 50}
+		cpp_entry { key: "E2C" value: 50}
+		cpp_entry { key: "E2D" value: 50}
+		cpp_entry { key: "E3A" value: 50}
+		cpp_entry { key: "E3B" value: 50}
+		cpp_entry { key: "E3C" value: 50}
+		cpp_entry { key: "E3D" value: 50}
+		cpp_entry { key: "E4A" value: 50}
+		cpp_entry { key: "E4B" value: 50}
+		cpp_entry { key: "E4C" value: 50}
+		cpp_entry { key: "E4D" value: 50}
+		cpp_entry { key: "E5A" value: 50}
+		cpp_entry { key: "E5B" value: 50}
+		cpp_entry { key: "E5C" value: 50}
+		cpp_entry { key: "E5D" value: 50}
+		cpp_entry { key: "EW" value: 50}		
+		cpp_entry { key: "EW1" value: 50}
+		cpp_entry { key: "EW2" value: 50}
+		cpp_entry { key: "EW3" value: 50}
+		cpp_entry { key: "EW4" value: 50}
 		cpp_entry { key: "ES1" value: 50 }
 		cpp_entry { key: "ES2" value: 50 }
 		cpp_entry { key: "ESB" value: 1000 }
@@ -1001,6 +1026,31 @@ CppConfig
 		cpp_entry { key: "NOO" value: 8900 }
 		cpp_entry { key: "NPG" value: 2500 }
 		cpp_entry { key: "NQ" value: 20 }
+      	cpp_entry {key: "Q1A" value: 20}
+      	cpp_entry {key: "Q1B" value: 20}
+      	cpp_entry {key: "Q1C" value: 20}
+      	cpp_entry {key: "Q1D" value: 20}
+      	cpp_entry {key: "Q2A" value: 20}
+      	cpp_entry {key: "Q2B" value: 20}
+      	cpp_entry {key: "Q2C" value: 20}
+      	cpp_entry {key: "Q2D" value: 20}
+      	cpp_entry {key: "Q3A" value: 20}
+      	cpp_entry {key: "Q3B" value: 20}
+      	cpp_entry {key: "Q3C" value: 20}
+      	cpp_entry {key: "Q3D" value: 20}
+      	cpp_entry {key: "Q4A" value: 20}
+      	cpp_entry {key: "Q4B" value: 20}
+      	cpp_entry {key: "Q4C" value: 20}
+      	cpp_entry {key: "Q4D" value: 20}
+      	cpp_entry {key: "Q5A" value: 20}
+      	cpp_entry {key: "Q5B" value: 20}
+      	cpp_entry {key: "Q5C" value: 20}
+      	cpp_entry {key: "Q5D" value: 20}
+      	cpp_entry {key: "QNE" value: 20}
+      	cpp_entry {key: "QN1" value: 20}
+      	cpp_entry {key: "QN2" value: 20}
+      	cpp_entry {key: "QN3" value: 20}
+      	cpp_entry {key: "QN4" value: 20}
 		cpp_entry { key: "NQQ" value: 20 }
 		cpp_entry { key: "NQT" value: 20 }
 		cpp_entry { key: "NR" value: 2500 }
@@ -1506,12 +1556,12 @@ CppConfig
 		cpp_entry { key: "ZT" value: 2000 }
 		cpp_entry { key: "ZW" value: 50 }
 		cpp_entry { key: "ZWT" value: 50 }
-	}
-}
+	}
+}
 
-CmeSymbolMics
-{
-	value
+CmeSymbolMics
+{
+	value
 	{
 		cme_mic { key: "0A" value: NYMEX }
 		cme_mic { key: "0B" value: NYMEX }
@@ -3012,5 +3062,5 @@ CmeSymbolMics
 		cme_mic { key: "ZT" value: CBT }
 		cme_mic { key: "ZW" value: CBT }
 		cme_mic { key: "ZWT" value: CBT }
-	}
+	}
 }

src/lib_spark/spark/config/profiles/default/orderman_cfg.conf

@@ -30,5 +30,6 @@ OrdermanCfg {
 	    poll: false 
       hosted_actor: "*"
     }
+    fallback_cust_order_handling_inst: "Y"
   }
 }

src/lib_spark/spark/config/profiles/default/toggles.conf

@@ -5,9 +5,15 @@ FillsSoftCleanup {
 }
 
 OrdersSoftCleanup {
-   value {
-      enabled: true
-   }
+  value {
+     enabled: true
+  }
+}
+
+Mdp3OptionExpirationCutoff {
+  value {
+    max_days_until_expiration: 14
+  }
 }
 PlacementInQueue {
   value{

src/lib_spark/spark/config/protos/toggles.proto

@@ -32,11 +32,17 @@ message SoundNotifications {
   bool enabled = 1;
 }
 
-message FilterExpiredFills {
-  bool enabled = 1;
-}
-
-message VolumeAtPrice {
+message FilterExpiredFills {
+  bool enabled = 1;
+}
+
+// MDP3 will not publish CME options whose maturity is beyond this many days.
+// A value of zero disables this cutoff.
+message Mdp3OptionExpirationCutoff {
+  uint32 max_days_until_expiration = 1;
+}
+
+message VolumeAtPrice {
   bool enabled = 1;
 }
 
@@ -124,3 +130,8 @@ message VolSurfaceUsageUnit{
 message VolSurfaceUsage{
   repeated VolSurfaceUsageUnit items = 1;
 }
+
+message LogTrades{
+  string sec_name = 1;
+  bool log = 2;
+}

src/lib_spark/spark/jetfix/gateway/CoreGateway.cpp

@@ -10,6 +10,7 @@
 #include "spark/orderman/orderman_utils/orderman_common.h"
 #include "spark/tracing/bts_trace.h"
 #include "spark/utils/TimeStamp.h"
+#include "spark/utils/bts_assert.h"
 #include "spark/utils/bts_chrono.h"
 #include "spark/utils/datetime.h"
 #include "spark/utils/thread_manager.h"
@@ -97,7 +98,8 @@ struct CoreGateway::PImpl {
        : cfg(cfg), observer(observer), make_connection(conn_factory),
          make_store(store_factory), state(State::OFF), start_of_pause(),
          session_info(), event_queue(), timer_thread(), destroy(false),
-         is_on(false), has_replayed(false), send_batch(), recv_batch(),
+         is_on(false), has_replayed(false), gave_up_permanently(false),
+         send_batch(), recv_batch(),
          connection_attempts(0), perf_log(l) {
       send_batch.reserve(1024);
       recv_batch.reserve(1024);
@@ -116,6 +118,7 @@ struct CoreGateway::PImpl {
    std::atomic<bool> destroy;
    bool is_on;
    bool has_replayed;
+   bool gave_up_permanently;
    std::vector<EventData> send_batch;
    std::vector<EventData> recv_batch;
    uint32_t connection_attempts;
@@ -329,6 +332,11 @@ void CoreGateway::sync_session_ready(EventData const &) {
    case State::WSESSION_TERMINATE:
       break;
    case State::CONNECTING:
+      LOG(Jetfix, Info,
+          "Jetfix session {} is now READY after socket connect. attempts={} "
+          "max_attempts={}",
+          me->cfg.name(), me->connection_attempts,
+          me->cfg.max_connection_attempts());
       transition(State::READY);
       me->observer.ready();
       break;
@@ -346,10 +354,18 @@ void CoreGateway::sync_session_terminated(EventData const &) {
       break;
 
    case State::WSESSION_TERMINATE:
+      LOG(Jetfix, Warn,
+          "Session terminated callback received for {} while waiting for "
+          "reader disconnect. attempts_so_far={}",
+          me->cfg.name(), me->connection_attempts);
       pause_or_shutdown();
       break;
 
    case State::READY:
+      LOG(Jetfix, Warn,
+          "Session terminated for {} from READY. Closing socket and waiting "
+          "for reader disconnect.",
+          me->cfg.name());
       me->session_info->session.reset();
       me->session_info->connection->shutdown();
       transition(State::WREADER_DISCONNECT);
@@ -357,6 +373,10 @@ void CoreGateway::sync_session_terminated(EventData const &) {
       break;
 
    case State::CONNECTING:
+      LOG(Jetfix, Warn,
+          "Session terminated for {} during CONNECTING before READY. "
+          "Closing socket and waiting for reader disconnect.",
+          me->cfg.name());
       me->session_info->session.reset();
       me->session_info->connection->shutdown();
       transition(State::WREADER_DISCONNECT);
@@ -376,17 +396,28 @@ void CoreGateway::sync_reader_disconnected(EventData const &) {
       break;
 
    case State::READY:
+      LOG(Jetfix, Warn,
+          "Socket reader disconnected for {} from READY. "
+          "The exchange or network closed the TCP session.",
+          me->cfg.name());
       me->session_info->session.reset();
       me->observer.disconnect();
       transition(State::WSESSION_TERMINATE);
       break;
 
    case State::CONNECTING:
+      LOG(Jetfix, Warn,
+          "Socket reader disconnected for {} during CONNECTING before READY. "
+          "The TCP socket connected, but the session never completed logon.",
+          me->cfg.name());
       me->session_info->session.reset();
       transition(State::WSESSION_TERMINATE);
       break;
 
    case State::WREADER_DISCONNECT:
+      LOG(Jetfix, Info,
+          "Reader disconnect completed for {} after session shutdown.",
+          me->cfg.name());
       pause_or_shutdown();
       break;
 
@@ -430,27 +461,46 @@ void CoreGateway::sync_in_msg_sequenced(EventData const &ev) {
       bts_assert(false);
    }
 }
-
 void CoreGateway::sync_time(EventData const &ev) {
    switch (me->state) {
    case State::OFF:
       break;
    case State::PAUSED:
+      if (me->gave_up_permanently)
+         break;
+
       if (datetime::is_at_least_secs_old(
               me->start_of_pause, me->cfg.seconds_to_pause())) {
-         LOG(Jetfix, Info, "Pause period ended for jetfix connection {}",
-             me->cfg.name());
+         LOG(Jetfix, Info,
+             "Pause ended for jetfix connection {}. pause_secs={} "
+             "attempts_so_far={} max_attempts={} host={}:{} segment={} "
+             "session_window={}-{} {}-{} replayed={}",
+             me->cfg.name(), me->cfg.seconds_to_pause(),
+             me->connection_attempts, me->cfg.max_connection_attempts(),
+             me->cfg.socket_connect_host_ip(), me->cfg.socket_connect_port(),
+             me->cfg.cme_market_segment_id(), me->cfg.start_time(),
+             me->cfg.end_time(), me->cfg.start_day(), me->cfg.end_day(),
+             me->has_replayed);
          me->session_info = std::make_unique<SessionInfo>();
-
          me->session_info->store.reset(me->make_store(me->cfg));
-
-         if (me->session_info->store->session_active()) {
+         auto const session_active = me->session_info->store->session_active();
+         LOG(Jetfix, Info,
+             "Loaded FIX store for {}. session_active={} next_attempt={} "
+             "max_attempts={}",
+             me->cfg.name(), session_active, me->connection_attempts + 1,
+             me->cfg.max_connection_attempts());
+         if (session_active) {
             me->connection_attempts++;
-
+            LOG(Jetfix, Info,
+                "Attempting socket connect for jetfix connection {}. "
+                "attempt={} max_attempts={}",
+                me->cfg.name(), me->connection_attempts,
+                me->cfg.max_connection_attempts());
             if (!me->has_replayed) {
                auto msgs_to_replay = me->session_info->store->replay();
-               LOG(Jetfix, Warn, "Replaying {} FIX msgs",
-                   msgs_to_replay.size());
+               LOG(Jetfix, Warn,
+                   "Replaying {} FIX msgs for {} before reconnect",
+                   msgs_to_replay.size(), me->cfg.name());
                me->observer.replay(msgs_to_replay);
                me->has_replayed = true;
             }
@@ -458,14 +508,20 @@ void CoreGateway::sync_time(EventData const &ev) {
             if (me->connection_attempts > me->cfg.max_connection_attempts()) {
                me->session_info.reset();
                me->start_of_pause = datetime::now();
+               me->gave_up_permanently = true;
                LOG(Jetfix, Error,
                    "Failed to connect to exchange gateway {} too many "
-                   "times. Giving up permanently. You must bounce "
-                   "the server to recover.",
-                   me->cfg.name());
+                   "times. Giving up permanently. attempts={} "
+                   "max_attempts={} pause_secs={} host={}:{} segment={}. "
+                   "You must bounce the server to recover.",
+                   me->cfg.name(), me->connection_attempts,
+                   me->cfg.max_connection_attempts(),
+                   me->cfg.seconds_to_pause(),
+                   me->cfg.socket_connect_host_ip(),
+                   me->cfg.socket_connect_port(),
+                   me->cfg.cme_market_segment_id());
                break;
             }
-
             me->session_info->connection.reset(me->make_connection(me->cfg));
             LOG(Jetfix, Warn, "Trying to connect to exchange for {} ({}/{})",
                 me->cfg.name(), me->connection_attempts,
@@ -489,11 +545,18 @@ void CoreGateway::sync_time(EventData const &ev) {
                me->session_info.reset();
                me->start_of_pause = datetime::now();
                LOG(Jetfix, Warn,
-                   "Failed to connect to exchange for {}. ({}/{}) ",
+                   "Failed to connect to exchange for {}. ({}/{}) "
+                   "Will retry after {} seconds.",
                    me->cfg.name(), me->connection_attempts,
-                   me->cfg.max_connection_attempts());
+                   me->cfg.max_connection_attempts(),
+                   me->cfg.seconds_to_pause());
             }
          } else {
+            LOG(Jetfix, Warn,
+                "FIX store for {} is not active yet. Deferring reconnect. "
+                "session_window={}-{} {}-{}",
+                me->cfg.name(), me->cfg.start_time(), me->cfg.end_time(),
+                me->cfg.start_day(), me->cfg.end_day());
             me->session_info.reset();
             me->start_of_pause = datetime::now();
          }
@@ -516,6 +579,12 @@ void CoreGateway::pause_or_shutdown() {
 
    if (me->is_on) {
       me->start_of_pause = datetime::now();
+      LOG(Jetfix, Info,
+          "Pausing jetfix gateway {}. pause_secs={} attempts_so_far={} "
+          "max_attempts={} gave_up_permanently={}",
+          me->cfg.name(), me->cfg.seconds_to_pause(),
+          me->connection_attempts, me->cfg.max_connection_attempts(),
+          me->gave_up_permanently);
       transition(State::PAUSED);
    } else
       transition(State::OFF);

src/lib_spark/spark/jetfix/gateway/Session.cpp

@@ -114,6 +114,15 @@ void Session::start() {
    me->start_of_logon = me->now();
    me->last_seq_num = me->begin_of_this_session;
    me->last_seq_num_processed = me->store.last_seq_num_processed();
+   LOG(Jetfix, Info,
+       "Starting jetfix session {}. sender_compid={} sender_subid={} "
+       "target_compid={} target_subid={} secure_logon={} raw_data_len={} "
+       "user_name_present={} seq_num={} last_processed={}",
+       me->cfg.name(), me->cfg.sender_compid(), me->cfg.sender_subid(),
+       me->cfg.target_compid(), me->cfg.target_subid(),
+       me->cfg.use_cme_secure_logon(), me->cfg.raw_data().size(),
+       !me->cfg.user_name().empty(), me->last_seq_num + 1,
+       me->last_seq_num_processed);
    transition(State::WLOGON);
    internal_send(make_logon(
        me->last_seq_num + 1, me->last_seq_num_processed, *me->current_time,

src/lib_spark/spark/md/GeneralMarketDataListener.cpp

@@ -54,6 +54,7 @@
 #include <google/protobuf/timestamp.pb.h>
 #include <google/protobuf/util/time_util.h>
 #include <memory>
+#include <mutex>
 #include <optional>
 #include <sstream>
 #include <string>
@@ -280,8 +281,9 @@ struct GeneralMarketDataListener::PImpl {
    // if this is set to true, then the security key passed on in the
    // on_securities function is used. If this is not set to true, then a new key
    // is generated
-   const bool use_given_sec_key =
-       true; // with true, replay works, with orders goign in ((did not push it in, since it was always false - need to test))
+   // Normally the source provides Spark keys.  MDP3 explicitly opts into
+   // external CME SecurityID resolution before it begins delivery.
+   bool use_given_sec_key{true};
    const std::string cc_name_prefix = "cc_security";
    size_t n_bk_updates = 0;
    bts::persistence::IDGenerator *id_gen;
@@ -295,6 +297,14 @@ struct GeneralMarketDataListener::PImpl {
    // (a flag is used to tell if bts::domain::SecurityID can be used as bts id
    // as well)
    std::unordered_map<bts::domain::SecurityID, uint64_t> cc_sec_id_map = {};
+   // CME can define a calendar/inter-commodity spread before every leg
+   // definition reaches this channel.  Retain it until its external leg IDs
+   // have been resolved to the GMDL security namespace.
+   std::vector<bts::domain::Security> deferred_spreads;
+   // MDP3 invokes one listener per CME channel.  Serialize only the simulated
+   // route at the GMDL boundary so definition/index/book updates remain
+   // ordered with respect to the single GMDL producer.
+   std::mutex simulation_callback_mutex;
    MDArray bids = {};
    MDArray asks = {};
    IncrementalBookData inc_book_data = {};
@@ -385,9 +395,29 @@ GeneralMarketDataListener::GeneralMarketDataListener(
 
 GeneralMarketDataListener::~GeneralMarketDataListener() {}
 
+void GeneralMarketDataListener::set_source_security_ids_are_external(
+    bool value) {
+   // Sources configure this during start-up, before any callback can access
+   // the index.  Keeping it source-scoped avoids changing Databento GLBX.
+   //currently all other data sources set it to true only mdp3 based cme data sets it as false
+   me->use_given_sec_key = !value;
+}
+
 void GeneralMarketDataListener::on_securities(
     const std::vector<domain::Security const *> &complete_security_list) {
+   std::unique_lock<std::mutex> simulation_lock(
+       me->simulation_callback_mutex, std::defer_lock);
+   if (me->sim_ex_gwy != nullptr) simulation_lock.lock();
    LOG_SECURITIES_EVENT(Info, complete_security_list.size());
+   const auto spread_legs_resolved = [this](const domain::Security &sec) {
+      if (sec.type() != SEC_TYPE(SPREAD)) return true;
+      const auto &spread = std::get<domain::SpreadSecurity>(sec.security);
+      return std::all_of(
+          spread.legs.begin(), spread.legs.end(), [this](const auto &leg) {
+             return me->cc_sec_id_map.find(leg.security_id) !=
+                    me->cc_sec_id_map.end();
+          });
+   };
    for (auto s_ptr : complete_security_list) {
       const domain::Security &sec = *s_ptr;
       CCSecSearchResult search_result = _find_sec(sec, true);
@@ -409,12 +439,28 @@ void GeneralMarketDataListener::on_securities(
          _add_option(sec);
          break;
       case SEC_TYPE(SPREAD):
-         _add_spread(sec);
+         if (spread_legs_resolved(sec)) {
+            _add_spread(sec);
+         } else {
+            me->deferred_spreads.push_back(sec);
+            LOG(md, Info,
+                "Deferring GLBX spread {} until its CME leg definitions arrive",
+                sec.desc());
+         }
          break;
       default:
          break;
       }
    }
+   for (auto it = me->deferred_spreads.begin();
+        it != me->deferred_spreads.end();) {
+      if (spread_legs_resolved(*it)) {
+         _add_spread(*it);
+         it = me->deferred_spreads.erase(it);
+      } else {
+         ++it;
+      }
+   }
 }
 
 void GeneralMarketDataListener::on_security_state(
@@ -442,6 +488,9 @@ void GeneralMarketDataListener::on_security_statistics(
 
 void GeneralMarketDataListener::on_book(
     const bts::domain::Book &book, std::optional<utils::TimeStamp> time) {
+   std::unique_lock<std::mutex> simulation_lock(
+       me->simulation_callback_mutex, std::defer_lock);
+   if (me->sim_ex_gwy != nullptr) simulation_lock.lock();
    LOG_BOOK(Diag, book, BookLog_Status::PROCESSING);
    // in case this is called externally in conjunction iwth inc book udpates
    me->inc_book_data.overwrite_book(book);
@@ -488,6 +537,9 @@ co2::AggressorSide try_fill_agg_side(
 }
 void GeneralMarketDataListener::on_trades(
     const std::vector<domain::TradeRecord const *> &trades) {
+   std::unique_lock<std::mutex> simulation_lock(
+       me->simulation_callback_mutex, std::defer_lock);
+   if (me->sim_ex_gwy != nullptr) simulation_lock.lock();
    utils::Stopwatch sw;
    for (auto tp : trades) {
       sw.lap();
@@ -504,7 +556,7 @@ void GeneralMarketDataListener::on_trades(
       aggressor_side = try_fill_agg_side(tp->trade_price, aggressor_side, bk);
       const auto &sec = handle->handle.security();
       LOG_TRADE_RECORD(
-          Info, t, sec.exchange, sec.type, sec.security_desc(),
+          Diag, t, sec.exchange, sec.type, sec.security_desc(),
           TradeRecordLog_Status::PROCESSING);
 
       const auto trade_ts = tp->time_stamp.to_ll();
@@ -790,14 +842,30 @@ void GeneralMarketDataListener::_on_book(
    me->n_bk_updates++;
    auto handle = _find_handle(book.security_id());
    if (handle == nullptr) {
-      LOG_BOOK(Error, book, BookLog_Status::ABORTED, "Sec handle not found");
+      const auto mapping = me->cc_sec_id_map.find(book.security_id());
+      const bool mapped = mapping != me->cc_sec_id_map.end();
+      LOG_BOOK(
+          Error, book, domain::DomainSecurityUtils::convert(me->venue),
+          api::SECURITYTYPE_UNSET, "", BookLog_Status::ABORTED,
+          fmt::format(
+              "Sec handle not found; reason={}; mapped_spark_key={}; "
+              "key_mode={}; registered_mappings={}; deferred_spreads={}; "
+              "simulation={}",
+              mapped ? "mapped_key_missing_from_index" : "source_id_not_mapped",
+              mapped ? std::to_string(mapping->second) : "unmapped",
+              me->use_given_sec_key ? "source_key" : "persisted_spark_key",
+              me->cc_sec_id_map.size(), me->deferred_spreads.size(),
+              me->sim_ex_gwy != nullptr));
       return;
    }
    copy_books(book, me->bids, bts::domain::Side::Value::Buy);
    copy_books(book, me->asks, bts::domain::Side::Value::Sell);
-   _possibly_match_with_sim_orders(
-       bts::md::SecurityKey(book.security_id().value()));
    const auto &sec = handle->handle.security();
+   // `book.security_id()` is the source identity for MDP3 (the CME
+   // SecurityID).  Simulation orders are keyed by the persisted Spark key
+   // owned by this GMDL handle.
+   _possibly_match_with_sim_orders(
+       bts::md::SecurityKey(handle->handle.key()));
    LOG_BOOK(
        Diag, book, sec.exchange, sec.type, sec.security_desc(),
        BookLog_Status::PROCESSING);

src/lib_spark/spark/md/GeneralMarketDataListener.h

@@ -23,6 +23,9 @@ class GeneralMarketDataListener : public IMarketDataListener {
        api::Venue venue,
        persistence::IDGenerator *sec_id_store);
    virtual ~GeneralMarketDataListener();
+   // MDP3 identifies securities in callbacks by CME SecurityID, not by a
+   // persisted Spark key.  Sources opt into this before they start.
+   void set_source_security_ids_are_external(bool value = true);
    void on_securities(const std::vector<domain::Security const *>
                           &complete_security_list) override;
    void on_security_state(const domain::SecurityState &security_state) override;

src/lib_spark/spark/md/MDProducerHandle.cpp

@@ -1,4 +1,5 @@
 #include "spark/logging/logging.h"
+#include "spark/md/log_messages/trade_latency_log.h"
 #ifndef NO_CPP0X
 #include "MDProducerHandle.h"
 #include "common.h"
@@ -133,6 +134,10 @@ void MDProducerHandle::add_trade(
       trade.set_key(key());
       _channel.add_trade(trade);
    }
+
+   log_trade_latency(
+       TradeLatencyStage::SHM_WRITTEN, security(), tradetime, price, agside,
+       volume, from_spread);
 }
 void MDProducerHandle::set_last_trade(
     TimeStamp time,

src/lib_spark/spark/md/log_messages/book_log.h

@@ -42,15 +42,21 @@ struct BookLog {
        api::Venue venue,
        api::SecurityType sec_type,
        std::string sec_name,
-       BookLog_Status status)
+       BookLog_Status status,
+       std::string message = "")
        : book(std::move(book)), venue(venue), sec_type(sec_type),
-         sec_name(std::move(sec_name)), status(status), message(),
-         timestamp(core::TimeStamp::now()) {}
+         sec_name(std::move(sec_name)), status(status),
+         message(std::move(message)), timestamp(core::TimeStamp::now()) {}
 
    explicit BookLog(
        domain::Book book, BookLog_Status status, std::string message)
-       : book(std::move(book)), status(status), message(std::move(message)),
-         timestamp(core::TimeStamp::now()) {}
+       : BookLog(
+             std::move(book),
+             api::VENUE_UNSET,
+             api::SECURITYTYPE_UNSET,
+             "",
+             status,
+             std::move(message)) {}
 };
 
 } // namespace bts::md

src/lib_spark/spark/md/log_messages/trade_latency_log.h

@@ -0,0 +1,124 @@
+#pragma once
+
+#include "spark/config/config.h"
+#include "spark/config/protos/toggles.pb.h"
+#include "spark/core/FMT_IMPL.h"
+#include "spark/core/time_stamp.h"
+#include "spark/logging/logging.h"
+#include "spark/md/PriceSize.h"
+#include "spark/md/Shm/FixedSecurity.h"
+#include "spark/md/protos/aggressor_side.h"
+#include "spark/utils/datetime.h"
+
+#include <string>
+#include <cstdint>
+#include <utility>
+
+namespace bts::md {
+
+enum class TradeLatencyStage {
+   MDP3_RECEIVED,
+   SHM_WRITTEN,
+};
+
+inline const char *TradeLatencyStage_Name(TradeLatencyStage stage) {
+   switch (stage) {
+   case TradeLatencyStage::MDP3_RECEIVED:
+      return "mdp3_received";
+   case TradeLatencyStage::SHM_WRITTEN:
+      return "shm_written";
+   }
+   return "unknown";
+}
+
+inline std::string format_ll_datetime(datetime::LLDateTime ts) {
+   return datetime::iso_date_time_string(datetime::ll_to_ptime(ts));
+}
+
+inline bool should_log_trade_latency(const FixedSecurity &security) {
+   const auto &cfg = config::get<config::toggles::LogTrades>();
+   if (!cfg.log() || cfg.sec_name().empty()) {
+      return false;
+   }
+   return cfg.sec_name() == security.security_desc() ||
+          cfg.sec_name() == security.symbol();
+}
+
+struct TradeLatencyLog {
+   TradeLatencyStage stage;
+   std::string sec_name;
+   uint32_t security_key;
+   int64_t trade_ts_unix_ns;
+   std::string trade_ts_iso;
+   int64_t log_ts_unix_ns;
+   std::string log_ts_iso;
+   uint64_t monotonic_log_ts_ns;
+   int64_t exchange_to_log_us;
+   double price;
+   double quantity;
+   std::string aggressor_side;
+   uint32_t total_volume;
+   bool from_spread;
+
+   TradeLatencyLog(
+       TradeLatencyStage stage,
+       const FixedSecurity &security,
+       datetime::LLDateTime trade_ts,
+       core::TimeStamp log_ts,
+       utils::TimeStamp monotonic_log_ts,
+       const PriceSize &price_size,
+       co2::AggressorSide aggressor_side,
+       uint32_t total_volume,
+       bool from_spread)
+       : stage(stage), sec_name(security.security_desc()),
+         security_key(security.key), trade_ts_unix_ns(trade_ts),
+         trade_ts_iso(format_ll_datetime(trade_ts)),
+         log_ts_unix_ns(log_ts.nanos()), log_ts_iso(log_ts.to_iso_string()),
+         monotonic_log_ts_ns(monotonic_log_ts.to_uint64()),
+         exchange_to_log_us(
+             (log_ts - core::TimeStamp::from_ptime(datetime::ll_to_ptime(trade_ts)))
+                 .total_micros()),
+         price(price_size.price()), quantity(price_size.size()),
+         aggressor_side(co2::AggressorSide_Name(aggressor_side)),
+         total_volume(total_volume), from_spread(from_spread) {}
+};
+
+inline void log_trade_latency(
+    TradeLatencyStage stage,
+    const FixedSecurity &security,
+    datetime::LLDateTime trade_ts,
+    const PriceSize &price_size,
+    co2::AggressorSide aggressor_side,
+    uint32_t total_volume,
+    bool from_spread) {
+   if (!should_log_trade_latency(security)) {
+      return;
+   }
+   const auto log_ts = core::TimeStamp::now();
+   const auto monotonic_log_ts = utils::TimeStamp::now();
+   LOG(md, Info, "[trade_latency.json]{}",
+       TradeLatencyLog(
+           stage, security, trade_ts, log_ts, monotonic_log_ts, price_size,
+           aggressor_side, total_volume, from_spread));
+}
+
+} // namespace bts::md
+
+FMT_IMPL(
+    bts::md::TradeLatencyLog,
+    l,
+    R"({{ "stage": "{}", "sec": "{}", "key": {}, "tradeTsUnixNs": {}, "tradeTsIso": "{}", "logTsUnixNs": {}, "logTsIso": "{}", "logMonoNs": {}, "exchangeToLogUs": {}, "price": {}, "qty": {}, "agg": "{}", "totalVolume": {}, "fromSpread": {} }})",
+    bts::md::TradeLatencyStage_Name(l.stage),
+    l.sec_name,
+    l.security_key,
+    l.trade_ts_unix_ns,
+    l.trade_ts_iso,
+    l.log_ts_unix_ns,
+    l.log_ts_iso,
+    l.monotonic_log_ts_ns,
+    l.exchange_to_log_us,
+    l.price,
+    l.quantity,
+    l.aggressor_side,
+    l.total_volume,
+    l.from_spread);

src/lib_spark/spark/orderman/actors/FillStore.cpp

@@ -23,6 +23,7 @@
 #include "spark/utils/overloaded.h"
 
 #include <boost/date_time/posix_time/posix_time.hpp>
+#include <cctype>
 #include <google/protobuf/timestamp.pb.h>
 #include <string>
 #include <sys/time.h>
@@ -308,15 +309,60 @@ void FillStore::on_execution_report(
 
                auto exchange_ts = std::visit(
                    overloaded{
-                       [](const orderman::ExchangeTimeStampStr &ts) {
-                          return utils::pb::to_timestamp(
+                       [&](const orderman::ExchangeTimeStampStr &ts) {
+                          auto gts = utils::pb::to_timestamp(
                               datetime::parse_FIX_timestamp(to_str(ts)));
+                          const std::string ts_str = to_str(ts);
+                          try {
+                             const auto dot_pos = ts_str.find('.');
+                             if (dot_pos != std::string::npos) {
+                                std::string frac_digits;
+                                for (std::size_t i = dot_pos + 1;
+                                     i < ts_str.size() &&
+                                     std::isdigit(static_cast<unsigned char>(
+                                         ts_str[i]));
+                                     ++i) {
+                                   frac_digits.push_back(ts_str[i]);
+                                }
+
+                                if (!frac_digits.empty()) {
+                                   if (frac_digits.size() > 9) {
+                                      frac_digits.resize(9);
+                                   }
+                                   while (frac_digits.size() < 9) {
+                                      frac_digits.push_back('0');
+                                   }
+                                   gts.set_nanos(
+                                       static_cast<int32_t>(
+                                           std::stoul(frac_digits)));
+                                }
+                             }
+                          } catch (const std::exception &e) {
+                             WARN_FN(
+                                 "on_execution_report",
+                                 "fill {} transact_time(str) raw='{}' "
+                                 "failed to inspect fractional nanos: {}",
+                                 fk, ts_str, std::string(e.what()));
+                          }
+                          INFO_FN(
+                              "on_execution_report",
+                              "fill {} transact_time(str) raw='{}' "
+                              "converted_seconds={} converted_nanos={}",
+                              fk, ts_str, gts.seconds(), gts.nanos());
+                          return gts;
                        },
-                       [](ExchangeTimeStamp64 ts) {
+                       [&](ExchangeTimeStamp64 ts) {
                           google::protobuf::Timestamp gts;
-                          auto seconds = ts.value() / 1E9;
+                          static constexpr std::uint64_t nanos_per_second =
+                              1'000'000'000ULL;
+                          auto seconds = ts.value() / nanos_per_second;
                           gts.set_seconds(seconds);
-                          gts.set_nanos(ts.value() - (seconds * 1E9));
+                          gts.set_nanos(ts.value() % nanos_per_second);
+                          INFO_FN(
+                              "on_execution_report",
+                              "fill {} transact_time(u64) raw_ns={} "
+                              "converted_seconds={} converted_nanos={}",
+                              fk, ts.value(), gts.seconds(), gts.nanos());
                           return gts;
                        }},
                    er.transact_time);

src/lib_spark/spark/orderman/ilink3/IL3GW.cpp

@@ -47,7 +47,7 @@ IL3GW::IL3GW(
     : _config(conf), _x_state(State::INIT), _buffer(buffer),
       _on_state_change([](auto) {}), _on_exe_report([](auto) {}),
       _on_session_change([](auto) {}), _connection(con), _session(session),
-      _msg_count(0), _time(), _pool(), _exe_parser(cfg, db), _retransmit() {
+      _msg_count(0), _time(), _pool("IL3GW_rprts", 1024*2, 100), _exe_parser(cfg, db), _retransmit() {
    LOG(OrderMan, Info,
        "IL3GW::IL3GW cme_session: {} uuid: {} to_cme: {} from_cme: {}",
        _config.cme_session, _session.uuid, _session.to_cme, _session.from_cme);
@@ -179,15 +179,10 @@ uint64_t IL3GW::send_party_details(const api::orderman::RoutingRecord &route) {
    using namespace iLinkBinary;
 
    auto msg = _buffer.frame<PartyDetailsDefinitionRequest518>(
-       // 2 repeating group header.
-       // 1 for party details and 1 for trade reg
-       // This seems unsafe but the api doesn't seem to want you to know how to
-       // size buffers ahead of time
-       2 * PartyDetailsDefinitionRequest518::NoPartyDetails::sbeHeaderSize() +
-       // N * size of each block
-       3 * PartyDetailsDefinitionRequest518::NoPartyDetails::sbeBlockLength() +
-       0 * PartyDetailsDefinitionRequest518::NoTrdRegPublications::
-               sbeBlockLength());
+       // Use the generated helper so group sizing stays in sync with the
+       // active schema when CME updates the template layout.
+       PartyDetailsDefinitionRequest518::computeLength(3, 0) -
+       PartyDetailsDefinitionRequest518::sbeBlockLength());
    msg.partyDetailsListReqID(0);
    msg.sendingTimeEpoch(_time.now());
    msg.listUpdateAction(ListUpdAct::Add);
@@ -637,8 +632,13 @@ void IL3GW::handle(
    handle_common(msg);
 }
 void IL3GW::handle(const iLinkBinary::BusinessReject521 &msg) {
-   LOG(iLink3, Error, "Business Reject: {} '{}'.", msg.businessRejectReason(),
-       string{translate_business_reject_reason(msg.businessRejectReason())});
+   LOG(iLink3, Error,
+       "Business Reject: reason={} '{}' ref_seq={} ref_tag={} ref_msg_type='{}' "
+       "ref_id={} text='{}'",
+       msg.businessRejectReason(),
+       string{translate_business_reject_reason(msg.businessRejectReason())},
+       msg.refSeqNum(), msg.refTagID(), msg.getRefMsgTypeAsStringView(),
+       msg.businessRejectRefID(), msg.getTextAsStringView());
    handle_common(msg);
    // A business reject can de-sync the sequence number because it rejects
    // the sequence number of the rejected message. Send a sequence message

src/lib_spark/spark/orderman/ilink3/IL3Parser.cpp

@@ -70,9 +70,35 @@ bool IL3Parser::parse_common(const T &msg, ExecutionReport *data) {
    data->ice_cl_ord_id = 0;
    data->ice_company_id = 0;
    data->cfg = cfg;
+   data->fill_agg_flag = api::orderman::PASSIVE;
    return true;
 }
 
+template <class T>
+void IL3Parser::parse_fill_aggressor(const T &, ExecutionReport *data) {
+   data->fill_agg_flag = api::orderman::PASSIVE;
+}
+
+template <>
+void IL3Parser::parse_fill_aggressor(
+    const iLinkBinary::ExecutionReportTradeOutright525 &msg,
+    ExecutionReport *data) {
+   data->fill_agg_flag =
+       msg.aggressorIndicator() == iLinkBinary::BooleanFlag::True
+           ? api::orderman::AGGRESSOR
+           : api::orderman::PASSIVE;
+}
+
+template <>
+void IL3Parser::parse_fill_aggressor(
+    const iLinkBinary::ExecutionReportTradeSpread526 &msg,
+    ExecutionReport *data) {
+   data->fill_agg_flag =
+       msg.aggressorIndicator() == iLinkBinary::BooleanFlag::True
+           ? api::orderman::AGGRESSOR
+           : api::orderman::PASSIVE;
+}
+
 template <class T>
 void IL3Parser::parse_trade(const T &msg, ExecutionReport *data) {
    data->exchange_fill_id->copy_from(msg.getExecIDAsStringView());
@@ -103,6 +129,7 @@ void IL3Parser::parse_trade(const T &msg, ExecutionReport *data) {
           Price{price.mantissa() * md::pow10(price.exponent())});
    }
    data->filled_quantity = Quantity{msg.cumQty()};
+   parse_fill_aggressor(msg, data);
 }
 
 bool IL3Parser::has_account() const {

src/lib_spark/spark/orderman/ilink3/IL3Parser.h

@@ -47,6 +47,8 @@ class IL3Parser {
  private:
    template <class T> bool parse_common(const T &, ExecutionReport *);
    template <class T> void parse_trade(const T &, ExecutionReport *);
+   template <class T>
+   void parse_fill_aggressor(const T &, ExecutionReport *);
 };
 
 bool parse_report(const std::string &, IL3Parser &, ExecutionReport *);

src/lib_spark/spark/orderman/ilink3/generated/BusinessReject521.h

@@ -127,7 +127,9 @@
 #include "QuoteAckStatus.h"
 #include "ExpCycle.h"
 #include "MassCancelTIF.h"
+#include "Side.h"
 #include "OrderStatus.h"
+#include "CrossTypeEnum.h"
 #include "ShortSaleType.h"
 #include "PartyDetailRole.h"
 #include "MassActionScope.h"
@@ -168,7 +170,7 @@ public:
     static const std::uint16_t SBE_BLOCK_LENGTH = static_cast<std::uint16_t>(330);
     static const std::uint16_t SBE_TEMPLATE_ID = static_cast<std::uint16_t>(521);
     static const std::uint16_t SBE_SCHEMA_ID = static_cast<std::uint16_t>(8);
-    static const std::uint16_t SBE_SCHEMA_VERSION = static_cast<std::uint16_t>(8);
+    static const std::uint16_t SBE_SCHEMA_VERSION = static_cast<std::uint16_t>(9);
 
     enum MetaAttribute
     {
@@ -242,7 +244,7 @@ public:
 
     SBE_NODISCARD static SBE_CONSTEXPR std::uint16_t sbeSchemaVersion() SBE_NOEXCEPT
     {
-        return static_cast<std::uint16_t>(8);
+        return static_cast<std::uint16_t>(9);
     }
 
     SBE_NODISCARD static SBE_CONSTEXPR const char *sbeSemanticType() SBE_NOEXCEPT

src/lib_spark/spark/orderman/ilink3/generated/CmtaGiveUpCD.h

@@ -32,7 +32,7 @@ public:
     {
         GiveUp = static_cast<char>(71),
         SGXoffset = static_cast<char>(83),
-        NULL_VALUE = static_cast<char>(0x00)
+        NULL_VALUE = static_cast<char>(0)
     };
 
     static CmtaGiveUpCD::Value get(const char value)
@@ -41,7 +41,7 @@ public:
         {
             case static_cast<char>(71): return GiveUp;
             case static_cast<char>(83): return SGXoffset;
-            case static_cast<char>(0x00): return NULL_VALUE;
+            case static_cast<char>(0): return NULL_VALUE;
         }
 
         throw std::runtime_error("unknown value for enum CmtaGiveUpCD [E103]");