added more detail missing security log in gmdl

The old name was unnecessarily vague

.dotnet-cli/.nuget/packages/microsoft.net.test.sdk/17.10.0/build/netcoreapp3.1/Microsoft.NET.Test.Sdk.Program.cs

@@ -1,4 +0,0 @@
-// <auto-generated> This file has been auto generated. </auto-generated>
-using System;
-[Microsoft.VisualStudio.TestPlatform.TestSDKAutoGeneratedCode]
-class AutoGeneratedProgram {static void Main(string[] args){}}

.env

@@ -29,6 +29,9 @@
 #     tools/deploy.sh --help
 #
 
+### Host/Runtime ###
+TZ=Etc/UTC
+
 ### Sparkd ###
 # The internal image for sparkd to use. For more info, view:
 #   src/server/sparkd/Dockerfile
@@ -38,6 +41,9 @@ SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
 # Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
 # Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
 # SPARKD__DATA_DIR=/home/bts/sparkdata/data
+# Mount volume or dir to "/md_recording" inside the docker. Defaults to volume
+# "sparkd_md_recording_<config_port>" (e.g., sparkd_md_recording_8000).
+# SPARKD__MD_RECORDING_DIR=/home/bts/md_recording
 SPARKD__SOURCE_DIR=install/sparkd-release-gcc
 SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
 SPARKD__CONFIG_PORT=8000
@@ -45,6 +51,8 @@ SPARKD__QUICKBUS_PORT=8001
 SPARKD__SIM_FIX_PORT=8002
 SPARKD__SHM_SIZE=4gb
 SPARKD__PYTHON_REQUIREMENTS=requirements.txt
+SPARKD__SPARKD_ENTRYPOINT=src/app_sparkd/sparkd-entrypoint.sh
+SPARKD__MAIN_ENTRYPOINT=./sparkd-entrypoint.sh
 
 ### PubSubServer ###
 PUBSUB__CONFIG_DIR=src/DataStreamSuite/PubSubServer/Configurations/Development

.gitignore

@@ -64,9 +64,3 @@ trace_data/
 *.lst
 .env.local
 .secrets
-.dotnet-cli/
-.dotnet-cli/
-.dotnet-cli /
-.dotnet/
-.codex_restore*.log
-src/cs/Bts.Spark.Core/.codexobj/

.vscode/launch.json

@@ -183,11 +183,10 @@
       "windows": {
         "program": "PubSubServer.exe",
       },
-      "args": [
-        "--config",
-        "${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
-      ],
       "cwd": "build/Debug/bin/net8.0",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
+      },
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -203,71 +202,9 @@
         "program": "PubSubServer.exe",
       },
       "cwd": "build/Debug/bin/net8.0",
-      // "env": {
-      //   "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
-      // },
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "DSPublisher",
-      "type": "coreclr",
-      "request": "launch",
-      "preLaunchTask": "Build DataStream (Debug)",
-      "program": "",
-      "linux": {
-        "program": "DSPublisher",
+      "env": {
+        "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
       },
-      "windows": {
-        "program": "DSPublisher.exe",
-      },
-      "cwd": "build/Debug/bin/net8.0",
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "DSPublisher (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "DSPublisher",
-      },
-      "windows": {
-        "program": "DSPublisher.exe",
-      },
-      "cwd": "build/Debug/bin/net8.0",
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "PersistenceServer",
-      "type": "coreclr",
-      "request": "launch",
-      "preLaunchTask": "Build DataStream (Debug)",
-      "program": "",
-      "linux": {
-        "program": "PersistenceServer",
-      },
-      "windows": {
-        "program": "PersistenceServer.exe",
-      },
-      "cwd": "build/Debug/bin/net8.0",
-      "stopAtEntry": false,
-      "console": "integratedTerminal"
-    },
-    {
-      "name": "PersistenceServer (Skip Build)",
-      "type": "coreclr",
-      "request": "launch",
-      "program": "",
-      "linux": {
-        "program": "PersistenceServer",
-      },
-      "windows": {
-        "program": "PersistenceServer.exe",
-      },
-      "cwd": "build/Debug/bin/net8.0",
       "stopAtEntry": false,
       "console": "integratedTerminal"
     },
@@ -276,7 +213,7 @@
       "type": "coreclr",
       "request": "launch",
       "program": "",
-      "preLaunchTask": "Build Api Server (Debug)",
+      "preLaunchTask": "Build Spark Sdk (Debug)",
       "windows": {
         "program": "Bts.Spark.Sdk.Demo.exe",
       },
@@ -348,7 +285,7 @@
       "windows": {
         "program": "Spark.exe",
         "args": [
-          "--sparkd",
+          "--Spark:ConfigServer",
           "${input:sparkConfigServer}"
         ],
         "cwd": "build/Debug/bin/net8.0-windows",
@@ -511,14 +448,15 @@
       "default": "Local",
     },
     {
-      "id": "pubSubServerConfig",
+      "id": "pubSubServerEnvironment",
       "type": "pickString",
-      "description": "PubSub Configuration",
+      "description": "PubSubServer Configuration",
       "options": [
-        "PubSubServer",
-        "SimpleEchoClient",
+        "Development",
+        "EchoServer",
+        "Local",
       ],
-      "default": "PubSubServer",
+      "default": "Development",
     },
     {
       "id": "sdkDemoEnvironment",

compose.yaml

@@ -2,6 +2,7 @@ services:
   sparkd:
     volumes:
       - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
     build:
       dockerfile: src/app_sparkd/Dockerfile
       args:
@@ -9,24 +10,35 @@ services:
         - SOURCE_DIR=${SPARKD__SOURCE_DIR}
         - CONFIG_DIR=${SPARKD__CONFIG_DIR}
         - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
     shm_size: "${SPARKD__SHM_SIZE:-512mb}"
-    command: >
-      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd/data
-        if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
-          mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
-        fi
-        if [ -e "/home/bts/.local/share/bts/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/conf/.env.local'
-          set -a
-          . /home/bts/.local/share/bts/sparkd/conf/.env.local
-          set +a
-        fi
-        if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then
-          echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)'
-          cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/
-        fi
-        ./sparkd run"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
+    ports:
+      - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
+      - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
+      - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
+    deploy:
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-4gb}"
+
+  sparkd-shell:
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
+    build:
+      dockerfile: docker/images/bluetradingsystems/linux_shell/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
+    shm_size: "${SPARKD__SHM_SIZE:-512mb}"
+    command: "do sleep 1; done"
     env_file:
       - .env
       - ${SPARKD__CONFIG_DIR}/.env
@@ -42,6 +54,7 @@ services:
   sparkd-symbols:
     volumes:
       - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
       - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
     build:
       dockerfile: src/app_sparkd/Dockerfile
@@ -50,26 +63,8 @@ services:
         - SOURCE_DIR=${SPARKD__SOURCE_DIR}
         - CONFIG_DIR=${SPARKD__CONFIG_DIR}
         - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
     shm_size: "${SPARKD__SHM_SIZE:-512mb}"
-    command: >
-      sh -c "
-        echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd
-        if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
-          mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
-        fi
-        ls -alh /home/bts/.local/share/bts/sparkd/conf/
-        cat /home/bts/.local/share/bts/sparkd/conf/.env.local
-        if [ -e "/home/bts/.local/share/bts/sparkd/conf/.env.local" ]; then
-          echo 'Sourcing /home/bts/.local/share/bts/sparkd/conf/.env.local'
-          set -a
-          . /home/bts/.local/share/bts/sparkd/conf/.env.local
-          set +a
-        fi
-        if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then
-          echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)'
-          cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/
-        fi
-        ./sparkd run"
     env_file:
       - .env
       - ${SPARKD__CONFIG_DIR}/.env
@@ -77,6 +72,7 @@ services:
       - "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
       - "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
       - "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
+    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
     deploy:
       restart_policy:
         condition: any
@@ -84,7 +80,65 @@ services:
       resources:
         limits:
           memory: "${SPARKD__MEMORY:-4gb}"
-
+  sparkd-multicast:
+    environment:
+      - TZ=America/Chicago
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
+      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
+    build:
+      dockerfile: src/app_sparkd/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
+    shm_size: "${SPARKD__SHM_SIZE:-8192mb}"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
+    network_mode: "host"
+    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
+    deploy:
+      restart_policy:
+        condition: any
+        delay: 3s
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-16gb}"
+  
+  
+  sparkd-multicast-cert:
+    environment:
+      - TZ=America/Chicago
+    volumes:
+      - ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
+      - ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
+      - ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
+    build:
+      dockerfile: src/app_sparkd/Dockerfile
+      args:
+        - IMAGE=${SPARKD__IMAGE}
+        - SOURCE_DIR=${SPARKD__SOURCE_DIR}
+        - CONFIG_DIR=${SPARKD__CONFIG_DIR}
+        - PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
+        - SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
+    shm_size: "${SPARKD__SHM_SIZE:-8192mb}"
+    env_file:
+      - .env
+      - ${SPARKD__CONFIG_DIR}/.env
+    network_mode: "host"
+    entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
+    deploy:
+      restart_policy:
+        condition: any
+        delay: 3s
+      resources:
+        limits:
+          memory: "${SPARKD__MEMORY:-16gb}"
+  
   sparkd-tail:
     image: bluetradingsystems/linux_dotnet8
     volumes:
@@ -195,3 +249,6 @@ volumes:
     name: "md_publisher_22019"
   historic_md:
     name: "historic_md"
+  md_recording:
+    name: "sparkd_md_recording_${SPARKD__CONFIG_PORT:-8000}"
+    external: true

doc/sparkd-security-id-lifecycle.md

@@ -0,0 +1,942 @@
+# Security ID handling in `sparkd`
+
+This document traces how security identifiers are created, translated, persisted,
+indexed, and published by the `sparkd` market-data paths. It focuses on the
+legacy CME MDP3/OnixS pipeline and the newer generalized market-data pipeline.
+
+## Executive summary
+
+`sparkd` uses two identifiers that must not be confused:
+
+| Identifier | Meaning | Typical field | Lifetime |
+|---|---|---|---|
+| Exchange/feed ID | The venue's identifier used to route incoming definitions, books, and trades | `FixedSecurity::numeric_id`, `ExchangeKey`, or `domain::Security::security_common.exchange_sec_id` | Feed/session-specific unless the source guarantees stability |
+| Spark security key | Spark's internal non-zero identifier used in shared memory, APIs, orders, positions, and downstream services | `FixedSecurity::key`, `api::Security::id()`, `SecurityKey`; historically also called `security_id` | Intended to be stable across restarts |
+
+`invalid_security_key` is `0`; a security with key `0` is not yet publicly
+discoverable through the shared-memory consumer path.
+
+The two principal paths differ:
+
+- **MDP3/OnixS:** the definition callback sets only the CME ID in
+  `numeric_id`. `MDProducer::add_security()` allocates the shared-memory entry
+  with key `0`. `SecurityIDRecorder`, driven by a 200 ms dispatcher timer,
+  resolves or inserts a stable key in `key_ids.sqlite`; its callback finally
+  calls `MDProducerHandle::set_key()`.
+- **Generalized/newer pipeline:** `GeneralMarketDataListener::on_securities()`
+  currently uses the ID already present on the incoming domain security
+  (`use_given_sec_key == true`). Live adapters normally obtain that ID
+  synchronously from `IDGenerator` first. `_copy_security_common()` then copies
+  it to both `FixedSecurity::key` and `numeric_id`, so
+  `MDProducer::add_security()` publishes it synchronously. Only the listener's
+  own fallback `IDGenerator` call is bypassed by the current constant.
+
+## End-to-end diagrams
+
+### CME MDP3 / OnixS
+
+```mermaid
+sequenceDiagram
+    participant FE as OnixS feed-engine thread
+    participant L as MDP3Listener
+    participant P as MDProducer / shared memory
+    participant Q as SecurityIDRecorder queues
+    participant D as sparkd dispatcher thread
+    participant DB as key_ids.sqlite
+
+    FE->>L: instrument definition (CME SecurityID)
+    L->>L: set_security_common() - numeric_id = CME SecurityID, key remains 0
+    L->>P: add_security(FixedSecurity)
+    P->>P: allocate hot + cold entries with key 0
+    P->>Q: ensure_key(handle)
+    L->>L: index[CME SecurityID] = handle
+    Note over FE,L: Books can route by CME ID while the Spark key is pending
+    D->>Q: every 200 ms - work_spreads(), work_handles(), work_db()
+    Q->>DB: find (namespace, descriptor), or insert a new row
+    DB-->>Q: SQLite rowid = Spark security key
+    Q->>P: handle.set_key(rowid)
+    P->>P: hot.key first, then cold security.key, then increment security_count
+    Note over P: The security is now public to shared-memory consumers
+```
+
+### Generalized/newer pipeline (current behavior)
+
+```mermaid
+sequenceDiagram
+    participant A as Venue adapter / plugin
+    participant DB as IDGenerator / key_ids.sqlite
+    participant G as GeneralMarketDataListener
+    participant I as GeneralSecurityIndex
+    participant P as MDProducer / shared memory
+
+    A->>DB: get_or_add(venue, type, adapter key)
+    DB-->>A: SQLite rowid as domain SecurityID
+    A->>G: on_securities(domain Security list with ID)
+    G->>G: _find_sec() - use incoming security_common.id
+    G->>G: _copy_security_common() - key = id, numeric_id = id
+    G->>P: add_security(FixedSecurity with non-zero key)
+    P->>P: set_key() during add_security()
+    G->>I: index[id] = handle
+    Note over G,P: No asynchronous ID assignment in the current configuration
+```
+
+## Data-flow charts
+
+The sequence diagrams above emphasize timing and thread transitions. These
+flowcharts emphasize which identifier is carried through each representation
+and where data is retained.
+
+### MDP3 / legacy data flow
+
+```mermaid
+flowchart TD
+    CME["CME MDP3 packets<br/>CME SecurityID"] --> ONIXS["OnixS feed engine<br/>decode definition and updates"]
+
+    ONIXS --> DEF["MDP3Listener definition callback"]
+    DEF --> FIXED["FixedSecurity<br/>numeric_id = CME SecurityID<br/>key = 0"]
+
+    FIXED --> PRODUCER["MDProducer::add_security"]
+    PRODUCER --> PENDING["Hot and cold shared memory<br/>allocated but not public<br/>key = 0"]
+    PRODUCER --> QUEUE["SecurityIDRecorder<br/>pending_handle queue"]
+
+    FIXED --> CMEINDEX["CMESecurityIndex<br/>CME SecurityID to handle"]
+
+    QUEUE --> TIMER["Dispatcher timer<br/>every 200 ms"]
+    TIMER --> LEGS{"Spread legs<br/>all resolved?"}
+    LEGS -- No --> UNRESOLVED["unresolved_legs<br/>exchange IDs awaiting keys"]
+    UNRESOLVED --> CMEINDEX
+    CMEINDEX --> LEGS
+    LEGS -- Yes or outright --> LOGICAL["Logical identity<br/>venue + type + security_desc"]
+
+    LOGICAL --> SELECT["SQLite SELECT<br/>key_ids.sqlite"]
+    SELECT -- Existing row --> ROWID["SQLite rowid<br/>Spark security key"]
+    SELECT -- Missing row --> INSERT["Batched SQLite INSERT"]
+    INSERT --> ROWID
+
+    ROWID --> SETKEY["MDProducerHandle::set_key"]
+    SETKEY --> HOT["Write hot-entry key"]
+    HOT --> COLD["Write cold security key"]
+    COLD --> COUNT["Increment security_count"]
+    COUNT --> PUBLIC["Public security<br/>visible to MDConsumer"]
+
+    ONIXS --> UPDATE["Book, trade, status, or RFQ<br/>carrying CME SecurityID"]
+    UPDATE --> CMEINDEX
+    CMEINDEX --> HANDLE["MDProducerHandle"]
+    HANDLE --> LIVE["Update live shared-memory data"]
+    PUBLIC --> LIVE
+```
+
+The CME index is usable before the Spark key is assigned, so feed updates can
+reach the internal handle while the security is still hidden from consumers.
+The persistent rowid crosses the publication boundary only at `set_key()`.
+
+### Generalized/newer data flow
+
+```mermaid
+flowchart TD
+    FEED["Venue feed, Databento,<br/>Python plugin, or replay"] --> ADAPTER["Source adapter<br/>build domain::Security"]
+
+    ADAPTER --> IDTYPE{"Live definition or<br/>recorded/pre-keyed input?"}
+    IDTYPE -- Live --> IDINPUT["Logical identity<br/>venue + type + adapter-chosen name"]
+    IDINPUT --> MUTEX["IDGenerator global mutex"]
+    MUTEX --> SELECT["Synchronous SQLite SELECT<br/>key_ids.sqlite"]
+    SELECT -- Existing row --> ROWID["SQLite rowid"]
+    SELECT -- Missing row --> INSERT["Synchronous SQLite INSERT<br/>autocommit"]
+    INSERT --> ROWID
+    ROWID --> DOMAINID["domain::SecurityID<br/>used as Spark ID"]
+    IDTYPE -- Recorded or pre-keyed --> DOMAINID
+
+    DOMAINID --> DOMAINSEC["SecurityCommon::id = Spark ID"]
+    DOMAINSEC --> LISTENER["GeneralMarketDataListener::on_securities"]
+    LISTENER --> CCMAP["cc_sec_id_map<br/>incoming ID to Spark ID"]
+    LISTENER --> CONVERT["_copy_security_common<br/>FixedSecurity.key = ID<br/>numeric_id = ID"]
+
+    CONVERT --> PRODUCER["MDProducer::add_security"]
+    PRODUCER --> SETKEY["Immediate set_key<br/>no recorder timer"]
+    SETKEY --> SHM["Hot and cold shared memory<br/>immediately public"]
+    CONVERT --> GENERALINDEX["GeneralSecurityIndex<br/>ID to handle"]
+
+    FEED --> EVENT["Book, trade, or statistics<br/>with exchange/source identity"]
+    EVENT --> EVENTMAP["Adapter map<br/>source identity to domain::SecurityID"]
+    EVENTMAP --> EVENTID["Event carries domain::SecurityID"]
+    EVENT --> PYHOT["Python bridge only<br/>synchronous IDGenerator lookup"]
+    PYHOT --> EVENTID
+    PYHOT -. uses same mutex and SQLite path .-> MUTEX
+    EVENTID --> CCMAP
+    CCMAP --> GENERALINDEX
+    GENERALINDEX --> HANDLE["MDProducerHandle"]
+    HANDLE --> SHMUPDATE["Update book, trade,<br/>statistics, or status"]
+    SHM --> SHMUPDATE
+
+```
+
+In the normal newer live path, persistence occurs before the definition enters
+the generalized listener. The DB call therefore adds synchronous adapter-side
+latency, but publication needs no later timer. Most adapters use an in-memory
+map for subsequent events; the dotted Python path is the documented exception.
+
+## The publication boundary
+
+`MDProducerHandle::set_key()` is the definitive publication point for the
+legacy shared-memory representation. Its write ordering is intentional:
+
+1. Write the key into the hot book entry (`HotEntry` or `HotDepthEntry`).
+2. Write the key into the cold security entry.
+3. Increment the cold header's public `security_count`.
+
+The cold security key is written last because consumers treat a non-zero cold
+key as visibility. At that moment the corresponding hot entry must already be
+addressable. `set_key()` only permits the transition from `0` to one key; it is
+not a key-update mechanism.
+
+## Identifier representations
+
+The names are historically inconsistent. In particular, many protobuf fields
+named `security_id` contain the **Spark key**, not the exchange's ID.
+
+| Layer | Exchange/feed identifier | Spark identifier |
+|---|---|---|
+| MDP3 wire/OnixS | `securityId()` | not present |
+| Legacy fixed security | `FixedSecurity::numeric_id` (`uint64_t`) | `FixedSecurity::key` / `security_key` (`uint32_t`) |
+| Legacy API security | venue-specific `exchange_id` | top-level `Security.id()` |
+| New domain model | `SecurityCommon::exchange_sec_id` (string) is descriptive; `SecurityCommon::id` is also used to correlate source events | `domain::SecurityID` (`uint64_t`), currently copied into the Spark key |
+| Shared-memory hot book | none | `HotEntry::key` or `HotDepthEntry::key` |
+| Shared-memory cold metadata | `FixedSecurity::numeric_id` | `FixedSecurity::key` |
+| CME runtime lookup | `CMESecurityIndex` key | `SecurityData::handle.key()` |
+| General runtime lookup | incoming/canonical `domain::SecurityID` | `GeneralSecurityIndex` key and `handle.key()` (currently the same value) |
+
+The generalized model declares `domain::SecurityID` as 64-bit, but the legacy
+shared-memory `security_key` remains 32-bit. `_copy_security_common()` assigns a
+64-bit value to both `numeric_id` and the 32-bit `key`. This is safe only while
+allocated IDs fit in 32 bits; unlike the legacy lookup path, this assignment has
+no explicit checked conversion.
+
+## Which parts use exchange IDs, descriptors, or Spark security IDs?
+
+The three identifier classes serve different jobs:
+
+- **Exchange/external ID:** routes data inside a venue or vendor feed adapter.
+- **Descriptor/natural key:** recognizes a definition logically at the
+  persistence, discovery, or human-facing boundary.
+- **Spark security ID/key:** joins the instrument to books, orders, fills,
+  positions, and services after publication.
+
+The core downstream system does not repeatedly identify a security from its
+description. The description is primarily used to obtain a Spark key; after
+that boundary, most joins use the numeric Spark key.
+
+### Component-level usage matrix
+
+| Component or data path | Exchange/external ID | Descriptor or name | Spark security ID/key |
+|---|---|---|---|
+| CME MDP3 definition decoder | Reads `securityId()` and stores it in `FixedSecurity::numeric_id` | Copies definition symbol/tag 55 into `security_desc` | Leaves `FixedSecurity::key` at `0` initially |
+| `CMESecurityIndex` | Primary map key; CME ID resolves to a handle | Not used for update routing | Returns `handle.key()` when the Spark key has been assigned |
+| MDP3 books, trades, status, statistics, and RFQs | Incoming CME ID selects the handle through `CMESecurityIndex` | Not used to locate each update | The selected handle writes keyed shared-memory/trade data after publication |
+| MDP3 spread definitions | Each leg initially contains `legSecurityId()` as `exchange_key` | Parent `security_desc` identifies the spread's DB row | Each leg's Spark key and then the parent Spark key must be resolved before publication |
+| Legacy ICE OnixS | Uses ICE `marketId` as `numeric_id` and index key | ICE security description becomes the logical lookup name | Uses the same recorder-assigned Spark key pattern as MDP3 |
+| `SecurityIDRecorder` | Does not allocate from the external ID | Uses `(venue, security type, securities::descriptor())` to find/create a row | Treats the SQLite rowid as the resulting Spark key |
+| Native generalized adapters | Use exchange strings/numeric IDs to parse definitions and correlate updates | Usually pass a description or exchange instrument name to `IDGenerator` | Put the returned ID into `SecurityCommon::id` and later events |
+| Databento adapter | Uses `msg.hd.instrument_id` for feed correlation and its in-memory map | Uses the decimal instrument ID string—not the human description—as the `IDGenerator` name | Caches `instrument_id -> domain::SecurityID` and places that ID on books/trades |
+| Python bridge | Reads `exchange_security_id` from definitions, books, and trades | Uses that external-ID string as the `IDGenerator` name | Resolves it to `domain::SecurityID`; currently performs this lookup for every event |
+| `IDGenerator` | Does not interpret whether `name` came from an exchange ID | Uses exactly `(venue, type, caller-supplied name)` | Returns the SQLite rowid as a 64-bit ID |
+| `GeneralMarketDataListener` | Does not route on `exchange_sec_id` directly | Copies `sec.desc()` into fixed metadata; its disabled fallback allocator would use `sec.desc()` | Uses incoming `SecurityCommon::id`, maps events through `cc_sec_id_map`, and copies the value into both `key` and `numeric_id` |
+| `GeneralSecurityIndex` | No separate external-ID concept in the current use-given path | Not used | Maps the generalized ID/Spark key to `MDProducerHandle` |
+| `MDProducer` / shared memory | Retains `numeric_id`; in legacy feeds this is the exchange ID, but in the generalized path it is currently the Spark ID | Stores `security_desc` in the cold definition | Uses the key in hot/cold entries; non-zero key is the visibility boundary |
+| `MDConsumer` | Not normally used to find live books | Can enumerate/filter converted security definitions | Finds handles and book data by `security_key` |
+| `SecurityCache` / `securities.sqlite` | Preserves venue-specific exchange data inside the serialized security blob | Maintains descriptor/name lookup indexes and supports venue/type/name queries | Primary persisted definition key is `api::Security.id()` |
+| MBO processor and MBO service | May retain exchange **order** IDs, which are separate from security exchange IDs | Not used to find the instrument | Indexes snapshots and requests by `SecurityKey`; MBO protobuf `security_id` means Spark key |
+| Order manager and algorithms | Execution gateways eventually use venue-specific fields from the resolved security to encode orders | Descriptions appear in logs and errors, not as the primary join | Order definitions, subscriptions, book events, native orders, spread legs, and fills use `security_key` / `Security.id()` |
+| Position, PNL, and risk services | Not used as their position join | May resolve metadata for display/reporting | Positions, fills, overrides, and PNL handles are keyed by Spark security ID |
+| Relay and market-data recordings | Serialized security definitions may preserve venue exchange IDs | Preserve full descriptors in recorded definitions | Books, updates, trades, and replay correlation preserve the recorded Spark key |
+| Synthetic security service | No venue exchange ID; synthetics use `VENUE_BTS` | Synthetic name is the recorder's logical descriptor | Recorder assigns the SQLite rowid and synthetic metadata is keyed by it |
+
+### Exchange/external IDs: where they stop
+
+In MDP3 the separation is explicit:
+
+```text
+CME SecurityID
+    -> FixedSecurity.numeric_id
+    -> CMESecurityIndex lookup
+    -> MDProducerHandle
+    -> Spark security_key for downstream publication
+```
+
+The external CME ID continues to route feed messages but does not become the
+order/position key. This permits the same downstream Spark key to survive a
+restart as long as the logical descriptor mapping remains in
+`key_ids.sqlite`, even if the feed must rediscover the CME routing ID.
+
+The generalized path collapses more of these roles:
+
+```text
+exchange/vendor identity
+    -> adapter calls IDGenerator
+    -> domain::SecurityID
+    -> FixedSecurity.key and FixedSecurity.numeric_id
+    -> GeneralSecurityIndex and downstream Spark key
+```
+
+`SecurityCommon::exchange_sec_id` may still contain the original textual
+external identifier, but `GeneralMarketDataListener` does not use it to route
+events and does not put it in `numeric_id`. The adapter must translate later
+events to `domain::SecurityID` before invoking the listener.
+
+### Descriptor/name users
+
+Descriptors are used in three main places:
+
+1. **Key allocation:** `SecurityIDRecorder` uses the legacy venue-specific
+   descriptor, while `IDGenerator` uses its caller-provided `name`.
+2. **Definition lookup and discovery:** `SecurityCache` supports
+   venue/type/name lookup; symbol filters and administrative tools use security
+   metadata to select or display instruments.
+3. **Diagnostics:** logs include symbols and descriptions so humans can
+   understand a numeric key.
+
+Descriptions generally do not drive the high-rate book, trade, order, fill, or
+position paths. Those paths use numeric IDs after definition resolution.
+
+It is important that “descriptor” does not mean one universal canonical field:
+
+- Legacy CME uses Globex `security_desc` from tag 55.
+- Many generalized adapters use `sec.desc()` or an exchange instrument name.
+- Databento uses its numeric instrument ID converted to a string.
+- Python uses `exchange_security_id`.
+
+Therefore, two adapters may describe the same economic contract but request
+different logical rows and receive different Spark IDs.
+
+### Spark security-ID users
+
+Once a security is published, its Spark key is the dominant system identifier:
+
+```mermaid
+flowchart LR
+    KEY["Spark security key"] --> SHM["Hot/cold shared memory"]
+    KEY --> MDC["MDConsumer and book subscriptions"]
+    KEY --> MBO["MBO snapshots and queries"]
+    KEY --> OMS["Orders, algorithms, and spread legs"]
+    KEY --> FILLS["Fills and execution records"]
+    KEY --> POS["Positions, PNL, and risk"]
+    KEY --> API["Services and client APIs"]
+    KEY --> REC["Relay and market-data recordings"]
+    KEY --> SECDB["Serialized security definitions"]
+```
+
+Fields named `security_id`, `security_key`, or top-level `Security.id()` in
+these downstream areas almost always mean the Spark key. They should not be
+compared directly with CME `SecurityID`, Databento `instrument_id`, or another
+venue/vendor identifier without going through the relevant adapter/index
+mapping.
+
+## Persistence and allocation
+
+### The authoritative key mapping: `key_ids.sqlite`
+
+Both allocators use `SqliteStore("key_ids")`. With the normal configuration the
+file is:
+
+```text
+<SystemConfig.data_path>/databases/key_ids/key_ids.sqlite
+```
+
+The database contains one table and two indexes:
+
+```sql
+CREATE TABLE store (ns TEXT, key TEXT, value TEXT);
+CREATE UNIQUE INDEX key_index ON store(ns, key);
+CREATE INDEX ns_index ON store(ns);
+```
+
+The SQLite `rowid` is the generated Spark key. `ns` is normally
+`<VENUE>.<SECURITY_TYPE>` (for example `GLBX.FUTURE`), and `key` is the security
+descriptor chosen by the caller. `value` is normally empty. Therefore ID
+stability depends on all three of these remaining stable:
+
+- the configured `data_path` and preservation of this database;
+- the venue/type namespace;
+- the descriptor supplied by the feed adapter.
+
+The two allocator implementations share the file and convention:
+
+- `SecurityIDRecorder` (legacy): derives the namespace from legacy API enums and
+  uses `securities::descriptor()`. For CME this is the Globex
+  `security_desc`, populated from the definition's symbol/tag 55.
+- `IDGenerator` (newer): derives the namespace from new domain enums and uses
+  the caller's `name`. Most adapters pass `sec.desc()` or an exchange instrument
+  identifier; Databento deliberately passes the decimal instrument ID, and the
+  Python bridge uses the exchange security ID.
+
+### Logical-row examples
+
+The following names are illustrative. Production uses the exact string emitted
+by the selected feed/adapter, including its case, spacing, and weekly-series
+notation.
+
+#### Legacy `SecurityIDRecorder`
+
+For CME, `set_security_common()` copies the MDP3 definition's symbol/tag 55 into
+Globex `security_desc`. After converting the fixed security to
+`api::Security`, `securities::descriptor()` returns that same
+`glbx.security_desc()` string.
+
+| Instrument | Legacy enums | Example descriptor from `security_desc` | Row searched/inserted in `store` |
+|---|---|---|---|
+| Future | `VENUE_GLBX`, `SECURITYTYPE_FUTURE` | `ESZ6` | `ns='GLBX.FUTURE', key='ESZ6'` |
+| Standard option | `VENUE_GLBX`, `SECURITYTYPE_OPTION` | `ESZ6 C6500` | `ns='GLBX.OPTION', key='ESZ6 C6500'` |
+| Weekly option | `VENUE_GLBX`, `SECURITYTYPE_OPTION` | `ESZ6-W1 C6500` | `ns='GLBX.OPTION', key='ESZ6-W1 C6500'` |
+| Spot from a legacy-compatible venue path | `VENUE_COINBASE`, `SECURITYTYPE_SPOT` | `BTC-USD` | `ns='COINBASE.SPOT', key='BTC-USD'` |
+
+A weekly option does not get a special namespace. Standard and weekly options
+are both `SECURITYTYPE_OPTION`, so both live in `GLBX.OPTION`; the exact
+`security_desc` distinguishes the weekly series. For example, if the database
+assigned rowids `42001` through `42004` to the rows above, those integers—not
+the descriptor strings—would be the Spark security keys.
+
+Conceptually, legacy allocation is:
+
+```text
+future:        lookup_or_insert("GLBX.FUTURE",  "ESZ6")
+option:        lookup_or_insert("GLBX.OPTION",  "ESZ6 C6500")
+weekly option: lookup_or_insert("GLBX.OPTION",  "ESZ6-W1 C6500")
+spot:          lookup_or_insert("COINBASE.SPOT", "BTC-USD")
+```
+
+#### Newer `IDGenerator`
+
+The namespace result is similar, but the third argument is chosen by the
+adapter. Examples using a description-oriented adapter are:
+
+```cpp
+get_or_add(VENUE_GLBX, SECURITY_TYPE_FUTURE, "ESZ6");
+// store: ns="GLBX.FUTURE", key="ESZ6"
+
+get_or_add(VENUE_GLBX, SECURITY_TYPE_OPTION, "ESZ6 C6500");
+// store: ns="GLBX.OPTION", key="ESZ6 C6500"
+
+get_or_add(VENUE_GLBX, SECURITY_TYPE_OPTION, "ESZ6-W1 C6500");
+// store: ns="GLBX.OPTION", key="ESZ6-W1 C6500"
+
+get_or_add(VENUE_COINBASE, SECURITY_TYPE_SPOT, "BTC-USD");
+// store: ns="COINBASE.SPOT", key="BTC-USD"
+```
+
+Databento intentionally produces a different logical row even when the human
+description is `ESZ6`. If its instrument ID were `12345678`, it would call the
+equivalent of:
+
+```cpp
+get_or_add(VENUE_GLBX, SECURITY_TYPE_FUTURE, "12345678");
+// store: ns="GLBX.FUTURE", key="12345678"
+```
+
+Likewise, the Python bridge uses `exchange_security_id`. A spot definition with
+`exchange_security_id="BTC-USD"` produces the `COINBASE.SPOT / BTC-USD` row,
+while a provider that sends `exchange_security_id="BTCUSD"` produces a
+different row and therefore a different Spark key.
+
+These examples show why the tuple is only logical when the adapter's naming
+contract is also known. `GLBX.FUTURE / ESZ6` and `GLBX.FUTURE / 12345678` may
+describe the same contract to a human but are distinct rows to `key_ids.sqlite`.
+
+Both enum-prefix removals currently produce the same namespace spelling (for
+example, `GLBX.OPTION`), so both mechanisms allocate from one global SQLite
+rowid space. Live rows cannot share a rowid. The table does not declare
+`AUTOINCREMENT`, however, so deleting rows—especially the highest row—can allow
+SQLite to reuse a number later. Security-key rows should therefore be treated
+as append-only operational data.
+
+`SqliteStore` uses WAL mode and `PRAGMA synchronous = 0`. This favors startup
+and feed latency over maximum durability against a host/power failure. The
+unique `(ns, key)` index prevents two different rows for the same logical
+security.
+
+### Other places the key is stored
+
+| Location | Purpose | Authoritative for allocation? |
+|---|---|---|
+| Hot and cold shared-memory segments | Live publication and fast consumer lookup | No; rebuilt at process startup |
+| `CMESecurityIndex` / `GeneralSecurityIndex` and adapter maps | In-process routing from an incoming ID to a producer handle | No; in-memory only |
+| `<data_path>/databases/md/securities.sqlite` | Serialized legacy `api::Security` definitions keyed by Spark key; used by security cache and MDP3 Sunday preload | No; it stores metadata after a key exists |
+| Relay/market-data recordings | Preserve the key embedded in recorded securities, books, and trades for deterministic replay | No; replay normally reuses the recorded value |
+| Orders, fills, positions, APIs | Reference a security by the published Spark key | No |
+
+The `SecurityIDRecorder::work_db()` comment says a published security is also
+recorded in the security database, but that function does **not** write
+`securities.sqlite` in the current code. Full security definitions are persisted
+separately by `SecurityCache::flush_to_disk()` through
+`db::insert_or_replace()`.
+
+## CME MDP3 / OnixS pipeline in detail
+
+### 1. Construction and thread setup
+
+`MDP3Main::PImpl` owns one `CMESecurityIndex`, one `SecurityIDRecorder`, and one
+`MDProducer`. It constructs the recorder with:
+
+- a lookup callback that maps `ExchangeKey` through the CME index to the
+  handle's current Spark key; and
+- `Dispatcher::current()`, which drives a 200 ms recorder timer.
+
+In the current `MarketDataGateway` route, `OnixsGlbxSubscription` creates a
+dedicated `dispatcher_thread`, constructs and starts `MDP3Main` on that thread,
+then calls that thread's dispatcher `run()`. The older `md run onixs` command
+uses its command/dispatcher thread similarly.
+
+The OnixS feed engine has its own configured thread pool. Instrument and market
+data callbacks therefore originate on OnixS feed-engine threads, while recorder
+database work and final key assignment execute on the MDP3 dispatcher thread.
+
+| MDP3 stage | Executing thread | DB call? |
+|---|---|---|
+| Decode definition and set CME `numeric_id` | OnixS feed-engine callback thread | No |
+| Allocate hot/cold entry and queue handle | Same OnixS callback thread | No |
+| Add CME-ID index entry | Same OnixS callback thread | No |
+| Resolve spread legs | MDP3 dispatcher thread, on 200 ms timer | No |
+| Select/insert Spark key | MDP3 dispatcher thread, on 200 ms timer | Yes |
+| `handle.set_key()` and public visibility | MDP3 dispatcher thread, in DB callback | No additional call |
+
+### 2. Definition callback: exchange ID is set, Spark key is not
+
+For futures, options, and spreads, `MDP3Listener::set_security_common()` does:
+
+```cpp
+fsec.numeric_id = def.securityId();
+```
+
+It also copies symbol/descriptor/venue and pricing fields. A newly constructed
+`FixedSecurity` already has `key == invalid_security_key` (`0`), and this helper
+does not change it.
+
+For a spread, every leg starts with:
+
+```cpp
+leg.exchange_key = ExchangeKey(definition_leg.legSecurityId());
+leg.key = invalid_security_key;
+```
+
+Thus both the parent and legs initially carry exchange-routing identity, but no
+Spark identity.
+
+### 3. Shared-memory allocation and exchange indexing
+
+`MDP3Listener::add_security()` performs these operations in order:
+
+1. `_producer.add_security(security)` allocates a hot entry and a variable-sized
+   cold entry and copies the security into the cold segment.
+2. Because the key is `0`, `MDProducer` queues the handle through
+   `SecurityIDRecorder::ensure_key()`.
+3. `_index.add_security(security.numeric_id, handle)` immediately creates the
+   CME-ID-to-handle mapping.
+4. The initial exchange status is written through the handle.
+
+This ordering means incoming books can already be routed by CME ID while the
+Spark key is pending. Shared-memory consumers cannot discover the security yet.
+Also, `MDProducerHandle::add_trade()` only appends to the public trade queue
+when `key() != 0`, so a trade received in this short window updates the cold
+last-trade state but is not added to the keyed trade log.
+
+### 4. Recorder tick: spreads, handles, then database
+
+Every 200 ms the recorder calls, in this exact order:
+
+1. `work_spreads()` tries to fill previously unresolved spread leg keys by
+   asking `CMESecurityIndex` for each leg exchange ID.
+2. `work_handles()` drains new handles. Outrights are converted to an API
+   security and queued for DB lookup immediately. A spread without all leg keys
+   is retained in `pending_handle` and its leg pointers are registered in
+   `unresolved_legs`.
+3. `work_db()` resolves queued `(venue, type, descriptor)` requests.
+
+Because spread work precedes new-handle work, a newly seen unresolved spread
+normally waits until at least the next timer tick to resolve legs. It waits
+longer if any outright leg has not yet received its Spark key. A spread's own
+key is not requested until every leg key is non-zero.
+
+### 5. Lookup or generation
+
+`work_db()` first queries `(ns, key)`:
+
+- Existing row: use its `rowid` as the key and invoke the callback.
+- Missing row: batch it into `insert_many_resolve_ids()`. SQLite assigns a new
+  rowid inside a transaction, the row object receives that ID, and the callback
+  runs after the transaction completes.
+
+New inserts are throttled to roughly 200 per timer pass (the current `> 200`
+check can admit 201) to avoid delaying heartbeats during startup. The queues are
+TBB concurrent queues, but all SQLite access is serialized onto the recorder's
+dispatcher thread. `check_access()` asserts this single-thread rule after the
+first work pass records the dispatcher thread ID.
+
+### 6. Final assignment and visibility
+
+The queued callback captures a copy of `MDProducerHandle` and calls
+`handle.set_key(id)` on the dispatcher thread. `set_key()` writes hot, then cold,
+then increments `security_count`. From this point:
+
+- shared-memory scans report the security;
+- `CMESecurityIndex::get_key(CME_ID)` returns the Spark key;
+- trades and RFQs can be emitted with a non-zero key;
+- resolved spreads can copy this key into their leg structures.
+
+### 7. Sunday preload is a fast path
+
+For configured symbols on Sunday, `MDP3Main::prepare_listener()` loads stored
+definitions from `securities.sqlite`. It converts them with
+`set_security_key=true`, so both the security key and spread leg keys come from
+the serialized API security. `MDProducer::add_security()` calls `set_key()`
+immediately, and no recorder lookup is required for that preloaded handle. The
+CME exchange ID is still restored into `numeric_id`, allowing the normal CME
+index to be built.
+
+## Generalized/newer pipeline in detail
+
+### 1. Topology and ownership
+
+`Run::run()` starts a detached thread named `MarketDataGateway`. On that thread,
+`run_md_gwy()` obtains the process-wide `IDGenerator`, constructs
+`MarketDataGateway`, and starts the configured providers. For each `(venue,
+source)` returned by a provider, the gateway creates an `MdSink` containing:
+
+- a venue-local `GeneralSecurityIndex`;
+- a default `MDProducer` (not wired to a `SecurityIDRecorder`); and
+- a `GeneralMarketDataListener` connecting the source to that producer/index.
+
+The source decides its callback thread. Some sources call the listener while
+starting; others own network or replay threads. The generalized listener calls
+the producer inline on that source thread. The process-wide `IDGenerator`
+serializes its own calls with a mutex, but the listener's transient
+`cc_sec_id_map` is an ordinary `unordered_map`, so each source/listener pair is
+expected to serialize security/event callbacks.
+
+| Generalized stage | Executing thread | DB call? |
+|---|---|---|
+| Build/parse domain definition | Source/adaptor thread | No |
+| `IDGenerator::get_or_add()` | Same source/adaptor thread | Yes, synchronously |
+| `GeneralMarketDataListener::on_securities()` | Same source callback thread | No in current use-given mode |
+| Copy ID to `FixedSecurity::key` and `numeric_id` | Same source callback thread | No |
+| `MDProducer::add_security()` / `set_key()` | Same source callback thread | No additional call |
+| Add `GeneralSecurityIndex` entry | Same source callback thread | No |
+
+There are two important compatibility exceptions inside the provider framework:
+
+- `OnixsGlbxSubscription::start()` ignores the supplied generalized listener
+  and starts `MDP3Main`; CME/OnixS therefore still follows the asynchronous
+  legacy flow documented above.
+- `EmumbaDeribitSubscription::start()` also ignores that listener and uses
+  `EmMDListener`, which allocates via `IDGenerator` and writes through its own
+  producer/index. Its assignment is still synchronous and pre-populated, so it
+  resembles the newer ID flow even though it does not pass through
+  `GeneralMarketDataListener`.
+
+### 2. How a new ID is constructed
+
+For a normal live generalized adapter, ID creation happens **in the adapter,
+before** `GeneralMarketDataListener::on_securities()`:
+
+```text
+logical identity = (domain venue, domain security type, adapter-chosen name)
+Spark ID         = SQLite rowid for that identity in key_ids.sqlite
+```
+
+`IDGenerator::get_or_add(venue, type, name)` formats a namespace by stripping
+the enum prefixes:
+
+```text
+VENUE_TAIFEX + SECURITY_TYPE_FUTURE -> TAIFEX.FUTURE
+```
+
+It then calls `retrieve_one(ns, name)`. If the row exists, it returns that
+row's `rowid`. If absent, it inserts `(ns, name, "")`, and SQLite assigns the
+next rowid. That integer is wrapped in `domain::SecurityID` and placed in
+`SecurityCommon::id`.
+
+The third identity component is adapter-specific, and this matters when trying
+to reproduce a mapping:
+
+| Source/adaptor family | Typical `name` sent to `IDGenerator` |
+|---|---|
+| Binance, Coinbase, Deribit, OKX and similar native/CCAPI paths | Usually `sec.desc()` / exchange instrument name |
+| Taifex | product/instrument ID string |
+| Databento | `std::to_string(msg.hd.instrument_id)`, not the human-readable description |
+| Python bridge | `exchange_security_id` (and it asserts Python supplied ID is zero for definitions) |
+| Emumba Deribit | extracted Emumba security descriptor |
+| Replay | no new allocation for a recorded definition; it reuses the recorded `SecurityCommon::id` |
+
+So “security name” alone is not a universal logical key. Reproduction requires
+the exact venue enum, exact security-type enum, and the exact adapter-specific
+name used at insertion time.
+
+### 3. What `GeneralMarketDataListener` does with that ID
+
+`on_securities()` first calls `_find_sec(sec, true)`. Its current, compile-time
+behavior is controlled by:
+
+```cpp
+const bool use_given_sec_key = true;
+```
+
+Consequently, for a newly seen incoming `domain::SecurityID` it does **not**
+call the DB. It copies `security_common.id.value()` into its transient
+`cc_sec_id_map`. `_add_future()`, `_add_option()`, `_add_spread()`, or
+`_add_spot()` then calls `_copy_security_common()`, which sets:
+
+```cpp
+to.key = incoming_id;
+to.numeric_id = incoming_id;
+```
+
+This collapses the generalized source/correlation ID and Spark shared-memory
+key into the same number. The source's text `exchange_sec_id` remains on the
+domain security but is not copied into `FixedSecurity::numeric_id` by this
+path.
+
+`_commit_security()` then:
+
+1. calls `MDProducer::add_security()` with an already non-zero key;
+2. causes `MDProducerHandle::set_key()` to run immediately during that call;
+3. inserts the handle in `GeneralSecurityIndex` under `numeric_id` (the same
+   number); and
+4. returns with the security already visible in shared memory.
+
+Because the sink's producer has no recorder, supplying `id == 0` would allocate
+the shared-memory entry but never assign or publish its key. Generalized live
+sources must therefore provide a valid non-zero ID before `on_securities()`.
+
+The listener contains an alternative branch for `use_given_sec_key == false`.
+That branch would call `IDGenerator::get_or_add(me->venue, sec.type(),
+sec.desc())` itself. It is currently disabled, and `_generate_security_key()` is
+also effectively unused. Turning the flag off is not a mechanical change: an
+adapter such as Databento allocates using numeric instrument ID while the
+listener branch would use `sec.desc()`, potentially creating a second key for
+the same instrument.
+
+There is also a spread failure edge case. `_find_sec(sec, true)` records the
+parent ID in `cc_sec_id_map` before `_add_spread()` resolves its legs. If any leg
+ID is absent, `_add_spread()` logs and returns without committing the spread. A
+later repeat of the parent definition is then classified as a duplicate and
+ignored, so definition ordering is a hard requirement in the current code.
+
+### 4. How later books, trades, and statistics find the security
+
+Every later domain event carries a `domain::SecurityID`:
+
+- `Book::security_id()`;
+- `TradeRecord::sec_id`; or
+- `SecurityMarketDataStatistics::security_id`.
+
+The listener first uses `cc_sec_id_map` to translate that event ID to the value
+stored in `GeneralSecurityIndex`, then obtains the `MDProducerHandle`. With
+`use_given_sec_key == true`, this mapping is numerically identity-to-identity,
+but it still serves as the admission check that the definition was seen.
+
+Most high-throughput adapters avoid SQLite on market-data events:
+
+- Databento allocates on the definition and caches
+  `instrument_id -> domain::SecurityID` in `sec_id_map` for books/trades.
+- Taifex keeps the resolved ID in its security-details mapper.
+- Other native subscriptions maintain comparable descriptor/ID maps.
+
+The Python bridge is a notable exception: `to_bts_security_()`,
+`to_bts_trade_()`, and `to_book_()` all call `get_sec_id_()`, which calls
+`IDGenerator::get_id(..., add_if_new=true)`. As written, that performs at least
+one SQLite `SELECT` for each converted Python definition, trade, or book. It
+returns the existing ID after the first insert, but it is still a DB call and
+takes the global `IDGenerator` mutex on every event.
+
+### 5. Persistence timing and latency
+
+The newer allocator is synchronous:
+
+- Existing security: mutex acquisition plus one SQLite `SELECT` on the caller's
+  thread.
+- New security: mutex acquisition, one `SELECT`, then one SQLite `INSERT`
+  (SQLite autocommit), also on the caller's thread.
+- Listener publication: no further DB call under the current
+  `use_given_sec_key` setting.
+
+The ID therefore exists in `key_ids.sqlite` before the adapter sends the
+definition to the listener. There is no intentional 200 ms publication delay,
+but the source callback/startup path directly pays DB and mutex latency. Bulk
+definition discovery performs one lookup and possibly one insert per security;
+`IDGenerator` does not batch those operations.
+
+WAL mode helps concurrency, while `synchronous=0` reduces fsync cost. It does
+not eliminate SQL parsing/stepping, mutex contention, page-cache misses, or
+occasional file-system work. The impact is usually a startup/definition cost
+when adapters cache the result, but it becomes hot-path latency for code such as
+the current Python bridge that resolves every event through the DB.
+
+`GeneralMarketDataListener` itself does not persist the full contract definition
+to `securities.sqlite`; that remains the separate security-cache/metadata
+persistence path described earlier.
+
+## Comparison: MDP3 recorder vs generalized allocator
+
+| Property | MDP3 / `SecurityIDRecorder` | Generalized / `IDGenerator` |
+|---|---|---|
+| Logical input | venue + type + legacy `security_desc` | venue + type + adapter-chosen name |
+| Numeric output | SQLite rowid, checked/read as 32-bit `security_key` | SQLite rowid returned as 64-bit `domain::SecurityID`, then narrowed into 32-bit shared-memory key |
+| Allocation point | After definition is copied to shared memory | Normally inside adapter before listener sees definition |
+| DB execution | Asynchronous on dispatcher timer | Synchronous on caller/source thread |
+| Existing ID cost | One deferred `SELECT` | One immediate `SELECT` per call |
+| New ID cost | Deferred, batched transaction (about 200/pass) | Immediate `SELECT` + individual autocommit `INSERT` |
+| Time until public | Usually 0–200+ ms; spreads may take extra ticks | Immediate after synchronous lookup/insert completes |
+| Feed callback blocking | Queue push only for ID work | Yes, if the adapter invokes `IDGenerator` there |
+| Spread legs | Parent waits asynchronously until all leg keys resolve by exchange ID | Definition processing requires every leg's incoming ID already exist in `cc_sec_id_map`; otherwise the spread is aborted |
+| Runtime ID relationship | Exchange ID and Spark key remain distinct | Incoming/canonical ID, `numeric_id`, index ID, and Spark key are currently the same number |
+| Persistence database | `key_ids.sqlite` | The same `key_ids.sqlite` |
+| Batching | Yes for new rows | No |
+| Failure behavior | Security has an internal handle with key 0 while normal queued work is pending; DB work does not block the feed callback | DB errors/zero results occur in the synchronous source path; a zero ID can leave an unpublishable entry, with no pending-key state machine |
+
+Neither algorithm generates a UUID or a content hash. Both ultimately use an
+opaque, database-assigned SQLite rowid. It is normally increasing and looks like
+a sequence, but this schema does not request SQLite `AUTOINCREMENT`. The tuple
+used to *look up* the row is logical; the resulting number is not derivable from
+that tuple.
+
+## Can another system reconstruct the same keys?
+
+Not from `(venue, type, name)` alone. Another system can reconstruct the
+**logical lookup tuple**, but it cannot calculate the numeric Spark key because
+the key depends on the historical insertion order of every row in
+`key_ids.sqlite`.
+
+For example, two empty systems can insert the exact same `GLBX.FUTURE / ESZ6`
+row and receive different IDs if other securities were inserted before it on
+one system. They are sequential opaque IDs, not deterministic hashes and not
+UUIDs.
+
+Ways to preserve cross-system identity are:
+
+1. Copy/replicate `key_ids.sqlite` (including rowids) and keep descriptor rules
+   identical. This preserves existing numeric keys but requires coordinated
+   allocation for new ones.
+2. Export an explicit mapping of `(namespace, adapter key) -> Spark key`, then
+   import rows with their original rowids before either system allocates new
+   securities.
+3. Treat a stable external identity (venue + type + exchange instrument ID, or
+   another canonical instrument identifier) as the cross-system key and carry a
+   translation table to each system's local Spark key.
+4. Introduce a deterministic ID scheme only as a versioned migration. A hash can
+   be computed independently, but collision handling, descriptor
+   canonicalization, aliases/renames, the existing 32-bit ABI, and compatibility
+   with orders/recordings/databases all need explicit design.
+
+Copying only `securities.sqlite` is insufficient as an allocator strategy. It
+contains security blobs keyed by already-assigned IDs, but `key_ids.sqlite` is
+the mapping used to find or allocate those IDs. Conversely, copying only
+`key_ids.sqlite` preserves the numbers but not all contract metadata.
+
+## Other assignment paths in `sparkd`
+
+The two main pipelines reduce to four ways a producer handle can acquire a key:
+
+1. **Already keyed input.** `MDProducer::add_security()` sees a non-zero
+   `FixedSecurity::key` and calls `set_key()` immediately. Generalized feeds,
+   Sunday preload, relay playback, and other replay paths use this mechanism.
+2. **Recorder-backed live input.** The input key is zero and the producer owns a
+   `SecurityIDRecorder`; `ensure_key()` queues asynchronous assignment. CME
+   MDP3 and legacy OnixS ICE use this mechanism. ICE uses `marketId` as
+   `numeric_id` and otherwise follows the same recorder/index design.
+3. **Service assignment.** `SyntheticSecurityService` calls
+   `SecurityIDRecorder::assign_key()` directly on a background dispatcher. The
+   callback receives a copied `api::Security` with its top-level `id` set, then
+   publishes it to synthetic subscribers. It uses the same `key_ids.sqlite`
+   rowid allocator.
+4. **Unkeyed input with no recorder.** The entry is allocated with key zero and
+   never becomes public. This is not a valid production assignment path, but it
+   is the failure mode if a generalized source omits its ID.
+
+Relay and replay deserve care when reasoning about persistence:
+
+- The generalized `MDReplayer` forwards the recorded domain security ID and
+  emits later books/statistics with the recorded `sec_key`, so the listener's
+  current use-given mode preserves the recording's numeric identity.
+- The legacy relay producer converts a recorded `api::Security` with
+  `set_security_key=true`, so its shared-memory publication also preserves the
+  embedded key.
+- The legacy relay recorder additionally calls `bulk_insert(..., "REPLAY: ")`
+  for logical mappings. Those rows use a `REPLAY: ` namespace prefix; this is a
+  separate mapping population step, not a deterministic derivation of the key.
+
+## Operational implications
+
+- Back up `key_ids.sqlite` and `securities.sqlite` together if numeric ID and
+  definition continuity matter.
+- Do not delete/rebuild `key_ids.sqlite` on one host and assume IDs will match
+  another host, even when all instruments are identical.
+- Before moving a feed between adapters, compare the exact `name` each adapter
+  supplies to `IDGenerator`. Matching venue/type is insufficient.
+- Cache generated IDs in the source after definition processing. Avoid calling
+  `IDGenerator` from book/trade hot paths; the Python bridge currently violates
+  this guideline.
+- Keep generated rowids below the 32-bit shared-memory ceiling until the
+  `security_key` ABI is widened or explicit bounds checking is added to the
+  generalized path.
+- Ensure spread legs are announced before generalized spread parents.
+- Treat `synchronous=0` as a durability decision. A copied DB should be taken
+  with SQLite-aware backup/checkpoint handling rather than by casually copying
+  active WAL files.
+
+Useful inspection queries (run against the configured DB path) are:
+
+```sql
+SELECT rowid AS spark_key, ns, key
+FROM store
+ORDER BY rowid;
+
+SELECT rowid AS spark_key, ns, key
+FROM store
+WHERE ns = 'GLBX.FUTURE' AND key = '<exact-adapter-key>';
+```
+
+## Source map
+
+- [`SecurityIDRecorder.cpp`](../src/lib_spark/spark/md/SecurityIDRecorder.cpp):
+  legacy queues, 200 ms work ordering, logical row construction, DB callbacks,
+  and spread-leg resolution.
+- [`IDGenerator.cpp`](../src/lib_spark/spark/persistence/IDGenerator.cpp):
+  synchronous generalized lookup/allocation and process-wide mutex.
+- [`SqliteStore.cpp`](../src/lib_spark/spark/persistence/SqliteStore.cpp): DB
+  location, schema, WAL/synchronous pragmas, individual inserts, and batch
+  inserts.
+- [`MDProducer.cpp`](../src/lib_spark/spark/md/MDProducer.cpp) and
+  [`MDProducerHandle.cpp`](../src/lib_spark/spark/md/MDProducerHandle.cpp):
+  shared-memory allocation, recorder handoff, and ordered publication.
+- [`FixedSecurity.h`](../src/lib_spark/spark/md/Shm/FixedSecurity.h) and
+  [`MDStructure.h`](../src/lib_spark/spark/md/Shm/MDStructure.h): identifier
+  widths and hot/cold storage fields.
+- [`MDP3Main.cpp`](../src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp)
+  and
+  [`MDP3Listener.h`](../src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h):
+  MDP3 construction, definition callbacks, CME ID extraction, and spread legs.
+- [`MDP3Listener.cpp`](../src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp):
+  producer/index insertion order.
+- [`OnixsGlbxSubscription.cpp`](../src/app_sparkd/sparkd/md/glbx/OnixsGlbxSubscription.cpp):
+  current dedicated MDP3 dispatcher-thread ownership.
+- [`MarketDataGateway.cpp`](../src/app_sparkd/sparkd/md/MarketDataGateway.cpp)
+  and
+  [`GeneralMarketDataListener.cpp`](../src/lib_spark/spark/md/GeneralMarketDataListener.cpp):
+  generalized source/sink topology, incoming-ID mapping, fixed-security
+  conversion, and publication.
+- [`DatabentoDataHandler.cpp`](../src/app_sparkd/sparkd/md/databento/DatabentoDataHandler.cpp),
+  [`TaifexDataProcessors.h`](../src/app_sparkd/sparkd/md/taifex/TaifexDataProcessors.h),
+  and
+  [`PyMarketDataSubscription.cpp`](../src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp):
+  representative adapter-side allocation and caching behavior.
+- [`SecurityDB.cpp`](../src/lib_spark/spark/md/SecurityDB.cpp) and
+  [`SecurityCache.cpp`](../src/lib_spark/spark/md/SecurityCache.cpp): separate
+  persistence of full security definitions.
+- [`SecurityIndex.h`](../src/lib_spark/spark/md/SecurityIndex.h),
+  [`MBOServiceServer.cpp`](../src/lib_spark/spark/md/MBOServiceServer.cpp), and
+  [`MDConsumer.cpp`](../src/lib_spark/spark/md/MDConsumer.cpp): exchange-index
+  translation followed by Spark-keyed market-data consumption and MBO access.
+- [`OrderFactory.cpp`](../src/lib_spark/spark/orderman/order_factory/OrderFactory.cpp),
+  [`FillStore.cpp`](../src/lib_spark/spark/orderman/actors/FillStore.cpp), and
+  [`PositionMap.cpp`](../src/app_sparkd/sparkd/position/PositionMap.cpp):
+  representative downstream use of Spark security keys.
+- [`MDReplayer.cpp`](../src/app_sparkd/sparkd/md/recording/MDReplayer.cpp) and
+  [`RelayClient.cpp`](../src/lib_spark/spark/md/relay/RelayClient.cpp): recorded
+  ID reuse.

docker/images/bluetradingsystems/linux_dotnet8/Dockerfile

@@ -8,6 +8,7 @@ RUN apt-get update && apt-get install -y ca-certificates
 RUN apt-get update && apt-get install -y libssh2-1-dev
 RUN apt-get update && apt-get install -y software-properties-common
 RUN apt-get update && apt-get install -y sudo
+RUN apt-get update && apt-get install -y tzdata
 
 # GCC
 ARG gcc_major=13
@@ -18,6 +19,11 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
 RUN apt-get update \
   && apt-get install -y dotnet8
 
+RUN apt-get update && apt-get install -y gdb
+RUN apt-get update && apt-get install -y fish
+RUN apt-get update && apt-get install -y vim
+RUN apt-get update && apt-get install -y less
+
 RUN locale-gen en_US.UTF-8
 
 ARG docker_user="bts"

docker/images/bluetradingsystems/linux_shell/Dockerfile

@@ -0,0 +1,11 @@
+FROM ubuntu:24.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y \
+  fish \
+  less \
+  vim
+
+USER bts
+WORKDIR /home/bts

spark_client.sln

@@ -1,433 +1,306 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 17
-VisualStudioVersion = 17.3.32922.545
-MinimumVisualStudioVersion = 15.0.26124.0
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{ECD78CC5-138F-4809-A764-2358C2AC2E82}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{51C40BA9-F226-4A35-939D-1ED6F5FF7299}"
-EndProject
+VisualStudioVersion = 17.0.31903.59
+MinimumVisualStudioVersion = 10.0.40219.1
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{684863A1-3396-4D31-A731-C0725CF07F4E}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{45292808-4EF3-4D4A-B3F5-B75C42EE5870}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\Bts.Net.csproj", "{7543386A-5EBA-4AF0-9E0F-35657B58557D}"
+EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\Bts.Net.csproj", "{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{90A48718-F6A7-4747-97EA-89632B451008}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{F7C6F079-67BA-4899-9B43-D3F8C121D264}"
 EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{1A5B8C5D-BADC-457E-819F-0050C44FE4D3}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{2A0FBDE0-D870-4567-85BD-F7793A5862C6}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{ECBAA198-EC93-4174-8156-BDA5F5E95042}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{B6070F41-8759-448B-B56D-A7D15D489569}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{B2D721AB-920C-4CD6-8DBF-6778A115A449}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "client", "client", "{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "spark", "spark", "{42C014A0-00C5-0620-8552-38912E891A7F}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark", "src\client\spark\Spark.csproj", "{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{2B8243FF-60C6-4607-8AAE-7762D29B0A48}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
-		Debug|x64 = Debug|x64
 		Debug|Any CPU = Debug|Any CPU
+		Debug|x64 = Debug|x64
 		Debug|x86 = Debug|x86
-		MinSizeRel|x64 = MinSizeRel|x64
-		MinSizeRel|Any CPU = MinSizeRel|Any CPU
-		MinSizeRel|x86 = MinSizeRel|x86
-		Release|x64 = Release|x64
 		Release|Any CPU = Release|Any CPU
+		Release|x64 = Release|x64
 		Release|x86 = Release|x86
-		RelWithDebInfo|x64 = RelWithDebInfo|x64
-		RelWithDebInfo|Any CPU = RelWithDebInfo|Any CPU
-		RelWithDebInfo|x86 = RelWithDebInfo|x86
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x86.Build.0 = Debug|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|Any CPU.Build.0 = Release|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x86.ActiveCfg = Release|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x86.Build.0 = Release|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x86.Build.0 = Debug|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|Any CPU.Build.0 = Release|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x86.ActiveCfg = Release|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x86.Build.0 = Release|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.ActiveCfg = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.Build.0 = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x86.Build.0 = Debug|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.Build.0 = Debug|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.ActiveCfg = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.Build.0 = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|Any CPU.Build.0 = Release|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x86.ActiveCfg = Release|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x86.Build.0 = Release|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x86.Build.0 = Debug|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|Any CPU.Build.0 = Release|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x86.ActiveCfg = Release|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x86.Build.0 = Release|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.ActiveCfg = Debug|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.Build.0 = Debug|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x86.Build.0 = Debug|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|Any CPU.Build.0 = Release|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x86.ActiveCfg = Release|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x86.Build.0 = Release|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.ActiveCfg = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.Build.0 = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x86.Build.0 = Debug|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.Build.0 = Debug|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.ActiveCfg = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.Build.0 = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|Any CPU.Build.0 = Release|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x86.ActiveCfg = Release|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x86.Build.0 = Release|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x86.Build.0 = Debug|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|Any CPU.Build.0 = Release|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x86.ActiveCfg = Release|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x86.Build.0 = Release|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x86.Build.0 = Debug|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|Any CPU.Build.0 = Release|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x86.ActiveCfg = Release|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x86.Build.0 = Release|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x86.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.Build.0 = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|Any CPU.Build.0 = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x86.ActiveCfg = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x86.Build.0 = Release|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x86.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.ActiveCfg = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.Build.0 = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|Any CPU.Build.0 = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x86.ActiveCfg = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x86.Build.0 = Release|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x86.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.ActiveCfg = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.Build.0 = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|Any CPU.Build.0 = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x86.ActiveCfg = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x86.Build.0 = Release|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x86.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.ActiveCfg = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.Build.0 = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|Any CPU.Build.0 = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x86.ActiveCfg = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x86.Build.0 = Release|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
-		{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|x64.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|x86.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|x64.ActiveCfg = Release|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|x64.Build.0 = Release|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|Any CPU.Build.0 = Release|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|x86.ActiveCfg = Release|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|x86.Build.0 = Release|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Debug|x64.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Debug|x86.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Release|x64.ActiveCfg = Release|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Release|x64.Build.0 = Release|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Release|Any CPU.Build.0 = Release|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Release|x86.ActiveCfg = Release|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.Release|x86.Build.0 = Release|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|x64.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|x86.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|x64.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|x86.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|x64.ActiveCfg = Release|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|x64.Build.0 = Release|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|Any CPU.Build.0 = Release|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|x86.ActiveCfg = Release|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|x86.Build.0 = Release|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|Any CPU.Build.0 = Debug|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x64.ActiveCfg = Debug|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x64.Build.0 = Debug|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x86.ActiveCfg = Debug|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x86.Build.0 = Debug|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|Any CPU.ActiveCfg = Release|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|Any CPU.Build.0 = Release|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x64.ActiveCfg = Release|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x64.Build.0 = Release|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x86.ActiveCfg = Release|x64
+		{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x86.Build.0 = Release|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|Any CPU.Build.0 = Debug|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x64.ActiveCfg = Debug|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x64.Build.0 = Debug|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x86.ActiveCfg = Debug|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x86.Build.0 = Debug|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|Any CPU.ActiveCfg = Release|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|Any CPU.Build.0 = Release|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x64.ActiveCfg = Release|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x64.Build.0 = Release|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x86.ActiveCfg = Release|x64
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x86.Build.0 = Release|x64
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x64.Build.0 = Debug|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x86.Build.0 = Debug|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x64.ActiveCfg = Release|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x64.Build.0 = Release|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x86.ActiveCfg = Release|Any CPU
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x86.Build.0 = Release|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x64.Build.0 = Debug|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x86.Build.0 = Debug|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|Any CPU.Build.0 = Release|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x64.ActiveCfg = Release|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x64.Build.0 = Release|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x86.ActiveCfg = Release|Any CPU
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x86.Build.0 = Release|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x64.Build.0 = Debug|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x86.Build.0 = Debug|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x64.ActiveCfg = Release|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x64.Build.0 = Release|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x86.ActiveCfg = Release|Any CPU
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x86.Build.0 = Release|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x64.Build.0 = Debug|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x86.Build.0 = Debug|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x64.ActiveCfg = Release|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x64.Build.0 = Release|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x86.ActiveCfg = Release|Any CPU
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x86.Build.0 = Release|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x64.Build.0 = Debug|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x86.Build.0 = Debug|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x64.ActiveCfg = Release|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x64.Build.0 = Release|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x86.ActiveCfg = Release|Any CPU
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x86.Build.0 = Release|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x64.Build.0 = Debug|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x86.Build.0 = Debug|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x64.ActiveCfg = Release|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x64.Build.0 = Release|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x86.ActiveCfg = Release|Any CPU
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x86.Build.0 = Release|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x64.Build.0 = Debug|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x86.Build.0 = Debug|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|Any CPU.Build.0 = Release|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x64.ActiveCfg = Release|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x64.Build.0 = Release|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x86.ActiveCfg = Release|Any CPU
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x86.Build.0 = Release|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x64.Build.0 = Debug|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x86.Build.0 = Debug|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x64.ActiveCfg = Release|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x64.Build.0 = Release|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x86.ActiveCfg = Release|Any CPU
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x86.Build.0 = Release|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x64.Build.0 = Debug|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x86.Build.0 = Debug|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x64.ActiveCfg = Release|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x64.Build.0 = Release|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x86.ActiveCfg = Release|Any CPU
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x86.Build.0 = Release|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x64.Build.0 = Debug|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x86.Build.0 = Debug|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|Any CPU.Build.0 = Release|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x64.ActiveCfg = Release|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x64.Build.0 = Release|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x86.ActiveCfg = Release|Any CPU
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x86.Build.0 = Release|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x64.Build.0 = Debug|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x86.Build.0 = Debug|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x64.ActiveCfg = Release|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x64.Build.0 = Release|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x86.ActiveCfg = Release|Any CPU
+		{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x86.Build.0 = Release|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x64.Build.0 = Debug|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x86.Build.0 = Debug|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x64.ActiveCfg = Release|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x64.Build.0 = Release|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x86.ActiveCfg = Release|Any CPU
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x86.Build.0 = Release|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x64.Build.0 = Debug|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x86.Build.0 = Debug|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|Any CPU.Build.0 = Release|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x64.ActiveCfg = Release|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x64.Build.0 = Release|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x86.ActiveCfg = Release|Any CPU
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x86.Build.0 = Release|Any CPU
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|Any CPU.Build.0 = Debug|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x64.ActiveCfg = Debug|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x64.Build.0 = Debug|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x86.ActiveCfg = Debug|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x86.Build.0 = Debug|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|Any CPU.ActiveCfg = Release|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|Any CPU.Build.0 = Release|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x64.ActiveCfg = Release|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x64.Build.0 = Release|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x86.ActiveCfg = Release|x64
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x86.Build.0 = Release|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|Any CPU.Build.0 = Debug|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x64.ActiveCfg = Debug|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x64.Build.0 = Debug|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x86.ActiveCfg = Debug|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x86.Build.0 = Debug|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|Any CPU.ActiveCfg = Release|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|Any CPU.Build.0 = Release|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x64.ActiveCfg = Release|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x64.Build.0 = Release|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x86.ActiveCfg = Release|x64
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x86.Build.0 = Release|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|Any CPU.ActiveCfg = Debug|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|Any CPU.Build.0 = Debug|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x64.ActiveCfg = Debug|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x64.Build.0 = Debug|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x86.ActiveCfg = Debug|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x86.Build.0 = Debug|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|Any CPU.ActiveCfg = Release|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|Any CPU.Build.0 = Release|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x64.ActiveCfg = Release|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x64.Build.0 = Release|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x86.ActiveCfg = Release|x64
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x86.Build.0 = Release|x64
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
+		{D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
+		{684863A1-3396-4D31-A731-C0725CF07F4E} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
+		{45292808-4EF3-4D4A-B3F5-B75C42EE5870} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
+		{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
+		{28A26DF7-5F7A-43A2-A548-B55FEBCC3958} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
+		{7543386A-5EBA-4AF0-9E0F-35657B58557D} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
 		{60E0D592-FFB3-0D0A-C4AA-E88919B48F9D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
-		{1A5B8C5D-BADC-457E-819F-0050C44FE4D3} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
-		{2A0FBDE0-D870-4567-85BD-F7793A5862C6} = {1A5B8C5D-BADC-457E-819F-0050C44FE4D3}
-	EndGlobalSection
-	GlobalSection(ExtensibilityGlobals) = postSolution
-		SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
-	EndGlobalSection
-	GlobalSection(Performance) = preSolution
-		HasPerformanceSessions = true
+		{C13A5ABD-F739-4BE8-A2FE-511E989A5B43} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{F7C6F079-67BA-4899-9B43-D3F8C121D264} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{ECBAA198-EC93-4174-8156-BDA5F5E95042} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{1E296FB3-D185-4A95-A1EB-AA5B37B85C60} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{B6070F41-8759-448B-B56D-A7D15D489569} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{B2D721AB-920C-4CD6-8DBF-6778A115A449} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{38EE36C6-892C-4C94-8BB0-1ACB164E1F20} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
+		{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
+		{42C014A0-00C5-0620-8552-38912E891A7F} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3} = {42C014A0-00C5-0620-8552-38912E891A7F}
+		{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
+		{2B8243FF-60C6-4607-8AAE-7762D29B0A48} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
 	EndGlobalSection
 EndGlobal

src/DataStreamSuite/.vscode/settings.json

@@ -1,3 +0,0 @@
-{
-   "dotnet.preferCSharpExtension": true
-}

src/DataStreamSuite/DSPublisher/Configurations/.gitkeep

@@ -1 +0,0 @@
-

src/DataStreamSuite/DSPublisher/DSLWSClient/dsl_websocket_client.py

@@ -1,168 +0,0 @@
-"""
-Simple DSPublisher DSL websocket client.
-
-Example:
-   python dsl_websocket_client.py --dsl "HeikinAshiStream(...)"
-   python dsl_websocket_client.py --dsl-file query.dsl
-"""
-
-import argparse
-import asyncio
-import json
-import os
-import subprocess
-import sys
-from pathlib import Path
-
-try:
-   import websockets
-except ImportError:
-   print("Installing missing dependency: websockets")
-   subprocess.check_call([sys.executable, "-m", "pip", "install", "websockets"])
-   import websockets
-
-
-DEFAULT_DSL = 'RSIUIStream("ES", "TIME", 5, 7)'
-DEFAULT_URL = "ws://localhost:19093/ws/"
-
-
-def load_dsl(dsl: str | None, dsl_file: str | None) -> str:
-   if dsl and dsl_file:
-      raise ValueError("Pass either --dsl or --dsl-file, not both.")
-
-   if dsl_file:
-      return Path(dsl_file).read_text(encoding="utf-8").strip()
-
-   if dsl:
-      return dsl.strip()
-
-   if DEFAULT_DSL.startswith("#"):
-      raise ValueError("No DSL provided. Fill in DEFAULT_DSL, or pass --dsl / --dsl-file.")
-
-   return DEFAULT_DSL
-
-
-def load_config_path(config_path: str | None) -> Path | None:
-   if config_path:
-      return Path(config_path)
-
-   environment = os.environ.get("DOTNET_ENVIRONMENT")
-   config_file_name = "appsettings.json" if not environment else f"appsettings.{environment}.json"
-   script_dir = Path(__file__).resolve().parent
-   candidate_dirs = [
-      script_dir,
-      script_dir.parent,
-      Path.cwd(),
-   ]
-
-   for candidate_dir in candidate_dirs:
-      candidate_path = candidate_dir / config_file_name
-      if candidate_path.exists():
-         return candidate_path
-
-   return None
-
-
-def load_url(url: str | None, config_path: str | None) -> str:
-   if url:
-      return url
-
-   resolved_config_path = load_config_path(config_path)
-   if not resolved_config_path:
-      print(f"Config file not found; using default url: {DEFAULT_URL}")
-      return DEFAULT_URL
-
-   config = json.loads(resolved_config_path.read_text(encoding="utf-8"))
-   dsl_api_server_config = config.get("DSLAPIServer", {})
-   ip = dsl_api_server_config.get("Ip", "localhost")
-   port = int(dsl_api_server_config.get("Port", 19093))
-   ws_url = f"ws://{ip}:{port}/ws/"
-   print(f"Loaded DSPublisher DSL websocket url from {resolved_config_path}: {ws_url}")
-   return ws_url
-
-
-async def send_subscription(websocket, dsl: str) -> None:
-   payload = {
-      "type": "subscribe",
-      "dsl": dsl,
-   }
-   await websocket.send(json.dumps(payload))
-   print("Sent subscribe request:")
-   print(json.dumps(payload, indent=2))
-
-
-async def send_unsubscribe(websocket, subscription_id: str) -> None:
-   payload = {
-      "type": "unsubscribe",
-      "subscriptionId": subscription_id,
-   }
-   await websocket.send(json.dumps(payload))
-   print("Sent unsubscribe request:")
-   print(json.dumps(payload, indent=2))
-
-
-def print_server_message(raw_message: str) -> str | None:
-   try:
-      message = json.loads(raw_message)
-   except json.JSONDecodeError:
-      print(raw_message)
-      return None
-
-   print("Received server message:")
-   print(json.dumps(message, indent=2))
-   return message.get("subscriptionId")
-
-
-async def run_client(url: str, dsl: str) -> None:
-   subscription_id: str | None = None
-
-   async with websockets.connect(url) as websocket:
-      print(f"Connected to DSPublisher DSL websocket: {url}")
-      await send_subscription(websocket, dsl)
-
-      try:
-         async for raw_message in websocket:
-            latest_subscription_id = print_server_message(raw_message)
-            subscription_id = subscription_id or latest_subscription_id
-      finally:
-         if subscription_id:
-            await send_unsubscribe(websocket, subscription_id)
-
-
-def main() -> int:
-   parser = argparse.ArgumentParser(
-      description="Subscribe to a DSPublisher DSL websocket query and print responses."
-   )
-   parser.add_argument(
-      "--url",
-      help="DSPublisher DSL websocket URL. Overrides appsettings DSLAPIServer config.",
-   )
-   parser.add_argument(
-      "--config",
-      help="Path to DSPublisher appsettings JSON. Defaults to appsettings[.$DOTNET_ENVIRONMENT].json near this script.",
-   )
-   parser.add_argument(
-      "--dsl",
-      help="DSL expression to subscribe to.",
-   )
-   parser.add_argument(
-      "--dsl-file",
-      help="Path to a text file containing the DSL expression.",
-   )
-   args = parser.parse_args()
-
-   try:
-      dsl = load_dsl(args.dsl, args.dsl_file)
-      url = load_url(args.url, args.config)
-      asyncio.run(run_client(url, dsl))
-      return 0
-   except KeyboardInterrupt:
-      print("stopped")
-      return 0
-   except Exception as ex:
-      print(f"error: {ex}")
-      return 1
-
-
-if __name__ == "__main__":
-   raise SystemExit(main())

src/DataStreamSuite/DSPublisher/DSLWSServer/DSLAPIWSListener.cs

@@ -1,264 +0,0 @@
-namespace DSPublisher.DSLWSServer;
-
-using System;
-using System.Collections.Concurrent;
-using System.Collections.Generic;
-using System.Net;
-using System.Net.WebSockets;
-using System.Text;
-using System.Text.Json;
-using System.Threading;
-using System.Threading.Tasks;
-
-using BCG;
-using Bts.DataStreamCore;
-
-/// <summary>
-/// Websocket-facing API boundary for DSL clients. It opens the listener port,
-/// accepts subscribe/unsubscribe messages, and sends DataPacket JSON back to clients.
-/// Instantiated by DSPAPIServer.
-/// </summary>
-public sealed class DSLAPIWSListener : IDisposable {
-   readonly HttpListener listener = new();
-   readonly IDSLBookkeeper bookkeeper;
-   readonly string prefix;
-   readonly ConcurrentDictionary<string, ClientConnection> clients_by_id = new();
-   CancellationTokenSource? cancellation_token_source;
-   Task? listener_task;
-   bool disposed;
-
-   sealed class ClientConnection {
-      public required string client_id { get; init; }
-      public required WebSocket web_socket { get; init; }
-      public HashSet<Guid> subscription_ids { get; } = new();
-      public SemaphoreSlim send_lock { get; } = new(1, 1);
-      public DataPacketJsonSerializer serializer { get; } = new();
-   }
-
-   public DSLAPIWSListener(string prefix, IDSLBookkeeper bookkeeper) {
-      ArgumentException.ThrowIfNullOrWhiteSpace(prefix);
-      ArgumentNullException.ThrowIfNull(bookkeeper);
-      this.prefix = prefix;
-      this.bookkeeper = bookkeeper;
-   }
-
-   public Task StartAsync(CancellationToken cancellation_token = default) {
-      _throw_if_disposed();
-      if (listener_task is not null) {
-         return Task.CompletedTask;
-      }
-
-      cancellation_token_source = CancellationTokenSource.CreateLinkedTokenSource(cancellation_token);
-      listener.Prefixes.Add(prefix);
-      listener.Start();
-      listener_task = Task.Run(() => _listen_loop(cancellation_token_source.Token), CancellationToken.None);
-      return Task.CompletedTask;
-   }
-
-   public async Task StopAsync() {
-      if (cancellation_token_source is null) {
-         return;
-      }
-
-      cancellation_token_source.Cancel();
-      if (listener.IsListening) {
-         listener.Stop();
-      }
-
-      foreach (var client_connection in clients_by_id.Values) {
-         await _cleanup_client(client_connection);
-      }
-
-      if (listener_task is not null) {
-         await listener_task;
-         listener_task = null;
-      }
-
-      cancellation_token_source.Dispose();
-      cancellation_token_source = null;
-   }
-
-   public void Dispose() {
-      if (disposed) {
-         return;
-      }
-
-      StopAsync().GetAwaiter().GetResult();
-      listener.Close();
-      disposed = true;
-   }
-
-   async Task _listen_loop(CancellationToken cancellation_token) {
-      while (!cancellation_token.IsCancellationRequested) {
-         HttpListenerContext context;
-         try {
-            context = await listener.GetContextAsync();
-         } catch (HttpListenerException) when (cancellation_token.IsCancellationRequested || !listener.IsListening) {
-            break;
-         } catch (ObjectDisposedException) {
-            break;
-         }
-
-         _ = Task.Run(() => _handle_request(context, cancellation_token), cancellation_token);
-      }
-   }
-
-   async Task _handle_request(HttpListenerContext context, CancellationToken cancellation_token) {
-      if (!context.Request.IsWebSocketRequest) {
-         context.Response.StatusCode = (int)HttpStatusCode.BadRequest;
-         context.Response.Close();
-         return;
-      }
-
-      HttpListenerWebSocketContext web_socket_context = await context.AcceptWebSocketAsync(null);
-      var client_connection = new ClientConnection {
-         client_id = Guid.NewGuid().ToString("N"),
-         web_socket = web_socket_context.WebSocket,
-      };
-
-      clients_by_id[client_connection.client_id] = client_connection;
-      try {
-         await _read_messages(client_connection, cancellation_token);
-      } finally {
-         await _cleanup_client(client_connection);
-      }
-   }
-
-   async Task _read_messages(ClientConnection client_connection, CancellationToken cancellation_token) {
-      var buffer = new byte[64 * 1024];
-      while (client_connection.web_socket.State == WebSocketState.Open && !cancellation_token.IsCancellationRequested) {
-         var result = await client_connection.web_socket.ReceiveAsync(buffer, cancellation_token);
-         if (result.MessageType == WebSocketMessageType.Close) {
-            await client_connection.web_socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "ok", cancellation_token);
-            break;
-         }
-
-         var json = Encoding.UTF8.GetString(buffer, 0, result.Count);
-         await _handle_message(client_connection, json, cancellation_token);
-      }
-   }
-
-   async Task _handle_message(ClientConnection client_connection, string json, CancellationToken cancellation_token) {
-      DSLWebSocketRequest? request;
-      try {
-         request = JsonSerializer.Deserialize<DSLWebSocketRequest>(json);
-      } catch (JsonException ex) {
-         await _send_error(client_connection, $"Invalid request JSON: {ex.Message}", cancellation_token);
-         return;
-      }
-
-      if (request?.type is null) {
-         await _send_error(client_connection, "Request type is required.", cancellation_token);
-         return;
-      }
-
-      if (string.Equals(request.type, "subscribe", StringComparison.OrdinalIgnoreCase)) {
-         await _handle_subscribe(client_connection, request, cancellation_token);
-         return;
-      }
-
-      if (string.Equals(request.type, "unsubscribe", StringComparison.OrdinalIgnoreCase)) {
-         await _handle_unsubscribe(client_connection, request, cancellation_token);
-         return;
-      }
-
-      await _send_error(client_connection, $"Unknown request type: {request.type}", cancellation_token);
-   }
-
-   async Task _handle_subscribe(ClientConnection client_connection, DSLWebSocketRequest request, CancellationToken cancellation_token) {
-      if (string.IsNullOrWhiteSpace(request.dsl)) {
-         await _send_error(client_connection, "Subscribe request requires dsl.", cancellation_token);
-         return;
-      }
-
-      Guid subscription_id = Guid.Empty;
-      subscription_id = bookkeeper.Subscribe(request.dsl, packet => {
-         if (subscription_id == Guid.Empty) {
-            return;
-         }
-         _ = _send_packet(client_connection, subscription_id, packet, CancellationToken.None);
-      }, client_connection.client_id);
-
-      client_connection.subscription_ids.Add(subscription_id);
-      await _send_response(client_connection, new DSLWebSocketResponse {
-         type = "subscribed",
-         subscription_id = subscription_id,
-      }, cancellation_token);
-   }
-
-   async Task _handle_unsubscribe(ClientConnection client_connection, DSLWebSocketRequest request, CancellationToken cancellation_token) {
-      if (request.subscription_id is null) {
-         await _send_error(client_connection, "Unsubscribe request requires subscriptionId.", cancellation_token);
-         return;
-      }
-
-      var subscription_id = request.subscription_id.Value;
-      if (!client_connection.subscription_ids.Remove(subscription_id)) {
-         await _send_error(client_connection, $"Subscription does not belong to this client: {subscription_id}", cancellation_token);
-         return;
-      }
-
-      bookkeeper.Unsubscribe(subscription_id);
-      await _send_response(client_connection, new DSLWebSocketResponse {
-         type = "unsubscribed",
-         subscription_id = subscription_id,
-      }, cancellation_token);
-   }
-
-   async Task _send_packet(ClientConnection client_connection, Guid subscription_id, DataPacket packet, CancellationToken cancellation_token) {
-      var packet_json = client_connection.serializer.Serialize(packet);
-      using var packet_document = JsonDocument.Parse(packet_json ?? "{}");
-      await _send_response(client_connection, new DSLWebSocketResponse {
-         type = "data",
-         subscription_id = subscription_id,
-         packet = packet_document.RootElement.Clone(),
-      }, cancellation_token);
-   }
-
-   Task _send_error(ClientConnection client_connection, string message, CancellationToken cancellation_token) =>
-      _send_response(client_connection, new DSLWebSocketResponse {
-         type = "error",
-         message = message,
-      }, cancellation_token);
-
-   async Task _send_response(ClientConnection client_connection, DSLWebSocketResponse response, CancellationToken cancellation_token) {
-      if (client_connection.web_socket.State != WebSocketState.Open) {
-         bookkeeper.DisposeClient(client_connection.client_id);
-         return;
-      }
-
-      var json = JsonSerializer.Serialize(response);
-      var bytes = Encoding.UTF8.GetBytes(json);
-      await client_connection.send_lock.WaitAsync(cancellation_token);
-      try {
-         await client_connection.web_socket.SendAsync(bytes, WebSocketMessageType.Text, true, cancellation_token);
-      } catch {
-         bookkeeper.DisposeClient(client_connection.client_id);
-         throw;
-      } finally {
-         client_connection.send_lock.Release();
-      }
-   }
-
-   async Task _cleanup_client(ClientConnection client_connection) {
-      if (clients_by_id.TryRemove(client_connection.client_id, out _)) {
-         bookkeeper.DisposeClient(client_connection.client_id);
-      }
-
-      if (client_connection.web_socket.State == WebSocketState.Open || client_connection.web_socket.State == WebSocketState.CloseReceived) {
-         try {
-            await client_connection.web_socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "cleanup", CancellationToken.None);
-         } catch {
-         }
-      }
-
-      client_connection.send_lock.Dispose();
-      client_connection.web_socket.Dispose();
-   }
-
-   void _throw_if_disposed() {
-      if (disposed) {
-         throw new ObjectDisposedException(nameof(DSLAPIWSListener));
-      }
-   }
-}

src/DataStreamSuite/DSPublisher/DSLWSServer/DSLBookkeeper.cs

@@ -1,216 +0,0 @@
-namespace DSPublisher.DSLWSServer;
-
-using System;
-using System.Collections.Generic;
-
-using BCG;
-using Bts.DataStreamCore;
-using bts.BlueOrb;
-using bts;
-
-/// <summary>
-/// Tracks active DSL subscriptions and shared DSLStream instances. Multiple clients
-/// subscribing to the same DSL reuse one stream, and DoWork dispatches queued packets.
-/// Instantiated by DSPAPIServer.
-/// </summary>
-public sealed class DSLBookkeeper : IDSLBookkeeper {
-   sealed class QueryState {
-      public required string dsl { get; init; }
-      public required DSLStream stream { get; init; }
-      public DataStream.ChangeDelegate? stream_handler { get; set; }
-      public HashSet<Guid> subscription_ids { get; } = new();
-      public Queue<DataPacket> pending_packets { get; } = new();
-   }
-
-   sealed class SubscriptionState {
-      public required Guid subscription_id { get; init; }
-      public required string client_id { get; init; }
-      public required string dsl_key { get; init; }
-      public required Action<DataPacket> on_data { get; init; }
-   }
-
-   readonly object lock_obj = new();
-   readonly Dictionary<string, QueryState> queries_by_dsl = new(StringComparer.Ordinal);
-   readonly Dictionary<Guid, SubscriptionState> subscriptions_by_id = new();
-   readonly Dictionary<string, HashSet<Guid>> subscriptions_by_client = new(StringComparer.Ordinal);
-   bool disposed;
-
-   public Guid Subscribe(string dsl, Action<DataPacket> on_data, string client_id) {
-      ArgumentException.ThrowIfNullOrWhiteSpace(dsl);
-      ArgumentNullException.ThrowIfNull(on_data);
-      ArgumentException.ThrowIfNullOrWhiteSpace(client_id);
-
-      lock (lock_obj) {
-         _throw_if_disposed();
-
-         var dsl_key = _normalize_dsl(dsl);
-         if (!queries_by_dsl.TryGetValue(dsl_key, out var query_state)) {
-            _log_lifecycle($"DSLBookkeeper.Subscribe client_id: {client_id} creating new DSLStream dsl: {dsl_key}");
-            query_state = _create_query_state(dsl_key);
-            queries_by_dsl.Add(dsl_key, query_state);
-         } else {
-            _log_lifecycle($"DSLBookkeeper.Subscribe client_id: {client_id} reusing existing DSLStream subscription_count: {query_state.subscription_ids.Count} dsl: {dsl_key}");
-         }
-
-         var subscription_id = Guid.NewGuid();
-         var subscription_state = new SubscriptionState {
-            subscription_id = subscription_id,
-            client_id = client_id,
-            dsl_key = dsl_key,
-            on_data = on_data,
-         };
-
-         subscriptions_by_id.Add(subscription_id, subscription_state);
-         query_state.subscription_ids.Add(subscription_id);
-
-         if (!subscriptions_by_client.TryGetValue(client_id, out var client_subscriptions)) {
-            client_subscriptions = new();
-            subscriptions_by_client.Add(client_id, client_subscriptions);
-         }
-         client_subscriptions.Add(subscription_id);
-
-         _log_lifecycle($"DSLBookkeeper.Subscribe client_id: {client_id} subscription_id: {subscription_id} active_subscriptions_for_dsl: {query_state.subscription_ids.Count} dsl: {dsl_key}");
-         return subscription_id;
-      }
-   }
-
-   public void Unsubscribe(Guid subscription_id) {
-      lock (lock_obj) {
-         if (disposed) {
-            return;
-         }
-         _unsubscribe_locked(subscription_id);
-      }
-   }
-
-   public void DisposeClient(string client_id) {
-      ArgumentException.ThrowIfNullOrWhiteSpace(client_id);
-
-      lock (lock_obj) {
-         if (disposed) {
-            return;
-         }
-
-         if (!subscriptions_by_client.TryGetValue(client_id, out var subscription_ids)) {
-            return;
-         }
-
-         foreach (var subscription_id in subscription_ids.ToArray()) {
-            _unsubscribe_locked(subscription_id);
-         }
-      }
-   }
-
-   public void DoWork() {
-      List<(Action<DataPacket> on_data, DataPacket packet)> callbacks = new();
-
-      lock (lock_obj) {
-         if (disposed) {
-            return;
-         }
-
-         foreach (var query_state in queries_by_dsl.Values) {
-            while (query_state.pending_packets.TryDequeue(out var packet)) {
-               foreach (var subscription_id in query_state.subscription_ids) {
-                  if (subscriptions_by_id.TryGetValue(subscription_id, out var subscription_state)) {
-                     callbacks.Add((subscription_state.on_data, packet));
-                  }
-               }
-            }
-         }
-      }
-
-      foreach (var callback in callbacks) {
-         callback.on_data(callback.packet);
-      }
-   }
-
-   public void Dispose() {
-      lock (lock_obj) {
-         if (disposed) {
-            return;
-         }
-
-         foreach (var query_state in queries_by_dsl.Values) {
-            _dispose_query_state(query_state);
-         }
-
-         queries_by_dsl.Clear();
-         subscriptions_by_id.Clear();
-         subscriptions_by_client.Clear();
-         disposed = true;
-      }
-   }
-
-   QueryState _create_query_state(string dsl_key) {
-      _log_lifecycle($"DSLBookkeeper._create_query_state creating DSLStream dsl: {dsl_key}");
-      var stream = new DSLStream(dsl_key);
-      var query_state = new QueryState {
-         dsl = dsl_key,
-         stream = stream,
-      };
-
-      DataStream.ChangeDelegate stream_handler = new_value => {
-         if (new_value is not DataPacket packet) {
-            return;
-         }
-
-         lock (lock_obj) {
-            if (!disposed) {
-               query_state.pending_packets.Enqueue(packet);
-            }
-         }
-      };
-
-      query_state.stream_handler = stream_handler;
-      stream.onChange += stream_handler;
-      return query_state;
-   }
-
-   void _unsubscribe_locked(Guid subscription_id) {
-      if (!subscriptions_by_id.Remove(subscription_id, out var subscription_state)) {
-         return;
-      }
-
-      if (subscriptions_by_client.TryGetValue(subscription_state.client_id, out var client_subscriptions)) {
-         client_subscriptions.Remove(subscription_id);
-         if (client_subscriptions.Count == 0) {
-            subscriptions_by_client.Remove(subscription_state.client_id);
-         }
-      }
-
-      if (queries_by_dsl.TryGetValue(subscription_state.dsl_key, out var query_state)) {
-         query_state.subscription_ids.Remove(subscription_id);
-         _log_lifecycle($"DSLBookkeeper.Unsubscribe subscription_id: {subscription_id} client_id: {subscription_state.client_id} remaining_subscriptions_for_dsl: {query_state.subscription_ids.Count} dsl: {subscription_state.dsl_key}");
-         if (query_state.subscription_ids.Count == 0) {
-            _log_lifecycle($"DSLBookkeeper.Unsubscribe garbage collecting DSLStream; last subscription removed dsl: {subscription_state.dsl_key}");
-            _dispose_query_state(query_state);
-            queries_by_dsl.Remove(subscription_state.dsl_key);
-         }
-      }
-   }
-
-   static string _normalize_dsl(string dsl) =>
-      dsl.Trim();
-
-   static void _dispose_query_state(QueryState query_state) {
-      _log_lifecycle($"DSLBookkeeper._dispose_query_state disposing DSLStream dsl: {query_state.dsl} pending_packets: {query_state.pending_packets.Count} subscription_count: {query_state.subscription_ids.Count}");
-      if (query_state.stream_handler is not null) {
-         query_state.stream.onChange -= query_state.stream_handler;
-         query_state.stream_handler = null;
-      }
-      query_state.subscription_ids.Clear();
-      query_state.pending_packets.Clear();
-   }
-
-   static void _log_lifecycle(string message) {
-      Console.WriteLine(message);
-      Log.Gen.info(message);
-   }
-
-   void _throw_if_disposed() {
-      if (disposed) {
-         throw new ObjectDisposedException(nameof(DSLBookkeeper));
-      }
-   }
-}

src/DataStreamSuite/DSPublisher/DSLWSServer/DSLWebSocketMessages.cs

@@ -1,39 +0,0 @@
-namespace DSPublisher.DSLWSServer;
-
-using System;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-/// <summary>
-/// Client-to-server websocket protocol message for subscribing and unsubscribing.
-/// Instantiated by System.Text.Json when DSLAPIWSListener deserializes client messages.
-/// </summary>
-public sealed class DSLWebSocketRequest {
-   [JsonPropertyName("type")]
-   public string? type { get; set; }
-
-   [JsonPropertyName("dsl")]
-   public string? dsl { get; set; }
-
-   [JsonPropertyName("subscriptionId")]
-   public Guid? subscription_id { get; set; }
-}
-
-/// <summary>
-/// Server-to-client websocket protocol message. Data responses carry serialized
-/// DataPacket JSON in the packet field.
-/// Instantiated by DSLAPIWSListener before serializing responses to clients.
-/// </summary>
-public sealed class DSLWebSocketResponse {
-   [JsonPropertyName("type")]
-   public string type { get; set; } = "";
-
-   [JsonPropertyName("subscriptionId")]
-   public Guid? subscription_id { get; set; }
-
-   [JsonPropertyName("message")]
-   public string? message { get; set; }
-
-   [JsonPropertyName("packet")]
-   public JsonElement? packet { get; set; }
-}

src/DataStreamSuite/DSPublisher/DSLWSServer/DSPAPIServer.cs

@@ -1,37 +0,0 @@
-namespace DSPublisher.DSLWSServer;
-
-using System;
-using System.Threading;
-using System.Threading.Tasks;
-
-/// <summary>
-/// Top-level DSPublisher API server. Owns the websocket listener and the DSL
-/// subscription bookkeeper, and exposes the small lifecycle surface Program.cs uses.
-/// Instantiated by Program.cs.
-/// </summary>
-public sealed class DSPAPIServer : IDisposable {
-   readonly DSLBookkeeper bookkeeper;
-   readonly DSLAPIWSListener web_socket_listener;
-
-   public DSPAPIServer(global::DSLAPIServerConfig config) {
-      ArgumentNullException.ThrowIfNull(config);
-      bookkeeper = new DSLBookkeeper();
-      var web_socket_prefix = $"http://{config.Ip}:{config.Port}/ws/";
-      web_socket_listener = new DSLAPIWSListener(web_socket_prefix, bookkeeper);
-      Console.WriteLine($"DSPAPIServer listening for websocket DSL API requests on {web_socket_prefix}");
-   }
-
-   public Task StartAsync(CancellationToken cancellation_token = default) =>
-      web_socket_listener.StartAsync(cancellation_token);
-
-   public Task StopAsync() =>
-      web_socket_listener.StopAsync();
-
-   public void DoWork() =>
-      bookkeeper.DoWork();
-
-   public void Dispose() {
-      web_socket_listener.Dispose();
-      bookkeeper.Dispose();
-   }
-}

src/DataStreamSuite/DSPublisher/DSLWSServer/IDSLBookkeeper.cs

@@ -1,18 +0,0 @@
-namespace DSPublisher.DSLWSServer;
-
-using System;
-
-using BCG;
-using Bts.DataStreamCore;
-
-/// <summary>
-/// Internal contract between the websocket listener and the DSL subscription owner.
-/// The listener handles transport; the bookkeeper owns stream lifetime and callbacks.
-/// Implemented by DSLBookkeeper and consumed by DSLAPIWSListener.
-/// </summary>
-public interface IDSLBookkeeper : IDisposable {
-   Guid Subscribe(string dsl, Action<DataPacket> on_data, string client_id);
-   void Unsubscribe(Guid subscription_id);
-   void DisposeClient(string client_id);
-   void DoWork();
-}

src/DataStreamSuite/DSPublisher/DSPublisher.csproj

@@ -1,31 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
-    <Nullable>enable</Nullable>
-    <ImplicitUsings>enable</ImplicitUsings>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\DataStream\DataStream.csproj" />
-    <ProjectReference Include="..\PubSub\PubSub.csproj" />
-    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ConfigurationProfile Include="appsettings*.json" />
-
-    <None Update="appsettings*.json">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-      <TargetPath>DSPublisher.%(Filename)%(Extension)</TargetPath>
-    </None>
-    <None Update="WSPublishers\**\*.py">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </None>
-    <None Update="DSLWSClient\**\*.py">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </None>
-  </ItemGroup>
-
-</Project>

src/DataStreamSuite/DSPublisher/DSPublisherConfig.cs

@@ -1,21 +0,0 @@
-using System.Collections.Generic;
-
-public sealed class DSPublisherConfig {
-   public List<StreamPublishingNode> StreamPublishingNodes { get; init; } = new();
-   public List<PythonPublishingNode> PythonPublishingNodes { get; init; } = new();
-}
-
-public sealed class DSLAPIServerConfig {
-   public string Ip { get; init; } = "localhost";
-   public int Port { get; init; } = 19093;
-}
-
-public sealed class StreamPublishingNode {
-   public string Name { get; init; } = string.Empty;
-   public string Expression { get; init; } = string.Empty;
-}
-
-public sealed class PythonPublishingNode {
-   public string PythonScriptLocation { get; init; } = string.Empty;
-   public string Args { get; init; } = string.Empty;
-}

src/DataStreamSuite/DSPublisher/Program.cs

@@ -1,125 +0,0 @@
-using System;
-using System.IO;
-using System.Text.Json;
-using System.Threading;
-
-using BCG;
-using bts;
-using Bts.PubSub;
-using bts.BlueOrb;
-using DSPublisher.DSLWSServer;
-
-Console.WriteLine("Starting DSPublisher");
-
-var configPath = load_config_path();
-Console.WriteLine($"Loading config from: {configPath}");
-if (!File.Exists(configPath)) {
-   Console.WriteLine($"Config file not found: {configPath}");
-   return;
-}
-
-var json = File.ReadAllText(configPath);
-using var doc = JsonDocument.Parse(json);
-
-if (!doc.RootElement.TryGetProperty("DSPublisher", out var dspPublisherElement)) {
-   Console.WriteLine("DSPublisher config section not found");
-   return;
-}
-
-var json_options = new JsonSerializerOptions {
-   PropertyNameCaseInsensitive = true,
-};
-
-var config = dspPublisherElement.Deserialize<DSPublisherConfig>(json_options) ?? new DSPublisherConfig();
-var dsl_api_server_config = load_dsl_api_server_config(doc, json_options);
-
-Console.WriteLine("StreamPublishingNodes:");
-foreach (var node in config.StreamPublishingNodes) {
-   Console.WriteLine($"  Name: {node.Name}");
-   Console.WriteLine($"  Expression: {node.Expression}");
-}
-
-Console.WriteLine("PythonPublishingNodes:");
-foreach (var node in config.PythonPublishingNodes) {
-   Console.WriteLine($"  PythonScriptLocation: {node.PythonScriptLocation}");
-   Console.WriteLine($"  Args: {node.Args}");
-}
-
-var pubsub_config = load_pubsub_config(doc, json_options);
-PubSubConfig.SetGlobalConfig(pubsub_config);
-using var view_stream_database = new ViewStreamDatabase(new SubscriberCoreFactory());
-ViewStreamDatabase.SetGlobalInstance(view_stream_database);
-
-//var pub_table_broker = new PubTableBroker();
-//var stream_publishers = _process_publishing_nodes(config.StreamPublishingNodes, pub_table_broker);
-
-using var dsp_api_server = new DSPAPIServer(dsl_api_server_config);
-await dsp_api_server.StartAsync();
-
-while (true) {
-   view_stream_database.doWork();
-   dsp_api_server.DoWork();
-   Thread.Sleep(TimeSpan.FromMilliseconds(10));
-}
-
-string load_config_path() {
-   var environment = Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT");
-   var baseFileName = string.IsNullOrWhiteSpace(environment)
-      ? "appsettings.json"
-      : $"appsettings.{environment}.json";
-   var dspPublisherFileName = $"DSPublisher.{baseFileName}";
-   var dspPublisherConfigPath = Path.Combine(AppContext.BaseDirectory, dspPublisherFileName);
-   if (File.Exists(dspPublisherConfigPath)) {
-      return dspPublisherConfigPath;
-   }
-
-   return Path.Combine(AppContext.BaseDirectory, baseFileName);
-}
-
-PubSubConfig load_pubsub_config(JsonDocument doc, JsonSerializerOptions json_options) {
-   if (!doc.RootElement.TryGetProperty("PubSub", out var pubsub_element)) {
-      Log.Gen.warn("PubSub config section not found; using PubSubConfig.Default().");
-      return PubSubConfig.Default();
-   }
-
-   var pubsub_config = pubsub_element.Deserialize<PubSubConfig>(json_options);
-   if (pubsub_config is null) {
-      Log.Gen.error("PubSub config section could not be deserialized; using PubSubConfig.Default().");
-      return PubSubConfig.Default();
-   }
-
-   Log.Gen.info("Loaded PubSub config from DSPublisher config file.");
-   return pubsub_config;
-}
-
-DSLAPIServerConfig load_dsl_api_server_config(JsonDocument doc, JsonSerializerOptions json_options) {
-   if (!doc.RootElement.TryGetProperty("DSLAPIServer", out var dsl_api_server_element)) {
-      Log.Gen.warn("DSLAPIServer config section not found; using DSLAPIServerConfig defaults.");
-      return new DSLAPIServerConfig();
-   }
-
-   var dsl_api_server_config = dsl_api_server_element.Deserialize<DSLAPIServerConfig>(json_options);
-   if (dsl_api_server_config is null) {
-      Log.Gen.error("DSLAPIServer config section could not be deserialized; using DSLAPIServerConfig defaults.");
-      return new DSLAPIServerConfig();
-   }
-
-   Log.Gen.info($"Loaded DSLAPIServer config ip: {dsl_api_server_config.Ip} port: {dsl_api_server_config.Port}");
-   return dsl_api_server_config;
-}
-
-List<IDisposable> _process_publishing_nodes(IEnumerable<StreamPublishingNode> publishing_nodes, PubTableBroker pub_table_broker) {
-   var stream_publishers = new List<IDisposable>();
-   foreach (var node in publishing_nodes) {
-      if (string.IsNullOrWhiteSpace(node.Name) || string.IsNullOrWhiteSpace(node.Expression)) {
-         Log.Gen.error("Skipping StreamPublishingNode because name or expression is empty.");
-         continue;
-      }
-
-      Log.Gen.info($"Creating ViewPublisher name: {node.Name} expression: {node.Expression}");
-      var stream = new DSLStream(node.Expression);
-      stream_publishers.Add(new ViewPublisher(node.Name, pub_table_broker, stream));
-   }
-
-   return stream_publishers;
-}

src/DataStreamSuite/DSPublisher/Services/.gitkeep

@@ -1 +0,0 @@
-

src/DataStreamSuite/DSPublisher/WSPublishers/DatabentoPublisher.py

@@ -1,146 +0,0 @@
-"""
-Requires:
-   pip install -U databento
-   pip install websockets
-
-Sources:
-   https://databento.com/docs/quickstart/set-up
-   https://databento.com/docs/schemas-and-data-formats/bbo
-"""
-
-import datetime as dt
-import subprocess
-import sys
-import time
-
-try:
-   import databento as db
-except ImportError:
-   print("Installing missing dependency: databento")
-   subprocess.check_call([sys.executable, "-m", "pip", "install", "-U", "databento"])
-   import databento as db
-
-from databento.common.error import BentoError
-
-try:
-   import websockets
-except ImportError:
-   print("Installing missing dependency: websockets")
-   subprocess.check_call([sys.executable, "-m", "pip", "install", "websockets"])
-   import websockets
-
-from databento_transformer import DatabentoTransformer
-from pubsub_client import PubSubClient
-
-
-API_KEY = "db-VksTJvhMmbLjbJu9Kg6GPS99B8FDN"
-DATASET = "GLBX.MDP3"
-SYMBOL = "ES.FUT"
-PUBSUB_WS_URL = "ws://localhost:19092/ws/"
-BBO_TABLE_NAME = "livebbo_glbx_future_es_continuouscontracts"
-TRADES_TABLE_NAME = "livetrades_glbx_future_es_continuouscontracts"
-DEFINITIONS_TABLE_NAME = "livecontracts_glbx_future_es"
-
-_pubsub_client = PubSubClient(PUBSUB_WS_URL)
-_transformer = DatabentoTransformer(
-   symbol=SYMBOL,
-   bbo_table_name=BBO_TABLE_NAME,
-   trades_table_name=TRADES_TABLE_NAME,
-   definitions_table_name=DEFINITIONS_TABLE_NAME,
-)
-
-
-def _publish_contract_definitions() -> None:
-   historical_client = db.Historical(API_KEY)
-   dataset_range = historical_client.metadata.get_dataset_range(DATASET)
-   schema_range = dataset_range.get("schema", {}).get("definition", {})
-   available_end = schema_range.get("end", dataset_range["end"])
-
-   if isinstance(available_end, str):
-      available_end_dt = dt.datetime.fromisoformat(available_end.replace("Z", "+00:00"))
-   else:
-      available_end_dt = available_end
-
-   # Databento's range end is exclusive. Back off slightly so the request stays
-   # strictly inside the available window.
-   end_dt = available_end_dt - dt.timedelta(seconds=1)
-   start_dt = max(end_dt - dt.timedelta(days=1), end_dt.replace(hour=0, minute=0, second=0, microsecond=0))
-   start = start_dt.isoformat()
-   end = end_dt.isoformat()
-
-   definition_data = historical_client.timeseries.get_range(
-      dataset=DATASET,
-      schema="definition",
-      stype_in="parent",
-      symbols=SYMBOL,
-      start=start,
-      end=end,
-   )
-
-   df = definition_data.to_df()
-   payload = _transformer.build_definitions_payload(df)
-   if payload is None:
-      print("No Databento contract definitions returned")
-      return
-
-   print(f"Publishing {len(payload['rows'])} Databento contract definition rows")
-   _pubsub_client.publish(payload)
-
-
-def on_record(record) -> None:
-   # Print every callback first so we can see whether Databento is flowing
-   # even if our row extraction logic misses a field shape.
-   print(record)
-
-   payloads = _transformer.transform_record(record)
-   if payloads:
-      for payload in payloads:
-         _pubsub_client.publish(payload)
-      return
-
-   print("Skipping record with no BBO/trade fields")
-
-
-def main() -> None:
-   _pubsub_client.start()
-   print(f"Starting Databento live client for {SYMBOL} on {DATASET}")
-   try:
-      _publish_contract_definitions()
-   except Exception as ex:
-      print(f"Contract definition publish failed: {ex}", flush=True)
-
-   while True:
-      try:
-         client = db.Live(key=API_KEY)
-
-         client.subscribe(
-            dataset=DATASET,
-            schema="bbo-1s",
-            stype_in="parent",
-            symbols=SYMBOL,
-         )
-         client.subscribe(
-            dataset=DATASET,
-            schema="trades",
-            stype_in="parent",
-            symbols=SYMBOL,
-         )
-
-         client.add_callback(on_record)
-         client.start()
-         print("Databento client started, waiting for BBO and trade records...", flush=True)
-         client.block_for_close()
-         print("Databento client closed cleanly", flush=True)
-         break
-      except BentoError as ex:
-         print(f"Databento live connection error: {ex}", flush=True)
-         print("Retrying Databento live connection in 5 seconds...", flush=True)
-         time.sleep(5)
-      except Exception as ex:
-         print(f"Databento publisher unexpected error: {ex}", flush=True)
-         print("Retrying Databento live connection in 5 seconds...", flush=True)
-         time.sleep(5)
-
-
-if __name__ == "__main__":
-   main()

src/DataStreamSuite/DSPublisher/WSPublishers/databento_transformer.py

@@ -1,244 +0,0 @@
-import datetime as dt
-
-
-class DatabentoTransformer:
-   def __init__(
-      self,
-      symbol: str,
-      bbo_table_name: str,
-      trades_table_name: str,
-      definitions_table_name: str,
-   ):
-      self._symbol = symbol
-      self._bbo_table_name = bbo_table_name
-      self._trades_table_name = trades_table_name
-      self._definitions_table_name = definitions_table_name
-      self._instrument_id_to_symbol: dict[int, str] = {}
-      self._last_bbo_row: dict | None = None
-      self._trade_row_id = 0
-      self._trade_table_created = False
-
-   def transform_record(self, record) -> list[dict]:
-      payload = self._handle_bbo_record(record)
-      if payload is not None:
-         return [payload]
-
-      payload = self._handle_trade_record(record)
-      if payload is not None:
-         return [payload]
-
-      return []
-
-   def build_definitions_payload(self, definition_df) -> dict | None:
-      if definition_df is None or definition_df.empty:
-         return None
-
-      df = definition_df.reset_index()
-      if "instrument_class" in df.columns:
-         df = df[df["instrument_class"].astype(str) == "F"]
-      if "security_type" in df.columns:
-         df = df[df["security_type"].astype(str) == "FUT"]
-      if "raw_symbol" in df.columns:
-         df = df.sort_values("ts_recv" if "ts_recv" in df.columns else "ts_event")
-         df = df.drop_duplicates(subset=["raw_symbol"], keep="last")
-
-      rows: list[dict] = []
-      for idx, row in enumerate(df.itertuples(index=False), start=1):
-         instrument_id = int(getattr(row, "instrument_id", 0) or 0)
-         security_name = str(getattr(row, "raw_symbol", ""))
-         if instrument_id > 0 and security_name:
-            self._instrument_id_to_symbol[instrument_id] = security_name
-
-         rows.append({
-            "id": idx,
-            "security_name": security_name,
-            "instrument_id": instrument_id,
-            "instrument_class": str(getattr(row, "instrument_class", "")),
-            "security_type": str(getattr(row, "security_type", "")),
-            "asset": str(getattr(row, "asset", "")),
-            "exchange": str(getattr(row, "exchange", "")),
-            "currency": str(getattr(row, "currency", "")),
-            "expiration": self._ns_to_iso(getattr(row, "expiration", 0)),
-            "activation": self._ns_to_iso(getattr(row, "activation", 0)),
-            "min_price_increment": self._nano_to_float(getattr(row, "min_price_increment", 0)),
-            "display_factor": self._nano_to_float(getattr(row, "display_factor", 0)),
-            "price_display_multiplier": self._nano_to_float(getattr(row, "display_factor", 0)),
-            "contract_multiplier": int(getattr(row, "contract_multiplier", 0) or 0),
-            "contract_multiplier_unit": int(getattr(row, "contract_multiplier_unit", 0) or 0),
-            "unit_of_measure": str(getattr(row, "unit_of_measure", "")),
-            "unit_of_measure_qty": self._nano_to_float(getattr(row, "unit_of_measure_qty", 0)),
-            "min_lot_size": int(getattr(row, "min_lot_size", 0) or 0),
-            "min_trade_vol": int(getattr(row, "min_trade_vol", 0) or 0),
-            "max_trade_vol": int(getattr(row, "max_trade_vol", 0) or 0),
-            "maturity_year": int(getattr(row, "maturity_year", 0) or 0),
-            "maturity_month": int(getattr(row, "maturity_month", 0) or 0),
-            "maturity_day": int(getattr(row, "maturity_day", 0) or 0),
-            "underlying": str(getattr(row, "underlying", "")),
-         })
-
-      if not rows:
-         return None
-
-      return {
-         "op": "create_table",
-         "table_name": self._definitions_table_name,
-         "rows": rows,
-      }
-
-   def _handle_bbo_record(self, record) -> dict | None:
-      row = self._to_bbo_row(record)
-      if row is None:
-         return None
-
-      if self._last_bbo_row is None:
-         payload = {
-            "op": "create_table",
-            "table_name": self._bbo_table_name,
-            "rows": [row],
-         }
-      else:
-         payload = {
-            "op": "multiple_remove_add",
-            "table_name": self._bbo_table_name,
-            "removed_rows": [self._last_bbo_row],
-            "added_rows": [row],
-         }
-
-      self._last_bbo_row = row
-      return payload
-
-   def _handle_trade_record(self, record) -> dict | None:
-      row = self._to_trade_row(record)
-      if row is None:
-         return None
-
-      row["id"] = self._trade_row_id
-      self._trade_row_id += 1
-
-      if not self._trade_table_created:
-         payload = {
-            "op": "create_table",
-            "table_name": self._trades_table_name,
-            "rows": [row],
-         }
-         self._trade_table_created = True
-         return payload
-
-      return {
-         "op": "multiple_remove_add",
-         "table_name": self._trades_table_name,
-         "removed_rows": [],
-         "added_rows": [row],
-      }
-
-   def _to_bbo_row(self, record) -> dict | None:
-      bid_px = self._get_level_attr(record, "bid_px", default=self._get_attr(record, "bid_px_00"))
-      ask_px = self._get_level_attr(record, "ask_px", default=self._get_attr(record, "ask_px_00"))
-      if bid_px is None and ask_px is None:
-         return None
-
-      return {
-         "id": 0,
-         "security_name": self._get_security_name(record),
-         "timestamp": self._get_timestamp(record),
-         "bid_px": float(bid_px or 0.0),
-         "ask_px": float(ask_px or 0.0),
-         "bid_sz": int(self._get_level_attr(record, "bid_sz", default=self._get_attr(record, "bid_sz_00", default=0)) or 0),
-         "ask_sz": int(self._get_level_attr(record, "ask_sz", default=self._get_attr(record, "ask_sz_00", default=0)) or 0),
-      }
-
-   def _to_trade_row(self, record) -> dict | None:
-      price = self._get_attr(record, "price", "px")
-      size = self._get_attr(record, "size")
-      if price is None and size is None:
-         return None
-
-      side = self._get_attr(record, "side", "aggressor_side", default="")
-
-      return {
-         "id": 0,
-         "security_name": self._get_security_name(record),
-         "timestamp": self._get_timestamp(record),
-         "price": float(price or 0.0),
-         "size": int(size or 0),
-         "side": str(side),
-      }
-
-   def _get_security_name(self, record) -> str:
-      instrument_id = self._get_attr(record, "instrument_id", default=0)
-      try:
-         instrument_id_int = int(instrument_id or 0)
-      except (TypeError, ValueError):
-         instrument_id_int = 0
-
-      if instrument_id_int > 0 and instrument_id_int in self._instrument_id_to_symbol:
-         return self._instrument_id_to_symbol[instrument_id_int]
-
-      value = self._get_attr(
-         record,
-         "symbol",
-         "raw_symbol",
-         "stype_out_symbol",
-         "stype_in_symbol",
-         default=self._symbol,
-      )
-      return str(value)
-
-   def _get_timestamp(self, record) -> int:
-      value = self._get_attr(record, "ts_event", "ts_recv", default=0)
-      try:
-         return int(value)
-      except (TypeError, ValueError):
-         return 0
-
-   def _get_attr(self, record, *names, default=None):
-      for name in names:
-         if hasattr(record, name):
-            value = getattr(record, name)
-            if value is not None:
-               return value
-      return default
-
-   def _get_level_attr(self, record, name, default=None):
-      levels = self._get_attr(record, "levels", default=None)
-      if levels:
-         level0 = levels[0]
-         if hasattr(level0, name):
-            value = getattr(level0, name)
-            if value is not None:
-               return value
-      return default
-
-   def _ns_to_iso(self, timestamp_ns) -> str:
-      if timestamp_ns is None:
-         return ""
-
-      if hasattr(timestamp_ns, "isoformat"):
-         try:
-            if hasattr(timestamp_ns, "to_pydatetime"):
-               timestamp_dt = timestamp_ns.to_pydatetime()
-            else:
-               timestamp_dt = timestamp_ns
-
-            if timestamp_dt is None:
-               return ""
-
-            if getattr(timestamp_dt, "tzinfo", None) is None:
-               timestamp_dt = timestamp_dt.replace(tzinfo=dt.timezone.utc)
-            return timestamp_dt.isoformat()
-         except (TypeError, ValueError, AttributeError):
-            pass
-
-      try:
-         ts_int = int(timestamp_ns)
-         if ts_int <= 0:
-            return ""
-         return dt.datetime.fromtimestamp(ts_int / 1_000_000_000, tz=dt.timezone.utc).isoformat()
-      except (TypeError, ValueError, OSError, OverflowError):
-         return ""
-
-   def _nano_to_float(self, value) -> float:
-      try:
-         return float(value) / 1_000_000_000.0
-      except (TypeError, ValueError):
-         return 0.0

src/DataStreamSuite/DSPublisher/WSPublishers/pubsub_client.py

@@ -1,58 +0,0 @@
-import asyncio
-import json
-import queue
-import threading
-
-import websockets
-
-
-class PubSubClient:
-   def __init__(self, ws_url: str):
-      self._ws_url = ws_url
-      self._outgoing_queue: "queue.Queue[dict]" = queue.Queue()
-      self._sender_thread: threading.Thread | None = None
-
-   def start(self) -> threading.Thread:
-      if self._sender_thread is None or not self._sender_thread.is_alive():
-         self._sender_thread = self._start_sender_thread()
-      return self._sender_thread
-
-   def publish(self, payload: dict) -> None:
-      self._outgoing_queue.put(payload)
-
-   def publish_create_table(self, table_name: str, rows: list[dict]) -> None:
-      self.publish({
-         "op": "create_table",
-         "table_name": table_name,
-         "rows": rows,
-      })
-
-   def publish_multiple_remove_add(
-      self,
-      table_name: str,
-      removed_rows: list[dict],
-      added_rows: list[dict],
-   ) -> None:
-      self.publish({
-         "op": "multiple_remove_add",
-         "table_name": table_name,
-         "removed_rows": removed_rows,
-         "added_rows": added_rows,
-      })
-
-   def _start_sender_thread(self) -> threading.Thread:
-      thread = threading.Thread(target=self._run_send_loop, daemon=True)
-      thread.start()
-      return thread
-
-   def _run_send_loop(self) -> None:
-      asyncio.run(self._ws_send_loop())
-
-   async def _ws_send_loop(self) -> None:
-      async with websockets.connect(self._ws_url) as ws:
-         print(f"Connected to PubSubServer websocket: {self._ws_url}")
-         while True:
-            payload = await asyncio.to_thread(self._outgoing_queue.get)
-            body = json.dumps(payload)
-            print(body)
-            await ws.send(body)

src/DataStreamSuite/DSPublisher/appsettings.Development.json

@@ -1,69 +0,0 @@
-{
-  "PubSub": {
-    "HeartbeatInterval": 15000,
-    "MissingHeartbeatsBeforeGc": 50,
-    "DisposedConnectionCheckPeriod": 1000,
-    "ServerPort": 22019,
-    "Routing": {
-      "PublisherRoutes": [
-        {
-          "Name": "default",
-          "ServerAddress": "bts-cloud://127.0.0.1:22019",
-          "TagRegExp": ".*"
-        }
-      ],
-      "SubscriberRoutes": [
-        {
-          "Name": "default",
-          "ServerAddress": "bts-cloud://127.0.0.1:22019",
-          "TagRegExp": ".*"
-        }
-      ]
-    },
-    "TagTypeToAssemblyMap": {
-      "Entries": [
-        {
-          "HandlerClass": "BCG.PubSubServerTagHandler",
-          "Assembly": "DataStreamCore.dll"
-        }
-      ]
-    },
-    "TagHandlerFactory": {
-      "Handlers": [
-        {
-          "TagType": "DataStream",
-          "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerStreamTagHandler"
-        },
-        {
-          "TagType": "DS_.*",
-          "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerStreamTagHandler"
-        },
-        {
-          "TagType": ".*",
-          "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerGenericDataHandler"
-        }
-      ]
-    }
-  },
-  "DSLAPIServer": {
-    "Ip": "localhost",
-    "Port": 19093
-  },
-  "DSPublisher": {
-    "StreamPublishingNodes": [
-      {
-        "Name": "sample_stream",
-        "Expression": "MillisecondStream(1000,5)"
-      }
-    ],
-    "PythonPublishingNodes": [
-      {
-        "PythonScriptLocation": "WSPublishers/DatabentoPublisher.py",
-        "Args": ""
-      }
-    ]
-  }
-}

src/DataStreamSuite/DSPublisher/appsettings.json

@@ -1,69 +0,0 @@
-{
-  "PubSub": {
-    "HeartbeatInterval": 15000,
-    "MissingHeartbeatsBeforeGc": 50,
-    "DisposedConnectionCheckPeriod": 1000,
-    "ServerPort": 22019,
-    "Routing": {
-      "PublisherRoutes": [
-        {
-          "Name": "default",
-          "ServerAddress": "bts-cloud://127.0.0.1:22019",
-          "TagRegExp": ".*"
-        }
-      ],
-      "SubscriberRoutes": [
-        {
-          "Name": "default",
-          "ServerAddress": "bts-cloud://127.0.0.1:22019",
-          "TagRegExp": ".*"
-        }
-      ]
-    },
-    "TagTypeToAssemblyMap": {
-      "Entries": [
-        {
-          "HandlerClass": "BCG.PubSubServerTagHandler",
-          "Assembly": "DataStreamCore.dll"
-        }
-      ]
-    },
-    "TagHandlerFactory": {
-      "Handlers": [
-        {
-          "TagType": "DataStream",
-          "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerStreamTagHandler"
-        },
-        {
-          "TagType": "DS_.*",
-          "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerStreamTagHandler"
-        },
-        {
-          "TagType": ".*",
-          "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerGenericDataHandler"
-        }
-      ]
-    }
-  },
-  "DSLAPIServer": {
-    "Ip": "localhost",
-    "Port": 19093
-  },
-  "DSPublisher": {
-    "StreamPublishingNodes": [
-      {
-        "Name": "sample_stream",
-        "Expression": "MillisecondStream(1000,5)"
-      }
-    ],
-    "PythonPublishingNodes": [
-      {
-        "PythonScriptLocation": "WSPublishers/DatabentoPublisher.py",
-        "Args": ""
-      }
-    ]
-  }
-}

src/DataStreamSuite/DataStream/BlueOrb/AddColumnsStream.cs

@@ -2,8 +2,7 @@
    using BCG;
 
    [DataStreamUserRespresentation("AddColumnsStream")]
-   // example: AddColumnsStream(l,"jc1:001")
-   public class AddColumnsStream : ViewStream {
+   public class AddColumnsStream: ViewStream {
       public AddColumnsStream(ViewStream parent_str, string add_col_expr) {
          ViewStream rs = new BCG.AddColumnsStream(parent_str, add_col_expr);
          mySetRealSource(rs);

src/DataStreamSuite/DataStream/BlueOrb/HideColumnsStream.cs

@@ -1,126 +0,0 @@
-namespace BlueOrb.Operators.Core {
-   using BCG;
-
-   using System;
-   using System.Collections;
-   using System.Collections.Generic;
-
-   [DataStreamUserRespresentation("HideColumnsStream")]
-   public class HideColumnsStream : ViewStream {
-      // Wrapper row to hide the columns
-      public class MyRow : IBlueRow {
-         #region IBlueRow members
-         private IBlueTableSchema _schema;
-         private IBlueRow _row;
-         public MyRow(IBlueTableSchema schema, IBlueRow row) {
-            _schema = schema;
-            _row = row;
-         }
-         public IBlueTableSchema schema { get { return _schema; } }
-         #endregion
-
-         #region ICollection members
-         public int Count { get { return schema.columnNames.Length; } }
-
-         public bool IsSynchronized { get { return false; } }
-
-         public object SyncRoot { get { return this; } }
-
-         public void CopyTo(Array array, int index) {
-            int i = index;
-            foreach (object o in this as IEnumerable)
-               array.SetValue(o, i++);
-         }
-         #endregion
-
-         #region IDictionary members
-         public bool IsFixedSize { get { return true; } }
-
-         public bool IsReadOnly { get { return true; } }
-
-         public object this[object key] {
-            get {
-               if (key is string)
-                  return this[(string)key];
-               return this[(int)key];
-            }
-            set { throw new NotSupportedException("The IBlueRow is readonly"); }
-         }
-
-         public object this[int index] {
-            get {
-               return this[schema.columnInfo[index].colName];
-            }
-         }
-
-         public object this[string col_name] {
-            get {
-               if (_schema.contains_column(col_name))
-                  return _row[col_name];
-               else
-                  throw new ApplicationException($"{col_name} not found");
-            }
-         }
-
-         public ICollection Keys { get { return schema.columnNames; } }
-
-         public ICollection Values {
-            get {
-               object[] retVal = new object[schema.columnNames.Length];
-               CopyTo(retVal, 0);
-               return retVal;
-            }
-         }
-
-         public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-         public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
-
-         public bool Contains(object key) { return schema[(string)key] != null; }
-
-         public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
-
-         IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
-
-         public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
-         #endregion
-
-      }
-
-      private readonly ViewStream _parent_str;
-      private readonly HashSet<string> _removed_cols;
-      private BlueTable _state;
-      public HideColumnsStream(ViewStream parent_str, string hide_cols_expr) {
-         _parent_str = parent_str;
-         _removed_cols = new HashSet<string>(hide_cols_expr.Split(',', ' '));
-         if (_removed_cols.Contains("id"))
-            _removed_cols.Remove("id");
-      }
-
-      protected override void myOnFirstSubscriber() {
-         _parent_str.onChange += _handle_parent_str_cb;
-      }
-
-      private void _handle_parent_str_cb(object dp_obj) {
-         // this could be made incremental and hence faster
-         DataPacket dp_in = dp_obj as DataPacket;
-         if (_state == null) {
-            List<ColumnInfo> cis = new List<ColumnInfo>();
-            foreach (ColumnInfo ci in dp_in.schema.columnInfo) {
-               if (!_removed_cols.Contains(ci.colName))
-                  cis.Add(new ColumnInfo(ci.colName, cis.Count, ci.colType));
-            }
-            _state = new BlueTable(new BlueTableSchema(dp_in.schema.tableName, cis.ToArray()));
-         }
-         _state.clear();
-         foreach (IBlueRow r in dp_in.newRows) {
-            _state.addRow(new MyRow(_state.schema, r));
-         }
-         var dp_out = BlueDataUtility.build_dp_inc(_state,
-            myLastValue == null ? new BlueTable(_state.schema) : (myLastValue as DataPacket).newRows);
-         myFireEvent(dp_out);
-      }
-   }
-
-
-}

src/DataStreamSuite/DataStream/BlueOrb/Join/CartesianProductStream.cs

@@ -1,14 +0,0 @@
-namespace bts.BlueOrb {
-   using BCG;
-
-   [DataStreamUserRespresentation("CartesianProductStream")]
-   public class CartesianProductStream : ViewStream {
-      public CartesianProductStream(ViewStream ls, ViewStream rs) {
-         ls = new AddColumnsStream(ls, "ljc: 001");
-         rs = new AddColumnsStream(rs, "rjc: 001");
-         ViewStream res = new SimpleJoinStream(ls, "ljc", rs, "rjc");
-         res = new global::BlueOrb.Operators.Core.HideColumnsStream(res, "ljc, rjc");
-         mySetRealSource(res);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/Join/FullJoinStream.cs

@@ -1,15 +0,0 @@
-namespace BlueOrb.Operators.Core {
-   using System;
-
-   using BCG;
-
-   [DataStreamUserRespresentation("FullJoinStream")]
-   // JoinStream is technically a LeftJoin. FullJoin only results in rows that have bpth left and right side
-   public class FullJoinStream : ViewStream {
-      public FullJoinStream(ViewStream l_str, string ljcs, ViewStream r_str, string rjcs) {
-         ViewStream js = new global::bts.BlueOrb.SimpleJoinStream(l_str, ljcs, r_str, rjcs);
-         ViewStream rs = new bts.BlueOrb.FilterStream(js, r => !DBNull.Value.Equals(r["right_id"]));
-         mySetRealSource(rs);
-      }
-   }
-}

src/DataStreamSuite/DataStream/BlueOrb/Join/SimpleJoinStream.cs

@@ -1,5 +1,4 @@
-namespace bts.BlueOrb
-{
+namespace bts.BlueOrb {
    using BCG;
 
    using Bts.DataStream;
@@ -14,10 +13,8 @@
    using RowHash = System.Int32;
 
    [DataStreamUserRespresentation("JoinStream")]
-   public class SimpleJoinStream : ViewStream
-   {
-      public class JoinBookkeepingData
-      {
+   public class SimpleJoinStream : ViewStream {
+      public class JoinBookkeepingData {
          // initialized in cons
          readonly string[] _join_col_names;
          private readonly BlueRowHasher _hasher;
@@ -26,8 +23,7 @@
          int[]? _join_cols;
          private IDictionary<RowHash, List<IBlueRow>> _hash_row_map = new Dictionary<RowHash, List<IBlueRow>>();
 
-         public JoinBookkeepingData(ViewStream vs, List<string> join_cols, BlueRowHasher row_hasher)
-         {
+         public JoinBookkeepingData(ViewStream vs, List<string> join_cols, BlueRowHasher row_hasher) {
             _join_col_names = join_cols.Where(c => !string.IsNullOrEmpty(c)).ToArray();
             _hasher = row_hasher;
          }
@@ -44,28 +40,22 @@
          public List<IBlueRow>? get_matching_rows(int hash) =>
             _hash_row_map.GetValueOrDefault(hash);
 
-         private RowHash _get_hash(IBlueRow r)
-         {
-            if (_join_cols == null || _join_cols.Length == 0)
-            {
+         private RowHash _get_hash(IBlueRow r) {
+            if (_join_cols == null || _join_cols.Length == 0) {
                return 0;
             }
             return _hasher.get_hash(r, _join_cols);
          }
 
-         public void update_data(IDataPacket dp)
-         {
-            if (!ready)
-            {
+         public void update_data(IDataPacket dp) {
+            if (!ready) {
                _schema = dp.newRows.schema;
                _join_cols = _schema.ResolveIndices(_join_col_names);
             }
             _hash_row_map.Clear();
-            foreach (IBlueRow r in dp.newRows)
-            {
+            foreach (IBlueRow r in dp.newRows) {
                int hash = _get_hash(r);
-               if (!_hash_row_map.TryGetValue(hash, out var hash_rows))
-               {
+               if (!_hash_row_map.TryGetValue(hash, out var hash_rows)) {
                   hash_rows = new List<IBlueRow>();
                   _hash_row_map.Add(hash, hash_rows);
                }
@@ -74,10 +64,8 @@
          }
 
          [MemberNotNullWhen(true, nameof(_schema))]
-         public bool ready
-         {
-            get
-            {
+         public bool ready {
+            get {
                return _schema != null;
             }
          }
@@ -89,11 +77,9 @@
       IBlueTableSchema? _schema;
 
       public SimpleJoinStream(ViewStream ls, string ljc, ViewStream rs, string rjc) :
-         this(ViewStreamDatabase.instance, ls, ljc, rs, rjc)
-      { }
+         this(ViewStreamDatabase.instance, ls, ljc, rs, rjc) { }
 
-      public SimpleJoinStream(ViewStreamDatabase vsd, ViewStream ls, string ljc, ViewStream rs, string rjc)
-      {
+      public SimpleJoinStream(ViewStreamDatabase vsd, ViewStream ls, string ljc, ViewStream rs, string rjc) {
          _ls = ls;
          _rs = rs;
          List<string> lj_col_list = ljc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
@@ -105,35 +91,30 @@
          _rdata = new JoinBookkeepingData(rs, rj_col_list, hasher);
       }
 
-      protected override void myOnFirstSubscriber()
-      {
+      protected override void myOnFirstSubscriber() {
          _ls.onChange += _ls_onchange;
          _rs.onChange += _rs_onchange;
       }
 
-      private void _ls_onchange(object dp_obj)
-      {
+      private void _ls_onchange(object dp_obj) {
          var dp = (IDataPacket)dp_obj;
          _ldata.update_data(dp);
          _form_and_fire();
       }
 
-      private void _rs_onchange(object dp_obj)
-      {
+      private void _rs_onchange(object dp_obj) {
          var dp = (IDataPacket)dp_obj;
          _rdata.update_data(dp);
          _form_and_fire();
       }
 
-      protected override void myOnLastSubscriber()
-      {
+      protected override void myOnLastSubscriber() {
          _ls.onChange -= _ls_onchange;
          _rs.onChange -= _rs_onchange;
       }
 
       [MemberNotNull(nameof(_schema))]
-      private void _build_schema()
-      {
+      private void _build_schema() {
          if (_schema != null)
             return;
          var lschema = Log.notnull(_ldata.schema);
@@ -141,64 +122,52 @@
          var cis = new List<ColumnInfo> {
             new("id", 0, TypeCode.Int32)
          };
-         foreach (ColumnInfo ci in lschema.columnInfo)
-         {
+         foreach (ColumnInfo ci in lschema.columnInfo) {
             string col_name = rschema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
             cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
          }
-         foreach (ColumnInfo ci in rschema.columnInfo)
-         {
+         foreach (ColumnInfo ci in rschema.columnInfo) {
             string col_name = lschema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
             cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
          }
          _schema = new BlueTableSchema($"{lschema.tableName}_join_{rschema.tableName}", cis.ToArray());
       }
 
-      private void _form_and_fire()
-      {
+      private void _form_and_fire() {
          if (!_ldata.ready || !_rdata.ready)
             return;
          _build_schema();
          List<object[]> res_rows = new List<object[]>();
 
          // this is where the join happens
-         foreach (var kv in _ldata.rows)
-         {
+         foreach (var kv in _ldata.rows) {
             RowHash hash = kv.Key;
             List<IBlueRow> l_rows = kv.Value;
             List<IBlueRow>? r_rows = _rdata.get_matching_rows(hash);
 
-            foreach (IBlueRow lr in l_rows)
-            {
-               if (r_rows != null)
-               {
-                  foreach (IBlueRow r in r_rows)
-                  {
+            foreach (IBlueRow lr in l_rows) {
+               if (r_rows != null) {
+                  foreach (IBlueRow r in r_rows) {
                      object[] rowvals = build_row_array(res_rows.Count, lr);
                      copy_row(rowvals, r, false);
                      res_rows.Add(rowvals);
                   }
-               }
-               else
-               {
+               } else {
                   object[] rowvals = build_row_array(res_rows.Count, lr);
                   res_rows.Add(rowvals);
                }
             }
          }
          BlueTable rv = new BlueTable(_schema);
-         foreach (object[] rval in res_rows)
-         {
+         foreach (object[] rval in res_rows) {
             rv.addRow(new BlueRow(_schema, rval));
          }
          var dp = BlueDataUtility.build_dp(rv, myLastValue as IDataPacket);
          myFireEvent(dp);
 
-         object[] build_row_array(int id, IBlueRow l_row)
-         {
+         object[] build_row_array(int id, IBlueRow l_row) {
             object[] rv = new object[_schema.columnNames.Length];
-            for (int i = 0; i < _schema.columnNames.Length; i++)
-            {
+            for (int i = 0; i < _schema.columnNames.Length; i++) {
                rv[i] = DBNull.Value;
             }
             rv[0] = id;
@@ -206,26 +175,30 @@
             return rv;
          }
 
-         void copy_row(object?[] rval, IBlueRow r, bool is_left)
-         {
+         void copy_row(object?[] rval, IBlueRow r, bool is_left) {
             int offset = 1;
             JoinBookkeepingData sdata = _ldata;
-            if (is_left)
-            {
+            if (is_left) {
                offset = 1;
                sdata = _ldata;
-            }
-            else
-            {
+            } else {
                sdata = _rdata;
                offset = 1 + _ldata.RequireSchema().columnInfo.Length;
             }
-            for (int i = 0; i < sdata.RequireSchema().columnInfo.Length; i++)
-            {
+            for (int i = 0; i < sdata.RequireSchema().columnInfo.Length; i++) {
                rval[offset + i] = r[i];
             }
          }
       }
    }
 
+   [DataStreamUserRespresentation("FullJoinStream")]
+   public class FullJoinStream : ViewStream {
+      public FullJoinStream(ViewStream l_str, string ljcs, ViewStream r_str, string rjcs) {
+         ViewStream js = new SimpleJoinStream(l_str, ljcs, r_str, rjcs);
+         ViewStream rs = new bts.BlueOrb.FilterStream(js, r => !DBNull.Value.Equals(r["right_id"]));
+         mySetRealSource(rs);
+      }
+   }
+
 }

src/DataStreamSuite/DataStream/BlueOrb/TimeSeriesStream.cs

@@ -29,26 +29,22 @@ namespace bts.BlueOrb {
          _stream.onChange -= new ChangeDelegate(_onstream_change);
       }
 
-      private bool _orig_schema_has_timestamp = false;
+
       private void _onstream_change(object o) {
          DataPacket dp = o as DataPacket;
          DateTime now = DateTime.Now;
          if (_table == null) {
             IBlueTableSchema schema = new BlueTableSchema(dp.newRows.schema.columnInfo, dp.newRows.schema.tableName);
-            if (!schema.contains_column("timestamp")) {
-               _orig_schema_has_timestamp = true;
-               schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
-            }
+            schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
             _table = new CircularBufferTable(schema, _size);
          }
          foreach (IBlueRow r in dp.addedRows) {
             MutableBlueRow ar = new MutableBlueRow(_table.schema, r);
-            if (!_orig_schema_has_timestamp)
-               ar.setValue("timestamp", now);
+            ar.setValue("timestamp", now);
             ar.setValue("id", _id++);
             _table.add_row(ar);
          }
-         var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
+         var dp_out =  BlueDataUtility.build_dp(_table, myLastValue as DataPacket);
          myFireEvent(dp_out);
       }
    }

src/DataStreamSuite/DataStream/DataStreamExtensions.cs

@@ -23,16 +23,16 @@ public static class DataStreamExtensions {
    public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source, string? name = null, ILogger? logger = null) where T : IBlueRow =>
       source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)), name, logger);
 
-   public static IObservable<DataPacket> AsObservable(this ViewStream source) {
-      return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
+   public static IObservable<IDataPacket> AsObservable(this ViewStream source) {
+      return source as IObservable<IDataPacket> ?? Observable.Create<IDataPacket>(Subscribe);
 
-      IDisposable Subscribe(IObserver<DataPacket> observer) {
+      IDisposable Subscribe(IObserver<IDataPacket> observer) {
          source.onChange += OnChange;
          return Disposable.Create(() => source.onChange -= OnChange);
 
          void OnChange(object? newValue) =>
-            observer.OnNext(newValue as DataPacket ?? throw new ArgumentException(
-               $"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
+            observer.OnNext(newValue as IDataPacket ?? throw new ArgumentException(
+               $"Expected IDataPacket, got {newValue?.GetType()}", nameof(newValue)));
       }
    }
 

src/DataStreamSuite/DataStream/Offloading/RemoteQueryManager.cs

@@ -88,8 +88,7 @@ namespace BCG {
          return Task.CompletedTask;
       }
 
-      private void _on_str(DataPacket dp)
-      {
+      void _on_str(IDataPacket dp) {
          foreach (IBlueRow r in dp.newRows)
          {
             string name = r.get<string>("name");

src/DataStreamSuite/DataStream/Primitives/CartesianProduct/CartesianProductStream.cs

@@ -0,0 +1,22 @@
+// using System;
+// using BCG;
+// using BCG.Utilities;
+
+// namespace BCG
+// {
+//    public class CartesianProductStream : ViewStream
+//    {
+//       public CartesianProductStream(ViewStream v1, ViewStream v2)
+//       {
+//          mySetRealSource(
+//          new JoinStream (new AddColumnsStream (v1,
+//             "hiddenCartesianColumn=1"),
+//             "hiddenCartesianColumn",
+//             "!id hiddenCartesianColumn dateTime:dateBoundary",
+//             new AddColumnsStream (v2,
+//             "hiddenCartesianColumn=1"),
+//             "hiddenCartesianColumn",
+//             "!id hiddenCartesianColumn"));
+//       }
+//    }
+// }

src/DataStreamSuite/DataStream/Primitives/Distinct/DistinctRowEvaluator.cs

@@ -14,10 +14,8 @@ namespace BCG
 		public DistinctRowEvaluator(string colMapExpression, string[] columnNames)
          : base (colMapExpression, null, columnNames) {}
 
-
-      public override IBlueRow evaluate (DataPacket packet)
-      {
-         myOutputTable = myOutputTable == null ? new GroupByOutputTable (packet.schema) : myOutputTable;
+      public override IBlueRow? evaluate(IDataPacket packet) {
+         myOutputTable ??= new GroupByOutputTable(packet.Schema);
          string hashString = GroupByStream.getRepString (packet, myGroupByCols);
          if (packet.newRows.Count == 0)
             //if there are no rows for this group we remove the group from the output table

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByEvaluator.cs

@@ -42,7 +42,7 @@ namespace BCG
          return retVal;
       }
 
-      public abstract IBlueRow evaluate (DataPacket dp) ;
+      public abstract IBlueRow? evaluate(IDataPacket dp);
       #endregion
 
       #region Helper methods

src/DataStreamSuite/DataStream/Primitives/GroupBy/GroupByStream.cs

@@ -8,12 +8,13 @@ namespace BCG {
    internal class GroupByStream : MethodDrivenStream {
       private class MyFuncGroupByEvaluator : IGroupByEvaluator {
          private string[] _groupby_cols;
-         private Func<DataPacket, IBlueRow> _groupby_func;
-         public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func) {
+         readonly Func<IDataPacket, IBlueRow> _groupby_func;
+         public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<IDataPacket, IBlueRow> groupby_func) {
             _groupby_cols = groupby_cols.ToArray();
             _groupby_func = groupby_func;
          }
-         public IBlueRow evaluate(DataPacket packet) { return _groupby_func(packet); }
+         public IBlueRow evaluate(IDataPacket packet) =>
+            _groupby_func(packet);
 
          public string[] groupByColumns { get { return _groupby_cols; } }
 
@@ -32,7 +33,7 @@ namespace BCG {
       private IBlueTableSchema myOutputSchema;
       private int myIdGenerator = 0;
 
-      public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
+      public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<IDataPacket, IBlueRow> groupby_func)
          : this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func)) { }
 
       public GroupByStream(ViewStream stream, IGroupByEvaluator groupByEvaluator)
@@ -187,11 +188,11 @@ namespace BCG {
          }
       }
 
-      public static string getRepString(DataPacket dp, string[] colNames) {
+      public static string getRepString(IDataPacket dp, string[] colNames) {
          return BlueDataUtility.rowToString(getRepRow(dp), colNames);
       }
 
-      public static IBlueRow getRepRow(DataPacket dp) {
+      public static IBlueRow getRepRow(IDataPacket dp) {
          if (dp.newRows.Count > 0)
             return BlueDataUtility.extractRow(dp.newRows, 0);
          else if (dp.addedRows.Count > 0)

src/DataStreamSuite/DataStream/Primitives/GroupBy/IGroupByEvaluator.cs

@@ -1,5 +1,3 @@
-using System;
-
 namespace BCG
 {
    /// <summary>
@@ -14,7 +12,7 @@ namespace BCG
       /// deletetion) and the added and deletedRows being the increments. If a group has
       /// to be totally removed then a null should be returned
       /// </summary>
-      IBlueRow evaluate (DataPacket packet);
+      IBlueRow? evaluate(IDataPacket packet);
 
       /// <summary>
       /// Returns the columns (belonging to the original schema) on which this IGroupByEvaluator

src/DataStreamSuite/DataStream/Primitives/GroupBy/MaxGroupByEvaluator.cs

@@ -14,9 +14,8 @@ namespace BCG {
         public MaxGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
            : base(colMapEpression, outputCol, groupByCols) { }
 
-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) :
-               myOutputTable;
+        public override IBlueRow? evaluate(IDataPacket dp) {
+            myOutputTable ??= new GroupByOutputTable(dp.Schema);
             string hashString = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
             IBlueRow currMaxRow = myOutputTable.getRow(hashString);
             IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);

src/DataStreamSuite/DataStream/Primitives/GroupBy/MinGroupByEvaluator.cs

@@ -1,7 +1,3 @@
-using System;
-
-using BCG.Utilities;
-
 namespace BCG {
     public class MinGroupByEvaluator : MaxGroupByEvaluator {
         public MinGroupByEvaluator(string outputCol, string[] groupByCols)
@@ -10,8 +6,8 @@ namespace BCG {
         public MinGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) :
            base(colMapEpression, outputCol, groupByCols) { }
 
-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+        public override IBlueRow? evaluate(IDataPacket dp) {
+            myOutputTable ??= new GroupByOutputTable(dp.Schema);
             string stringRep = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
             IBlueRow currMinRow = myOutputTable.getRow(stringRep);
             IBlueRow minRow = BlueDataUtility.computeMinRow(dp.deletedRows, myOutputColumn);

src/DataStreamSuite/DataStream/Primitives/GroupBy/SumGroupByEvaluator.cs

@@ -1,5 +1,3 @@
-using System;
-
 namespace BCG {
     /// <summary>
     /// Computes the sum for a given column
@@ -10,8 +8,8 @@ namespace BCG {
         public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
            : base(colMapEpression, outputCol, groupByCols) { }
 
-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
+        public override IBlueRow evaluate(IDataPacket dp) {
+            myOutputTable ??= new GroupByOutputTable(dp.Schema);
 
             IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
             return sumRow;
@@ -33,7 +31,7 @@ namespace BCG {
 
         #region IGroupByEvaluator Members
 
-        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum(packet.newRows, myOutputCols); }
+        public IBlueRow evaluate(IDataPacket packet) => BlueDataUtility.computeSum(packet.newRows, myOutputCols);
 
         public string[] groupByColumns { get { return myGroupByCols; } }
 
@@ -51,9 +49,8 @@ namespace BCG {
         public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
            : base(colMapEpression, outputCol, groupByCols) { }
 
-        public override IBlueRow evaluate(DataPacket dp) {
-            myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
-
+        public override IBlueRow evaluate(IDataPacket dp) {
+            myOutputTable ??= new GroupByOutputTable(dp.Schema);
             IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
             return sumRow;
         }
@@ -68,7 +65,7 @@ namespace BCG {
 
         #region IGroupByEvaluator Members
 
-        public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols); }
+        public IBlueRow evaluate(IDataPacket packet) => BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols);
 
         public string[] groupByColumns { get { return myGroupByCols; } }
 

src/DataStreamSuite/DataStream/Primitives/Map/GlobalMapStream.cs

@@ -2,6 +2,8 @@ using BCG;
 
 using MongoDB.Driver.Core.WireProtocol.Messages;
 
+using Spark.Core.Diagnostics;
+
 using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -37,7 +39,7 @@ namespace bts {
       }
 
       void _handle_stream_update(object dp_obj) {
-         DataPacket dp = dp_obj as DataPacket;
+         var dp = ExceptionAssert.Argument.Is<IDataPacket>(dp_obj);
          IDictionary<int, List<IBlueRow>> groups = !String.IsNullOrEmpty(_group_by_col) ?
          BlueDataUtility.group_together(dp.newRows, new string[] { _group_by_col })
          :

src/DataStreamSuite/DataStream/Primitives/Misc/PubsubEchoTestStream.cs

@@ -8,34 +8,48 @@ using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-using bts;
-using BCG;
 
-namespace BlueOrb.Pubsub.Diagnostics {
-   // This publishes data to the pubsub server, subscribes, gets it back and records the time it took for the round trip
+namespace bts
+{
    [DataStreamUserRespresentation("pss_echo")]
-   public class PubsubEchoTestStream : ViewStream, IWorker {
-      public class PayloadMessage {
+   public class PubsubEchoTestStream : ViewStream, IWorker
+   {
+      public class PayloadMessage
+      {
          public readonly string? msg_id;
          public readonly Int64 timestamp = 0;
          public readonly byte[]? payload = null;
 
-         public static byte[] create_data(int payload_size, byte payload_byte) {
+         public static byte[] create_data(int payload_size, byte payload_byte)
+         {
             var rv = new byte[payload_size];
-            for (int i = 0; i < payload_size; i++) {
+            for (int i = 0; i < payload_size; i++)
+            {
                rv[i] = payload_byte;
             }
             return rv;
          }
 
-         public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_) {
+         // public PayloadMessage(string guid_, Int64 timestamp_, int payload_size, byte payload_byte)
+         // {
+         //    guid = guid_;
+         //    timestamp = timestamp_;
+         //    payload = new byte[payload_size];
+         //    for (int i = 0; i < payload_size; i++)
+         //    {
+         //       payload[i] = payload_byte;
+         //    }
+         // }
+         public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_)
+         {
             msg_id = guid_;
             timestamp = timestamp_;
             payload = payload_;
          }
 
          private PayloadMessage() { }
-         public byte[] serialize() {
+         public byte[] serialize()
+         {
             var str = new MemoryStream();
             BinaryWriter bw = new BinaryWriter(str);
             bw.Write(msg_id ?? string.Empty);
@@ -47,7 +61,8 @@ namespace BlueOrb.Pubsub.Diagnostics {
             return bytes;
          }
 
-         public static PayloadMessage deserialize(byte[] ser_bytes) {
+         public static PayloadMessage deserialize(byte[] ser_bytes)
+         {
             var str = new MemoryStream(ser_bytes) {
                Position = 0
             };
@@ -72,7 +87,8 @@ namespace BlueOrb.Pubsub.Diagnostics {
       private readonly string _tag_name;
       private readonly int _payload_sz;
       private readonly IPubSubRouter _router;
-      private enum IOState {
+      private enum IOState
+      {
          SEND = 0,
          RECV = 1
       };
@@ -107,7 +123,8 @@ namespace BlueOrb.Pubsub.Diagnostics {
       public PubsubEchoTestStream(string server_url, int publish_period_msecs, int payload_size, string tag_name)
       : this(server_url, TimeSpan.FromMilliseconds(publish_period_msecs), payload_size, tag_name, ViewStreamDatabase.instance) { }
 
-      protected override void myOnFirstSubscriber() {
+      protected override void myOnFirstSubscriber()
+      {
          _pub_core = PublisherCore.Create(_router);
          _sub_core = new SubscriberCore(_router);
          _sub_core.subscribe(_tag_name);
@@ -115,7 +132,8 @@ namespace BlueOrb.Pubsub.Diagnostics {
 
       }
 
-      protected override void myOnLastSubscriber() {
+      protected override void myOnLastSubscriber()
+      {
          _vsd.removeWorker(this);
          Log.notnull(_sub_core).unsubscribe(_tag_name);
       }
@@ -126,7 +144,8 @@ namespace BlueOrb.Pubsub.Diagnostics {
             SubscriberRoutes = new() { new(serverUrl, ".*") },
          });
 
-      private void _fire_dp_after_send() {
+      private void _fire_dp_after_send()
+      {
          Log.notnull(_last_sent_msg);
          // as soon as the data is published, its reflected in the stream state
          BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value,
@@ -136,7 +155,8 @@ namespace BlueOrb.Pubsub.Diagnostics {
          myFireEvent(dp_out);
       }
 
-      private void _fire_dp_after_recv() {
+      private void _fire_dp_after_recv()
+      {
          Log.notnull(_last_recvd_msg);
          Log.notnull(_last_sent_msg);
          // update the last publish row
@@ -153,11 +173,13 @@ namespace BlueOrb.Pubsub.Diagnostics {
          }
       }
 
-      private string _get_next_id() {
+      private string _get_next_id()
+      {
          return _table.Count.ToString();
       }
 
-      private void _publish_next_packet() {
+      private void _publish_next_packet()
+      {
          Log.notnull(_pub_core);
          var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte);
          _last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes);
@@ -166,10 +188,13 @@ namespace BlueOrb.Pubsub.Diagnostics {
          _last_publish_time = DateTime.Now;
       }
 
-      public void doWork() {
-         switch (_state) {
+      public void doWork()
+      {
+         switch (_state)
+         {
             case IOState.SEND:
-               if (DateTime.Now - _last_publish_time >= _publish_period) {
+               if (DateTime.Now - _last_publish_time >= _publish_period)
+               {
                   _publish_next_packet();
                   _last_publish_time = DateTime.Now;
                   _fire_dp_after_send();
@@ -181,7 +206,8 @@ namespace BlueOrb.Pubsub.Diagnostics {
                var updates = Log.notnull(_sub_core).get_updates(TimeSpan.Zero);
                updates = updates.Where(u => u.tag == _tag_name);
                var recv_msg = find_packet(updates);
-               if (recv_msg != null) {
+               if (recv_msg != null)
+               {
                   _last_recvd_msg = recv_msg;
                   _fire_dp_after_recv();
                   _state = IOState.SEND;
@@ -204,7 +230,8 @@ namespace BlueOrb.Pubsub.Diagnostics {
          }
       }
 
-      public static long epoch_now() {
+      public static long epoch_now()
+      {
          return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
       }
 

src/DataStreamSuite/DataStream/Primitives/Project/New/NewProjectStream.cs

@@ -1,17 +1,12 @@
 namespace bts {
    using BCG;
 
-   using bts.collections;
-
-   using IronPython.Runtime.Operations;
-
    using ProtoBuf;
 
    using System;
    using System.Collections;
    using System.Collections.Generic;
    using System.Linq;
-   using System.Runtime.Intrinsics.Arm;
 
    public class ProjectData {
       // schema from the base stream

src/DataStreamSuite/DataStream/Publish/PublisherStream.cs

@@ -1,5 +1,4 @@
-namespace BlueOrb.PubSub {
-   using BCG;
+namespace BCG {
    // this for testing purposes mainly, its wraps a ViewPublisher around a stream, 
    // and hence epublishes it out to the PS
    public class PublisherStream : ViewStream {

src/DataStreamSuite/DataStreamCore/BlueData/BlueDataUtility.cs

@@ -15,7 +15,6 @@ using mbts;
 using BCG.Extensions;
 using System.Text.Json;
 using System.Text.Json.Nodes;
-using Newtonsoft.Json;
 using BCG;
 using System.Runtime.CompilerServices;
 using bts;
@@ -360,11 +359,11 @@ namespace BCG {
       }
 
       public static Dictionary<string, string> convert_json_to_dictionary(string json_string) {
-         Dictionary<string, string> dictionary = JsonConvert.DeserializeObject<Dictionary<string, string>>(json_string);
+         Dictionary<string, string> dictionary = JsonSerializer.Deserialize<Dictionary<string, string>>(json_string);
          return dictionary;
       }
       public static IBlueRow convert_json_to_row(string json_string, int id = 0) {
-         Dictionary<string, object> dictionary = JsonConvert.DeserializeObject<Dictionary<string, object>>(json_string);
+         Dictionary<string, object> dictionary = JsonSerializer.Deserialize<Dictionary<string, object>>(json_string);
          if (!dictionary.ContainsKey("id")) {
             dictionary.Add("id", id);
          }
@@ -1847,7 +1846,7 @@ namespace BCG {
          return sb.ToString();
       }
 
-      public static string rows_id_string(this IBlueRowCollection table) {
+      public static string id_string(IBlueRowCollection table) {
          string rv = String.Empty;
          foreach (IBlueRow r in table)
             rv += $"{r["id"]} ";
@@ -2246,7 +2245,7 @@ namespace BCG {
 
       public static T get_row_value<T>(IBlueRow r, string col_name, T error_val) {
          try {
-            object rv = (r != null && r.schema.contains_column(col_name)) ? r[col_name] : DBNull.Value;
+            object rv = (r != null && r.Contains(col_name)) ? r[col_name] : DBNull.Value;
             if (rv == null || rv == DBNull.Value)
                return error_val;
             else

src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfo.cs

@@ -1,4 +1,6 @@
 #nullable enable
+using Bts.DataStreamCore;
+
 using System;
 using System.Text.Json.Serialization;
 
@@ -7,6 +9,7 @@ namespace BCG {
    /// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable
    /// </summary>
    [Serializable]
+   [JsonConverter(typeof(ColumnInfoJsonConverter))]
    public class ColumnInfo : IComparable<ColumnInfo>, IComparable, IEquatable<ColumnInfo> {
 
       //the default column info does not cach values in a lookup table, therefore it is in group 0
@@ -67,16 +70,12 @@ namespace BCG {
       private Type myColType;
       private object? myColValue;
 
-      [JsonPropertyName("colNo")]
       public int colNo { get { return myColNo; } }
 
-      [JsonPropertyName("colName")]
       public string colName { get { return myColName; } }
 
-      [JsonPropertyName("colType")]
       public Type colType { get { return myColType; } }
 
-      [JsonIgnore]
       public object? colValue {get{return myColValue;}}
    }
 }

src/DataStreamSuite/DataStreamCore/BlueData/ColumnInfoJsonConverter.cs

@@ -0,0 +1,27 @@
+namespace Bts.DataStreamCore;
+
+using BCG;
+
+using Spark.Core.Json;
+
+using System;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using System.Text.Json.Serialization;
+
+public class ColumnInfoJsonConverter : JsonConverter<ColumnInfo> {
+   public override ColumnInfo Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+      var jobj = JsonSerializer.Deserialize<JsonObject>(ref reader);
+      return new(
+         jobj[nameof(ColumnInfo.colName)].GetValue<string>(),
+         jobj[nameof(ColumnInfo.colNo)].GetValue<int>(),
+         Type.GetType(jobj[nameof(ColumnInfo.colType)].GetValue<string>()));
+   }
+
+   public override void Write(Utf8JsonWriter writer, ColumnInfo value, JsonSerializerOptions options) {
+      using var _ = writer.WriteObject();
+      writer.WriteString(nameof(ColumnInfo.colName), value.colName);
+      writer.WriteNumber(nameof(ColumnInfo.colNo), value.colNo);
+      writer.WriteString(nameof(ColumnInfo.colType), value.colType.ToString());
+   }
+}

src/DataStreamSuite/DataStreamCore/BlueData/ExtensionFunctions/Comparisons.cs

@@ -1,142 +0,0 @@
-namespace BlueOrb.Data {
-   using BCG;
-
-   using bts;
-
-
-   using System;
-   using System.Linq;
-
-   public static class Comparisons {
-      public static int compare_row(this IBlueRow row, IBlueRow other_row, params string[] col_names) {
-         int comp_res = 0;
-         foreach (string col_name in col_names) {
-            object val1 = row[col_name];
-            object val2 = other_row[col_name];
-            comp_res = compare_row_values(val1, val2);
-            if (comp_res != 0)
-               break;
-         }
-         return comp_res;
-      }
-
-      public static void compare_tables_id(this IBlueRowCollection table1,
-                                           IBlueRowCollection table2,
-                                           out IBlueRowCollection rows_only_in_table1,
-                                           out IBlueRowCollection rows_only_in_table2) {
-         Log.assert(table1 != null);
-         table2 = table2 ?? new BlueTable(table1.schema);
-         var schema = get_output_schema(table1, table2);
-         var table1_only = new BlueTable(schema);
-         var table2_only = new BlueTable(schema);
-
-         if (table1 is not null) {
-            foreach (IBlueRow row in table1)
-               if (table2 is null || !table2.containsID(row.get<int>("id")))
-                  table1_only.addRow(row);
-         }
-
-         if (table2 is not null) {
-            foreach (IBlueRow row in table2)
-               if (table1 is null || !table1.containsID(row.get<int>("id")))
-                  table2_only.addRow(row);
-         }
-
-         rows_only_in_table1 = table1_only;
-         rows_only_in_table2 = table2_only;
-      }
-
-      public static void compare_tables_deep(this IBlueRowCollection table1,
-                                             IBlueRowCollection table2,
-                                             out IBlueRowCollection rows_only_in_table1,
-                                             out IBlueRowCollection rows_only_in_table2) {
-         Log.assert(table1 != null);
-         table2 = table2 ?? new BlueTable(table1.schema);
-         var schema = get_output_schema(table1, table2);
-         validate_matching_schemas(table1, table2);
-
-         var table1_only = new BlueTable(schema);
-         var table2_only = new BlueTable(schema);
-         var compare_cols = schema.columnNames.Where(c => c != "id").ToArray();
-
-         if (table1 is not null) {
-            foreach (IBlueRow row in table1) {
-               if (table2 is null || !table2.containsID(row.get<int>("id"))) {
-                  table1_only.addRow(row);
-                  continue;
-               }
-
-               var other_row = table2[row.get<int>("id")];
-               if (row.compare_row(other_row, compare_cols) != 0) {
-                  table1_only.addRow(row);
-                  table2_only.addRow(other_row);
-               }
-            }
-         }
-
-         if (table2 is not null) {
-            foreach (IBlueRow row in table2)
-               if (table1 is null || !table1.containsID(row.get<int>("id")))
-                  table2_only.addRow(row);
-         }
-
-         rows_only_in_table1 = table1_only;
-         rows_only_in_table2 = table2_only;
-      }
-
-      static int compare_row_values(object v1, object v2) {
-         if (is_null(v1))
-            return is_null(v2) ? 0 : -1;
-         if (is_null(v2))
-            return 1;
-         try {
-            var comparable1 = (IComparable)v1;
-            return comparable1.CompareTo(v2);
-         } catch (InvalidCastException) {
-            if (v1 is byte[] bytes1 && v2 is byte[] bytes2)
-               return compare_bytes(bytes1, bytes2) ? 0 : 1;
-            throw;
-         }
-      }
-
-      static bool compare_bytes(byte[] v1, byte[] v2) {
-         if (v1.Length != v2.Length)
-            return false;
-
-         for (int i = 0; i < v1.Length; i++)
-            if (v1[i] != v2[i])
-               return false;
-
-         return true;
-      }
-
-      static bool is_null(object o) {
-         return o == null || o == DBNull.Value;
-      }
-
-      static IBlueTableSchema get_output_schema(IBlueRowCollection table1, IBlueRowCollection table2) {
-         if (table1 is not null)
-            return table1.schema;
-         if (table2 is not null)
-            return table2.schema;
-         throw new ArgumentNullException(nameof(table1), "At least one table must be non-null.");
-      }
-
-      static void validate_matching_schemas(IBlueRowCollection table1, IBlueRowCollection table2) {
-         if (table1 is null || table2 is null)
-            return;
-
-         var schema1 = table1.schema;
-         var schema2 = table2.schema;
-         if (schema1.columnInfo.Length != schema2.columnInfo.Length)
-            throw new ArgumentException("Tables must have matching schemas.");
-
-         for (int i = 0; i < schema1.columnInfo.Length; i++) {
-            var c1 = schema1.columnInfo[i];
-            var c2 = schema2.columnInfo[i];
-            if (c1.colName != c2.colName || c1.colType != c2.colType)
-               throw new ArgumentException("Tables must have matching schemas.");
-         }
-      }
-   }
-}

src/DataStreamSuite/DataStreamCore/BlueData/ExtensionFunctions/ToStrings.cs

@@ -1,35 +0,0 @@
-namespace BlueOrb.Data {
-   using BCG;
-
-   using System;
-   using System.Text;
-
-public static class ToStrings {
-      public static string ToDebugString(this IBlueRow row) {
-         StringBuilder sb = new StringBuilder();
-         foreach (ColumnInfo c in row.schema.columnInfo) {
-            if (sb.Length != 0)
-               sb.Append(", ");
-            sb.Append(c.colName);
-            sb.Append(": ");
-            object val = row[c.colName];
-            sb.Append(is_null(val) ? "NULL" : val.ToString());
-         }
-         return sb.ToString();
-      }
-
-      public static string ToDebugString(this IBlueRowCollection rows) {
-         StringBuilder sb = new StringBuilder();
-         foreach (IBlueRow row in rows) {
-            if (sb.Length != 0)
-               sb.AppendLine();
-            sb.Append(row.ToDebugString());
-         }
-         return sb.ToString();
-      }
-
-      static bool is_null(object o) {
-         return o == null || o == DBNull.Value;
-      }
-   }
-}

src/DataStreamSuite/DataStreamCore/DataPacket.cs

@@ -41,7 +41,7 @@ namespace BCG {
       }
 
       public IBlueRowCollection newRows, addedRows, deletedRows;
-      public IDictionary<string, object> meta_data = null;
+      public Dictionary<string, object> meta_data = null;
       public DataPacket() { }
 
       public DataPacket(IBlueTableSchema schema)
@@ -53,7 +53,7 @@ namespace BCG {
          this.deletedRows = deletedRows;
       }
       public DataPacket(IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows,
-                  IDictionary<string, object> meta_data) {
+                  Dictionary<string, object> meta_data) {
          this.newRows = newRows;
          this.addedRows = addedRows;
          this.deletedRows = deletedRows;
@@ -63,6 +63,7 @@ namespace BCG {
       IBlueRowCollection IDataPacket.newRows => newRows;
       IBlueRowCollection IDataPacket.addedRows => addedRows;
       IBlueRowCollection IDataPacket.deletedRows => deletedRows;
+      IReadOnlyDictionary<string, object> IDataPacket.Metadata => meta_data;
 
       public void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows) {
          new_rows = newRows;

src/DataStreamSuite/DataStreamCore/DataStreamCore.csproj

@@ -8,7 +8,6 @@
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.24.3" />
-    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageReference Include="OpenAI" Version="2.0.0" />
     <PackageReference Include="SimMetrics.Net" Version="1.0.5" />
   </ItemGroup>

src/DataStreamSuite/DataStreamCore/DynamicLoading/PrimitiveManager.cs

@@ -6,89 +6,89 @@ namespace BCG {
    using System.Reflection;
    using System.IO;
    using Bts.DataStreamCore;
-   using System.Collections.Generic;
-   using System.Linq;
 
    public class PrimitiveManager {
       static MarketDataStreamFactory mdStreamFactory;
-      private Dictionary<string, Type> _dslopname_to_type_map = new Dictionary<string, Type>();
-      private static PrimitiveManager _self;
-      private HashSet<string> _loaded_assemblies = new HashSet<string>();
-      private static string[] _allowed_namespaces_matches = new string[] {
-         "blueorb"
-      };
+      private IDictionary myTypeLookupTable = new Hashtable();
+      private static PrimitiveManager mySelf;
+      private ArrayList myAssemblyNames = new ArrayList();
 
       private PrimitiveManager() {
-         _load_all_streams();
+         myLoadAllTypes();
       }
 
-      private static bool _is_loadable_stream(Type type) {
-         Func<bool> namespace_match = () => {
-            foreach (string ns in _allowed_namespaces_matches) {
-               if (type.Namespace.ToLower().Contains(ns.ToLower()))
-                  return true;
-            }
-            return false;
-         };
-         bool rv = type.IsSubclassOf(typeof(ViewStream))
-                   && type.IsPublic;
-         //rv = rv && (!type.IsInterface && !type.IsAbstract);
-         rv = rv && namespace_match();
-         return rv;
-      }
-
-      private void _load_all_streams() {
-         load_streams_from_assembly(this.GetType().Assembly.FullName);
+      private void myLoadAllTypes() {
+         AppDomain currentDomain = AppDomain.CurrentDomain;
+         currentDomain.AssemblyResolve += new ResolveEventHandler(myAssemblyResolver);
+         loadAssembly(this.GetType().Assembly.FullName);
          var assemblies = new string[] { "DataStream.dll", "SharedStreams.dll" };
-         foreach (string assembly_name in assemblies) {
-            bts.Log.Gen.info($"PrimitiveManager._load_all_streams assemblyName: {assembly_name}");
-            string resolved_path_name = assembly_name;
-            if (!System.IO.Path.IsPathRooted(assembly_name)) {
-               resolved_path_name = Path.GetFullPath(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, assembly_name));
+         foreach (string assemblyName in assemblies) {
+            bts.Log.Gen.info($"PrimitiveManager.myLoadAllTypes assemblyName: {assemblyName}");
+            string resolvedAssemblyPath = assemblyName;
+            if (!System.IO.Path.IsPathRooted(assemblyName)) {
+               resolvedAssemblyPath = Path.GetFullPath(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, assemblyName));
             }
-            load_streams_from_assembly(resolved_path_name);
+            loadAssembly(resolvedAssemblyPath);
          }
       }
 
-      public Type get_dsl_op_type(string dsl_op_name) {
-         Type rv;
-         if (!_dslopname_to_type_map.TryGetValue(dsl_op_name, out rv))
-            return null;
-         return rv;
+      private static Assembly myAssemblyResolver(object sender, ResolveEventArgs args) {
+         foreach (Assembly a in AppDomain.CurrentDomain.GetAssemblies())
+            if (args.Name.Equals(a.FullName))
+               return a;
+         return null;
       }
 
-      public void load_streams_from_assembly(string assembly_name) {
-         bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assembly_name}");
-         if (_loaded_assemblies.Contains(assembly_name))
+      /// <summary>
+      /// Loads all the Stream types associated with the assembly
+      /// </summary>
+      /// <param name="assemblyName"></param>
+      public void loadAssembly(string assemblyName) {
+         bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assemblyName}");
+         if (myAssemblyNames.Contains(assemblyName))
             return;
-         _loaded_assemblies.Add(assembly_name);
-         Assembly loaded_assembly = GetType().Assembly.FullName.Equals(assembly_name) ?
+         myAssemblyNames.Add(assemblyName);
+         Assembly loadedAssembly = GetType().Assembly.FullName.Equals(assemblyName) ?
             GetType().Assembly :
-            Assembly.LoadFrom(assembly_name);
+            Assembly.LoadFrom(assemblyName);
 
-         foreach (Type type in loaded_assembly.GetTypes())
-            if (_is_loadable_stream(type)) {
-               bool is_visible = true;
-               string dsl_name = type.Name; //createDefaultName(type.Name);
+         foreach (Type type in loadedAssembly.GetTypes())
+            if (!type.IsInterface
+               && !type.IsAbstract
+               && type.IsSubclassOf(typeof(ViewStream))
+               && type.IsPublic) {
+               bool isVisible = true;
+               string userRep = createDefaultName(type.Name);
                foreach (Attribute att in type.GetCustomAttributes(false))
                   if (att is DataStreamUserRespresentation) {
                      DataStreamUserRespresentation dsatt = att as DataStreamUserRespresentation;
-                     is_visible = dsatt.isVisible;
-                     dsl_name = is_visible ?
-                        dsatt.friendlyName == null ? dsl_name : dsatt.friendlyName
+                     isVisible = dsatt.isVisible;
+                     userRep = isVisible ?
+                        dsatt.friendlyName == null ? userRep : dsatt.friendlyName
                         : null;
                      break;
                   }
-               if (is_visible) {
-                  // Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName);
-                  _dslopname_to_type_map[dsl_name] = type;
+               if (isVisible) {
+                 // Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName);
+                  myTypeLookupTable[userRep] = type;
                }
             }
       }
 
+      public static string createDefaultName(string typeName) {
+         if (!typeName.EndsWith("Stream"))
+            return typeName;
+         string retVal = typeName.Remove(typeName.Length - "Stream".Length, "Stream".Length);
+         string startString = new String(new char[] { retVal[0] }).ToLower();
+         retVal = retVal.Remove(0, 1);
+         retVal = retVal.Insert(0, startString);
+         return retVal;
+      }
+
+
 
       public static PrimitiveManager instance {
-         get { return _self = _self == null ? new PrimitiveManager() : _self; }
+         get { return mySelf = mySelf == null ? new PrimitiveManager() : mySelf; }
       }
 
       public static void SetMarketDataClient(object value) {
@@ -96,20 +96,31 @@ namespace BCG {
          mdStreamFactory = new(value);
       }
 
-      public string dsl_to_type_name(string dsl_op_name) {
-         Type rv;
-         if (!_dslopname_to_type_map.TryGetValue(dsl_op_name, out rv))
-            throw new ApplicationException(dsl_op_name + " is not a loaded dsl op");
-         return rv.Name;
+      /// <summary>
+      /// Returns the Type name corresponding to the primitive name
+      /// </summary>
+      /// <param name="friendlyName"></param>
+      /// <returns></returns>
+      public string find(string friendlyName) {
+         if (myTypeLookupTable[friendlyName] == null)
+            throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types");
+         return (myTypeLookupTable[friendlyName] as Type).Name;
       }
 
+      /// <summary>
+      /// Constructs and returns a ViewStream of type corresponding to the friendly name
+      /// </summary>
+      /// <param name="friendlyName">friendly primitive name </param>
+      /// <param name="args">args can be null or an empty ICollection if the paramterless constructor
+      /// is the desired one</param>
+      /// <returns></returns>
       public ViewStream build(string friendlyName, ICollection args) {
          object[] constructorArgs = null;
          if (args != null && args.Count > 0) {
             constructorArgs = new object[args.Count];
             args.CopyTo(constructorArgs, 0);
          }
-         var type = _dslopname_to_type_map[friendlyName] as Type
+         var type = myTypeLookupTable[friendlyName] as Type
             ?? throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types");
          if (type.HasCustomAttribute<LiveMarketDataStreamAttribute>()) {
             if (mdStreamFactory is null) {
@@ -124,14 +135,30 @@ namespace BCG {
             Activator.CreateInstance(type, constructorArgs) as ViewStream;
       }
 
-      public string[] get_dsl_op_names() {
-         return _dslopname_to_type_map.Keys.ToArray();
+      /// <summary>
+      /// Returns the name of all the primitives 
+      /// </summary>
+      /// <returns></returns>
+      public string[] getPrimitiveNames() {
+         string[] retVal = new string[myTypeLookupTable.Count];
+         myTypeLookupTable.Keys.CopyTo(retVal, 0);
+         return retVal;
       }
 
+      /// <summary>
+      /// Return the types of all the streams
+      /// </summary>
+      /// <returns></returns>
+      public Type[] getStreamTypes() {
+         Type[] retVal = new Type[myTypeLookupTable.Count];
+         myTypeLookupTable.Values.CopyTo(retVal, 0);
+         return retVal;
+      }
 
-      public ICollection get_dsl_op_cons_args(string dsl_op_name) {
+      public ICollection getConstructorSigns(string primitiveName) {
          ArrayList retVal = new ArrayList();
-         Type t = get_dsl_op_type(dsl_op_name);
+         Log.assert(myTypeLookupTable[primitiveName] != null);
+         Type t = myTypeLookupTable[primitiveName] as Type;
          foreach (ConstructorInfo ci in t.GetConstructors()) {
             string sign = "";
             sign = t.Name + " (";
@@ -145,18 +172,9 @@ namespace BCG {
          return retVal;
       }
 
-      public string get_dslop_assembly_name(string dsl_op_name) {
-         Type rv;
-         if (!_dslopname_to_type_map.TryGetValue(dsl_op_name, out rv))
-            throw new ApplicationException(dsl_op_name + " is not a loaded dsl op");
-         return rv.Assembly.FullName;
+      public string getAssemblyName(string primitiveName) {
+         Log.assert(myTypeLookupTable[primitiveName] != null);
+         return (myTypeLookupTable[primitiveName] as Type).Assembly.GetName().Name;
       }
-
-      public string get_qualified_op_name(string dsl_op_name) {
-         Type dsl_op_type = get_dsl_op_type(dsl_op_name);
-         Log.assert(dsl_op_type != null);
-         return $"{dsl_op_type.Namespace}.{dsl_op_name}";
-      }
-
    }
 }

src/DataStreamSuite/DataStreamCore/IDataPacket.cs

@@ -13,6 +13,7 @@ namespace BCG {
       IBlueRowCollection newRows { get; }
       IBlueRowCollection addedRows { get; }
       IBlueRowCollection deletedRows { get; }
+      IReadOnlyDictionary<string, object> Metadata { get; }
       //this is an ICollection of IDictionary
       void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows);
    }

src/DataStreamSuite/DataStreamCore/Messaging/Subscription/TableAccumulator2.cs

@@ -3,23 +3,23 @@ using System.IO;
 using System.Collections.Generic;
 using System.Linq;
 
-using BlueOrb.Data;
 using bts;
-using System.Threading;
 
-namespace BCG {
-   // This class is responsible for mantaining the state for a table. It gets the
-   // bytes from the pubsubserver via the handle_updates call 
-   // (called by VSD, - VSD is polling the SubscriberCOre for udpates), deserializes,
+namespace BCG
+{
+   // This class is responsible for mantaining the state fo a table. It gets the
+   // bytes from the pubsubserver via the handle_updates call (called by VSD), deserializes
    // updates the state of the table, and fires the TableUpdateDelegate.
-   public class TableAccumulator2 : ITableAccumulator {
+   public class TableAccumulator2 : ITableAccumulator
+   {
       private readonly string _tag;
       private readonly ISubscriberCore _subscriber_core;
       IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
       private BlueTable _table;
       private TableUpdateDelegate _on_update_delegate;
 
-      public TableAccumulator2(string tag, ISubscriberCore sub_core) {
+      public TableAccumulator2(string tag, ISubscriberCore sub_core)
+      {
          _tag = tag;
          _subscriber_core = sub_core;
       }
@@ -28,54 +28,61 @@ namespace BCG {
 
       //public bool is_active() { return _on_update_delegate != null; }
 
-      public event TableUpdateDelegate on_update {
-         add {
+      public event TableUpdateDelegate on_update
+      {
+         add
+         {
             Log.Gen.info("TableAccumulator2.on_update.add {_tag} subscribed");
-            if (_on_update_delegate == null) {
+            if (_on_update_delegate == null)
+            {
                //ie first subscription
                Log.Gen.info("TableAccumulator2.on_update {_tag} subscribed for the first time");
                _subscriber_core.subscribe(_tag);
             }
             _on_update_delegate =
                Delegate.Combine(_on_update_delegate, value) as TableUpdateDelegate;
-            // a new subsriber should get a full callback immediatly (new rows, 0 deleted, all added)
             if (_table != null)
                value(_table, _table, new BlueTable(_table.schema));
          }
-         remove {
+         remove
+         {
             Log.Gen.info("TableAccumulator2.on_update.remove {_tag} unsubscribed");
-            if (_on_update_delegate != null) {
+            if (_on_update_delegate != null)
+            {
                _on_update_delegate =
                   Delegate.Remove(_on_update_delegate, value) as TableUpdateDelegate;
                int n_subs = _on_update_delegate == null ? 0 : _on_update_delegate.GetInvocationList().Length;
                Log.Gen.info("TableAccumulator2.on_update.remove {_tag} unsubscribed n_subs: {n_subs}");
-               if (_on_update_delegate == null) {
+               if (_on_update_delegate == null)
+               {
                   _subscriber_core.unsubscribe(_tag);
-                  _table = null;
+                  // TODO
+                  //_table = null;
                }
             }
 
          }
       }
       private int _update_no = 0;
-      public void handle_updates(IEnumerable<byte[]> msgs) {
+      public void handle_updates(IEnumerable<byte[]> msgs)
+      {
          if (msgs == null || msgs.Count() == 0)
             return;
          Log.Gen.info($"TableAccumulator2.handle_updates _update_no: {_update_no++} msgs: {msgs.Count()} msg_0: {msgs.ElementAt(0).Length}");
          IBlueRowCollection table_0 = _table == null ? null : new BlueTable(_table);
-         int i = 0;
-         foreach (var msg in msgs) {
-            bts.StreamIOMessage new_msg_txt = _deser.deserialize(msg);
+         foreach (var msg in msgs)
+         {
+            bts.StreamIOMessage str_msg = _deser.deserialize(msg);
             var state_txt = _table == null ? "NULL" : _table.Count.ToString();// BlueDataUtility.id_string(_table);
-            Log.Gen.info($"TableAccumulator2.handle_update tag: {_tag} object_hash: {GetHashCode()} thread_id: {Thread.CurrentThread.ManagedThreadId} msg: {i++ / msgs.Count()} msg_sz: {msg.Length} new_msg_txt: {new_msg_txt} pre_update_count: {state_txt}");
-
-            switch (new_msg_txt.op) {
+            Log.Gen.info($"TableAccumulator2.handle_update tag: {_tag} sz: {msg.Length} str_msg: {str_msg} _state: {state_txt}");
+            switch (str_msg.op)
+            {
                case bts.StreamIOMessage.StreamOp.CREATE:
-                  _table = new BlueTable(new_msg_txt.create_message.rows);
+                  _table = new BlueTable(str_msg.create_message.rows);
                   break;
                case bts.StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD:
-                  _table.deleteRows(new_msg_txt.multiple_remove_add_message.removed_rows);
-                  _table.addRows(new_msg_txt.multiple_remove_add_message.added_rows);
+                  _table.deleteRows(str_msg.multiple_remove_add_message.removed_rows);
+                  _table.addRows(str_msg.multiple_remove_add_message.added_rows);
                   break;
             }
          }
@@ -84,16 +91,11 @@ namespace BCG {
          IBlueRowCollection d_rows, a_rows;
          //BlueDataUtility.compute_differences_id_based(_table, table_0, out d_rows, out a_rows);
          BlueDataUtility.compute_differences_fast(_table, table_0, out d_rows, out a_rows);
-         Log.Gen.info($"TableAccumulator2.handle_updates firing update, _table: {_table.rows_id_string()} d_rows: {d_rows.rows_id_string()} a_rows: {a_rows.rows_id_string()}");
-         if (a_rows.Count == 0 && d_rows.Count == 0 && table_0 != null) {
-            Log.Gen.info($"TableAccmulator2.handle_updates ex! _table: {_table.ToDebugString()} table_0: {table_0.ToDebugString()} a_rows: {a_rows.ToDebugString()} d_rows: {d_rows.ToDebugString()}");
-            //same table was pubslished (maybe surpress it on the publisher or pubsubs)
-            return;
-         }
          _on_update_delegate?.Invoke(_table, a_rows, d_rows);
       }
 
-      public void update(Stream[] streams) {
+      public void update(Stream[] streams)
+      {
       }
 
       #endregion

src/DataStreamSuite/DataStreamCore/Messaging/v2/Receive/PubSubServer/PubsubServerStreamTagHandler.cs

@@ -30,7 +30,7 @@ namespace bts {
       }
    }
 
-   // This class is created by the SingleTagData for stream tag_type. Every DS tag has a PubsubServerSingleStreamTagHandler
+   // This class is created by the SingleTagData for stream tag_type. Every pub
 
    public class PubsubServerSingleStreamTagHandler : IPubSubServerTagHandler {
       // this is kept around to deserielize the incoming messages and keep the state of the table

src/DataStreamSuite/DataStreamExpressions/PrimitiveExpression.cs

@@ -20,7 +20,7 @@ public class PrimitiveExpression : StatementExpression {
    }
 
    public override string print(int startCol, IList<string> activeBindings) {
-      string prelude = "new " + PrimitiveManager.instance.dsl_to_type_name(myName) + " (";
+      string prelude = "new " + PrimitiveManager.instance.find(myName) + " (";
 
       return prelude + myPrintArgs(startCol + prelude.Length, activeBindings) + ")";
    }

src/DataStreamSuite/DataStreamPublisher/Services/DataStreamPublisherService.cs

@@ -8,11 +8,11 @@ using Bts.DataStreamCore;
 using Bts.DataStreamExpressions;
 using Bts.PubSub;
 
+using global::Spark.Core.Disposables;
+
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Options;
 
-using Spark.Core.Disposables;
-
 using System;
 using System.Collections.Generic;
 using System.Linq;

src/DataStreamSuite/DataStreamPublisher/WSPublishers/DatabentoPublisher.py

@@ -1,146 +0,0 @@
-"""
-Requires:
-   pip install -U databento
-   pip install websockets
-
-Sources:
-   https://databento.com/docs/quickstart/set-up
-   https://databento.com/docs/schemas-and-data-formats/bbo
-"""
-
-import datetime as dt
-import subprocess
-import sys
-import time
-
-try:
-   import databento as db
-except ImportError:
-   print("Installing missing dependency: databento")
-   subprocess.check_call([sys.executable, "-m", "pip", "install", "-U", "databento"])
-   import databento as db
-
-from databento.common.error import BentoError
-
-try:
-   import websockets
-except ImportError:
-   print("Installing missing dependency: websockets")
-   subprocess.check_call([sys.executable, "-m", "pip", "install", "websockets"])
-   import websockets
-
-from databento_transformer import DatabentoTransformer
-from pubsub_client import PubSubClient
-
-
-API_KEY = "db-VksTJvhMmbLjbJu9Kg6GPS99B8FDN"
-DATASET = "GLBX.MDP3"
-SYMBOL = "ES.FUT"
-PUBSUB_WS_URL = "ws://localhost:19092/ws/"
-BBO_TABLE_NAME = "livebbo_glbx_future_es_continuouscontracts"
-TRADES_TABLE_NAME = "livetrades_glbx_future_es_continuouscontracts"
-DEFINITIONS_TABLE_NAME = "livecontracts_glbx_future_es"
-
-_pubsub_client = PubSubClient(PUBSUB_WS_URL)
-_transformer = DatabentoTransformer(
-   symbol=SYMBOL,
-   bbo_table_name=BBO_TABLE_NAME,
-   trades_table_name=TRADES_TABLE_NAME,
-   definitions_table_name=DEFINITIONS_TABLE_NAME,
-)
-
-
-def _publish_contract_definitions() -> None:
-   historical_client = db.Historical(API_KEY)
-   dataset_range = historical_client.metadata.get_dataset_range(DATASET)
-   schema_range = dataset_range.get("schema", {}).get("definition", {})
-   available_end = schema_range.get("end", dataset_range["end"])
-
-   if isinstance(available_end, str):
-      available_end_dt = dt.datetime.fromisoformat(available_end.replace("Z", "+00:00"))
-   else:
-      available_end_dt = available_end
-
-   # Databento's range end is exclusive. Back off slightly so the request stays
-   # strictly inside the available window.
-   end_dt = available_end_dt - dt.timedelta(seconds=1)
-   start_dt = max(end_dt - dt.timedelta(days=1), end_dt.replace(hour=0, minute=0, second=0, microsecond=0))
-   start = start_dt.isoformat()
-   end = end_dt.isoformat()
-
-   definition_data = historical_client.timeseries.get_range(
-      dataset=DATASET,
-      schema="definition",
-      stype_in="parent",
-      symbols=SYMBOL,
-      start=start,
-      end=end,
-   )
-
-   df = definition_data.to_df()
-   payload = _transformer.build_definitions_payload(df)
-   if payload is None:
-      print("No Databento contract definitions returned")
-      return
-
-   print(f"Publishing {len(payload['rows'])} Databento contract definition rows")
-   _pubsub_client.publish(payload)
-
-
-def on_record(record) -> None:
-   # Print every callback first so we can see whether Databento is flowing
-   # even if our row extraction logic misses a field shape.
-   print(record)
-
-   payloads = _transformer.transform_record(record)
-   if payloads:
-      for payload in payloads:
-         _pubsub_client.publish(payload)
-      return
-
-   print("Skipping record with no BBO/trade fields")
-
-
-def main() -> None:
-   _pubsub_client.start()
-   print(f"Starting Databento live client for {SYMBOL} on {DATASET}")
-   try:
-      _publish_contract_definitions()
-   except Exception as ex:
-      print(f"Contract definition publish failed: {ex}", flush=True)
-
-   while True:
-      try:
-         client = db.Live(key=API_KEY)
-
-         client.subscribe(
-            dataset=DATASET,
-            schema="bbo-1s",
-            stype_in="parent",
-            symbols=SYMBOL,
-         )
-         client.subscribe(
-            dataset=DATASET,
-            schema="trades",
-            stype_in="parent",
-            symbols=SYMBOL,
-         )
-
-         client.add_callback(on_record)
-         client.start()
-         print("Databento client started, waiting for BBO and trade records...", flush=True)
-         client.block_for_close()
-         print("Databento client closed cleanly", flush=True)
-         break
-      except BentoError as ex:
-         print(f"Databento live connection error: {ex}", flush=True)
-         print("Retrying Databento live connection in 5 seconds...", flush=True)
-         time.sleep(5)
-      except Exception as ex:
-         print(f"Databento publisher unexpected error: {ex}", flush=True)
-         print("Retrying Databento live connection in 5 seconds...", flush=True)
-         time.sleep(5)
-
-
-if __name__ == "__main__":
-   main()

src/DataStreamSuite/DataStreamPublisher/WSPublishers/databento_transformer.py

@@ -1,244 +0,0 @@
-import datetime as dt
-
-
-class DatabentoTransformer:
-   def __init__(
-      self,
-      symbol: str,
-      bbo_table_name: str,
-      trades_table_name: str,
-      definitions_table_name: str,
-   ):
-      self._symbol = symbol
-      self._bbo_table_name = bbo_table_name
-      self._trades_table_name = trades_table_name
-      self._definitions_table_name = definitions_table_name
-      self._instrument_id_to_symbol: dict[int, str] = {}
-      self._last_bbo_row: dict | None = None
-      self._trade_row_id = 0
-      self._trade_table_created = False
-
-   def transform_record(self, record) -> list[dict]:
-      payload = self._handle_bbo_record(record)
-      if payload is not None:
-         return [payload]
-
-      payload = self._handle_trade_record(record)
-      if payload is not None:
-         return [payload]
-
-      return []
-
-   def build_definitions_payload(self, definition_df) -> dict | None:
-      if definition_df is None or definition_df.empty:
-         return None
-
-      df = definition_df.reset_index()
-      if "instrument_class" in df.columns:
-         df = df[df["instrument_class"].astype(str) == "F"]
-      if "security_type" in df.columns:
-         df = df[df["security_type"].astype(str) == "FUT"]
-      if "raw_symbol" in df.columns:
-         df = df.sort_values("ts_recv" if "ts_recv" in df.columns else "ts_event")
-         df = df.drop_duplicates(subset=["raw_symbol"], keep="last")
-
-      rows: list[dict] = []
-      for idx, row in enumerate(df.itertuples(index=False), start=1):
-         instrument_id = int(getattr(row, "instrument_id", 0) or 0)
-         security_name = str(getattr(row, "raw_symbol", ""))
-         if instrument_id > 0 and security_name:
-            self._instrument_id_to_symbol[instrument_id] = security_name
-
-         rows.append({
-            "id": idx,
-            "security_name": security_name,
-            "instrument_id": instrument_id,
-            "instrument_class": str(getattr(row, "instrument_class", "")),
-            "security_type": str(getattr(row, "security_type", "")),
-            "asset": str(getattr(row, "asset", "")),
-            "exchange": str(getattr(row, "exchange", "")),
-            "currency": str(getattr(row, "currency", "")),
-            "expiration": self._ns_to_iso(getattr(row, "expiration", 0)),
-            "activation": self._ns_to_iso(getattr(row, "activation", 0)),
-            "min_price_increment": self._nano_to_float(getattr(row, "min_price_increment", 0)),
-            "display_factor": self._nano_to_float(getattr(row, "display_factor", 0)),
-            "price_display_multiplier": self._nano_to_float(getattr(row, "display_factor", 0)),
-            "contract_multiplier": int(getattr(row, "contract_multiplier", 0) or 0),
-            "contract_multiplier_unit": int(getattr(row, "contract_multiplier_unit", 0) or 0),
-            "unit_of_measure": str(getattr(row, "unit_of_measure", "")),
-            "unit_of_measure_qty": self._nano_to_float(getattr(row, "unit_of_measure_qty", 0)),
-            "min_lot_size": int(getattr(row, "min_lot_size", 0) or 0),
-            "min_trade_vol": int(getattr(row, "min_trade_vol", 0) or 0),
-            "max_trade_vol": int(getattr(row, "max_trade_vol", 0) or 0),
-            "maturity_year": int(getattr(row, "maturity_year", 0) or 0),
-            "maturity_month": int(getattr(row, "maturity_month", 0) or 0),
-            "maturity_day": int(getattr(row, "maturity_day", 0) or 0),
-            "underlying": str(getattr(row, "underlying", "")),
-         })
-
-      if not rows:
-         return None
-
-      return {
-         "op": "create_table",
-         "table_name": self._definitions_table_name,
-         "rows": rows,
-      }
-
-   def _handle_bbo_record(self, record) -> dict | None:
-      row = self._to_bbo_row(record)
-      if row is None:
-         return None
-
-      if self._last_bbo_row is None:
-         payload = {
-            "op": "create_table",
-            "table_name": self._bbo_table_name,
-            "rows": [row],
-         }
-      else:
-         payload = {
-            "op": "multiple_remove_add",
-            "table_name": self._bbo_table_name,
-            "removed_rows": [self._last_bbo_row],
-            "added_rows": [row],
-         }
-
-      self._last_bbo_row = row
-      return payload
-
-   def _handle_trade_record(self, record) -> dict | None:
-      row = self._to_trade_row(record)
-      if row is None:
-         return None
-
-      row["id"] = self._trade_row_id
-      self._trade_row_id += 1
-
-      if not self._trade_table_created:
-         payload = {
-            "op": "create_table",
-            "table_name": self._trades_table_name,
-            "rows": [row],
-         }
-         self._trade_table_created = True
-         return payload
-
-      return {
-         "op": "multiple_remove_add",
-         "table_name": self._trades_table_name,
-         "removed_rows": [],
-         "added_rows": [row],
-      }
-
-   def _to_bbo_row(self, record) -> dict | None:
-      bid_px = self._get_level_attr(record, "bid_px", default=self._get_attr(record, "bid_px_00"))
-      ask_px = self._get_level_attr(record, "ask_px", default=self._get_attr(record, "ask_px_00"))
-      if bid_px is None and ask_px is None:
-         return None
-
-      return {
-         "id": 0,
-         "security_name": self._get_security_name(record),
-         "timestamp": self._get_timestamp(record),
-         "bid_px": float(bid_px or 0.0),
-         "ask_px": float(ask_px or 0.0),
-         "bid_sz": int(self._get_level_attr(record, "bid_sz", default=self._get_attr(record, "bid_sz_00", default=0)) or 0),
-         "ask_sz": int(self._get_level_attr(record, "ask_sz", default=self._get_attr(record, "ask_sz_00", default=0)) or 0),
-      }
-
-   def _to_trade_row(self, record) -> dict | None:
-      price = self._get_attr(record, "price", "px")
-      size = self._get_attr(record, "size")
-      if price is None and size is None:
-         return None
-
-      side = self._get_attr(record, "side", "aggressor_side", default="")
-
-      return {
-         "id": 0,
-         "security_name": self._get_security_name(record),
-         "timestamp": self._get_timestamp(record),
-         "price": float(price or 0.0),
-         "size": int(size or 0),
-         "side": str(side),
-      }
-
-   def _get_security_name(self, record) -> str:
-      instrument_id = self._get_attr(record, "instrument_id", default=0)
-      try:
-         instrument_id_int = int(instrument_id or 0)
-      except (TypeError, ValueError):
-         instrument_id_int = 0
-
-      if instrument_id_int > 0 and instrument_id_int in self._instrument_id_to_symbol:
-         return self._instrument_id_to_symbol[instrument_id_int]
-
-      value = self._get_attr(
-         record,
-         "symbol",
-         "raw_symbol",
-         "stype_out_symbol",
-         "stype_in_symbol",
-         default=self._symbol,
-      )
-      return str(value)
-
-   def _get_timestamp(self, record) -> int:
-      value = self._get_attr(record, "ts_event", "ts_recv", default=0)
-      try:
-         return int(value)
-      except (TypeError, ValueError):
-         return 0
-
-   def _get_attr(self, record, *names, default=None):
-      for name in names:
-         if hasattr(record, name):
-            value = getattr(record, name)
-            if value is not None:
-               return value
-      return default
-
-   def _get_level_attr(self, record, name, default=None):
-      levels = self._get_attr(record, "levels", default=None)
-      if levels:
-         level0 = levels[0]
-         if hasattr(level0, name):
-            value = getattr(level0, name)
-            if value is not None:
-               return value
-      return default
-
-   def _ns_to_iso(self, timestamp_ns) -> str:
-      if timestamp_ns is None:
-         return ""
-
-      if hasattr(timestamp_ns, "isoformat"):
-         try:
-            if hasattr(timestamp_ns, "to_pydatetime"):
-               timestamp_dt = timestamp_ns.to_pydatetime()
-            else:
-               timestamp_dt = timestamp_ns
-
-            if timestamp_dt is None:
-               return ""
-
-            if getattr(timestamp_dt, "tzinfo", None) is None:
-               timestamp_dt = timestamp_dt.replace(tzinfo=dt.timezone.utc)
-            return timestamp_dt.isoformat()
-         except (TypeError, ValueError, AttributeError):
-            pass
-
-      try:
-         ts_int = int(timestamp_ns)
-         if ts_int <= 0:
-            return ""
-         return dt.datetime.fromtimestamp(ts_int / 1_000_000_000, tz=dt.timezone.utc).isoformat()
-      except (TypeError, ValueError, OSError, OverflowError):
-         return ""
-
-   def _nano_to_float(self, value) -> float:
-      try:
-         return float(value) / 1_000_000_000.0
-      except (TypeError, ValueError):
-         return 0.0

src/DataStreamSuite/DataStreamPublisher/WSPublishers/pubsub_client.py

@@ -1,58 +0,0 @@
-import asyncio
-import json
-import queue
-import threading
-
-import websockets
-
-
-class PubSubClient:
-   def __init__(self, ws_url: str):
-      self._ws_url = ws_url
-      self._outgoing_queue: "queue.Queue[dict]" = queue.Queue()
-      self._sender_thread: threading.Thread | None = None
-
-   def start(self) -> threading.Thread:
-      if self._sender_thread is None or not self._sender_thread.is_alive():
-         self._sender_thread = self._start_sender_thread()
-      return self._sender_thread
-
-   def publish(self, payload: dict) -> None:
-      self._outgoing_queue.put(payload)
-
-   def publish_create_table(self, table_name: str, rows: list[dict]) -> None:
-      self.publish({
-         "op": "create_table",
-         "table_name": table_name,
-         "rows": rows,
-      })
-
-   def publish_multiple_remove_add(
-      self,
-      table_name: str,
-      removed_rows: list[dict],
-      added_rows: list[dict],
-   ) -> None:
-      self.publish({
-         "op": "multiple_remove_add",
-         "table_name": table_name,
-         "removed_rows": removed_rows,
-         "added_rows": added_rows,
-      })
-
-   def _start_sender_thread(self) -> threading.Thread:
-      thread = threading.Thread(target=self._run_send_loop, daemon=True)
-      thread.start()
-      return thread
-
-   def _run_send_loop(self) -> None:
-      asyncio.run(self._ws_send_loop())
-
-   async def _ws_send_loop(self) -> None:
-      async with websockets.connect(self._ws_url) as ws:
-         print(f"Connected to PubSubServer websocket: {self._ws_url}")
-         while True:
-            payload = await asyncio.to_thread(self._outgoing_queue.get)
-            body = json.dumps(payload)
-            print(body)
-            await ws.send(body)

src/DataStreamSuite/DataStreamPublisher/appsettings.json

@@ -59,9 +59,14 @@
           "Class": "bts.PubsubServerStreamTagHandler"
         },
         {
-          "TagType": "DS_.*",
-          "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerStreamTagHandler"
+          "TagType": "System_Sub",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemSubscriptionMapPublisher"
+        },
+        {
+          "TagType": "System_Stats",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemStatsPublisher"
         },
         {
           "TagType": ".*",
@@ -85,4 +90,4 @@
       }
     ]
   }
-}
+}

src/DataStreamSuite/DataStreamQueryAnalyzer/MainForm.cs

@@ -21,55 +21,11 @@ using System.Reactive.Subjects;
 using System.Text.RegularExpressions;
 using System.Threading;
 using System.Windows.Forms;
-using BlueOrb.Pubsub.Diagnostics;
 
 namespace BCG
 {
    public partial class MainForm : System.Windows.Forms.Form
    {
-      private const string LastQueryRegistryKey = "LastQuery";
-      private const string RecentSavedFilesRegistryKey = "RecentSavedFiles";
-      private const string DateTimeDisplayFormat = "M/d/yyyy h:mm:ss.fff tt";
-      private const int MaxRecentSavedFiles = 10;
-      private static readonly Color QueryEditorDefaultColor = Color.White;
-      private static readonly Color QueryEditorCommentColor = Color.FromArgb(106, 115, 125);
-      private static readonly Color QueryEditorOperatorColor = Color.FromArgb(86, 156, 214);
-      private static readonly Color QueryEditorAssignmentNameColor = Color.FromArgb(78, 201, 176);
-      private static readonly Color GridBackgroundColor = Color.FromArgb(37, 37, 38);
-      private static readonly Color GridAltRowColor = Color.FromArgb(43, 43, 48);
-      private static readonly Color GridHeaderColor = Color.FromArgb(62, 62, 66);
-      private static readonly Color GridGridLineColor = Color.FromArgb(58, 58, 61);
-      private static readonly Color GridSelectionColor = Color.FromArgb(9, 71, 113);
-      private static readonly Color GridSelectionForeColor = Color.White;
-      private static readonly Color GridTimestampColor = Color.FromArgb(206, 215, 222);
-      private static readonly Color GridIdentifierColor = Color.FromArgb(156, 163, 175);
-      private static readonly Color GridIndicatorColor = Color.FromArgb(156, 220, 254);
-
-      class PrimitiveTreeNodeComparer : IComparer
-      {
-         public int Compare(object? x, object? y)
-         {
-            var left = x as TreeNode;
-            var right = y as TreeNode;
-            if (left is null || right is null)
-               return 0;
-
-            string left_key = _sort_key(left);
-            string right_key = _sort_key(right);
-            return StringComparer.OrdinalIgnoreCase.Compare(left_key, right_key);
-         }
-
-         static string _sort_key(TreeNode node)
-         {
-            if (node.Parent is null)
-               return node.Text;
-
-            if (node.Tag is string primitive_name)
-               return primitive_name;
-
-            return node.Text;
-         }
-      }
       // static async Task _run() {
       //    var login = "bts";
       //    var passwd = String.Empty;
@@ -84,8 +40,6 @@ namespace BCG
       private string? myLastFileName;
       private ViewStream? myCurrentStream;
       private List<ToolStripMenuItem> mySiteItems = new List<ToolStripMenuItem>();
-      private readonly List<string> _recent_saved_files = new List<string>();
-      private ToolStripSeparator? _recent_saved_files_separator;
       // private string mySite = SitesCore.getMySiteName ();
       private string mySite = "TODO";
       private string myLastClassName = "MyView";
@@ -95,20 +49,11 @@ namespace BCG
       private TextWriter? myConsoleOutput;
       IDataPacket? pendingData;
       IDisposable? titleSub;
-      readonly ToolTip _primitive_tool_tip = new ToolTip();
-      readonly Dictionary<string, string?> _primitive_source_path_cache = new Dictionary<string, string?>();
-      Dictionary<string, string?>? _source_file_index;
-      string? _repo_root_path;
-      char? _last_primitive_search_char;
-      TreeNode? _last_primitive_search_assembly_node;
-      bool _is_applying_query_syntax_highlighting;
-      HashSet<string>? _dsl_operator_names;
 
       public MainForm()
       {
          //Task.Run(async () => await _run());
          InitializeComponent();
-         _apply_grid_theme();
          // myKQ = ConfigurationRequester.getApplicationQualifier();         
          myKQ = "TODO";
          var handleCreatedSubject = new Subject<bool>();
@@ -127,12 +72,13 @@ namespace BCG
          myWindowState = new PersistWindowState();
          myWindowState.parent = this;
          myWindowState.saveState = false;
-         myWindowState.LoadState += myLoadPersistedState;
-         myWindowState.SaveState += mySavePersistedState;
 
          this.Load += new EventHandler(myLoad);
          this.Closing += new CancelEventHandler(myClosing);
-         myQueryTextBox.TextChanged += (o, e) => _apply_query_syntax_highlighting();
+         myQueryTextBox.TextChanged += (o, e) =>
+         {
+            myQueryTextBox.ForeColor = Color.White;
+         };
       }
 
       event Action? titleInvalidated;
@@ -140,20 +86,14 @@ namespace BCG
       public string query_text
       {
          get { return myQueryTextBox.Text; }
-         set
-         {
-            myQueryTextBox.Text = value;
-            _apply_query_syntax_highlighting();
-         }
+         set { myQueryTextBox.Text = value; }
       }
 
       /// <summary>
       /// Clean up any resources being used.
       /// </summary>
-      protected override void Dispose(bool disposing)
-      {
-         if (disposing)
-         {
+      protected override void Dispose(bool disposing) {
+         if (disposing) {
             Disposable.Dispose(ref titleSub);
             components?.Dispose();
          }
@@ -164,7 +104,7 @@ namespace BCG
       {
          // TODO: Port PublishedTagsStream from edge (then we can uncomment impl below)
          //ViewStream vs = new PubSubServerStateStream();
-         ViewStream vs = new PubSubServerStateStream(ViewStreamDatabase.instance);
+         ViewStream vs = new PubSubServerStateStream(new SubscriberCoreFactory(), ViewStreamDatabase.instance);
          return new FilterStream(vs, "tag~='DS_'");
          //return new LiteralStream("tag/String", "DS_table1;DS_table2");
       }
@@ -174,8 +114,7 @@ namespace BCG
          titleInvalidated?.Invoke();
       }
 
-      void refreshTitle(IBlueRowCollection rows)
-      {
+      void refreshTitle(IBlueRowCollection rows) {
          string route_txt = String.Empty;
          foreach (IBlueRow r in rows)
          {
@@ -196,11 +135,11 @@ namespace BCG
       private IDictionary myGetLibraryData()
       {
          SortedList retVal = new SortedList();
-         string[] primitiveNames = PrimitiveManager.instance.get_dsl_op_names();
+         string[] primitiveNames = PrimitiveManager.instance.getPrimitiveNames();
          foreach (string pName in primitiveNames)
          {
-            string aName = PrimitiveManager.instance.get_dslop_assembly_name(pName);
-            ICollection cNames = PrimitiveManager.instance.get_dsl_op_cons_args(pName);
+            string aName = PrimitiveManager.instance.getAssemblyName(pName);
+            ICollection cNames = PrimitiveManager.instance.getConstructorSigns(pName);
             if (!retVal.Contains(aName))
                retVal.Add(aName, new SortedList());
             (retVal[aName] as IDictionary)[pName] = new ArrayList(cNames);
@@ -212,65 +151,6 @@ namespace BCG
          return retVal;
       }
 
-      HashSet<string> _get_dsl_operator_names()
-      {
-         if (_dsl_operator_names != null)
-            return _dsl_operator_names;
-
-         _dsl_operator_names = new HashSet<string>(
-            PrimitiveManager.instance.get_dsl_op_names().Cast<string>(),
-            StringComparer.OrdinalIgnoreCase);
-         return _dsl_operator_names;
-      }
-
-      void _apply_query_syntax_highlighting()
-      {
-         if (_is_applying_query_syntax_highlighting || myQueryTextBox.IsDisposed)
-            return;
-
-         _is_applying_query_syntax_highlighting = true;
-         try
-         {
-            int selection_start = myQueryTextBox.SelectionStart;
-            int selection_length = myQueryTextBox.SelectionLength;
-
-            myQueryTextBox.SuspendLayout();
-            myQueryTextBox.SelectAll();
-            myQueryTextBox.SelectionColor = QueryEditorDefaultColor;
-
-            foreach (Match match in Regex.Matches(myQueryTextBox.Text, @"^\s*([A-Za-z_]\w*)\s*=", RegexOptions.Multiline))
-            {
-               Group group = match.Groups[1];
-               myQueryTextBox.Select(group.Index, group.Length);
-               myQueryTextBox.SelectionColor = QueryEditorAssignmentNameColor;
-            }
-
-            foreach (Match match in Regex.Matches(myQueryTextBox.Text, @"\b([A-Za-z_]\w*)\s*\("))
-            {
-               Group group = match.Groups[1];
-               if (!_get_dsl_operator_names().Contains(group.Value))
-                  continue;
-               myQueryTextBox.Select(group.Index, group.Length);
-               myQueryTextBox.SelectionColor = QueryEditorOperatorColor;
-            }
-
-            foreach (Match match in Regex.Matches(myQueryTextBox.Text, @"//.*$", RegexOptions.Multiline))
-            {
-               myQueryTextBox.Select(match.Index, match.Length);
-               myQueryTextBox.SelectionColor = QueryEditorCommentColor;
-            }
-
-            myQueryTextBox.Select(selection_start, selection_length);
-            if (selection_length == 0)
-               myQueryTextBox.SelectionColor = QueryEditorDefaultColor;
-            myQueryTextBox.ResumeLayout();
-         }
-         finally
-         {
-            _is_applying_query_syntax_highlighting = false;
-         }
-      }
-
 
       private void myLoadAssemblies()
       {
@@ -281,18 +161,13 @@ namespace BCG
             TreeNode assemblyNode = new TreeNode(de.Key as String);
             foreach (DictionaryEntry innerDe in (de.Value as IDictionary))
             {
-               string primitive_name = innerDe.Key as string;
-               TreeNode primitiveNode = new TreeNode(primitive_name)
-               {
-                  Tag = primitive_name
-               };
+               TreeNode primitiveNode = new TreeNode(innerDe.Key as string);
                foreach (string con in (innerDe.Value as ICollection))
                   primitiveNode.Nodes.Add(con);
                assemblyNode.Nodes.Add(primitiveNode);
             }
             myTreeView.Nodes.Add(assemblyNode);
          }
-         myTreeView.Sort();
          /*
          myPrimitivesTextBox.Clear ();
          ArrayList al = new ArrayList(PrimitiveManager.instance.getPrimitiveNames());
@@ -307,8 +182,7 @@ namespace BCG
 
       private void myHandleTableListChange(object o)
       {
-         BeginInvoke(() =>
-         {
+         BeginInvoke(() => {
             tagRows = ((IDataPacket)o).newRows;
             myUpdateTableList();
          });
@@ -321,19 +195,6 @@ namespace BCG
          get { return myQueryTextBox.Height; }
          set { myQueryTextBox.Height = value; }
       }
-
-      private void myLoadPersistedState(object sender, BlueRegistry blueReg)
-      {
-         myQueryTextBox.Text = blueReg.getValue(RegistryRoot.HKCU, LastQueryRegistryKey, string.Empty)?.ToString() ?? string.Empty;
-         _load_recent_saved_files(blueReg.getValue(RegistryRoot.HKCU, RecentSavedFilesRegistryKey, string.Empty)?.ToString());
-         _refresh_recent_saved_files_menu();
-      }
-
-      private void mySavePersistedState(object sender, BlueRegistry blueReg)
-      {
-         blueReg.setValue(RegistryRoot.HKCU, LastQueryRegistryKey, myQueryTextBox.Text ?? string.Empty);
-         blueReg.setValue(RegistryRoot.HKCU, RecentSavedFilesRegistryKey, string.Join("|", _recent_saved_files));
-      }
       #endregion
 
       private Stopwatch _stopwatch = new Stopwatch();
@@ -426,7 +287,6 @@ namespace BCG
             }
             myLastFileName = fileName;
             this.Text = fileName;
-            _track_recent_saved_file(fileName);
          }
          catch (Exception exc)
          {
@@ -511,122 +371,6 @@ namespace BCG
          }
       }
 
-      private void myApplyDateTimeFormatting(IBlueTableSchema schema)
-      {
-         var dateTimeColumns = new HashSet<string>(
-            schema.columnInfo
-               .Where(ci => ci.colType == typeof(DateTime))
-               .Select(ci => ci.colName));
-
-         foreach (DataGridViewColumn column in myDataGrid.Columns)
-         {
-            if (!dateTimeColumns.Contains(column.DataPropertyName) && !dateTimeColumns.Contains(column.Name))
-               continue;
-
-            column.DefaultCellStyle.Format = DateTimeDisplayFormat;
-            column.Width = Math.Max(column.Width, 220);
-         }
-      }
-
-      void _apply_grid_theme()
-      {
-         myDataGrid.BackgroundColor = GridBackgroundColor;
-         myDataGrid.BorderStyle = BorderStyle.None;
-         myDataGrid.EnableHeadersVisualStyles = false;
-         myDataGrid.GridColor = GridGridLineColor;
-         myDataGrid.CellBorderStyle = DataGridViewCellBorderStyle.SingleHorizontal;
-         myDataGrid.SelectionMode = DataGridViewSelectionMode.FullRowSelect;
-         myDataGrid.MultiSelect = false;
-         myDataGrid.RowHeadersVisible = false;
-         myDataGrid.ColumnHeadersBorderStyle = DataGridViewHeaderBorderStyle.Single;
-         myDataGrid.RowHeadersBorderStyle = DataGridViewHeaderBorderStyle.None;
-         myDataGrid.ColumnHeadersHeight = 36;
-         myDataGrid.RowTemplate.Height = 24;
-         myDataGrid.Font = new Font("Consolas", 9F, FontStyle.Regular, GraphicsUnit.Point);
-
-         myDataGrid.DefaultCellStyle.BackColor = GridBackgroundColor;
-         myDataGrid.DefaultCellStyle.ForeColor = Color.White;
-         myDataGrid.DefaultCellStyle.SelectionBackColor = GridSelectionColor;
-         myDataGrid.DefaultCellStyle.SelectionForeColor = GridSelectionForeColor;
-         myDataGrid.DefaultCellStyle.Padding = new Padding(4, 1, 4, 1);
-
-         myDataGrid.AlternatingRowsDefaultCellStyle.BackColor = GridAltRowColor;
-         myDataGrid.AlternatingRowsDefaultCellStyle.ForeColor = Color.White;
-         myDataGrid.AlternatingRowsDefaultCellStyle.SelectionBackColor = GridSelectionColor;
-         myDataGrid.AlternatingRowsDefaultCellStyle.SelectionForeColor = GridSelectionForeColor;
-
-         myDataGrid.ColumnHeadersDefaultCellStyle.BackColor = GridHeaderColor;
-         myDataGrid.ColumnHeadersDefaultCellStyle.ForeColor = Color.White;
-         myDataGrid.ColumnHeadersDefaultCellStyle.SelectionBackColor = GridHeaderColor;
-         myDataGrid.ColumnHeadersDefaultCellStyle.SelectionForeColor = Color.White;
-         myDataGrid.ColumnHeadersDefaultCellStyle.Font = new Font("Consolas", 9F, FontStyle.Bold, GraphicsUnit.Point);
-         myDataGrid.ColumnHeadersDefaultCellStyle.Alignment = DataGridViewContentAlignment.MiddleLeft;
-         myDataGrid.ColumnHeadersDefaultCellStyle.Padding = new Padding(6, 2, 6, 2);
-      }
-
-      void _apply_grid_column_styling(IBlueTableSchema schema)
-      {
-         var column_types = schema.columnInfo.ToDictionary(ci => ci.colName, ci => ci.colType, StringComparer.OrdinalIgnoreCase);
-         foreach (DataGridViewColumn column in myDataGrid.Columns)
-         {
-            if (!column_types.TryGetValue(column.DataPropertyName, out var col_type) &&
-                !column_types.TryGetValue(column.Name, out col_type))
-            {
-               continue;
-            }
-
-            column.SortMode = DataGridViewColumnSortMode.NotSortable;
-
-            if (_is_numeric_type(col_type))
-            {
-               column.DefaultCellStyle.Alignment = DataGridViewContentAlignment.MiddleRight;
-            }
-            else
-            {
-               column.DefaultCellStyle.Alignment = DataGridViewContentAlignment.MiddleLeft;
-            }
-
-            if (col_type == typeof(DateTime))
-            {
-               column.DefaultCellStyle.ForeColor = GridTimestampColor;
-            }
-            else if (string.Equals(column.HeaderText, "id", StringComparison.OrdinalIgnoreCase))
-            {
-               column.DefaultCellStyle.ForeColor = GridIdentifierColor;
-            }
-            else if (_looks_like_indicator(column.HeaderText))
-            {
-               column.DefaultCellStyle.ForeColor = GridIndicatorColor;
-            }
-            else
-            {
-               column.DefaultCellStyle.ForeColor = Color.White;
-            }
-         }
-
-         static bool _is_numeric_type(Type type)
-         {
-            return type == typeof(int) ||
-                   type == typeof(long) ||
-                   type == typeof(float) ||
-                   type == typeof(double) ||
-                   type == typeof(decimal) ||
-                   type == typeof(short) ||
-                   type == typeof(uint) ||
-                   type == typeof(ulong);
-         }
-
-         static bool _looks_like_indicator(string column_name)
-         {
-            return column_name.Contains("rsi", StringComparison.OrdinalIgnoreCase) ||
-                   column_name.Contains("macd", StringComparison.OrdinalIgnoreCase) ||
-                   column_name.Contains("atr", StringComparison.OrdinalIgnoreCase) ||
-                   column_name.Contains("moving_average", StringComparison.OrdinalIgnoreCase) ||
-                   column_name.Contains("signal", StringComparison.OrdinalIgnoreCase) ||
-                   column_name.Contains("histogram", StringComparison.OrdinalIgnoreCase);
-         }
-      }
-
       // this is the stream callback
       private void myCallback(object o)
       {
@@ -642,8 +386,7 @@ namespace BCG
       private void myTimer_Tick(object sender, System.EventArgs e)
       {
          var dp = Interlocked.Exchange(ref pendingData, null);
-         if (dp is null)
-         {
+         if (dp is null) {
             return;
          }
          _stopwatch.Stop();
@@ -660,10 +403,6 @@ namespace BCG
          int hscrollval = myChangeStyle ? 0 : myDataGrid.HorizScrollBarValue;
 
          myDataGrid.DataSource = new SimpleBindableCollection(dp.newRows.schema, dp.newRows);
-         _apply_grid_theme();
-         myApplyDateTimeFormatting(dp.newRows.schema);
-         _apply_grid_column_styling(dp.newRows.schema);
-         myDataGrid.ClearSelection();
          var force = false;
          if (myChangeStyle && force)
          {
@@ -736,7 +475,7 @@ namespace BCG
       private void myMenuItemOpenClick(object sender, System.EventArgs e)
       {
          OpenFileDialog od = new OpenFileDialog();
-         od.Filter = "All files (*.*)|*.*|C# class (*.cs)|*.cs";
+         od.Filter = "C# class (*.cs)|*.cs";
 
          if (od.ShowDialog() == DialogResult.OK)
             loadFile(od.FileName);
@@ -748,7 +487,6 @@ namespace BCG
             w.Write(myCSharpText.Text);
          myLastFileName = fileName;
          this.Text = fileName;
-         _track_recent_saved_file(fileName);
       }
 
       private void myMenuItemSaveClick(object sender, System.EventArgs e)
@@ -771,75 +509,6 @@ namespace BCG
             mySave(sd.FileName);
       }
 
-      private void _load_recent_saved_files(string? serialized_paths)
-      {
-         _recent_saved_files.Clear();
-         if (string.IsNullOrWhiteSpace(serialized_paths))
-            return;
-
-         foreach (string path in serialized_paths.Split('|'))
-         {
-            if (string.IsNullOrWhiteSpace(path))
-               continue;
-            if (_recent_saved_files.Contains(path, StringComparer.OrdinalIgnoreCase))
-               continue;
-            _recent_saved_files.Add(path);
-            if (_recent_saved_files.Count >= MaxRecentSavedFiles)
-               break;
-         }
-      }
-
-      private void _track_recent_saved_file(string file_name)
-      {
-         _recent_saved_files.RemoveAll(path => string.Equals(path, file_name, StringComparison.OrdinalIgnoreCase));
-         _recent_saved_files.Insert(0, file_name);
-         if (_recent_saved_files.Count > MaxRecentSavedFiles)
-            _recent_saved_files.RemoveRange(MaxRecentSavedFiles, _recent_saved_files.Count - MaxRecentSavedFiles);
-         _refresh_recent_saved_files_menu();
-      }
-
-      private void _refresh_recent_saved_files_menu()
-      {
-         var file_menu = myToolStripMenuItemFile.DropDownItems;
-         for (int i = file_menu.Count - 1; i >= 0; i--)
-         {
-            if (file_menu[i] is ToolStripMenuItem menu_item && Equals(menu_item.Tag, "recent-saved-file"))
-               file_menu.RemoveAt(i);
-         }
-
-         if (_recent_saved_files_separator != null)
-         {
-            file_menu.Remove(_recent_saved_files_separator);
-            _recent_saved_files_separator.Dispose();
-            _recent_saved_files_separator = null;
-         }
-
-         if (_recent_saved_files.Count == 0)
-            return;
-
-         int insert_index = file_menu.IndexOf(myToolStripMenuItemExit);
-         if (insert_index < 0)
-            insert_index = file_menu.Count;
-
-         _recent_saved_files_separator = new ToolStripSeparator();
-         file_menu.Insert(insert_index, _recent_saved_files_separator);
-         insert_index++;
-
-         foreach (string path in _recent_saved_files)
-         {
-            var menu_item = new ToolStripMenuItem(Path.GetFileName(path))
-            {
-               Tag = "recent-saved-file",
-               ToolTipText = path,
-               BackColor = Color.FromArgb(36, 36, 49),
-               ForeColor = Color.Lime
-            };
-            menu_item.Click += (_, __) => loadFile(path);
-            file_menu.Insert(insert_index, menu_item);
-            insert_index++;
-         }
-      }
-
       private void myTableListBox_DoubleClick(object sender, System.EventArgs e)
       {
          if (myTableListBox.SelectedItem != null)
@@ -930,7 +599,6 @@ namespace BCG
          }
          Console.WriteLine("Redirecting console ouput to this window...");
 
-         myTreeView.TreeViewNodeSorter = new PrimitiveTreeNodeComparer();
          myLoadAssemblies();
 
          // ViewStreamDatabase.instance = new ViewStreamDatabase();
@@ -938,9 +606,6 @@ namespace BCG
          myAllTagsStream.onChange += new BCG.DataStream.ChangeDelegate(myHandleTableListChange);
          myTabControl.SelectedIndex = 3;
          myTreeView.AfterSelect += new TreeViewEventHandler(myHandleTreeNodeSelect);
-         myTreeView.KeyPress += myHandleTreeViewKeyPress;
-         myTreeView.NodeMouseHover += myHandleTreeNodeMouseHover;
-         myTreeView.MouseLeave += myHandleTreeViewMouseLeave;
          //do this lazily so we don't have to have bps, etc up to run dsqa
          //_library = new LibraryForm(this);
       }
@@ -981,138 +646,12 @@ namespace BCG
          {
             if (myTreeView.SelectedNode.LastNode == null)
             {
-               string primitive_name = myTreeView.SelectedNode.Parent.Tag as string ?? myTreeView.SelectedNode.Parent.Text;
-               string sn = primitive_name + "()";
+               string sn = myTreeView.SelectedNode.Parent.Text + "()";
                myQueryTextBox.Text = myQueryTextBox.Text == "" ? sn : myQueryTextBox.Text + "\n" + sn;
             }
          }
       }
 
-      private void myHandleTreeViewKeyPress(object? sender, KeyPressEventArgs e)
-      {
-         if (!Char.IsLetterOrDigit(e.KeyChar))
-            return;
-
-         char search_char = Char.ToLowerInvariant(e.KeyChar);
-         TreeNode? assembly_node = _get_selected_assembly_node();
-         if (assembly_node is null)
-            return;
-
-         int start_idx = 0;
-         if (_last_primitive_search_char == search_char &&
-             ReferenceEquals(_last_primitive_search_assembly_node, assembly_node) &&
-             myTreeView.SelectedNode?.Parent == assembly_node)
-         {
-            start_idx = myTreeView.SelectedNode.Index + 1;
-         }
-
-         TreeNode? match_node = _find_matching_primitive_node(assembly_node, search_char, start_idx) ??
-                                _find_matching_primitive_node(assembly_node, search_char, 0);
-
-         if (match_node is not null)
-         {
-            myTreeView.SelectedNode = match_node;
-            match_node.EnsureVisible();
-            _last_primitive_search_char = search_char;
-            _last_primitive_search_assembly_node = assembly_node;
-            e.Handled = true;
-         }
-      }
-
-      private TreeNode? _get_selected_assembly_node()
-      {
-         if (myTreeView.SelectedNode is null)
-            return myTreeView.Nodes.Count > 0 ? myTreeView.Nodes[0] : null;
-
-         return myTreeView.SelectedNode.Parent ?? myTreeView.SelectedNode;
-      }
-
-      private TreeNode? _find_matching_primitive_node(TreeNode assembly_node, char search_char, int start_idx)
-      {
-         for (int i = start_idx; i < assembly_node.Nodes.Count; i++)
-         {
-            TreeNode primitive_node = assembly_node.Nodes[i];
-            string primitive_name = primitive_node.Tag as string ?? primitive_node.Text;
-            if (primitive_name.Length > 0 && Char.ToLowerInvariant(primitive_name[0]) == search_char)
-               return primitive_node;
-         }
-         return null;
-      }
-
-      private void myHandleTreeNodeMouseHover(object? sender, TreeNodeMouseHoverEventArgs e)
-      {
-         if (e.Node?.Tag is not string primitive_name)
-         {
-            _primitive_tool_tip.Hide(myTreeView);
-            return;
-         }
-
-         Type? primitive_type = PrimitiveManager.instance.get_dsl_op_type(primitive_name);
-         if (primitive_type is null)
-         {
-            _primitive_tool_tip.Hide(myTreeView);
-            return;
-         }
-
-         string tooltip_text = $"Namespace: {primitive_type.Namespace ?? "(none)"}";
-         string? source_path = _get_source_file_path(primitive_type);
-         if (!String.IsNullOrWhiteSpace(source_path))
-            tooltip_text += $"\nSource: {source_path}";
-
-         Point mouse_point = myTreeView.PointToClient(Cursor.Position);
-         _primitive_tool_tip.Show(tooltip_text, myTreeView, mouse_point.X + 12, mouse_point.Y + 12, 8000);
-      }
-
-      private void myHandleTreeViewMouseLeave(object? sender, EventArgs e)
-      {
-         _primitive_tool_tip.Hide(myTreeView);
-      }
-
-      private string? _get_source_file_path(Type primitive_type)
-      {
-         string cache_key = primitive_type.FullName ?? primitive_type.Name;
-         if (_primitive_source_path_cache.TryGetValue(cache_key, out string? cached_path))
-            return cached_path;
-
-         Dictionary<string, string?> source_file_index = _source_file_index ??= _build_source_file_index();
-         if (!source_file_index.TryGetValue(primitive_type.Name, out string? source_path))
-            return null;
-         _primitive_source_path_cache[cache_key] = source_path;
-         return source_path;
-      }
-
-      private Dictionary<string, string?> _build_source_file_index()
-      {
-         Dictionary<string, string?> source_file_index = new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
-         string? repo_root = _repo_root_path ??= _find_repo_root();
-         if (String.IsNullOrWhiteSpace(repo_root))
-            return source_file_index;
-
-         string source_root = Path.Combine(repo_root, "src", "DataStreamSuite");
-         if (!Directory.Exists(source_root))
-            return source_file_index;
-
-         foreach (string file_path in Directory.EnumerateFiles(source_root, "*.cs", SearchOption.AllDirectories))
-         {
-            string type_name = Path.GetFileNameWithoutExtension(file_path);
-            if (!source_file_index.ContainsKey(type_name))
-               source_file_index[type_name] = file_path;
-         }
-         return source_file_index;
-      }
-
-      private string? _find_repo_root()
-      {
-         string? curr_path = AppDomain.CurrentDomain.BaseDirectory;
-         while (!String.IsNullOrWhiteSpace(curr_path))
-         {
-            if (Directory.Exists(Path.Combine(curr_path, "src", "DataStreamSuite")))
-               return curr_path;
-            curr_path = Directory.GetParent(curr_path)?.FullName;
-         }
-         return null;
-      }
-
       private void myToolStripMenuItemExit_Click(object sender, EventArgs e)
       {
          Close();
@@ -1142,7 +681,7 @@ namespace BCG
          DialogResult dr = ofd.ShowDialog(this);
          if (ofd.CheckFileExists && ofd.CheckPathExists && dr == DialogResult.OK)
          {
-            PrimitiveManager.instance.load_streams_from_assembly(ofd.FileName);
+            PrimitiveManager.instance.loadAssembly(ofd.FileName);
             myLoadAssemblies();
          }
       }
@@ -1229,8 +768,7 @@ namespace BCG
          s = new MutableAppendStream(s, s);
          s.onChange += OnChange;
 
-         void OnChange(object? obj)
-         {
+         void OnChange(object? obj) {
             s.onChange -= OnChange;
             //s = new MutableAppendStream(s, s);
             var rowCollection = ((IDataPacket)obj!).newRows;

src/DataStreamSuite/DataStreamQueryAnalyzer/Program.cs

@@ -8,6 +8,7 @@ using Bts.PubSub.Services;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 
+using Spark.Api.MarketData;
 using Spark.Api.Services;
 using Spark.Core.Services;
 
@@ -25,6 +26,10 @@ static class Program {
          host = HostBuilderExtensions.CreateBtsBuilder(args)
             .ConfigureServices(ConfigureServices)
             .Build();
+         var mdClient = host.Services.GetService<IMarketDataClient>();
+         if (mdClient is not null) {
+            PrimitiveManager.SetMarketDataClient(mdClient);
+         }
          await host.StartAsync();
          var winFormsThread = new Thread(RunWinFormsApp);
          winFormsThread.SetApartmentState(ApartmentState.STA);

src/DataStreamSuite/DataStreamQueryAnalyzer/appsettings.json

@@ -5,7 +5,8 @@
   "ViewStreamDatabase": {
     "Interval": "00:00:00.1"
   },
-  "Spark": {},
+  "Spark": {
+  },
   "SparkApi": {
     "IsEnabled": false,
     "Credentials": {
@@ -52,16 +53,21 @@
           "Class": "bts.PubsubServerStreamTagHandler"
         },
         {
-          "TagType": "DS_.*",
-          "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerStreamTagHandler"
+          "TagType": "System_Sub",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemSubscriptionMapPublisher"
+        },
+        {
+          "TagType": "System_Stats",
+          "Assembly": "PubSub",
+          "Class": "BCG.SystemStatsPublisher"
         },
         {
           "TagType": ".*",
           "Assembly": "DataStreamCore",
-          "Class": "bts.PubsubServerGenericDataHandler"
+         "Class": "bts.PubsubServerGenericDataHandler"
         }
       ]
     }
   }
-}
+}

src/DataStreamSuite/DataStreamSuite.sln

@@ -13,8 +13,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "PubSub\PubSub.csp
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSubServer", "PubSubServer\PubSubServer.csproj", "{0DBD814A-E17F-4836-8EEE-6E91B00C578F}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PersistenceServer", "PersistenceServer\PersistenceServer.csproj", "{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}"
-EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "BooleanExpressions\BooleanExpressions.csproj", "{EE53687E-BB62-45C6-A1B0-9B8F16B4E2C1}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Core", "..\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{88E70136-91AE-4F56-B243-8F4642B46D6A}"
@@ -45,10 +43,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "..\cs\Bts.Net\Bt
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "..\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DSPublisher", "DSPublisher\DSPublisher.csproj", "{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "APIBridgeServer", "APIBridgeServer\APIBridgeServer.csproj", "{3B14C417-531A-4E39-80C1-A4C14BD000D5}"
-EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -119,18 +113,6 @@ Global
 		{0DBD814A-E17F-4836-8EEE-6E91B00C578F}.Release|x64.Build.0 = Release|Any CPU
 		{0DBD814A-E17F-4836-8EEE-6E91B00C578F}.Release|x86.ActiveCfg = Release|Any CPU
 		{0DBD814A-E17F-4836-8EEE-6E91B00C578F}.Release|x86.Build.0 = Release|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|x64.Build.0 = Debug|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|x86.Build.0 = Debug|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|Any CPU.Build.0 = Release|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|x64.ActiveCfg = Release|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|x64.Build.0 = Release|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|x86.ActiveCfg = Release|Any CPU
-		{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|x86.Build.0 = Release|Any CPU
 		{EE53687E-BB62-45C6-A1B0-9B8F16B4E2C1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{EE53687E-BB62-45C6-A1B0-9B8F16B4E2C1}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{EE53687E-BB62-45C6-A1B0-9B8F16B4E2C1}.Debug|x64.ActiveCfg = Debug|Any CPU
@@ -311,30 +293,6 @@ Global
 		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x64.Build.0 = Release|Any CPU
 		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x86.ActiveCfg = Release|Any CPU
 		{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x86.Build.0 = Release|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|x64.Build.0 = Debug|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|x86.Build.0 = Debug|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|Any CPU.Build.0 = Release|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|x64.ActiveCfg = Release|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|x64.Build.0 = Release|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|x86.ActiveCfg = Release|Any CPU
-		{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|x86.Build.0 = Release|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|x64.Build.0 = Debug|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|x86.Build.0 = Debug|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|Any CPU.Build.0 = Release|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|x64.ActiveCfg = Release|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|x64.Build.0 = Release|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|x86.ActiveCfg = Release|Any CPU
-		{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

src/DataStreamSuite/DataStreamTests/DataStreamTests.csproj

@@ -27,7 +27,6 @@
     <ProjectReference Include="..\PubSub\PubSub.csproj" />
     <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
     <ProjectReference Include="..\DataStream\DataStream.csproj" />
-    <ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
   </ItemGroup>
 
 

src/DataStreamSuite/DataStreamTests/HeikinAshiStreamTests.cs

@@ -1,121 +0,0 @@
-namespace Bts.DataStreamTests;
-
-using BCG;
-
-using BlueOrb.Operators.Signal;
-
-using Bts.DataStreamCore;
-
-[TestFixture]
-public class HeikinAshiStreamTests {
-   readonly BlueTableSchema schema = new(
-      new ColumnInfo[] {
-         new("id", 0, TypeCode.Int32),
-         new("ts_event", 1, TypeCode.DateTime),
-         new("open", 2, TypeCode.Double),
-         new("high", 3, TypeCode.Double),
-         new("low", 4, TypeCode.Double),
-         new("close", 5, TypeCode.Double),
-         new("volume", 6, TypeCode.Double),
-      }, "bars");
-
-   [Test]
-   public void AppendsHistoryAndCalculatesRecursiveHeikinAshiValues() {
-      var driver = new TestDriverStream(schema, null);
-      var heikinAshi = new HeikinAshiStream(driver);
-      var packets = new List<IDataPacket>();
-      heikinAshi.onChange += nv => packets.Add((IDataPacket)nv);
-
-      driver.Reset(
-         Row(1, new DateTime(2026, 1, 1, 9, 30, 0, DateTimeKind.Utc), 100.0, 110.0, 90.0, 105.0, 10.0),
-         Row(2, new DateTime(2026, 1, 1, 9, 31, 0, DateTimeKind.Utc), 106.0, 112.0, 104.0, 110.0, 20.0)
-      );
-
-      Assert.That(packets, Has.Count.EqualTo(2));
-
-      Assert.That(packets[0].newRows.Count, Is.EqualTo(1));
-      Assert.That(packets[0].addedRows.Count, Is.EqualTo(1));
-      Assert.That(packets[0].deletedRows.Count, Is.EqualTo(0));
-
-      var first = packets[0].newRows[0];
-      Assert.That(first.get<double>("open"), Is.EqualTo(100.0).Within(1e-9));
-      Assert.That(first.get<double>("high"), Is.EqualTo(110.0).Within(1e-9));
-      Assert.That(first.get<double>("low"), Is.EqualTo(90.0).Within(1e-9));
-      Assert.That(first.get<double>("close"), Is.EqualTo(105.0).Within(1e-9));
-      Assert.That(first.get<double>("ha_open"), Is.EqualTo(102.5).Within(1e-9));
-      Assert.That(first.get<double>("ha_high"), Is.EqualTo(110.0).Within(1e-9));
-      Assert.That(first.get<double>("ha_low"), Is.EqualTo(90.0).Within(1e-9));
-      Assert.That(first.get<double>("ha_close"), Is.EqualTo(101.25).Within(1e-9));
-
-      Assert.That(packets[1].newRows.Count, Is.EqualTo(2));
-      Assert.That(packets[1].addedRows.Count, Is.EqualTo(1));
-      Assert.That(packets[1].deletedRows.Count, Is.EqualTo(0));
-
-      var second = packets[1].newRows[1];
-      Assert.That(second.get<double>("open"), Is.EqualTo(106.0).Within(1e-9));
-      Assert.That(second.get<double>("high"), Is.EqualTo(112.0).Within(1e-9));
-      Assert.That(second.get<double>("low"), Is.EqualTo(104.0).Within(1e-9));
-      Assert.That(second.get<double>("close"), Is.EqualTo(110.0).Within(1e-9));
-      Assert.That(second.get<double>("ha_open"), Is.EqualTo(101.875).Within(1e-9));
-      Assert.That(second.get<double>("ha_high"), Is.EqualTo(112.0).Within(1e-9));
-      Assert.That(second.get<double>("ha_low"), Is.EqualTo(101.875).Within(1e-9));
-      Assert.That(second.get<double>("ha_close"), Is.EqualTo(108.0).Within(1e-9));
-      Assert.That(second.get<double>("volume"), Is.EqualTo(20.0).Within(1e-9));
-   }
-
-   [Test]
-   public void RebuildsHistoryWhenSourceSnapshotChanges() {
-      var driver = new TestDriverStream(schema, null);
-      var heikinAshi = new HeikinAshiStream(driver);
-      var packets = new List<IDataPacket>();
-      heikinAshi.onChange += nv => packets.Add((IDataPacket)nv);
-
-      driver.Reset(
-         Row(1, new DateTime(2026, 1, 1, 9, 30, 0, DateTimeKind.Utc), 100.0, 110.0, 90.0, 105.0, 10.0),
-         Row(2, new DateTime(2026, 1, 1, 9, 31, 0, DateTimeKind.Utc), 106.0, 112.0, 104.0, 110.0, 20.0)
-      );
-
-      driver.Reset(
-         Row(1, new DateTime(2026, 1, 1, 9, 30, 0, DateTimeKind.Utc), 100.0, 110.0, 90.0, 105.0, 10.0),
-         Row(2, new DateTime(2026, 1, 1, 9, 31, 0, DateTimeKind.Utc), 106.0, 114.0, 103.0, 111.0, 21.0)
-      );
-
-      Assert.That(packets, Has.Count.EqualTo(3));
-
-      Assert.That(packets[2].newRows.Count, Is.EqualTo(2));
-      Assert.That(packets[2].addedRows.Count, Is.EqualTo(1));
-      Assert.That(packets[2].deletedRows.Count, Is.EqualTo(1));
-
-      var updated = packets[2].newRows[1];
-      Assert.That(updated.get<double>("open"), Is.EqualTo(106.0).Within(1e-9));
-      Assert.That(updated.get<double>("high"), Is.EqualTo(114.0).Within(1e-9));
-      Assert.That(updated.get<double>("low"), Is.EqualTo(103.0).Within(1e-9));
-      Assert.That(updated.get<double>("close"), Is.EqualTo(111.0).Within(1e-9));
-      Assert.That(updated.get<double>("ha_open"), Is.EqualTo(101.875).Within(1e-9));
-      Assert.That(updated.get<double>("ha_high"), Is.EqualTo(114.0).Within(1e-9));
-      Assert.That(updated.get<double>("ha_low"), Is.EqualTo(101.875).Within(1e-9));
-      Assert.That(updated.get<double>("ha_close"), Is.EqualTo(108.5).Within(1e-9));
-      Assert.That(updated.get<double>("volume"), Is.EqualTo(21.0).Within(1e-9));
-   }
-
-   static IBlueRow Row(int id, DateTime tsEvent, double open, double high, double low, double close, double volume) {
-      var row = new MutableBlueRow(
-         new BlueTableSchema(new[] {
-            new ColumnInfo("id", 0, TypeCode.Int32),
-            new ColumnInfo("ts_event", 1, TypeCode.DateTime),
-            new ColumnInfo("open", 2, TypeCode.Double),
-            new ColumnInfo("high", 3, TypeCode.Double),
-            new ColumnInfo("low", 4, TypeCode.Double),
-            new ColumnInfo("close", 5, TypeCode.Double),
-            new ColumnInfo("volume", 6, TypeCode.Double),
-         }, "bars"));
-      row.setValue("id", id);
-      row.setValue("ts_event", tsEvent);
-      row.setValue("open", open);
-      row.setValue("high", high);
-      row.setValue("low", low);
-      row.setValue("close", close);
-      row.setValue("volume", volume);
-      return row;
-   }
-}

src/DataStreamSuite/DataStreamTests/TestDriverStream.cs

@@ -5,7 +5,7 @@ using BCG.Utilities.General;
 
 using Bts.DataStreamCore;
 
-using Spark.Core.Collections;
+using global::Spark.Core.Collections;
 
 public class TestDriverStream : ViewStream, IWorker {
    readonly IBlueTableSchema schema;

src/DataStreamSuite/DataStreamTests/VWAPStreamTests.cs

@@ -1,53 +0,0 @@
-namespace Bts.DataStreamTests;
-
-using BCG;
-
-using BlueOrb.Operators.Signal;
-
-using Bts.DataStreamCore;
-
-[TestFixture]
-public class VWAPStreamTests {
-   readonly BlueTableSchema schema = new(
-      new ColumnInfo[] {
-         new("id", 0, TypeCode.Int32),
-         new("ts_event", 1, TypeCode.DateTime),
-         new("close", 2, TypeCode.Double),
-         new("volume", 3, TypeCode.Double),
-      }, "bars");
-
-   [Test]
-   public void EmitsRowsAndCalculatesRollingVWAP() {
-      var driver = new TestDriverStream(schema, null);
-      var vwap = new VWAPStream(driver, "close", "volume", 2);
-      var packets = new List<IDataPacket>();
-      vwap.onChange += nv => packets.Add((IDataPacket)nv);
-
-      driver.Reset(
-         Row(1, new DateTime(2026, 1, 1, 9, 30, 0, DateTimeKind.Utc), 100.0, 10.0),
-         Row(2, new DateTime(2026, 1, 1, 9, 30, 5, DateTimeKind.Utc), 110.0, 20.0)
-      );
-
-      Assert.That(packets, Has.Count.EqualTo(1));
-      Assert.That(packets[0].newRows.Count, Is.EqualTo(2));
-
-      var rows = packets[0].newRows.to_list().OrderBy(r => r.get<int>("id")).ToList();
-      Assert.That(rows[0].get<double>("vwap"), Is.EqualTo(100.0).Within(1e-9));
-      Assert.That(rows[1].get<double>("vwap"), Is.EqualTo((100.0 * 10.0 + 110.0 * 20.0) / 30.0).Within(1e-9));
-   }
-
-   static IBlueRow Row(int id, DateTime tsEvent, double close, double volume) {
-      var row = new MutableBlueRow(
-         new BlueTableSchema(new[] {
-            new ColumnInfo("id", 0, TypeCode.Int32),
-            new ColumnInfo("ts_event", 1, TypeCode.DateTime),
-            new ColumnInfo("close", 2, TypeCode.Double),
-            new ColumnInfo("volume", 3, TypeCode.Double),
-         }, "bars"));
-      row.setValue("id", id);
-      row.setValue("ts_event", tsEvent);
-      row.setValue("close", close);
-      row.setValue("volume", volume);
-      return row;
-   }
-}

src/DataStreamSuite/PersistenceServer/ConfigLoader.cs

@@ -1,33 +0,0 @@
-using System;
-using System.IO;
-using System.Text.Json;
-
-public static class ConfigLoader {
-   public static PersistenceServerConfig load() {
-      var config = new PersistenceServerConfig();
-      merge_from_file(config, Path.Combine(AppContext.BaseDirectory, "appsettings.json"));
-      merge_from_file(config, Path.Combine(AppContext.BaseDirectory, "appsettings.Development.json"));
-      return config;
-   }
-
-   private static void merge_from_file(PersistenceServerConfig target, string path) {
-      if (!File.Exists(path)) {
-         return;
-      }
-
-      var json = File.ReadAllText(path);
-      var loaded = JsonSerializer.Deserialize<PersistenceServerConfig>(json, new JsonSerializerOptions {
-         PropertyNameCaseInsensitive = true
-      });
-
-      if (loaded?.pub_sub is null) {
-         return;
-      }
-
-      if (target.pub_sub is null) {
-         target.pub_sub = loaded.pub_sub;
-      } else {
-         target.pub_sub.MergeWith(loaded.pub_sub);
-      }
-   }
-}

src/DataStreamSuite/PersistenceServer/PeristenceHelper.cs

@@ -1,148 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Data.SQLite;
-using System.IO;
-using System.Linq;
-
-public class PeristenceHelper {
-   private readonly string _connectionString;
-   public string database_path { get; }
-
-   public PeristenceHelper(string? databasePath = null) {
-      var resolvedPath = databasePath ?? Path.Combine(AppContext.BaseDirectory, "PersistenceServer.sqlite");
-      database_path = resolvedPath;
-      Directory.CreateDirectory(Path.GetDirectoryName(resolvedPath)!);
-      _connectionString = $"Data Source={resolvedPath};Version=3";
-   }
-
-   public void InsertRow(string tableName, IReadOnlyDictionary<string, string> row) {
-      ValidateRow(tableName, row);
-      using var connection = OpenConnection();
-      EnsureTable(connection, tableName, row.Keys);
-
-      var columns = row.Keys.Select(QuoteIdentifier).ToArray();
-      var parameterNames = row.Keys.Select(CreateParameterName).ToArray();
-      var commandText =
-         $"INSERT INTO {QuoteIdentifier(tableName)} ({string.Join(", ", columns)}) VALUES ({string.Join(", ", parameterNames)})";
-
-      using var command = new SQLiteCommand(commandText, connection);
-      AddParameters(command, row);
-      command.ExecuteNonQuery();
-   }
-
-   public void UpdateRow(string tableName, IReadOnlyDictionary<string, string> row) {
-      ValidateRow(tableName, row);
-      using var connection = OpenConnection();
-      EnsureTable(connection, tableName, row.Keys);
-
-      var keyColumns = GetKeyColumns(row);
-      var valueColumns = row.Keys.Where(column => !keyColumns.Contains(column, StringComparer.OrdinalIgnoreCase)).ToArray();
-      if (valueColumns.Length == 0) {
-         throw new InvalidOperationException("UpdateRow requires at least one non-key column to update.");
-      }
-
-      var setClause = string.Join(", ", valueColumns.Select(column => $"{QuoteIdentifier(column)} = {CreateParameterName(column)}"));
-      var whereClause = string.Join(" AND ", keyColumns.Select(column => $"{QuoteIdentifier(column)} = {CreateWhereParameterName(column)}"));
-      var commandText = $"UPDATE {QuoteIdentifier(tableName)} SET {setClause} WHERE {whereClause}";
-
-      using var command = new SQLiteCommand(commandText, connection);
-      foreach (var column in valueColumns) {
-         command.Parameters.AddWithValue(CreateParameterName(column), row[column]);
-      }
-
-      foreach (var column in keyColumns) {
-         command.Parameters.AddWithValue(CreateWhereParameterName(column), row[column]);
-      }
-
-      command.ExecuteNonQuery();
-   }
-
-   public void RemoveRow(string tableName, IReadOnlyDictionary<string, string> row) {
-      ValidateRow(tableName, row);
-      using var connection = OpenConnection();
-      EnsureTable(connection, tableName, row.Keys);
-
-      var keyColumns = GetKeyColumns(row);
-      var whereClause = string.Join(" AND ", keyColumns.Select(column => $"{QuoteIdentifier(column)} = {CreateWhereParameterName(column)}"));
-      var commandText = $"DELETE FROM {QuoteIdentifier(tableName)} WHERE {whereClause}";
-
-      using var command = new SQLiteCommand(commandText, connection);
-      foreach (var column in keyColumns) {
-         command.Parameters.AddWithValue(CreateWhereParameterName(column), row[column]);
-      }
-
-      command.ExecuteNonQuery();
-   }
-
-   private SQLiteConnection OpenConnection() {
-      var connection = new SQLiteConnection(_connectionString);
-      connection.Open();
-      return connection;
-   }
-
-   private static void ValidateRow(string tableName, IReadOnlyDictionary<string, string> row) {
-      if (string.IsNullOrWhiteSpace(tableName)) {
-         throw new ArgumentException("Table name is required.", nameof(tableName));
-      }
-
-      if (row.Count == 0) {
-         throw new ArgumentException("Row must contain at least one column.", nameof(row));
-      }
-   }
-
-   private static string[] GetKeyColumns(IReadOnlyDictionary<string, string> row) {
-      if (row.ContainsKey("namespace") && row.ContainsKey("key")) {
-         return new[] { "namespace", "key" };
-      }
-
-      if (row.ContainsKey("key")) {
-         return new[] { "key" };
-      }
-
-      throw new InvalidOperationException("Row must contain 'key', or both 'namespace' and 'key', for update/delete.");
-   }
-
-   private static void EnsureTable(SQLiteConnection connection, string tableName, IEnumerable<string> columns) {
-      var existingColumns = GetExistingColumns(connection, tableName);
-      if (existingColumns.Count == 0) {
-         var createColumns = string.Join(", ", columns.Select(column => $"{QuoteIdentifier(column)} TEXT"));
-         using var createCommand =
-            new SQLiteCommand($"CREATE TABLE IF NOT EXISTS {QuoteIdentifier(tableName)} ({createColumns})", connection);
-         createCommand.ExecuteNonQuery();
-         existingColumns = GetExistingColumns(connection, tableName);
-      }
-
-      foreach (var column in columns) {
-         if (existingColumns.Contains(column)) {
-            continue;
-         }
-
-         using var alterCommand =
-            new SQLiteCommand($"ALTER TABLE {QuoteIdentifier(tableName)} ADD COLUMN {QuoteIdentifier(column)} TEXT", connection);
-         alterCommand.ExecuteNonQuery();
-      }
-   }
-
-   private static HashSet<string> GetExistingColumns(SQLiteConnection connection, string tableName) {
-      var columns = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
-      using var command = new SQLiteCommand($"PRAGMA table_info({QuoteIdentifier(tableName)})", connection);
-      using var reader = command.ExecuteReader();
-      while (reader.Read()) {
-         columns.Add(reader["name"].ToString()!);
-      }
-
-      return columns;
-   }
-
-   private static void AddParameters(SQLiteCommand command, IReadOnlyDictionary<string, string> row) {
-      foreach (var pair in row) {
-         command.Parameters.AddWithValue(CreateParameterName(pair.Key), pair.Value);
-      }
-   }
-
-   private static string QuoteIdentifier(string identifier) => $"\"{identifier.Replace("\"", "\"\"")}\"";
-
-   private static string CreateParameterName(string columnName) => $"@p_{columnName.Replace("\"", string.Empty)}";
-
-   private static string CreateWhereParameterName(string columnName) => $"@w_{columnName.Replace("\"", string.Empty)}";
-}

src/DataStreamSuite/PersistenceServer/PersistenceCallHandler.cs

@@ -1,155 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Net;
-using System.Text.Json;
-
-public class PersistenceCallHandler {
-   private const string KeyValueTableName = "key_value_store";
-   private readonly PeristenceHelper _persistenceHelper;
-   private readonly PubSubSync _pub_sub_sync;
-
-   public PersistenceCallHandler(PeristenceHelper persistenceHelper, PubSubSync pub_sub_sync) {
-      _persistenceHelper = persistenceHelper;
-      _pub_sub_sync = pub_sub_sync;
-   }
-
-   public void Handle(HttpListenerRequest request, string payload) {
-      Console.WriteLine("----- Incoming REST call -----");
-      Console.WriteLine($"Timestamp: {DateTimeOffset.Now:O}");
-      Console.WriteLine($"Method: {request.HttpMethod}");
-      Console.WriteLine($"Url: {request.Url}");
-      Console.WriteLine($"RawUrl: {request.RawUrl}");
-      Console.WriteLine($"RemoteEndPoint: {request.RemoteEndPoint}");
-      Console.WriteLine($"UserAgent: {request.UserAgent}");
-      Console.WriteLine($"ContentType: {request.ContentType}");
-      Console.WriteLine($"ContentLength: {request.ContentLength64}");
-
-      if (request.QueryString.Count > 0) {
-         Console.WriteLine("Query:");
-         foreach (var key in request.QueryString.AllKeys) {
-            if (key is null) {
-               continue;
-            }
-
-            Console.WriteLine($"  {key} = {request.QueryString[key]}");
-         }
-      }
-
-      if (request.Headers.Count > 0) {
-         Console.WriteLine("Headers:");
-         foreach (var headerKey in request.Headers.AllKeys) {
-            if (headerKey is null) {
-               continue;
-            }
-
-            Console.WriteLine($"  {headerKey}: {request.Headers[headerKey]}");
-         }
-      }
-
-      Console.WriteLine("Body:");
-      Console.WriteLine(string.IsNullOrWhiteSpace(payload) ? "  <empty>" : payload);
-
-      var rpcRequest = ParseRpcRequest(payload);
-      HandleRpc(rpcRequest.FunctionName, rpcRequest.Arguments);
-      Console.WriteLine("------------------------------");
-   }
-
-   private static PersistenceRpcRequest ParseRpcRequest(string payload) {
-      if (string.IsNullOrWhiteSpace(payload)) {
-         throw new InvalidOperationException("RPC payload is empty.");
-      }
-
-      using var document = JsonDocument.Parse(payload);
-      var root = document.RootElement;
-
-      if (!root.TryGetProperty("function_name", out var functionNameElement) ||
-          functionNameElement.ValueKind != JsonValueKind.String) {
-         throw new InvalidOperationException("RPC payload must contain a string 'function_name'.");
-      }
-
-      if (!root.TryGetProperty("arguments", out var argumentsElement) ||
-          argumentsElement.ValueKind != JsonValueKind.Array) {
-         throw new InvalidOperationException("RPC payload must contain an 'arguments' array.");
-      }
-
-      var arguments = new List<string>();
-      foreach (var argument in argumentsElement.EnumerateArray()) {
-         if (argument.ValueKind != JsonValueKind.String) {
-            throw new InvalidOperationException("RPC arguments must all be strings.");
-         }
-
-         arguments.Add(argument.GetString()!);
-      }
-
-      return new PersistenceRpcRequest(functionNameElement.GetString()!, arguments.ToArray());
-   }
-
-   private void HandleRpc(string functionName, params string[] arguments) {
-      switch (functionName) {
-         case "keyvalue_insert":
-            RequireArgumentCount(functionName, arguments, 3);
-            HandleKeyValueInsert(arguments[0], arguments[1], arguments[2]);
-            break;
-         case "keyvalue_update":
-            RequireArgumentCount(functionName, arguments, 3);
-            HandleKeyValueUpdate(arguments[0], arguments[1], arguments[2]);
-            break;
-         case "keyvalue_delete":
-            RequireArgumentCount(functionName, arguments, 2);
-            HandleKeyValueDelete(arguments[0], arguments[1]);
-            break;
-         default:
-            throw new InvalidOperationException($"Unknown RPC function '{functionName}'.");
-      }
-   }
-
-   private static void RequireArgumentCount(string functionName, string[] arguments, int expectedCount) {
-      if (arguments.Length != expectedCount) {
-         throw new InvalidOperationException(
-            $"RPC function '{functionName}' expected {expectedCount} arguments but received {arguments.Length}.");
-      }
-   }
-
-   private void HandleKeyValueInsert(string itemNamespace, string key, string value) {
-      var row = new Dictionary<string, string> {
-         ["namespace"] = itemNamespace,
-         ["key"] = key,
-         ["value"] = value,
-      };
-      _persistenceHelper.InsertRow(KeyValueTableName, row);
-      _pub_sub_sync.publish_table(KeyValueTableName);
-      Console.WriteLine($"RPC: keyvalue_insert(namespace='{itemNamespace}', key='{key}', value='{value}')");
-   }
-
-   private void HandleKeyValueUpdate(string itemNamespace, string key, string value) {
-      var row = new Dictionary<string, string> {
-         ["namespace"] = itemNamespace,
-         ["key"] = key,
-         ["value"] = value,
-      };
-      _persistenceHelper.UpdateRow(KeyValueTableName, row);
-      _pub_sub_sync.publish_table(KeyValueTableName);
-      Console.WriteLine($"RPC: keyvalue_update(namespace='{itemNamespace}', key='{key}', value='{value}')");
-   }
-
-   private void HandleKeyValueDelete(string itemNamespace, string key) {
-      var row = new Dictionary<string, string> {
-         ["namespace"] = itemNamespace,
-         ["key"] = key,
-      };
-      _persistenceHelper.RemoveRow(KeyValueTableName, row);
-      _pub_sub_sync.publish_table(KeyValueTableName);
-      Console.WriteLine($"RPC: keyvalue_delete(namespace='{itemNamespace}', key='{key}')");
-   }
-}
-
-public record PersistenceRpcRequest {
-   public PersistenceRpcRequest(string functionName, string[] arguments) {
-      FunctionName = functionName;
-      Arguments = arguments;
-      Console.WriteLine($"PersistenceRpcRequest created for function '{FunctionName}' with {Arguments.Length} argument(s).");
-   }
-
-   public string FunctionName { get; init; }
-   public string[] Arguments { get; init; }
-}

src/DataStreamSuite/PersistenceServer/PersistenceServer.csproj

@@ -1,23 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <OutputType>Exe</OutputType>
-    <TargetFramework>net8.0</TargetFramework>
-    <Nullable>enable</Nullable>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="System.Data.SQLite" Version="1.0.118" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <Content Include="appsettings.json">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </Content>
-  </ItemGroup>
-
-</Project>

src/DataStreamSuite/PersistenceServer/PersistenceServerConfig.cs

@@ -1,5 +0,0 @@
-using Bts.PubSub;
-
-public class PersistenceServerConfig {
-   public PubSubConfig? pub_sub { get; set; }
-}

src/DataStreamSuite/PersistenceServer/Program.cs

@@ -1,113 +0,0 @@
-using Bts.DataStreamCore;
-using BCG;
-
-using System;
-using System.IO;
-using System.Net;
-using System.Text;
-using System.Threading;
-using System.Threading.Tasks;
-
-const int Port = 5079;
-var prefix = $"http://localhost:{Port}/";
-
-var listener = new HttpListener();
-listener.Prefixes.Add(prefix);
-var config = ConfigLoader.load();
-var persistenceHelper = new PeristenceHelper();
-var pub_sub_sync = new PubSubSync(persistenceHelper.database_path, config.pub_sub);
-var callHandler = new PersistenceCallHandler(persistenceHelper, pub_sub_sync);
-
-Console.WriteLine("PersistenceServer starting.");
-Console.WriteLine($"Listening on {prefix}");
-Console.WriteLine($"DataStreamCore loaded from: {typeof(IDataPacket).Assembly.FullName}");
-Console.WriteLine("Press Ctrl+C to stop.");
-
-var shutdown = new CancellationTokenSource();
-
-Console.CancelKeyPress += (_, eventArgs) => {
-   eventArgs.Cancel = true;
-   Console.WriteLine("Shutdown requested.");
-   shutdown.Cancel();
-   if (listener.IsListening) {
-      listener.Stop();
-   }
-};
-
-try {
-   pub_sub_sync.publish_all_tables();
-   listener.Start();
-   await RunAsync(listener, callHandler, shutdown.Token);
-   return 0;
-} catch (HttpListenerException ex) when (shutdown.IsCancellationRequested) {
-   Console.WriteLine($"Listener stopped: {ex.Message}");
-   return 0;
-} catch (Exception ex) {
-   Console.Error.WriteLine($"PersistenceServer crashed: {ex}");
-   return 1;
-} finally {
-   listener.Close();
-}
-
-static async Task RunAsync(
-   HttpListener listener,
-   PersistenceCallHandler callHandler,
-   CancellationToken cancellationToken) {
-   while (!cancellationToken.IsCancellationRequested) {
-      HttpListenerContext context;
-      try {
-         context = await listener.GetContextAsync();
-      } catch (HttpListenerException) when (cancellationToken.IsCancellationRequested) {
-         break;
-      } catch (ObjectDisposedException) when (cancellationToken.IsCancellationRequested) {
-         break;
-      }
-
-      _ = Task.Run(() => HandleRequestAsync(context, callHandler, cancellationToken), cancellationToken);
-   }
-}
-
-static async Task HandleRequestAsync(
-   HttpListenerContext context,
-   PersistenceCallHandler callHandler,
-   CancellationToken cancellationToken) {
-   var request = context.Request;
-   var response = context.Response;
-
-   try {
-      var body = await ReadBodyAsync(request, cancellationToken);
-      callHandler.Handle(request, body);
-
-      var responseText =
-         $$"""
-         {"status":"ok","method":"{{request.HttpMethod}}","path":"{{request.RawUrl ?? "/"}}"}
-         """;
-
-      var bytes = Encoding.UTF8.GetBytes(responseText);
-      response.StatusCode = (int)HttpStatusCode.OK;
-      response.ContentType = "application/json";
-      response.ContentEncoding = Encoding.UTF8;
-      response.ContentLength64 = bytes.LongLength;
-      await response.OutputStream.WriteAsync(bytes, cancellationToken);
-   } catch (Exception ex) {
-      Console.Error.WriteLine($"Request handling failed: {ex}");
-      response.StatusCode = (int)HttpStatusCode.InternalServerError;
-   } finally {
-      response.OutputStream.Close();
-      response.Close();
-   }
-}
-
-static async Task<string> ReadBodyAsync(HttpListenerRequest request, CancellationToken cancellationToken) {
-   if (!request.HasEntityBody) {
-      return string.Empty;
-   }
-
-   using var reader = new StreamReader(
-      request.InputStream,
-      request.ContentEncoding ?? Encoding.UTF8,
-      detectEncodingFromByteOrderMarks: true,
-      leaveOpen: true);
-
-   return await reader.ReadToEndAsync(cancellationToken);
-}

src/DataStreamSuite/PersistenceServer/PubSubSync.cs

@@ -1,116 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Data;
-using System.Data.SQLite;
-
-using BCG;
-using Bts.DataStreamCore;
-using Bts.PubSub;
-using Bts.PubSub.Routing;
-using Microsoft.Extensions.DependencyInjection;
-
-using bts;
-
-public class PubSubSync {
-   private readonly string _connectionString;
-   private readonly Dictionary<string, IPubTableBroker> _brokers = new(StringComparer.OrdinalIgnoreCase);
-   private readonly object _sync_root = new();
-   private readonly IServiceProvider _serviceProvider;
-   private readonly PubSubRouterConfig _router_config;
-   private readonly PubSubTagHandlerFactoryConfig _tag_handler_factory_config;
-
-   public PubSubSync(string databasePath, PubSubConfig? pubSubConfig = null) {
-      ArgumentException.ThrowIfNullOrWhiteSpace(databasePath);
-      _connectionString = $"Data Source={databasePath};Version=3";
-      var resolved_pub_sub_config = get_or_initialize_pub_sub_config(pubSubConfig);
-      _router_config = resolved_pub_sub_config.Routing ?? throw new InvalidOperationException("PubSub routing config is required.");
-      _tag_handler_factory_config = resolved_pub_sub_config.TagHandlerFactory ?? PubSubTagHandlerFactoryConfig.Default();
-      _serviceProvider = new ServiceCollection().BuildServiceProvider();
-   }
-
-   public void publish_all_tables() {
-      lock (_sync_root) {
-         foreach (var table_name in get_table_names()) {
-            publish_table_core(table_name);
-         }
-      }
-   }
-
-   public void publish_table(string table_name) {
-      ArgumentException.ThrowIfNullOrWhiteSpace(table_name);
-
-      lock (_sync_root) {
-         publish_table_core(table_name);
-      }
-   }
-
-   private void publish_table_core(string table_name) {
-      var rows = load_table(table_name);
-      var broker = get_or_create_broker(table_name);
-      broker.create_table(rows);
-      Console.WriteLine($"PubSubSync published table '{table_name}' with {rows.Count} row(s).");
-   }
-
-   private IPubTableBroker get_or_create_broker(string table_name) {
-      if (_brokers.TryGetValue(table_name, out var broker)) {
-         return broker;
-      }
-
-      var publisher_core = new PublisherCore(
-         new PubSubRouter(_router_config),
-         new PubsubTagHandlerFactory(_serviceProvider, _tag_handler_factory_config));
-      broker = new PubTableBroker(publisher_core);
-      _brokers[table_name] = broker;
-      return broker;
-   }
-
-   private string[] get_table_names() {
-      using var connection = open_connection();
-      using var command = new SQLiteCommand(
-         "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name",
-         connection);
-      using var reader = command.ExecuteReader();
-
-      var table_names = new List<string>();
-      while (reader.Read()) {
-         table_names.Add(reader.GetString(0));
-      }
-
-      return table_names.ToArray();
-   }
-
-   private IBlueRowCollection load_table(string table_name) {
-      using var connection = open_connection();
-      using var adapter = new SQLiteDataAdapter($"SELECT * FROM {quote_identifier(table_name)}", connection);
-      var table = new DataTable(table_name);
-      adapter.Fill(table);
-      table.TableName = table_name;
-      return BlueDataUtility.toIBlueRowCollection(table);
-   }
-
-   private SQLiteConnection open_connection() {
-      var connection = new SQLiteConnection(_connectionString);
-      connection.Open();
-      return connection;
-   }
-
-   private static PubSubConfig get_or_initialize_pub_sub_config(PubSubConfig? configured_pub_sub) {
-      try {
-         PubSubConfig.SetGlobalConfig(PubSubConfig.Default());
-         var existing = PubSubConfig.Instance;
-         if (configured_pub_sub is not null) {
-            existing.MergeWith(configured_pub_sub);
-         }
-         return existing;
-      } catch (InvalidOperationException) {
-         var config = PubSubConfig.Default();
-         if (configured_pub_sub is not null) {
-            config.MergeWith(configured_pub_sub);
-         }
-         PubSubConfig.SetGlobalConfig(config);
-         return config;
-      }
-   }
-
-   private static string quote_identifier(string identifier) => $"\"{identifier.Replace("\"", "\"\"")}\"";
-}

src/DataStreamSuite/PersistenceServer/appsettings.json

@@ -1,20 +0,0 @@
-{
-  "PubSub": {
-    "Routing": {
-      "PublisherRoutes": [
-        {
-          "Name": "default",
-          "ServerAddress": "bts-cloud://localhost:22019",
-          "TagRegExp": ".*"
-        }
-      ],
-      "SubscriberRoutes": [
-        {
-          "Name": "default",
-          "ServerAddress": "bts-cloud://localhost:22019",
-          "TagRegExp": ".*"
-        }
-      ]
-    }
-  }
-}

src/DataStreamSuite/PersistenceServer/client/persistence_client.py

@@ -1,43 +0,0 @@
-import json
-import sys
-import urllib.error
-import urllib.request
-
-
-SERVER_URL = "http://localhost:5079/save?source=python-client"
-
-
-def main() -> int:
-   payload = {
-      "function_name": "keyvalue_insert",
-      "arguments": [
-         "example-namespace",
-         "example-key",
-         "hello from python",
-      ],
-   }
-
-   data = json.dumps(payload).encode("utf-8")
-   request = urllib.request.Request(
-      SERVER_URL,
-      data=data,
-      headers={
-         "Content-Type": "application/json",
-         "X-Client": "PersistenceServerPythonClient",
-      },
-      method="POST",
-   )
-
-   try:
-      with urllib.request.urlopen(request) as response:
-         body = response.read().decode("utf-8")
-         print(f"status: {response.status}")
-         print(f"body: {body}")
-         return 0
-   except urllib.error.URLError as error:
-      print(f"request failed: {error}", file=sys.stderr)
-      return 1
-
-
-if __name__ == "__main__":
-   raise SystemExit(main())

src/DataStreamSuite/PubSub/Client/Communication/BlutoTCP/ClientToServerConnectionWrapper.cs

@@ -1,44 +1,44 @@
-// using System;
-// using System.Collections.Generic;
-// using System.Text;
-// using System.Runtime.InteropServices;
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Runtime.InteropServices;
 
-// namespace BCG
-// {
-//    public class ClientToServerConnectionWrapper
-//    {
-//       [DllImport("BlutoInterop.dll")]
-//       public unsafe static extern void init();
+namespace BCG
+{
+   public class ClientToServerConnectionWrapper
+   {
+      [DllImport("BlutoInterop.dll")]
+      public unsafe static extern void init();
 
-//       [DllImport("BlutoInterop.dll")]
-//       public unsafe static extern void initServerConnection(byte[] ip, int port);
+      [DllImport("BlutoInterop.dll")]
+      public unsafe static extern void initServerConnection(byte[] ip, int port);
 
-//       [DllImport("BlutoInterop.dll")]
-//       public static extern void sendDataToServer(byte[] ip, int port, byte[] message, int size);
+      [DllImport("BlutoInterop.dll")]
+      public static extern void sendDataToServer(byte[] ip, int port, byte[] message, int size);
 
-//       [DllImport("BlutoInterop.dll")]
-//       public unsafe static extern void getServerData(int timeout, int maxElems, BlutoTCPClientMessage*** retpts, out int npts);
+      [DllImport("BlutoInterop.dll")]
+      public unsafe static extern void getServerData(int timeout, int maxElems, BlutoTCPClientMessage*** retpts, out int npts);
 
-//       public static byte[] stringToBytes(string s)
-//       {
-//          byte[] rv = new byte[s.Length + 1];
-//          for (int i = 0; i < s.Length; i++)
-//             rv[i] = Convert.ToByte(s[i]);
-//          rv[s.Length] = Convert.ToByte ('\0');
-//          return rv;
-//       }
+      public static byte[] stringToBytes(string s)
+      {
+         byte[] rv = new byte[s.Length + 1];
+         for (int i = 0; i < s.Length; i++)
+            rv[i] = Convert.ToByte(s[i]);
+         rv[s.Length] = Convert.ToByte ('\0');
+         return rv;
+      }
 
-//       public static string intPtrToString(IntPtr iptr)
-//       {
-//          return Marshal.PtrToStringAnsi(iptr);
-//       }
+      public static string intPtrToString(IntPtr iptr)
+      {
+         return Marshal.PtrToStringAnsi(iptr);
+      }
 
-//       public static byte[] intPtrToByteArray(IntPtr iptr, int sz)
-//       {
-//          byte[] data = new byte[sz];
-//          Marshal.Copy(iptr, data, 0, data.Length);
-//          return data;
-//       }
+      public static byte[] intPtrToByteArray(IntPtr iptr, int sz)
+      {
+         byte[] data = new byte[sz];
+         Marshal.Copy(iptr, data, 0, data.Length);
+         return data;
+      }
 
-//    }
-// }
+   }
+}

src/DataStreamSuite/PubSub/Client/MetaData/PubSubClientMessage.cs

@@ -39,6 +39,8 @@ public static class PubSubMessageUtils {
    }
 }
 
+// Subscribe //////////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////////////////////
 public class SubscribeMsg : IPubSubMessage, IEquatable<SubscribeMsg> {
    public string? connection;
    public string subscribing_app = ClientInfo.thisClient.app;
@@ -84,6 +86,8 @@ public class SubscribeMsg : IPubSubMessage, IEquatable<SubscribeMsg> {
       $"{GetType().Name} connection: {connection} subscribing_app: {subscribing_app} tags: {string.Concat(tags.empty_if_null())}";
 }
 
+// Unsubscribe ////////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////////////////////
 public class UnsubscribeMsg : SubscribeMsg, IEquatable<UnsubscribeMsg> {
    public bool Equals(UnsubscribeMsg? other) =>
       other is not null && base.Equals(other);
@@ -95,6 +99,8 @@ public class UnsubscribeMsg : SubscribeMsg, IEquatable<UnsubscribeMsg> {
       base.GetHashCode();
 }
 
+// Trasnaction /////////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////////////////////
 public class TransactionMsg : IPubSubMessage, IEquatable<TransactionMsg> {
    uint n_msgs;
 
@@ -121,12 +127,15 @@ public class TransactionMsg : IPubSubMessage, IEquatable<TransactionMsg> {
       n_msgs.GetHashCode();
 }
 
+// Publish ////////////////////////////////////////////////////////////////
+/// ///////////////////////////////////////////////////////////////////////
 public class PublishMsg : IPubSubMessage, IEquatable<PublishMsg> {
    public string? connection;
    public string publishing_app = ClientInfo.thisClient.app;
    public string? tag;
    public string? tag_type;
    public bool is_compressed = false;
+   // this is the size that of the msg, regarless of it being serialized or not
    public long publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
    public byte[]? data_to_publish;
 
@@ -135,6 +144,8 @@ public class PublishMsg : IPubSubMessage, IEquatable<PublishMsg> {
       Log.assert(!string.IsNullOrEmpty(tag_type));
       Log.assert(!string.IsNullOrEmpty(tag));
       Log.assert(!string.IsNullOrEmpty(connection));
+      // The 'ThrowIfNullOrEmpty' calls are redundant (given the assert statements above), but
+      // necessary for the compiler to recognize the strings won't be null when this method exits.
       ArgumentException.ThrowIfNullOrEmpty(tag_type);
       ArgumentException.ThrowIfNullOrEmpty(tag);
       ArgumentException.ThrowIfNullOrEmpty(connection);
@@ -170,7 +181,7 @@ public class PublishMsg : IPubSubMessage, IEquatable<PublishMsg> {
    }
 
    public bool Equals(PublishMsg? other) =>
-      other is not null
+      other is not null 
          && connection == other.connection
          && publishing_app == other.publishing_app
          && tag == other.tag
@@ -209,6 +220,7 @@ public enum PubSubMessageType {
 
 public class PubSubClientMessage : IPubSubMessage {
    public PubSubMessageType msg_type;
+   // only one of these wil be non null
    public SubscribeMsg? sub_msg;
    public UnsubscribeMsg? unsub_msg;
    public TransactionMsg? transaction_msg;
@@ -253,7 +265,7 @@ public class PubSubClientMessage : IPubSubMessage {
             ValidateMsg(pub_msg).serialize_into_str(str);
             break;
 
-         T ValidateMsg<T>([NotNull] T? msg, [CallerArgumentExpression(nameof(msg))] string name = "") {
+         T ValidateMsg<T>([NotNull]T? msg, [CallerArgumentExpression(nameof(msg))]string name = "") {
             if (msg is null) {
                throw new InvalidOperationException($"msg type is {msg_type} but {name} is null");
             }
@@ -371,7 +383,7 @@ public class PubSubServerMessage : IPubSubMessage, IEquatable<PubSubServerMessag
 
    [MemberNotNull(nameof(tag), nameof(tag_type))]
    public void construct_deserialize(byte[] msg) =>
-      construct_deserialize(new MemoryStream(msg) { Position = 0 });
+      construct_deserialize(new MemoryStream(msg) { Position = 0});
 
    public bool Equals(PubSubServerMessage? other) =>
       other is not null

src/DataStreamSuite/PubSub/Config/PubSubConfig.cs

@@ -48,16 +48,16 @@ public class PubSubConfig {
          HeartbeatInterval = 15000,
          MissingHeartbeatsBeforeGc = 50,
          DisposedConnectionCheckPeriod = 1000,
-         ServerPort = 22019,
+         ServerPort = 12345,
          TagTypeToAssemblyMap = new(),
          TagHandlerFactory = PubSubTagHandlerFactoryConfig.Default(),
          SimpleEchoServer = false,
          Routing = new() {
             PublisherRoutes = new() {
-               new("bts-cloud://127.0.0.1:22019", ".*")
+               new("bts-cloud://127.0.0.1:12345", ".*")
             },
             SubscriberRoutes = new() {
-               new("bts-cloud://127.0.0.1:22019", ".*")
+               new("bts-cloud://127.0.0.1:12345", ".*")
             }
          }
       };

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_APM.cs

@@ -12,7 +12,6 @@ using System.Net.Sockets;
 
 // opens a TCP port and starts listening to connections
 public class AsyncTCPConnectionAcceptor_APM {
-   // This acceptor exists as an alternate implementation and is not the one currently used by the pubsub server.
    private readonly int _port;
    Socket? _accept_socket;
    readonly ITCPConnectionHandler _connection_handler;

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnectionAcceptor_AsyncWrapper.cs

@@ -13,7 +13,6 @@ namespace bts {
 
    // opens a TCP port and starts listening to connections. Unlike AsyncTCPConnectionAcceptor_APM,
    // it uses TcpListener, instead of raw sockets.
-   // This is the acceptor currently used by the pubsub server via ManagedTCPMessageHub.
    public class AsyncTCPConnectionAcceptor_AsyncWrapper {
       private readonly int _accept_port;
       // private Socket _accept_socket;

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnector_APM.cs

@@ -10,7 +10,6 @@ namespace bts {
     // Unlike AsyncTCPConnectionAcceptor_APM this class is static since the requests to connect are stand alone requests
     // whereas AsyncTCPConnectionAcceptor_APM is always listening for incoming connections which lends itself to more
     // statefullness and hence non static
-    // This connector exists as an alternate implementation and is not the one currently used by pubsub clients.
     public static class AsyncTCPConnector_APM {
 
         internal class SocketConnectionArgs {

src/DataStreamSuite/PubSub/ManagedTCP/AsyncTCPConnector_AsyncWrapper.cs

@@ -13,18 +13,17 @@ using System.Diagnostics.CodeAnalysis;
 
 
 namespace bts {
-   // Unlike AsyncTCPConnectionAcceptor_AsyncWrappe this class is static since the requests to connect are stand alone requests
-   // whereas AsyncTCPConnectionAcceptor_AsyncWrappe is always listening for incoming connections which lends itself to more
+   // Unlike AsyncTCPConnectionAcceptor_APM this class is static since the requests to connect are stand alone requests
+   // whereas AsyncTCPConnectionAcceptor_APM is always listening for incoming connections which lends itself to more
    // statefullness and hence non static
-   // This is the connector currently used by pubsub clients via PubSubServerManagedTCPConnection.
    public static class AsyncTCPConnector_AsyncWrapper {
       public static void connect_to(RemoteConnectionInfo server_address, ITCPConnectionHandler handler) {
          bts.Log.Gen.info($"AsyncTCPConnector_AsyncWrapper.connect_to server: {server_address}");
          //Task.Run(() => _connect_to(server_address, handler));
          ThreadPool.QueueUserWorkItem(_ => _keep_trying_blocking_function(
-            _connect, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5)));
+            Connect, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5)));
 
-         bool _connect() {
+         bool Connect() {
             try {
                return _connect_to(server_address, handler);
             } catch (Exception ex) {
@@ -50,7 +49,7 @@ namespace bts {
          handler.onConnection(conn);
          return true;
 
-         bool CreateTcpClient([NotNullWhen(true)] out TcpClient? tcpClient) {
+         bool CreateTcpClient([NotNullWhen(true)]out TcpClient? tcpClient) {
             try {
                tcpClient = new(server_rci.ip, server_rci.port);
                return true;

src/DataStreamSuite/PubSub/Misc/ClientInfo.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Text;
 
 namespace BCG
@@ -24,8 +24,8 @@ namespace BCG
       {
          get { return myApp; }
          set { myApp = value; }
-      }
 
+      }
       private string myKeyQualifier;
       public string keyQualifier
       {
@@ -38,8 +38,8 @@ namespace BCG
       {
          get { return myStartTime; }
          set { myStartTime = value; }
-      }
 
+      }
       string? myProcessString;
       public string processString
       {
@@ -49,8 +49,10 @@ namespace BCG
                myProcessString = getProcessString(";");
             return myProcessString;
          }
+
       }
 
+      
       public ClientInfo(string theMachineName, int thePid, string theProcessName, string theKeyQualifier, DateTime theStartTime)
       {
          myMachineName = theMachineName;
@@ -66,8 +68,10 @@ namespace BCG
          myMachineName = Environment.MachineName;
          myPid = System.Diagnostics.Process.GetCurrentProcess().Id;
          myApp = System.Diagnostics.Process.GetCurrentProcess().ProcessName.ToUpper();
-         string[] args = Environment.GetCommandLineArgs();
+         string[] args = System.Environment.GetCommandLineArgs();
 
+         //the key qualifier is the last argument given. If they didn't give any arguments
+         //but the program itself, then the KQ is empty. 
          if (args.Length > 1)
             myKeyQualifier = args[args.Length - 1];
          else
@@ -82,13 +86,13 @@ namespace BCG
 
       public static ClientInfo thisClient
       {
-         get
+         get 
          {
             if (myLocalClient == null)
             {
-               lock (myLock)
+               lock(myLock)
                {
-                  if (myLocalClient == null)
+                  if(myLocalClient == null)
                      myLocalClient = new ClientInfo();
                }
             }
@@ -98,6 +102,7 @@ namespace BCG
 
       public static ClientInfo fromProcessString(string procString)
       {
+
          ClientInfo outputGuy = new ClientInfo();
 
          string[] info = procString.Split(';');
@@ -107,7 +112,9 @@ namespace BCG
          outputGuy.myKeyQualifier = info[2];
          outputGuy.myPid = Int32.Parse(info[3]);
 
+
          return outputGuy;
+
       }
 
       private string getProcessString(string seperator)

src/DataStreamSuite/PubSub/Misc/IWorker.cs

@@ -3,7 +3,7 @@ using System;
 namespace BCG.Utilities.General
 {
    public interface IWorker
-   {
+	{
       void doWork();
-   }
+	}
 }

src/DataStreamSuite/PubSub/Misc/Pair.cs

@@ -79,4 +79,5 @@ namespace mbts
          return ToString().GetHashCode();
       }
    }
+
 }

src/DataStreamSuite/PubSub/Misc/StreamUtilityMethods.cs

@@ -9,7 +9,7 @@ namespace BCG {
    /// <summary>
    /// Collection of utility methods used throughout
    /// </summary>
-   public static class StreamUtilityMethods {
+	public static class StreamUtilityMethods {
 
       // extension method to write a string C++ style (len prefixed)
       public static void write_string_cpp(this BinaryWriter bw, string? str) {
@@ -98,7 +98,7 @@ namespace BCG {
          byte[] secondStreamBytes = new byte[stream.Length - splitLocation];
          for (int i = 0; i < stream.Length; ++i) {
             //for some resaon, the 'getBytesFromStream' function prepends the length of the byte array to the bytes. This means
-            //we have to ignore those bytes here.
+            //we have to ignore those bytes here. 
             if (i < splitLocation)
                firstStreamBytes[i] = streamBytes[i + 4];
             else
@@ -107,6 +107,8 @@ namespace BCG {
          }
 
          return new Stream[] { new MemoryStream(firstStreamBytes), new MemoryStream(secondStreamBytes) };
+
+
       }
    }
 }

src/DataStreamSuite/PubSub/PublishedCache.cs

@@ -4,8 +4,7 @@ using System;
 using System.Collections.Generic;
 
 namespace bts {
-   // This caches all the published data, on the pubsliher (client) side. 
-   // This class is created and held by
+   // This caches all the published data. This class is created and held by
    // PublisherCore. When data is publsihed , PublisherCore passes it to this 
    // class via the publish call. On a reconnection to the PubSubServer, 
    // PublisherCore calls get_content to get the data to re-publish, to 

src/DataStreamSuite/PubSub/PublisherCore.cs

@@ -43,20 +43,20 @@ namespace BCG {
 
       #region constructors
 
-      public PublisherCore(IPubSubRouter router, IPubSubTagHandlerFactory tag_handler_factory) {
+      public PublisherCore(IPubSubRouter router, IPubSubTagHandlerFactory tagHandlerFactory) {
          _router = router;
-         _state = new(tag_handler_factory);
+         _state = new(tagHandlerFactory);
          Thread pump_thread = new Thread(new ThreadStart(_message_pump));
          pump_thread.IsBackground = true;
          pump_thread.Start();
       }
 
-      public static PublisherCore Create(IPubSubRouter? router = null, IPubSubTagHandlerFactory? tag_handler_factory = null, IServiceProvider? service_provider = null) {
+      public static PublisherCore Create(IPubSubRouter? router = null, IPubSubTagHandlerFactory? tagHandlerFactory = null, IServiceProvider? serviceProvider = null) {
          router ??= new PubSubRouter();
-         tag_handler_factory ??= new PubsubTagHandlerFactory(
-            service_provider ?? PubSubServices.ServiceProvider,
+         tagHandlerFactory ??= new PubsubTagHandlerFactory(
+            serviceProvider ?? PubSubServices.ServiceProvider,
             PubSubConfig.Instance.TagHandlerFactory ?? PubSubTagHandlerFactoryConfig.Default());
-         return new(router, tag_handler_factory);
+         return new(router, tagHandlerFactory);
       }
 
       public void Dispose() { }
@@ -119,7 +119,7 @@ namespace BCG {
       #endregion
 
       // static void setGlobalInstance() {
-
+         
       // }
 
       #region Message Pump Thread - this is where messages are dequeued and the action performed

src/DataStreamSuite/PubSub/Server/IInternalPublisher.cs

@@ -2,6 +2,6 @@ namespace Bts.PubSub.Server;
 
 using bts;
 
-public interface IInternalPublisher {
+interface IInternalPublisher {
    void Publish(PublishMsg message);
 }

src/DataStreamSuite/PubSub/Server/IInternalPublisherQueue.cs

@@ -10,7 +10,7 @@ using System.Collections.Generic;
 /// messages to be published. It is also passed to the <see cref="BCG.PubSubMessageReactor"/>, so it
 /// can dequeue these messages and send them over the wire.
 /// </summary>
-public interface IInternalPublisherQueue {
+interface IInternalPublisherQueue {
    void Enqueue(PublishMsg message);
    List<PublishMsg> DequeueMultiple(TimeSpan ts, int maxElements);
 }

src/DataStreamSuite/PubSub/Server/IPubSubInternalPublisher.cs

@@ -1,9 +0,0 @@
-namespace BlueOrb.PubSub.Server;
-
-using BCG.Utilities.General;
-
-using System;
-
-public interface IPubSubInternalPublisher : IWorker {
-   event Action<string, string, byte[]>? onNewData;
-}

src/DataStreamSuite/PubSub/Server/IPubSubState.cs

@@ -10,16 +10,6 @@ public delegate void PubSubStateSingleTagChangeDelegate(PubSubSingleTagStateEven
 public delegate void PubSubStateChangeDelegate(string[] tags);
 public delegate void PubSubStatsDelegate(string tag, int size);
 
-public class PubSubServerTagStats {
-   public int n_updates { get; set; }
-   public long total_bytes { get; set; }
-   public int min_size { get; set; }
-   public int max_size { get; set; }
-   public int last_update_siz { get; set; }
-   public int average_size { get; set; }
-   public long last_update_time { get; set; }
-}
-
 public interface IPubSubState {
    event PubSubStateChangeDelegate? onTagMapChanged;
    event PubSubStatsDelegate? onPublishStats;
@@ -28,7 +18,6 @@ public interface IPubSubState {
 
    byte[] get_content(string tag);
    string getTagType(string tag);
-   PubSubServerTagStats get_stats(string tag);
    void get_all_connections(
       string tag,
       out Dictionary<string, string> subscribers,

src/DataStreamSuite/PubSub/Server/InternalPublisherQueue.cs

@@ -7,7 +7,7 @@ using bts;
 using System;
 using System.Collections.Generic;
 
-public class InternalPublisherQueue : IInternalPublisherQueue {
+class InternalPublisherQueue : IInternalPublisherQueue {
    readonly SyncQueue<PublishMsg> queue = new();
    readonly object lockObj = new();
 

src/DataStreamSuite/PubSub/Server/InternalTagHandling/SystemHeartbeatPublisher.cs

@@ -1,76 +1,72 @@
-// using System;
-// using BCG.Utilities.General;
-// using System.IO;
-// using bts;
-// using Bts.PubSub;
-// using Bts.PubSub.Server;
-// using Spark.Core;
+using System;
+using System.Collections.Generic;
+using System.Text;
+using BCG.Utilities.General;
+using System.IO;
+using bts;
+using Bts.PubSub;
+using Bts.PubSub.Server;
+using Spark.Core;
 
-// namespace BCG {
-//    class Pubsubheartbeat {
-//       public Int64 hb_time_epoch;
-//       public byte[] serialize() {
-//          MemoryStream str = new MemoryStream();
-//          BinaryWriter bw = new BinaryWriter(str);
-//          bw.Write(hb_time_epoch);
-//          str.Position = 0;
-//          return str.ToArray();
-//       }
+namespace BCG {
+   class Pubsubheartbeat {
+      public Int64 hb_time_epoch;
+      public byte[] serialize() {
+         MemoryStream str = new MemoryStream();
+         BinaryWriter bw = new BinaryWriter(str);
+         bw.Write(hb_time_epoch);
+         str.Position = 0;
+         return str.ToArray();
+      }
 
-//       public Pubsubheartbeat() {
-//          hb_time_epoch = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
-//       }
+      public Pubsubheartbeat() {
+         hb_time_epoch = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+      }
 
-//       public static Pubsubheartbeat deserialize(byte[] data) {
-//          long hb_time = BitConverter.ToInt64(data);
-//          return new Pubsubheartbeat() {
-//             hb_time_epoch = hb_time,
-//          };
-//       }
-//    }
-//    /// <summary>
-//    /// Periodicaly publishes a heartbeat, under the tag name System_Heartbeat.
-//    /// A subscriber can subscribe to it, and find out if a PubSubServer is alive 
-//    /// or not. Infact more so, if the subsriptions are working or not.
-//    /// </summary>
-//    internal class SystemHeartbeatPublisher : VirtualSingleton<SystemHeartbeatPublisher>, IWorker, IParameterizedWorker {
-//       public static string tag_type = "System_Heartbeat";
-//       public static string tag_name = "System_Heartbeat";
+      public static Pubsubheartbeat deserialize(byte[] data) {
+         long hb_time = BitConverter.ToInt64(data);
+         return new Pubsubheartbeat() {
+            hb_time_epoch = hb_time,
+         };
+      }
+   }
+   /// <summary>
+   /// Periodicaly publishes a heartbeat, under the tag name System_Heartbeat.
+   /// A subscriber can subscribe to it, and find out of a PubSub is alive or 
+   /// not. Infact more so, if the subsriptions are working or not.
+   /// </summary>
+   internal class SystemHeartbeatPublisher : VirtualSingleton<SystemHeartbeatPublisher>, IWorker, IParameterizedWorker {
+      public static string tag_type = "System_Heartbeat";
+      public static string tag_name = "System_Heartbeat";
+      readonly IInternalPublisher internalPublisher;
+      readonly IWorker workerImpl;
 
-//       readonly IInternalPublisher internalPublisher;
-//       readonly IWorker workerImpl;
+      public SystemHeartbeatPublisher(IInternalPublisher internalPublisher) {
+         this.internalPublisher = internalPublisher;
+         workerImpl = new SingleThreadedWorker(new AnonymousWorker(doWorkImpl));
+      }
 
-//       public SystemHeartbeatPublisher(IInternalPublisher internalPublisher) {
-//          this.internalPublisher = internalPublisher;
-//          workerImpl = new SingleThreadedWorker(new AnonymousWorker(doWorkImpl));
-//       }
+      TimeSpan IParameterizedWorker.Interval => TimeSpan.FromMilliseconds(500);
 
-//       TimeSpan IParameterizedWorker.Interval => TimeSpan.FromMilliseconds(500);
+      #region IWorker Members
 
-//       #region IWorker Members
+      public void doWork() =>
+         workerImpl.doWork();
 
-//       public void doWork() =>
-//          workerImpl.doWork();
+      void doWorkImpl() {
+         Pubsubheartbeat hb = new Pubsubheartbeat();
+         PublishMsg msg = new PublishMsg() {
+            connection = "Internal",
+            publishing_app = ClientInfo.thisClient.app,
+            tag = "pss_hb",
+            tag_type = "Internal",
+            is_compressed = false,
+            publish_time_epoch_ms = hb.hb_time_epoch,
+            data_to_publish = hb.serialize()
+         };
+         internalPublisher.Publish(msg);
+      }
 
-//       void doWorkImpl() {
-//          // why not publish a table
-//          Pubsubheartbeat hb = new Pubsubheartbeat();
-//          PublishMsg msg = new PublishMsg() {
-//             connection = "Internal",
-//             publishing_app = ClientInfo.thisClient.app,
-//             tag = "pss_hb",
-//             tag_type = "Internal",
-//             is_compressed = false,
-//             publish_time_epoch_ms = hb.hb_time_epoch,
-//             data_to_publish = hb.serialize()
-//          };
-//          internalPublisher.Publish(msg);
-//       }
-
-//       #endregion
-
-//       private void _publish_hb() {
-
-//       }
-//    }
-// }
+      #endregion
+   }
+}