Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Taha
|
7cff8b625d | ||
|
Taha
|
6950de82ff | ||
|
Taha
|
9140888b54 | ||
|
Taha
|
32a851954c |
No files matched your search
+4
@@ -0,0 +1,4 @@
|
||||
// <auto-generated> This file has been auto generated. </auto-generated>
|
||||
using System;
|
||||
[Microsoft.VisualStudio.TestPlatform.TestSDKAutoGeneratedCode]
|
||||
class AutoGeneratedProgram {static void Main(string[] args){}}
|
||||
@@ -29,9 +29,6 @@
|
||||
# tools/deploy.sh --help
|
||||
#
|
||||
|
||||
### Host/Runtime ###
|
||||
TZ=Etc/UTC
|
||||
|
||||
### Sparkd ###
|
||||
# The internal image for sparkd to use. For more info, view:
|
||||
# src/server/sparkd/Dockerfile
|
||||
@@ -41,9 +38,6 @@ SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
|
||||
# Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
|
||||
# Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
|
||||
# SPARKD__DATA_DIR=/home/bts/sparkdata/data
|
||||
# Mount volume or dir to "/md_recording" inside the docker. Defaults to volume
|
||||
# "sparkd_md_recording_<config_port>" (e.g., sparkd_md_recording_8000).
|
||||
# SPARKD__MD_RECORDING_DIR=/home/bts/md_recording
|
||||
SPARKD__SOURCE_DIR=install/sparkd-release-gcc
|
||||
SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
|
||||
SPARKD__CONFIG_PORT=8000
|
||||
@@ -51,8 +45,6 @@ SPARKD__QUICKBUS_PORT=8001
|
||||
SPARKD__SIM_FIX_PORT=8002
|
||||
SPARKD__SHM_SIZE=4gb
|
||||
SPARKD__PYTHON_REQUIREMENTS=requirements.txt
|
||||
SPARKD__SPARKD_ENTRYPOINT=src/app_sparkd/sparkd-entrypoint.sh
|
||||
SPARKD__MAIN_ENTRYPOINT=./sparkd-entrypoint.sh
|
||||
|
||||
### PubSubServer ###
|
||||
PUBSUB__CONFIG_DIR=src/DataStreamSuite/PubSubServer/Configurations/Development
|
||||
|
||||
@@ -64,3 +64,9 @@ trace_data/
|
||||
*.lst
|
||||
.env.local
|
||||
.secrets
|
||||
.dotnet-cli/
|
||||
.dotnet-cli/
|
||||
.dotnet-cli /
|
||||
.dotnet/
|
||||
.codex_restore*.log
|
||||
src/cs/Bts.Spark.Core/.codexobj/
|
||||
Vendored
+75
-13
@@ -183,10 +183,11 @@
|
||||
"windows": {
|
||||
"program": "PubSubServer.exe",
|
||||
},
|
||||
"args": [
|
||||
"--config",
|
||||
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
|
||||
],
|
||||
"cwd": "build/Debug/bin/net8.0",
|
||||
"env": {
|
||||
"DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
|
||||
},
|
||||
"stopAtEntry": false,
|
||||
"console": "integratedTerminal"
|
||||
},
|
||||
@@ -202,9 +203,71 @@
|
||||
"program": "PubSubServer.exe",
|
||||
},
|
||||
"cwd": "build/Debug/bin/net8.0",
|
||||
"env": {
|
||||
"DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
|
||||
// "env": {
|
||||
// "DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
|
||||
// },
|
||||
"stopAtEntry": false,
|
||||
"console": "integratedTerminal"
|
||||
},
|
||||
{
|
||||
"name": "DSPublisher",
|
||||
"type": "coreclr",
|
||||
"request": "launch",
|
||||
"preLaunchTask": "Build DataStream (Debug)",
|
||||
"program": "",
|
||||
"linux": {
|
||||
"program": "DSPublisher",
|
||||
},
|
||||
"windows": {
|
||||
"program": "DSPublisher.exe",
|
||||
},
|
||||
"cwd": "build/Debug/bin/net8.0",
|
||||
"stopAtEntry": false,
|
||||
"console": "integratedTerminal"
|
||||
},
|
||||
{
|
||||
"name": "DSPublisher (Skip Build)",
|
||||
"type": "coreclr",
|
||||
"request": "launch",
|
||||
"program": "",
|
||||
"linux": {
|
||||
"program": "DSPublisher",
|
||||
},
|
||||
"windows": {
|
||||
"program": "DSPublisher.exe",
|
||||
},
|
||||
"cwd": "build/Debug/bin/net8.0",
|
||||
"stopAtEntry": false,
|
||||
"console": "integratedTerminal"
|
||||
},
|
||||
{
|
||||
"name": "PersistenceServer",
|
||||
"type": "coreclr",
|
||||
"request": "launch",
|
||||
"preLaunchTask": "Build DataStream (Debug)",
|
||||
"program": "",
|
||||
"linux": {
|
||||
"program": "PersistenceServer",
|
||||
},
|
||||
"windows": {
|
||||
"program": "PersistenceServer.exe",
|
||||
},
|
||||
"cwd": "build/Debug/bin/net8.0",
|
||||
"stopAtEntry": false,
|
||||
"console": "integratedTerminal"
|
||||
},
|
||||
{
|
||||
"name": "PersistenceServer (Skip Build)",
|
||||
"type": "coreclr",
|
||||
"request": "launch",
|
||||
"program": "",
|
||||
"linux": {
|
||||
"program": "PersistenceServer",
|
||||
},
|
||||
"windows": {
|
||||
"program": "PersistenceServer.exe",
|
||||
},
|
||||
"cwd": "build/Debug/bin/net8.0",
|
||||
"stopAtEntry": false,
|
||||
"console": "integratedTerminal"
|
||||
},
|
||||
@@ -213,7 +276,7 @@
|
||||
"type": "coreclr",
|
||||
"request": "launch",
|
||||
"program": "",
|
||||
"preLaunchTask": "Build Spark Sdk (Debug)",
|
||||
"preLaunchTask": "Build Api Server (Debug)",
|
||||
"windows": {
|
||||
"program": "Bts.Spark.Sdk.Demo.exe",
|
||||
},
|
||||
@@ -285,7 +348,7 @@
|
||||
"windows": {
|
||||
"program": "Spark.exe",
|
||||
"args": [
|
||||
"--Spark:ConfigServer",
|
||||
"--sparkd",
|
||||
"${input:sparkConfigServer}"
|
||||
],
|
||||
"cwd": "build/Debug/bin/net8.0-windows",
|
||||
@@ -448,15 +511,14 @@
|
||||
"default": "Local",
|
||||
},
|
||||
{
|
||||
"id": "pubSubServerEnvironment",
|
||||
"id": "pubSubServerConfig",
|
||||
"type": "pickString",
|
||||
"description": "PubSubServer Configuration",
|
||||
"description": "PubSub Configuration",
|
||||
"options": [
|
||||
"Development",
|
||||
"EchoServer",
|
||||
"Local",
|
||||
"PubSubServer",
|
||||
"SimpleEchoClient",
|
||||
],
|
||||
"default": "Development",
|
||||
"default": "PubSubServer",
|
||||
},
|
||||
{
|
||||
"id": "sdkDemoEnvironment",
|
||||
|
||||
+37
-94
@@ -2,7 +2,6 @@ services:
|
||||
sparkd:
|
||||
volumes:
|
||||
- ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
|
||||
- ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
|
||||
build:
|
||||
dockerfile: src/app_sparkd/Dockerfile
|
||||
args:
|
||||
@@ -10,35 +9,24 @@ services:
|
||||
- SOURCE_DIR=${SPARKD__SOURCE_DIR}
|
||||
- CONFIG_DIR=${SPARKD__CONFIG_DIR}
|
||||
- PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
|
||||
- SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
|
||||
shm_size: "${SPARKD__SHM_SIZE:-512mb}"
|
||||
env_file:
|
||||
- .env
|
||||
- ${SPARKD__CONFIG_DIR}/.env
|
||||
ports:
|
||||
- "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
|
||||
- "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
|
||||
- "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
|
||||
entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: "${SPARKD__MEMORY:-4gb}"
|
||||
|
||||
sparkd-shell:
|
||||
volumes:
|
||||
- ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
|
||||
- ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
|
||||
build:
|
||||
dockerfile: docker/images/bluetradingsystems/linux_shell/Dockerfile
|
||||
args:
|
||||
- IMAGE=${SPARKD__IMAGE}
|
||||
- SOURCE_DIR=${SPARKD__SOURCE_DIR}
|
||||
- CONFIG_DIR=${SPARKD__CONFIG_DIR}
|
||||
- PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
|
||||
- SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
|
||||
shm_size: "${SPARKD__SHM_SIZE:-512mb}"
|
||||
command: "do sleep 1; done"
|
||||
command: >
|
||||
sh -c "
|
||||
echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd/data
|
||||
if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
|
||||
mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
|
||||
fi
|
||||
if [ -e "/home/bts/.local/share/bts/conf/.env.local" ]; then
|
||||
echo 'Sourcing /home/bts/.local/share/bts/conf/.env.local'
|
||||
set -a
|
||||
. /home/bts/.local/share/bts/sparkd/conf/.env.local
|
||||
set +a
|
||||
fi
|
||||
if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then
|
||||
echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)'
|
||||
cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/
|
||||
fi
|
||||
./sparkd run"
|
||||
env_file:
|
||||
- .env
|
||||
- ${SPARKD__CONFIG_DIR}/.env
|
||||
@@ -54,7 +42,6 @@ services:
|
||||
sparkd-symbols:
|
||||
volumes:
|
||||
- ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
|
||||
- ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
|
||||
- ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
|
||||
build:
|
||||
dockerfile: src/app_sparkd/Dockerfile
|
||||
@@ -63,8 +50,26 @@ services:
|
||||
- SOURCE_DIR=${SPARKD__SOURCE_DIR}
|
||||
- CONFIG_DIR=${SPARKD__CONFIG_DIR}
|
||||
- PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
|
||||
- SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
|
||||
shm_size: "${SPARKD__SHM_SIZE:-512mb}"
|
||||
command: >
|
||||
sh -c "
|
||||
echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd
|
||||
if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
|
||||
mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
|
||||
fi
|
||||
ls -alh /home/bts/.local/share/bts/sparkd/conf/
|
||||
cat /home/bts/.local/share/bts/sparkd/conf/.env.local
|
||||
if [ -e "/home/bts/.local/share/bts/sparkd/conf/.env.local" ]; then
|
||||
echo 'Sourcing /home/bts/.local/share/bts/sparkd/conf/.env.local'
|
||||
set -a
|
||||
. /home/bts/.local/share/bts/sparkd/conf/.env.local
|
||||
set +a
|
||||
fi
|
||||
if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then
|
||||
echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)'
|
||||
cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/
|
||||
fi
|
||||
./sparkd run"
|
||||
env_file:
|
||||
- .env
|
||||
- ${SPARKD__CONFIG_DIR}/.env
|
||||
@@ -72,7 +77,6 @@ services:
|
||||
- "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
|
||||
- "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
|
||||
- "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
|
||||
entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
|
||||
deploy:
|
||||
restart_policy:
|
||||
condition: any
|
||||
@@ -80,65 +84,7 @@ services:
|
||||
resources:
|
||||
limits:
|
||||
memory: "${SPARKD__MEMORY:-4gb}"
|
||||
sparkd-multicast:
|
||||
environment:
|
||||
- TZ=America/Chicago
|
||||
volumes:
|
||||
- ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
|
||||
- ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
|
||||
- ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
|
||||
build:
|
||||
dockerfile: src/app_sparkd/Dockerfile
|
||||
args:
|
||||
- IMAGE=${SPARKD__IMAGE}
|
||||
- SOURCE_DIR=${SPARKD__SOURCE_DIR}
|
||||
- CONFIG_DIR=${SPARKD__CONFIG_DIR}
|
||||
- PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
|
||||
- SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
|
||||
shm_size: "${SPARKD__SHM_SIZE:-8192mb}"
|
||||
env_file:
|
||||
- .env
|
||||
- ${SPARKD__CONFIG_DIR}/.env
|
||||
network_mode: "host"
|
||||
entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
|
||||
deploy:
|
||||
restart_policy:
|
||||
condition: any
|
||||
delay: 3s
|
||||
resources:
|
||||
limits:
|
||||
memory: "${SPARKD__MEMORY:-16gb}"
|
||||
|
||||
|
||||
sparkd-multicast-cert:
|
||||
environment:
|
||||
- TZ=America/Chicago
|
||||
volumes:
|
||||
- ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
|
||||
- ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
|
||||
- ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
|
||||
build:
|
||||
dockerfile: src/app_sparkd/Dockerfile
|
||||
args:
|
||||
- IMAGE=${SPARKD__IMAGE}
|
||||
- SOURCE_DIR=${SPARKD__SOURCE_DIR}
|
||||
- CONFIG_DIR=${SPARKD__CONFIG_DIR}
|
||||
- PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
|
||||
- SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
|
||||
shm_size: "${SPARKD__SHM_SIZE:-8192mb}"
|
||||
env_file:
|
||||
- .env
|
||||
- ${SPARKD__CONFIG_DIR}/.env
|
||||
network_mode: "host"
|
||||
entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
|
||||
deploy:
|
||||
restart_policy:
|
||||
condition: any
|
||||
delay: 3s
|
||||
resources:
|
||||
limits:
|
||||
memory: "${SPARKD__MEMORY:-16gb}"
|
||||
|
||||
|
||||
sparkd-tail:
|
||||
image: bluetradingsystems/linux_dotnet8
|
||||
volumes:
|
||||
@@ -249,6 +195,3 @@ volumes:
|
||||
name: "md_publisher_22019"
|
||||
historic_md:
|
||||
name: "historic_md"
|
||||
md_recording:
|
||||
name: "sparkd_md_recording_${SPARKD__CONFIG_PORT:-8000}"
|
||||
external: true
|
||||
@@ -1,942 +0,0 @@
|
||||
# Security ID handling in `sparkd`
|
||||
|
||||
This document traces how security identifiers are created, translated, persisted,
|
||||
indexed, and published by the `sparkd` market-data paths. It focuses on the
|
||||
legacy CME MDP3/OnixS pipeline and the newer generalized market-data pipeline.
|
||||
|
||||
## Executive summary
|
||||
|
||||
`sparkd` uses two identifiers that must not be confused:
|
||||
|
||||
| Identifier | Meaning | Typical field | Lifetime |
|
||||
|---|---|---|---|
|
||||
| Exchange/feed ID | The venue's identifier used to route incoming definitions, books, and trades | `FixedSecurity::numeric_id`, `ExchangeKey`, or `domain::Security::security_common.exchange_sec_id` | Feed/session-specific unless the source guarantees stability |
|
||||
| Spark security key | Spark's internal non-zero identifier used in shared memory, APIs, orders, positions, and downstream services | `FixedSecurity::key`, `api::Security::id()`, `SecurityKey`; historically also called `security_id` | Intended to be stable across restarts |
|
||||
|
||||
`invalid_security_key` is `0`; a security with key `0` is not yet publicly
|
||||
discoverable through the shared-memory consumer path.
|
||||
|
||||
The two principal paths differ:
|
||||
|
||||
- **MDP3/OnixS:** the definition callback sets only the CME ID in
|
||||
`numeric_id`. `MDProducer::add_security()` allocates the shared-memory entry
|
||||
with key `0`. `SecurityIDRecorder`, driven by a 200 ms dispatcher timer,
|
||||
resolves or inserts a stable key in `key_ids.sqlite`; its callback finally
|
||||
calls `MDProducerHandle::set_key()`.
|
||||
- **Generalized/newer pipeline:** `GeneralMarketDataListener::on_securities()`
|
||||
currently uses the ID already present on the incoming domain security
|
||||
(`use_given_sec_key == true`). Live adapters normally obtain that ID
|
||||
synchronously from `IDGenerator` first. `_copy_security_common()` then copies
|
||||
it to both `FixedSecurity::key` and `numeric_id`, so
|
||||
`MDProducer::add_security()` publishes it synchronously. Only the listener's
|
||||
own fallback `IDGenerator` call is bypassed by the current constant.
|
||||
|
||||
## End-to-end diagrams
|
||||
|
||||
### CME MDP3 / OnixS
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant FE as OnixS feed-engine thread
|
||||
participant L as MDP3Listener
|
||||
participant P as MDProducer / shared memory
|
||||
participant Q as SecurityIDRecorder queues
|
||||
participant D as sparkd dispatcher thread
|
||||
participant DB as key_ids.sqlite
|
||||
|
||||
FE->>L: instrument definition (CME SecurityID)
|
||||
L->>L: set_security_common() - numeric_id = CME SecurityID, key remains 0
|
||||
L->>P: add_security(FixedSecurity)
|
||||
P->>P: allocate hot + cold entries with key 0
|
||||
P->>Q: ensure_key(handle)
|
||||
L->>L: index[CME SecurityID] = handle
|
||||
Note over FE,L: Books can route by CME ID while the Spark key is pending
|
||||
D->>Q: every 200 ms - work_spreads(), work_handles(), work_db()
|
||||
Q->>DB: find (namespace, descriptor), or insert a new row
|
||||
DB-->>Q: SQLite rowid = Spark security key
|
||||
Q->>P: handle.set_key(rowid)
|
||||
P->>P: hot.key first, then cold security.key, then increment security_count
|
||||
Note over P: The security is now public to shared-memory consumers
|
||||
```
|
||||
|
||||
### Generalized/newer pipeline (current behavior)
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant A as Venue adapter / plugin
|
||||
participant DB as IDGenerator / key_ids.sqlite
|
||||
participant G as GeneralMarketDataListener
|
||||
participant I as GeneralSecurityIndex
|
||||
participant P as MDProducer / shared memory
|
||||
|
||||
A->>DB: get_or_add(venue, type, adapter key)
|
||||
DB-->>A: SQLite rowid as domain SecurityID
|
||||
A->>G: on_securities(domain Security list with ID)
|
||||
G->>G: _find_sec() - use incoming security_common.id
|
||||
G->>G: _copy_security_common() - key = id, numeric_id = id
|
||||
G->>P: add_security(FixedSecurity with non-zero key)
|
||||
P->>P: set_key() during add_security()
|
||||
G->>I: index[id] = handle
|
||||
Note over G,P: No asynchronous ID assignment in the current configuration
|
||||
```
|
||||
|
||||
## Data-flow charts
|
||||
|
||||
The sequence diagrams above emphasize timing and thread transitions. These
|
||||
flowcharts emphasize which identifier is carried through each representation
|
||||
and where data is retained.
|
||||
|
||||
### MDP3 / legacy data flow
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
CME["CME MDP3 packets<br/>CME SecurityID"] --> ONIXS["OnixS feed engine<br/>decode definition and updates"]
|
||||
|
||||
ONIXS --> DEF["MDP3Listener definition callback"]
|
||||
DEF --> FIXED["FixedSecurity<br/>numeric_id = CME SecurityID<br/>key = 0"]
|
||||
|
||||
FIXED --> PRODUCER["MDProducer::add_security"]
|
||||
PRODUCER --> PENDING["Hot and cold shared memory<br/>allocated but not public<br/>key = 0"]
|
||||
PRODUCER --> QUEUE["SecurityIDRecorder<br/>pending_handle queue"]
|
||||
|
||||
FIXED --> CMEINDEX["CMESecurityIndex<br/>CME SecurityID to handle"]
|
||||
|
||||
QUEUE --> TIMER["Dispatcher timer<br/>every 200 ms"]
|
||||
TIMER --> LEGS{"Spread legs<br/>all resolved?"}
|
||||
LEGS -- No --> UNRESOLVED["unresolved_legs<br/>exchange IDs awaiting keys"]
|
||||
UNRESOLVED --> CMEINDEX
|
||||
CMEINDEX --> LEGS
|
||||
LEGS -- Yes or outright --> LOGICAL["Logical identity<br/>venue + type + security_desc"]
|
||||
|
||||
LOGICAL --> SELECT["SQLite SELECT<br/>key_ids.sqlite"]
|
||||
SELECT -- Existing row --> ROWID["SQLite rowid<br/>Spark security key"]
|
||||
SELECT -- Missing row --> INSERT["Batched SQLite INSERT"]
|
||||
INSERT --> ROWID
|
||||
|
||||
ROWID --> SETKEY["MDProducerHandle::set_key"]
|
||||
SETKEY --> HOT["Write hot-entry key"]
|
||||
HOT --> COLD["Write cold security key"]
|
||||
COLD --> COUNT["Increment security_count"]
|
||||
COUNT --> PUBLIC["Public security<br/>visible to MDConsumer"]
|
||||
|
||||
ONIXS --> UPDATE["Book, trade, status, or RFQ<br/>carrying CME SecurityID"]
|
||||
UPDATE --> CMEINDEX
|
||||
CMEINDEX --> HANDLE["MDProducerHandle"]
|
||||
HANDLE --> LIVE["Update live shared-memory data"]
|
||||
PUBLIC --> LIVE
|
||||
```
|
||||
|
||||
The CME index is usable before the Spark key is assigned, so feed updates can
|
||||
reach the internal handle while the security is still hidden from consumers.
|
||||
The persistent rowid crosses the publication boundary only at `set_key()`.
|
||||
|
||||
### Generalized/newer data flow
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
FEED["Venue feed, Databento,<br/>Python plugin, or replay"] --> ADAPTER["Source adapter<br/>build domain::Security"]
|
||||
|
||||
ADAPTER --> IDTYPE{"Live definition or<br/>recorded/pre-keyed input?"}
|
||||
IDTYPE -- Live --> IDINPUT["Logical identity<br/>venue + type + adapter-chosen name"]
|
||||
IDINPUT --> MUTEX["IDGenerator global mutex"]
|
||||
MUTEX --> SELECT["Synchronous SQLite SELECT<br/>key_ids.sqlite"]
|
||||
SELECT -- Existing row --> ROWID["SQLite rowid"]
|
||||
SELECT -- Missing row --> INSERT["Synchronous SQLite INSERT<br/>autocommit"]
|
||||
INSERT --> ROWID
|
||||
ROWID --> DOMAINID["domain::SecurityID<br/>used as Spark ID"]
|
||||
IDTYPE -- Recorded or pre-keyed --> DOMAINID
|
||||
|
||||
DOMAINID --> DOMAINSEC["SecurityCommon::id = Spark ID"]
|
||||
DOMAINSEC --> LISTENER["GeneralMarketDataListener::on_securities"]
|
||||
LISTENER --> CCMAP["cc_sec_id_map<br/>incoming ID to Spark ID"]
|
||||
LISTENER --> CONVERT["_copy_security_common<br/>FixedSecurity.key = ID<br/>numeric_id = ID"]
|
||||
|
||||
CONVERT --> PRODUCER["MDProducer::add_security"]
|
||||
PRODUCER --> SETKEY["Immediate set_key<br/>no recorder timer"]
|
||||
SETKEY --> SHM["Hot and cold shared memory<br/>immediately public"]
|
||||
CONVERT --> GENERALINDEX["GeneralSecurityIndex<br/>ID to handle"]
|
||||
|
||||
FEED --> EVENT["Book, trade, or statistics<br/>with exchange/source identity"]
|
||||
EVENT --> EVENTMAP["Adapter map<br/>source identity to domain::SecurityID"]
|
||||
EVENTMAP --> EVENTID["Event carries domain::SecurityID"]
|
||||
EVENT --> PYHOT["Python bridge only<br/>synchronous IDGenerator lookup"]
|
||||
PYHOT --> EVENTID
|
||||
PYHOT -. uses same mutex and SQLite path .-> MUTEX
|
||||
EVENTID --> CCMAP
|
||||
CCMAP --> GENERALINDEX
|
||||
GENERALINDEX --> HANDLE["MDProducerHandle"]
|
||||
HANDLE --> SHMUPDATE["Update book, trade,<br/>statistics, or status"]
|
||||
SHM --> SHMUPDATE
|
||||
|
||||
```
|
||||
|
||||
In the normal newer live path, persistence occurs before the definition enters
|
||||
the generalized listener. The DB call therefore adds synchronous adapter-side
|
||||
latency, but publication needs no later timer. Most adapters use an in-memory
|
||||
map for subsequent events; the dotted Python path is the documented exception.
|
||||
|
||||
## The publication boundary
|
||||
|
||||
`MDProducerHandle::set_key()` is the definitive publication point for the
|
||||
legacy shared-memory representation. Its write ordering is intentional:
|
||||
|
||||
1. Write the key into the hot book entry (`HotEntry` or `HotDepthEntry`).
|
||||
2. Write the key into the cold security entry.
|
||||
3. Increment the cold header's public `security_count`.
|
||||
|
||||
The cold security key is written last because consumers treat a non-zero cold
|
||||
key as visibility. At that moment the corresponding hot entry must already be
|
||||
addressable. `set_key()` only permits the transition from `0` to one key; it is
|
||||
not a key-update mechanism.
|
||||
|
||||
## Identifier representations
|
||||
|
||||
The names are historically inconsistent. In particular, many protobuf fields
|
||||
named `security_id` contain the **Spark key**, not the exchange's ID.
|
||||
|
||||
| Layer | Exchange/feed identifier | Spark identifier |
|
||||
|---|---|---|
|
||||
| MDP3 wire/OnixS | `securityId()` | not present |
|
||||
| Legacy fixed security | `FixedSecurity::numeric_id` (`uint64_t`) | `FixedSecurity::key` / `security_key` (`uint32_t`) |
|
||||
| Legacy API security | venue-specific `exchange_id` | top-level `Security.id()` |
|
||||
| New domain model | `SecurityCommon::exchange_sec_id` (string) is descriptive; `SecurityCommon::id` is also used to correlate source events | `domain::SecurityID` (`uint64_t`), currently copied into the Spark key |
|
||||
| Shared-memory hot book | none | `HotEntry::key` or `HotDepthEntry::key` |
|
||||
| Shared-memory cold metadata | `FixedSecurity::numeric_id` | `FixedSecurity::key` |
|
||||
| CME runtime lookup | `CMESecurityIndex` key | `SecurityData::handle.key()` |
|
||||
| General runtime lookup | incoming/canonical `domain::SecurityID` | `GeneralSecurityIndex` key and `handle.key()` (currently the same value) |
|
||||
|
||||
The generalized model declares `domain::SecurityID` as 64-bit, but the legacy
|
||||
shared-memory `security_key` remains 32-bit. `_copy_security_common()` assigns a
|
||||
64-bit value to both `numeric_id` and the 32-bit `key`. This is safe only while
|
||||
allocated IDs fit in 32 bits; unlike the legacy lookup path, this assignment has
|
||||
no explicit checked conversion.
|
||||
|
||||
## Which parts use exchange IDs, descriptors, or Spark security IDs?
|
||||
|
||||
The three identifier classes serve different jobs:
|
||||
|
||||
- **Exchange/external ID:** routes data inside a venue or vendor feed adapter.
|
||||
- **Descriptor/natural key:** recognizes a definition logically at the
|
||||
persistence, discovery, or human-facing boundary.
|
||||
- **Spark security ID/key:** joins the instrument to books, orders, fills,
|
||||
positions, and services after publication.
|
||||
|
||||
The core downstream system does not repeatedly identify a security from its
|
||||
description. The description is primarily used to obtain a Spark key; after
|
||||
that boundary, most joins use the numeric Spark key.
|
||||
|
||||
### Component-level usage matrix
|
||||
|
||||
| Component or data path | Exchange/external ID | Descriptor or name | Spark security ID/key |
|
||||
|---|---|---|---|
|
||||
| CME MDP3 definition decoder | Reads `securityId()` and stores it in `FixedSecurity::numeric_id` | Copies definition symbol/tag 55 into `security_desc` | Leaves `FixedSecurity::key` at `0` initially |
|
||||
| `CMESecurityIndex` | Primary map key; CME ID resolves to a handle | Not used for update routing | Returns `handle.key()` when the Spark key has been assigned |
|
||||
| MDP3 books, trades, status, statistics, and RFQs | Incoming CME ID selects the handle through `CMESecurityIndex` | Not used to locate each update | The selected handle writes keyed shared-memory/trade data after publication |
|
||||
| MDP3 spread definitions | Each leg initially contains `legSecurityId()` as `exchange_key` | Parent `security_desc` identifies the spread's DB row | Each leg's Spark key and then the parent Spark key must be resolved before publication |
|
||||
| Legacy ICE OnixS | Uses ICE `marketId` as `numeric_id` and index key | ICE security description becomes the logical lookup name | Uses the same recorder-assigned Spark key pattern as MDP3 |
|
||||
| `SecurityIDRecorder` | Does not allocate from the external ID | Uses `(venue, security type, securities::descriptor())` to find/create a row | Treats the SQLite rowid as the resulting Spark key |
|
||||
| Native generalized adapters | Use exchange strings/numeric IDs to parse definitions and correlate updates | Usually pass a description or exchange instrument name to `IDGenerator` | Put the returned ID into `SecurityCommon::id` and later events |
|
||||
| Databento adapter | Uses `msg.hd.instrument_id` for feed correlation and its in-memory map | Uses the decimal instrument ID string—not the human description—as the `IDGenerator` name | Caches `instrument_id -> domain::SecurityID` and places that ID on books/trades |
|
||||
| Python bridge | Reads `exchange_security_id` from definitions, books, and trades | Uses that external-ID string as the `IDGenerator` name | Resolves it to `domain::SecurityID`; currently performs this lookup for every event |
|
||||
| `IDGenerator` | Does not interpret whether `name` came from an exchange ID | Uses exactly `(venue, type, caller-supplied name)` | Returns the SQLite rowid as a 64-bit ID |
|
||||
| `GeneralMarketDataListener` | Does not route on `exchange_sec_id` directly | Copies `sec.desc()` into fixed metadata; its disabled fallback allocator would use `sec.desc()` | Uses incoming `SecurityCommon::id`, maps events through `cc_sec_id_map`, and copies the value into both `key` and `numeric_id` |
|
||||
| `GeneralSecurityIndex` | No separate external-ID concept in the current use-given path | Not used | Maps the generalized ID/Spark key to `MDProducerHandle` |
|
||||
| `MDProducer` / shared memory | Retains `numeric_id`; in legacy feeds this is the exchange ID, but in the generalized path it is currently the Spark ID | Stores `security_desc` in the cold definition | Uses the key in hot/cold entries; non-zero key is the visibility boundary |
|
||||
| `MDConsumer` | Not normally used to find live books | Can enumerate/filter converted security definitions | Finds handles and book data by `security_key` |
|
||||
| `SecurityCache` / `securities.sqlite` | Preserves venue-specific exchange data inside the serialized security blob | Maintains descriptor/name lookup indexes and supports venue/type/name queries | Primary persisted definition key is `api::Security.id()` |
|
||||
| MBO processor and MBO service | May retain exchange **order** IDs, which are separate from security exchange IDs | Not used to find the instrument | Indexes snapshots and requests by `SecurityKey`; MBO protobuf `security_id` means Spark key |
|
||||
| Order manager and algorithms | Execution gateways eventually use venue-specific fields from the resolved security to encode orders | Descriptions appear in logs and errors, not as the primary join | Order definitions, subscriptions, book events, native orders, spread legs, and fills use `security_key` / `Security.id()` |
|
||||
| Position, PNL, and risk services | Not used as their position join | May resolve metadata for display/reporting | Positions, fills, overrides, and PNL handles are keyed by Spark security ID |
|
||||
| Relay and market-data recordings | Serialized security definitions may preserve venue exchange IDs | Preserve full descriptors in recorded definitions | Books, updates, trades, and replay correlation preserve the recorded Spark key |
|
||||
| Synthetic security service | No venue exchange ID; synthetics use `VENUE_BTS` | Synthetic name is the recorder's logical descriptor | Recorder assigns the SQLite rowid and synthetic metadata is keyed by it |
|
||||
|
||||
### Exchange/external IDs: where they stop
|
||||
|
||||
In MDP3 the separation is explicit:
|
||||
|
||||
```text
|
||||
CME SecurityID
|
||||
-> FixedSecurity.numeric_id
|
||||
-> CMESecurityIndex lookup
|
||||
-> MDProducerHandle
|
||||
-> Spark security_key for downstream publication
|
||||
```
|
||||
|
||||
The external CME ID continues to route feed messages but does not become the
|
||||
order/position key. This permits the same downstream Spark key to survive a
|
||||
restart as long as the logical descriptor mapping remains in
|
||||
`key_ids.sqlite`, even if the feed must rediscover the CME routing ID.
|
||||
|
||||
The generalized path collapses more of these roles:
|
||||
|
||||
```text
|
||||
exchange/vendor identity
|
||||
-> adapter calls IDGenerator
|
||||
-> domain::SecurityID
|
||||
-> FixedSecurity.key and FixedSecurity.numeric_id
|
||||
-> GeneralSecurityIndex and downstream Spark key
|
||||
```
|
||||
|
||||
`SecurityCommon::exchange_sec_id` may still contain the original textual
|
||||
external identifier, but `GeneralMarketDataListener` does not use it to route
|
||||
events and does not put it in `numeric_id`. The adapter must translate later
|
||||
events to `domain::SecurityID` before invoking the listener.
|
||||
|
||||
### Descriptor/name users
|
||||
|
||||
Descriptors are used in three main places:
|
||||
|
||||
1. **Key allocation:** `SecurityIDRecorder` uses the legacy venue-specific
|
||||
descriptor, while `IDGenerator` uses its caller-provided `name`.
|
||||
2. **Definition lookup and discovery:** `SecurityCache` supports
|
||||
venue/type/name lookup; symbol filters and administrative tools use security
|
||||
metadata to select or display instruments.
|
||||
3. **Diagnostics:** logs include symbols and descriptions so humans can
|
||||
understand a numeric key.
|
||||
|
||||
Descriptions generally do not drive the high-rate book, trade, order, fill, or
|
||||
position paths. Those paths use numeric IDs after definition resolution.
|
||||
|
||||
It is important that “descriptor” does not mean one universal canonical field:
|
||||
|
||||
- Legacy CME uses Globex `security_desc` from tag 55.
|
||||
- Many generalized adapters use `sec.desc()` or an exchange instrument name.
|
||||
- Databento uses its numeric instrument ID converted to a string.
|
||||
- Python uses `exchange_security_id`.
|
||||
|
||||
Therefore, two adapters may describe the same economic contract but request
|
||||
different logical rows and receive different Spark IDs.
|
||||
|
||||
### Spark security-ID users
|
||||
|
||||
Once a security is published, its Spark key is the dominant system identifier:
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
KEY["Spark security key"] --> SHM["Hot/cold shared memory"]
|
||||
KEY --> MDC["MDConsumer and book subscriptions"]
|
||||
KEY --> MBO["MBO snapshots and queries"]
|
||||
KEY --> OMS["Orders, algorithms, and spread legs"]
|
||||
KEY --> FILLS["Fills and execution records"]
|
||||
KEY --> POS["Positions, PNL, and risk"]
|
||||
KEY --> API["Services and client APIs"]
|
||||
KEY --> REC["Relay and market-data recordings"]
|
||||
KEY --> SECDB["Serialized security definitions"]
|
||||
```
|
||||
|
||||
Fields named `security_id`, `security_key`, or top-level `Security.id()` in
|
||||
these downstream areas almost always mean the Spark key. They should not be
|
||||
compared directly with CME `SecurityID`, Databento `instrument_id`, or another
|
||||
venue/vendor identifier without going through the relevant adapter/index
|
||||
mapping.
|
||||
|
||||
## Persistence and allocation
|
||||
|
||||
### The authoritative key mapping: `key_ids.sqlite`
|
||||
|
||||
Both allocators use `SqliteStore("key_ids")`. With the normal configuration the
|
||||
file is:
|
||||
|
||||
```text
|
||||
<SystemConfig.data_path>/databases/key_ids/key_ids.sqlite
|
||||
```
|
||||
|
||||
The database contains one table and two indexes:
|
||||
|
||||
```sql
|
||||
CREATE TABLE store (ns TEXT, key TEXT, value TEXT);
|
||||
CREATE UNIQUE INDEX key_index ON store(ns, key);
|
||||
CREATE INDEX ns_index ON store(ns);
|
||||
```
|
||||
|
||||
The SQLite `rowid` is the generated Spark key. `ns` is normally
|
||||
`<VENUE>.<SECURITY_TYPE>` (for example `GLBX.FUTURE`), and `key` is the security
|
||||
descriptor chosen by the caller. `value` is normally empty. Therefore ID
|
||||
stability depends on all three of these remaining stable:
|
||||
|
||||
- the configured `data_path` and preservation of this database;
|
||||
- the venue/type namespace;
|
||||
- the descriptor supplied by the feed adapter.
|
||||
|
||||
The two allocator implementations share the file and convention:
|
||||
|
||||
- `SecurityIDRecorder` (legacy): derives the namespace from legacy API enums and
|
||||
uses `securities::descriptor()`. For CME this is the Globex
|
||||
`security_desc`, populated from the definition's symbol/tag 55.
|
||||
- `IDGenerator` (newer): derives the namespace from new domain enums and uses
|
||||
the caller's `name`. Most adapters pass `sec.desc()` or an exchange instrument
|
||||
identifier; Databento deliberately passes the decimal instrument ID, and the
|
||||
Python bridge uses the exchange security ID.
|
||||
|
||||
### Logical-row examples
|
||||
|
||||
The following names are illustrative. Production uses the exact string emitted
|
||||
by the selected feed/adapter, including its case, spacing, and weekly-series
|
||||
notation.
|
||||
|
||||
#### Legacy `SecurityIDRecorder`
|
||||
|
||||
For CME, `set_security_common()` copies the MDP3 definition's symbol/tag 55 into
|
||||
Globex `security_desc`. After converting the fixed security to
|
||||
`api::Security`, `securities::descriptor()` returns that same
|
||||
`glbx.security_desc()` string.
|
||||
|
||||
| Instrument | Legacy enums | Example descriptor from `security_desc` | Row searched/inserted in `store` |
|
||||
|---|---|---|---|
|
||||
| Future | `VENUE_GLBX`, `SECURITYTYPE_FUTURE` | `ESZ6` | `ns='GLBX.FUTURE', key='ESZ6'` |
|
||||
| Standard option | `VENUE_GLBX`, `SECURITYTYPE_OPTION` | `ESZ6 C6500` | `ns='GLBX.OPTION', key='ESZ6 C6500'` |
|
||||
| Weekly option | `VENUE_GLBX`, `SECURITYTYPE_OPTION` | `ESZ6-W1 C6500` | `ns='GLBX.OPTION', key='ESZ6-W1 C6500'` |
|
||||
| Spot from a legacy-compatible venue path | `VENUE_COINBASE`, `SECURITYTYPE_SPOT` | `BTC-USD` | `ns='COINBASE.SPOT', key='BTC-USD'` |
|
||||
|
||||
A weekly option does not get a special namespace. Standard and weekly options
|
||||
are both `SECURITYTYPE_OPTION`, so both live in `GLBX.OPTION`; the exact
|
||||
`security_desc` distinguishes the weekly series. For example, if the database
|
||||
assigned rowids `42001` through `42004` to the rows above, those integers—not
|
||||
the descriptor strings—would be the Spark security keys.
|
||||
|
||||
Conceptually, legacy allocation is:
|
||||
|
||||
```text
|
||||
future: lookup_or_insert("GLBX.FUTURE", "ESZ6")
|
||||
option: lookup_or_insert("GLBX.OPTION", "ESZ6 C6500")
|
||||
weekly option: lookup_or_insert("GLBX.OPTION", "ESZ6-W1 C6500")
|
||||
spot: lookup_or_insert("COINBASE.SPOT", "BTC-USD")
|
||||
```
|
||||
|
||||
#### Newer `IDGenerator`
|
||||
|
||||
The namespace result is similar, but the third argument is chosen by the
|
||||
adapter. Examples using a description-oriented adapter are:
|
||||
|
||||
```cpp
|
||||
get_or_add(VENUE_GLBX, SECURITY_TYPE_FUTURE, "ESZ6");
|
||||
// store: ns="GLBX.FUTURE", key="ESZ6"
|
||||
|
||||
get_or_add(VENUE_GLBX, SECURITY_TYPE_OPTION, "ESZ6 C6500");
|
||||
// store: ns="GLBX.OPTION", key="ESZ6 C6500"
|
||||
|
||||
get_or_add(VENUE_GLBX, SECURITY_TYPE_OPTION, "ESZ6-W1 C6500");
|
||||
// store: ns="GLBX.OPTION", key="ESZ6-W1 C6500"
|
||||
|
||||
get_or_add(VENUE_COINBASE, SECURITY_TYPE_SPOT, "BTC-USD");
|
||||
// store: ns="COINBASE.SPOT", key="BTC-USD"
|
||||
```
|
||||
|
||||
Databento intentionally produces a different logical row even when the human
|
||||
description is `ESZ6`. If its instrument ID were `12345678`, it would call the
|
||||
equivalent of:
|
||||
|
||||
```cpp
|
||||
get_or_add(VENUE_GLBX, SECURITY_TYPE_FUTURE, "12345678");
|
||||
// store: ns="GLBX.FUTURE", key="12345678"
|
||||
```
|
||||
|
||||
Likewise, the Python bridge uses `exchange_security_id`. A spot definition with
|
||||
`exchange_security_id="BTC-USD"` produces the `COINBASE.SPOT / BTC-USD` row,
|
||||
while a provider that sends `exchange_security_id="BTCUSD"` produces a
|
||||
different row and therefore a different Spark key.
|
||||
|
||||
These examples show why the tuple is only logical when the adapter's naming
|
||||
contract is also known. `GLBX.FUTURE / ESZ6` and `GLBX.FUTURE / 12345678` may
|
||||
describe the same contract to a human but are distinct rows to `key_ids.sqlite`.
|
||||
|
||||
Both enum-prefix removals currently produce the same namespace spelling (for
|
||||
example, `GLBX.OPTION`), so both mechanisms allocate from one global SQLite
|
||||
rowid space. Live rows cannot share a rowid. The table does not declare
|
||||
`AUTOINCREMENT`, however, so deleting rows—especially the highest row—can allow
|
||||
SQLite to reuse a number later. Security-key rows should therefore be treated
|
||||
as append-only operational data.
|
||||
|
||||
`SqliteStore` uses WAL mode and `PRAGMA synchronous = 0`. This favors startup
|
||||
and feed latency over maximum durability against a host/power failure. The
|
||||
unique `(ns, key)` index prevents two different rows for the same logical
|
||||
security.
|
||||
|
||||
### Other places the key is stored
|
||||
|
||||
| Location | Purpose | Authoritative for allocation? |
|
||||
|---|---|---|
|
||||
| Hot and cold shared-memory segments | Live publication and fast consumer lookup | No; rebuilt at process startup |
|
||||
| `CMESecurityIndex` / `GeneralSecurityIndex` and adapter maps | In-process routing from an incoming ID to a producer handle | No; in-memory only |
|
||||
| `<data_path>/databases/md/securities.sqlite` | Serialized legacy `api::Security` definitions keyed by Spark key; used by security cache and MDP3 Sunday preload | No; it stores metadata after a key exists |
|
||||
| Relay/market-data recordings | Preserve the key embedded in recorded securities, books, and trades for deterministic replay | No; replay normally reuses the recorded value |
|
||||
| Orders, fills, positions, APIs | Reference a security by the published Spark key | No |
|
||||
|
||||
The `SecurityIDRecorder::work_db()` comment says a published security is also
|
||||
recorded in the security database, but that function does **not** write
|
||||
`securities.sqlite` in the current code. Full security definitions are persisted
|
||||
separately by `SecurityCache::flush_to_disk()` through
|
||||
`db::insert_or_replace()`.
|
||||
|
||||
## CME MDP3 / OnixS pipeline in detail
|
||||
|
||||
### 1. Construction and thread setup
|
||||
|
||||
`MDP3Main::PImpl` owns one `CMESecurityIndex`, one `SecurityIDRecorder`, and one
|
||||
`MDProducer`. It constructs the recorder with:
|
||||
|
||||
- a lookup callback that maps `ExchangeKey` through the CME index to the
|
||||
handle's current Spark key; and
|
||||
- `Dispatcher::current()`, which drives a 200 ms recorder timer.
|
||||
|
||||
In the current `MarketDataGateway` route, `OnixsGlbxSubscription` creates a
|
||||
dedicated `dispatcher_thread`, constructs and starts `MDP3Main` on that thread,
|
||||
then calls that thread's dispatcher `run()`. The older `md run onixs` command
|
||||
uses its command/dispatcher thread similarly.
|
||||
|
||||
The OnixS feed engine has its own configured thread pool. Instrument and market
|
||||
data callbacks therefore originate on OnixS feed-engine threads, while recorder
|
||||
database work and final key assignment execute on the MDP3 dispatcher thread.
|
||||
|
||||
| MDP3 stage | Executing thread | DB call? |
|
||||
|---|---|---|
|
||||
| Decode definition and set CME `numeric_id` | OnixS feed-engine callback thread | No |
|
||||
| Allocate hot/cold entry and queue handle | Same OnixS callback thread | No |
|
||||
| Add CME-ID index entry | Same OnixS callback thread | No |
|
||||
| Resolve spread legs | MDP3 dispatcher thread, on 200 ms timer | No |
|
||||
| Select/insert Spark key | MDP3 dispatcher thread, on 200 ms timer | Yes |
|
||||
| `handle.set_key()` and public visibility | MDP3 dispatcher thread, in DB callback | No additional call |
|
||||
|
||||
### 2. Definition callback: exchange ID is set, Spark key is not
|
||||
|
||||
For futures, options, and spreads, `MDP3Listener::set_security_common()` does:
|
||||
|
||||
```cpp
|
||||
fsec.numeric_id = def.securityId();
|
||||
```
|
||||
|
||||
It also copies symbol/descriptor/venue and pricing fields. A newly constructed
|
||||
`FixedSecurity` already has `key == invalid_security_key` (`0`), and this helper
|
||||
does not change it.
|
||||
|
||||
For a spread, every leg starts with:
|
||||
|
||||
```cpp
|
||||
leg.exchange_key = ExchangeKey(definition_leg.legSecurityId());
|
||||
leg.key = invalid_security_key;
|
||||
```
|
||||
|
||||
Thus both the parent and legs initially carry exchange-routing identity, but no
|
||||
Spark identity.
|
||||
|
||||
### 3. Shared-memory allocation and exchange indexing
|
||||
|
||||
`MDP3Listener::add_security()` performs these operations in order:
|
||||
|
||||
1. `_producer.add_security(security)` allocates a hot entry and a variable-sized
|
||||
cold entry and copies the security into the cold segment.
|
||||
2. Because the key is `0`, `MDProducer` queues the handle through
|
||||
`SecurityIDRecorder::ensure_key()`.
|
||||
3. `_index.add_security(security.numeric_id, handle)` immediately creates the
|
||||
CME-ID-to-handle mapping.
|
||||
4. The initial exchange status is written through the handle.
|
||||
|
||||
This ordering means incoming books can already be routed by CME ID while the
|
||||
Spark key is pending. Shared-memory consumers cannot discover the security yet.
|
||||
Also, `MDProducerHandle::add_trade()` only appends to the public trade queue
|
||||
when `key() != 0`, so a trade received in this short window updates the cold
|
||||
last-trade state but is not added to the keyed trade log.
|
||||
|
||||
### 4. Recorder tick: spreads, handles, then database
|
||||
|
||||
Every 200 ms the recorder calls, in this exact order:
|
||||
|
||||
1. `work_spreads()` tries to fill previously unresolved spread leg keys by
|
||||
asking `CMESecurityIndex` for each leg exchange ID.
|
||||
2. `work_handles()` drains new handles. Outrights are converted to an API
|
||||
security and queued for DB lookup immediately. A spread without all leg keys
|
||||
is retained in `pending_handle` and its leg pointers are registered in
|
||||
`unresolved_legs`.
|
||||
3. `work_db()` resolves queued `(venue, type, descriptor)` requests.
|
||||
|
||||
Because spread work precedes new-handle work, a newly seen unresolved spread
|
||||
normally waits until at least the next timer tick to resolve legs. It waits
|
||||
longer if any outright leg has not yet received its Spark key. A spread's own
|
||||
key is not requested until every leg key is non-zero.
|
||||
|
||||
### 5. Lookup or generation
|
||||
|
||||
`work_db()` first queries `(ns, key)`:
|
||||
|
||||
- Existing row: use its `rowid` as the key and invoke the callback.
|
||||
- Missing row: batch it into `insert_many_resolve_ids()`. SQLite assigns a new
|
||||
rowid inside a transaction, the row object receives that ID, and the callback
|
||||
runs after the transaction completes.
|
||||
|
||||
New inserts are throttled to roughly 200 per timer pass (the current `> 200`
|
||||
check can admit 201) to avoid delaying heartbeats during startup. The queues are
|
||||
TBB concurrent queues, but all SQLite access is serialized onto the recorder's
|
||||
dispatcher thread. `check_access()` asserts this single-thread rule after the
|
||||
first work pass records the dispatcher thread ID.
|
||||
|
||||
### 6. Final assignment and visibility
|
||||
|
||||
The queued callback captures a copy of `MDProducerHandle` and calls
|
||||
`handle.set_key(id)` on the dispatcher thread. `set_key()` writes hot, then cold,
|
||||
then increments `security_count`. From this point:
|
||||
|
||||
- shared-memory scans report the security;
|
||||
- `CMESecurityIndex::get_key(CME_ID)` returns the Spark key;
|
||||
- trades and RFQs can be emitted with a non-zero key;
|
||||
- resolved spreads can copy this key into their leg structures.
|
||||
|
||||
### 7. Sunday preload is a fast path
|
||||
|
||||
For configured symbols on Sunday, `MDP3Main::prepare_listener()` loads stored
|
||||
definitions from `securities.sqlite`. It converts them with
|
||||
`set_security_key=true`, so both the security key and spread leg keys come from
|
||||
the serialized API security. `MDProducer::add_security()` calls `set_key()`
|
||||
immediately, and no recorder lookup is required for that preloaded handle. The
|
||||
CME exchange ID is still restored into `numeric_id`, allowing the normal CME
|
||||
index to be built.
|
||||
|
||||
## Generalized/newer pipeline in detail
|
||||
|
||||
### 1. Topology and ownership
|
||||
|
||||
`Run::run()` starts a detached thread named `MarketDataGateway`. On that thread,
|
||||
`run_md_gwy()` obtains the process-wide `IDGenerator`, constructs
|
||||
`MarketDataGateway`, and starts the configured providers. For each `(venue,
|
||||
source)` returned by a provider, the gateway creates an `MdSink` containing:
|
||||
|
||||
- a venue-local `GeneralSecurityIndex`;
|
||||
- a default `MDProducer` (not wired to a `SecurityIDRecorder`); and
|
||||
- a `GeneralMarketDataListener` connecting the source to that producer/index.
|
||||
|
||||
The source decides its callback thread. Some sources call the listener while
|
||||
starting; others own network or replay threads. The generalized listener calls
|
||||
the producer inline on that source thread. The process-wide `IDGenerator`
|
||||
serializes its own calls with a mutex, but the listener's transient
|
||||
`cc_sec_id_map` is an ordinary `unordered_map`, so each source/listener pair is
|
||||
expected to serialize security/event callbacks.
|
||||
|
||||
| Generalized stage | Executing thread | DB call? |
|
||||
|---|---|---|
|
||||
| Build/parse domain definition | Source/adaptor thread | No |
|
||||
| `IDGenerator::get_or_add()` | Same source/adaptor thread | Yes, synchronously |
|
||||
| `GeneralMarketDataListener::on_securities()` | Same source callback thread | No in current use-given mode |
|
||||
| Copy ID to `FixedSecurity::key` and `numeric_id` | Same source callback thread | No |
|
||||
| `MDProducer::add_security()` / `set_key()` | Same source callback thread | No additional call |
|
||||
| Add `GeneralSecurityIndex` entry | Same source callback thread | No |
|
||||
|
||||
There are two important compatibility exceptions inside the provider framework:
|
||||
|
||||
- `OnixsGlbxSubscription::start()` ignores the supplied generalized listener
|
||||
and starts `MDP3Main`; CME/OnixS therefore still follows the asynchronous
|
||||
legacy flow documented above.
|
||||
- `EmumbaDeribitSubscription::start()` also ignores that listener and uses
|
||||
`EmMDListener`, which allocates via `IDGenerator` and writes through its own
|
||||
producer/index. Its assignment is still synchronous and pre-populated, so it
|
||||
resembles the newer ID flow even though it does not pass through
|
||||
`GeneralMarketDataListener`.
|
||||
|
||||
### 2. How a new ID is constructed
|
||||
|
||||
For a normal live generalized adapter, ID creation happens **in the adapter,
|
||||
before** `GeneralMarketDataListener::on_securities()`:
|
||||
|
||||
```text
|
||||
logical identity = (domain venue, domain security type, adapter-chosen name)
|
||||
Spark ID = SQLite rowid for that identity in key_ids.sqlite
|
||||
```
|
||||
|
||||
`IDGenerator::get_or_add(venue, type, name)` formats a namespace by stripping
|
||||
the enum prefixes:
|
||||
|
||||
```text
|
||||
VENUE_TAIFEX + SECURITY_TYPE_FUTURE -> TAIFEX.FUTURE
|
||||
```
|
||||
|
||||
It then calls `retrieve_one(ns, name)`. If the row exists, it returns that
|
||||
row's `rowid`. If absent, it inserts `(ns, name, "")`, and SQLite assigns the
|
||||
next rowid. That integer is wrapped in `domain::SecurityID` and placed in
|
||||
`SecurityCommon::id`.
|
||||
|
||||
The third identity component is adapter-specific, and this matters when trying
|
||||
to reproduce a mapping:
|
||||
|
||||
| Source/adaptor family | Typical `name` sent to `IDGenerator` |
|
||||
|---|---|
|
||||
| Binance, Coinbase, Deribit, OKX and similar native/CCAPI paths | Usually `sec.desc()` / exchange instrument name |
|
||||
| Taifex | product/instrument ID string |
|
||||
| Databento | `std::to_string(msg.hd.instrument_id)`, not the human-readable description |
|
||||
| Python bridge | `exchange_security_id` (and it asserts Python supplied ID is zero for definitions) |
|
||||
| Emumba Deribit | extracted Emumba security descriptor |
|
||||
| Replay | no new allocation for a recorded definition; it reuses the recorded `SecurityCommon::id` |
|
||||
|
||||
So “security name” alone is not a universal logical key. Reproduction requires
|
||||
the exact venue enum, exact security-type enum, and the exact adapter-specific
|
||||
name used at insertion time.
|
||||
|
||||
### 3. What `GeneralMarketDataListener` does with that ID
|
||||
|
||||
`on_securities()` first calls `_find_sec(sec, true)`. Its current, compile-time
|
||||
behavior is controlled by:
|
||||
|
||||
```cpp
|
||||
const bool use_given_sec_key = true;
|
||||
```
|
||||
|
||||
Consequently, for a newly seen incoming `domain::SecurityID` it does **not**
|
||||
call the DB. It copies `security_common.id.value()` into its transient
|
||||
`cc_sec_id_map`. `_add_future()`, `_add_option()`, `_add_spread()`, or
|
||||
`_add_spot()` then calls `_copy_security_common()`, which sets:
|
||||
|
||||
```cpp
|
||||
to.key = incoming_id;
|
||||
to.numeric_id = incoming_id;
|
||||
```
|
||||
|
||||
This collapses the generalized source/correlation ID and Spark shared-memory
|
||||
key into the same number. The source's text `exchange_sec_id` remains on the
|
||||
domain security but is not copied into `FixedSecurity::numeric_id` by this
|
||||
path.
|
||||
|
||||
`_commit_security()` then:
|
||||
|
||||
1. calls `MDProducer::add_security()` with an already non-zero key;
|
||||
2. causes `MDProducerHandle::set_key()` to run immediately during that call;
|
||||
3. inserts the handle in `GeneralSecurityIndex` under `numeric_id` (the same
|
||||
number); and
|
||||
4. returns with the security already visible in shared memory.
|
||||
|
||||
Because the sink's producer has no recorder, supplying `id == 0` would allocate
|
||||
the shared-memory entry but never assign or publish its key. Generalized live
|
||||
sources must therefore provide a valid non-zero ID before `on_securities()`.
|
||||
|
||||
The listener contains an alternative branch for `use_given_sec_key == false`.
|
||||
That branch would call `IDGenerator::get_or_add(me->venue, sec.type(),
|
||||
sec.desc())` itself. It is currently disabled, and `_generate_security_key()` is
|
||||
also effectively unused. Turning the flag off is not a mechanical change: an
|
||||
adapter such as Databento allocates using numeric instrument ID while the
|
||||
listener branch would use `sec.desc()`, potentially creating a second key for
|
||||
the same instrument.
|
||||
|
||||
There is also a spread failure edge case. `_find_sec(sec, true)` records the
|
||||
parent ID in `cc_sec_id_map` before `_add_spread()` resolves its legs. If any leg
|
||||
ID is absent, `_add_spread()` logs and returns without committing the spread. A
|
||||
later repeat of the parent definition is then classified as a duplicate and
|
||||
ignored, so definition ordering is a hard requirement in the current code.
|
||||
|
||||
### 4. How later books, trades, and statistics find the security
|
||||
|
||||
Every later domain event carries a `domain::SecurityID`:
|
||||
|
||||
- `Book::security_id()`;
|
||||
- `TradeRecord::sec_id`; or
|
||||
- `SecurityMarketDataStatistics::security_id`.
|
||||
|
||||
The listener first uses `cc_sec_id_map` to translate that event ID to the value
|
||||
stored in `GeneralSecurityIndex`, then obtains the `MDProducerHandle`. With
|
||||
`use_given_sec_key == true`, this mapping is numerically identity-to-identity,
|
||||
but it still serves as the admission check that the definition was seen.
|
||||
|
||||
Most high-throughput adapters avoid SQLite on market-data events:
|
||||
|
||||
- Databento allocates on the definition and caches
|
||||
`instrument_id -> domain::SecurityID` in `sec_id_map` for books/trades.
|
||||
- Taifex keeps the resolved ID in its security-details mapper.
|
||||
- Other native subscriptions maintain comparable descriptor/ID maps.
|
||||
|
||||
The Python bridge is a notable exception: `to_bts_security_()`,
|
||||
`to_bts_trade_()`, and `to_book_()` all call `get_sec_id_()`, which calls
|
||||
`IDGenerator::get_id(..., add_if_new=true)`. As written, that performs at least
|
||||
one SQLite `SELECT` for each converted Python definition, trade, or book. It
|
||||
returns the existing ID after the first insert, but it is still a DB call and
|
||||
takes the global `IDGenerator` mutex on every event.
|
||||
|
||||
### 5. Persistence timing and latency
|
||||
|
||||
The newer allocator is synchronous:
|
||||
|
||||
- Existing security: mutex acquisition plus one SQLite `SELECT` on the caller's
|
||||
thread.
|
||||
- New security: mutex acquisition, one `SELECT`, then one SQLite `INSERT`
|
||||
(SQLite autocommit), also on the caller's thread.
|
||||
- Listener publication: no further DB call under the current
|
||||
`use_given_sec_key` setting.
|
||||
|
||||
The ID therefore exists in `key_ids.sqlite` before the adapter sends the
|
||||
definition to the listener. There is no intentional 200 ms publication delay,
|
||||
but the source callback/startup path directly pays DB and mutex latency. Bulk
|
||||
definition discovery performs one lookup and possibly one insert per security;
|
||||
`IDGenerator` does not batch those operations.
|
||||
|
||||
WAL mode helps concurrency, while `synchronous=0` reduces fsync cost. It does
|
||||
not eliminate SQL parsing/stepping, mutex contention, page-cache misses, or
|
||||
occasional file-system work. The impact is usually a startup/definition cost
|
||||
when adapters cache the result, but it becomes hot-path latency for code such as
|
||||
the current Python bridge that resolves every event through the DB.
|
||||
|
||||
`GeneralMarketDataListener` itself does not persist the full contract definition
|
||||
to `securities.sqlite`; that remains the separate security-cache/metadata
|
||||
persistence path described earlier.
|
||||
|
||||
## Comparison: MDP3 recorder vs generalized allocator
|
||||
|
||||
| Property | MDP3 / `SecurityIDRecorder` | Generalized / `IDGenerator` |
|
||||
|---|---|---|
|
||||
| Logical input | venue + type + legacy `security_desc` | venue + type + adapter-chosen name |
|
||||
| Numeric output | SQLite rowid, checked/read as 32-bit `security_key` | SQLite rowid returned as 64-bit `domain::SecurityID`, then narrowed into 32-bit shared-memory key |
|
||||
| Allocation point | After definition is copied to shared memory | Normally inside adapter before listener sees definition |
|
||||
| DB execution | Asynchronous on dispatcher timer | Synchronous on caller/source thread |
|
||||
| Existing ID cost | One deferred `SELECT` | One immediate `SELECT` per call |
|
||||
| New ID cost | Deferred, batched transaction (about 200/pass) | Immediate `SELECT` + individual autocommit `INSERT` |
|
||||
| Time until public | Usually 0–200+ ms; spreads may take extra ticks | Immediate after synchronous lookup/insert completes |
|
||||
| Feed callback blocking | Queue push only for ID work | Yes, if the adapter invokes `IDGenerator` there |
|
||||
| Spread legs | Parent waits asynchronously until all leg keys resolve by exchange ID | Definition processing requires every leg's incoming ID already exist in `cc_sec_id_map`; otherwise the spread is aborted |
|
||||
| Runtime ID relationship | Exchange ID and Spark key remain distinct | Incoming/canonical ID, `numeric_id`, index ID, and Spark key are currently the same number |
|
||||
| Persistence database | `key_ids.sqlite` | The same `key_ids.sqlite` |
|
||||
| Batching | Yes for new rows | No |
|
||||
| Failure behavior | Security has an internal handle with key 0 while normal queued work is pending; DB work does not block the feed callback | DB errors/zero results occur in the synchronous source path; a zero ID can leave an unpublishable entry, with no pending-key state machine |
|
||||
|
||||
Neither algorithm generates a UUID or a content hash. Both ultimately use an
|
||||
opaque, database-assigned SQLite rowid. It is normally increasing and looks like
|
||||
a sequence, but this schema does not request SQLite `AUTOINCREMENT`. The tuple
|
||||
used to *look up* the row is logical; the resulting number is not derivable from
|
||||
that tuple.
|
||||
|
||||
## Can another system reconstruct the same keys?
|
||||
|
||||
Not from `(venue, type, name)` alone. Another system can reconstruct the
|
||||
**logical lookup tuple**, but it cannot calculate the numeric Spark key because
|
||||
the key depends on the historical insertion order of every row in
|
||||
`key_ids.sqlite`.
|
||||
|
||||
For example, two empty systems can insert the exact same `GLBX.FUTURE / ESZ6`
|
||||
row and receive different IDs if other securities were inserted before it on
|
||||
one system. They are sequential opaque IDs, not deterministic hashes and not
|
||||
UUIDs.
|
||||
|
||||
Ways to preserve cross-system identity are:
|
||||
|
||||
1. Copy/replicate `key_ids.sqlite` (including rowids) and keep descriptor rules
|
||||
identical. This preserves existing numeric keys but requires coordinated
|
||||
allocation for new ones.
|
||||
2. Export an explicit mapping of `(namespace, adapter key) -> Spark key`, then
|
||||
import rows with their original rowids before either system allocates new
|
||||
securities.
|
||||
3. Treat a stable external identity (venue + type + exchange instrument ID, or
|
||||
another canonical instrument identifier) as the cross-system key and carry a
|
||||
translation table to each system's local Spark key.
|
||||
4. Introduce a deterministic ID scheme only as a versioned migration. A hash can
|
||||
be computed independently, but collision handling, descriptor
|
||||
canonicalization, aliases/renames, the existing 32-bit ABI, and compatibility
|
||||
with orders/recordings/databases all need explicit design.
|
||||
|
||||
Copying only `securities.sqlite` is insufficient as an allocator strategy. It
|
||||
contains security blobs keyed by already-assigned IDs, but `key_ids.sqlite` is
|
||||
the mapping used to find or allocate those IDs. Conversely, copying only
|
||||
`key_ids.sqlite` preserves the numbers but not all contract metadata.
|
||||
|
||||
## Other assignment paths in `sparkd`
|
||||
|
||||
The two main pipelines reduce to four ways a producer handle can acquire a key:
|
||||
|
||||
1. **Already keyed input.** `MDProducer::add_security()` sees a non-zero
|
||||
`FixedSecurity::key` and calls `set_key()` immediately. Generalized feeds,
|
||||
Sunday preload, relay playback, and other replay paths use this mechanism.
|
||||
2. **Recorder-backed live input.** The input key is zero and the producer owns a
|
||||
`SecurityIDRecorder`; `ensure_key()` queues asynchronous assignment. CME
|
||||
MDP3 and legacy OnixS ICE use this mechanism. ICE uses `marketId` as
|
||||
`numeric_id` and otherwise follows the same recorder/index design.
|
||||
3. **Service assignment.** `SyntheticSecurityService` calls
|
||||
`SecurityIDRecorder::assign_key()` directly on a background dispatcher. The
|
||||
callback receives a copied `api::Security` with its top-level `id` set, then
|
||||
publishes it to synthetic subscribers. It uses the same `key_ids.sqlite`
|
||||
rowid allocator.
|
||||
4. **Unkeyed input with no recorder.** The entry is allocated with key zero and
|
||||
never becomes public. This is not a valid production assignment path, but it
|
||||
is the failure mode if a generalized source omits its ID.
|
||||
|
||||
Relay and replay deserve care when reasoning about persistence:
|
||||
|
||||
- The generalized `MDReplayer` forwards the recorded domain security ID and
|
||||
emits later books/statistics with the recorded `sec_key`, so the listener's
|
||||
current use-given mode preserves the recording's numeric identity.
|
||||
- The legacy relay producer converts a recorded `api::Security` with
|
||||
`set_security_key=true`, so its shared-memory publication also preserves the
|
||||
embedded key.
|
||||
- The legacy relay recorder additionally calls `bulk_insert(..., "REPLAY: ")`
|
||||
for logical mappings. Those rows use a `REPLAY: ` namespace prefix; this is a
|
||||
separate mapping population step, not a deterministic derivation of the key.
|
||||
|
||||
## Operational implications
|
||||
|
||||
- Back up `key_ids.sqlite` and `securities.sqlite` together if numeric ID and
|
||||
definition continuity matter.
|
||||
- Do not delete/rebuild `key_ids.sqlite` on one host and assume IDs will match
|
||||
another host, even when all instruments are identical.
|
||||
- Before moving a feed between adapters, compare the exact `name` each adapter
|
||||
supplies to `IDGenerator`. Matching venue/type is insufficient.
|
||||
- Cache generated IDs in the source after definition processing. Avoid calling
|
||||
`IDGenerator` from book/trade hot paths; the Python bridge currently violates
|
||||
this guideline.
|
||||
- Keep generated rowids below the 32-bit shared-memory ceiling until the
|
||||
`security_key` ABI is widened or explicit bounds checking is added to the
|
||||
generalized path.
|
||||
- Ensure spread legs are announced before generalized spread parents.
|
||||
- Treat `synchronous=0` as a durability decision. A copied DB should be taken
|
||||
with SQLite-aware backup/checkpoint handling rather than by casually copying
|
||||
active WAL files.
|
||||
|
||||
Useful inspection queries (run against the configured DB path) are:
|
||||
|
||||
```sql
|
||||
SELECT rowid AS spark_key, ns, key
|
||||
FROM store
|
||||
ORDER BY rowid;
|
||||
|
||||
SELECT rowid AS spark_key, ns, key
|
||||
FROM store
|
||||
WHERE ns = 'GLBX.FUTURE' AND key = '<exact-adapter-key>';
|
||||
```
|
||||
|
||||
## Source map
|
||||
|
||||
- [`SecurityIDRecorder.cpp`](../src/lib_spark/spark/md/SecurityIDRecorder.cpp):
|
||||
legacy queues, 200 ms work ordering, logical row construction, DB callbacks,
|
||||
and spread-leg resolution.
|
||||
- [`IDGenerator.cpp`](../src/lib_spark/spark/persistence/IDGenerator.cpp):
|
||||
synchronous generalized lookup/allocation and process-wide mutex.
|
||||
- [`SqliteStore.cpp`](../src/lib_spark/spark/persistence/SqliteStore.cpp): DB
|
||||
location, schema, WAL/synchronous pragmas, individual inserts, and batch
|
||||
inserts.
|
||||
- [`MDProducer.cpp`](../src/lib_spark/spark/md/MDProducer.cpp) and
|
||||
[`MDProducerHandle.cpp`](../src/lib_spark/spark/md/MDProducerHandle.cpp):
|
||||
shared-memory allocation, recorder handoff, and ordered publication.
|
||||
- [`FixedSecurity.h`](../src/lib_spark/spark/md/Shm/FixedSecurity.h) and
|
||||
[`MDStructure.h`](../src/lib_spark/spark/md/Shm/MDStructure.h): identifier
|
||||
widths and hot/cold storage fields.
|
||||
- [`MDP3Main.cpp`](../src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Main.cpp)
|
||||
and
|
||||
[`MDP3Listener.h`](../src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Listener.h):
|
||||
MDP3 construction, definition callbacks, CME ID extraction, and spread legs.
|
||||
- [`MDP3Listener.cpp`](../src/app_sparkd/sparkd/md/platforms/mdp3_onixs/MDP3Listener.cpp):
|
||||
producer/index insertion order.
|
||||
- [`OnixsGlbxSubscription.cpp`](../src/app_sparkd/sparkd/md/glbx/OnixsGlbxSubscription.cpp):
|
||||
current dedicated MDP3 dispatcher-thread ownership.
|
||||
- [`MarketDataGateway.cpp`](../src/app_sparkd/sparkd/md/MarketDataGateway.cpp)
|
||||
and
|
||||
[`GeneralMarketDataListener.cpp`](../src/lib_spark/spark/md/GeneralMarketDataListener.cpp):
|
||||
generalized source/sink topology, incoming-ID mapping, fixed-security
|
||||
conversion, and publication.
|
||||
- [`DatabentoDataHandler.cpp`](../src/app_sparkd/sparkd/md/databento/DatabentoDataHandler.cpp),
|
||||
[`TaifexDataProcessors.h`](../src/app_sparkd/sparkd/md/taifex/TaifexDataProcessors.h),
|
||||
and
|
||||
[`PyMarketDataSubscription.cpp`](../src/app_sparkd/sparkd/md/PyMarketDataSubscription.cpp):
|
||||
representative adapter-side allocation and caching behavior.
|
||||
- [`SecurityDB.cpp`](../src/lib_spark/spark/md/SecurityDB.cpp) and
|
||||
[`SecurityCache.cpp`](../src/lib_spark/spark/md/SecurityCache.cpp): separate
|
||||
persistence of full security definitions.
|
||||
- [`SecurityIndex.h`](../src/lib_spark/spark/md/SecurityIndex.h),
|
||||
[`MBOServiceServer.cpp`](../src/lib_spark/spark/md/MBOServiceServer.cpp), and
|
||||
[`MDConsumer.cpp`](../src/lib_spark/spark/md/MDConsumer.cpp): exchange-index
|
||||
translation followed by Spark-keyed market-data consumption and MBO access.
|
||||
- [`OrderFactory.cpp`](../src/lib_spark/spark/orderman/order_factory/OrderFactory.cpp),
|
||||
[`FillStore.cpp`](../src/lib_spark/spark/orderman/actors/FillStore.cpp), and
|
||||
[`PositionMap.cpp`](../src/app_sparkd/sparkd/position/PositionMap.cpp):
|
||||
representative downstream use of Spark security keys.
|
||||
- [`MDReplayer.cpp`](../src/app_sparkd/sparkd/md/recording/MDReplayer.cpp) and
|
||||
[`RelayClient.cpp`](../src/lib_spark/spark/md/relay/RelayClient.cpp): recorded
|
||||
ID reuse.
|
||||
@@ -8,7 +8,6 @@ RUN apt-get update && apt-get install -y ca-certificates
|
||||
RUN apt-get update && apt-get install -y libssh2-1-dev
|
||||
RUN apt-get update && apt-get install -y software-properties-common
|
||||
RUN apt-get update && apt-get install -y sudo
|
||||
RUN apt-get update && apt-get install -y tzdata
|
||||
|
||||
# GCC
|
||||
ARG gcc_major=13
|
||||
@@ -19,11 +18,6 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y dotnet8
|
||||
|
||||
RUN apt-get update && apt-get install -y gdb
|
||||
RUN apt-get update && apt-get install -y fish
|
||||
RUN apt-get update && apt-get install -y vim
|
||||
RUN apt-get update && apt-get install -y less
|
||||
|
||||
RUN locale-gen en_US.UTF-8
|
||||
|
||||
ARG docker_user="bts"
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
FROM ubuntu:24.04
|
||||
|
||||
ARG DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt-get update && apt-get install -y \
|
||||
fish \
|
||||
less \
|
||||
vim
|
||||
|
||||
USER bts
|
||||
WORKDIR /home/bts
|
||||
+1
-1
Submodule externals updated: 2755fd6b8f...c7478ef5cd.
+406
-279
@@ -1,306 +1,433 @@
|
||||
|
||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||
# Visual Studio Version 17
|
||||
VisualStudioVersion = 17.0.31903.59
|
||||
MinimumVisualStudioVersion = 10.0.40219.1
|
||||
VisualStudioVersion = 17.3.32922.545
|
||||
MinimumVisualStudioVersion = 15.0.26124.0
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{ECD78CC5-138F-4809-A764-2358C2AC2E82}"
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{51C40BA9-F226-4A35-939D-1ED6F5FF7299}"
|
||||
EndProject
|
||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}"
|
||||
EndProject
|
||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Core", "src\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{684863A1-3396-4D31-A731-C0725CF07F4E}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Api", "src\cs\Bts.Spark.Api\Bts.Spark.Api.csproj", "{45292808-4EF3-4D4A-B3F5-B75C42EE5870}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\Bts.Net.csproj", "{7543386A-5EBA-4AF0-9E0F-35657B58557D}"
|
||||
EndProject
|
||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}"
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\Bts.Net.csproj", "{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{F7C6F079-67BA-4899-9B43-D3F8C121D264}"
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\cs\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{90A48718-F6A7-4747-97EA-89632B451008}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}"
|
||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "cs", "cs", "{1A5B8C5D-BADC-457E-819F-0050C44FE4D3}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{ECBAA198-EC93-4174-8156-BDA5F5E95042}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{B6070F41-8759-448B-B56D-A7D15D489569}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{B2D721AB-920C-4CD6-8DBF-6778A115A449}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}"
|
||||
EndProject
|
||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "client", "client", "{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}"
|
||||
EndProject
|
||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "spark", "spark", "{42C014A0-00C5-0620-8552-38912E891A7F}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark", "src\client\spark\Spark.csproj", "{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{2B8243FF-60C6-4607-8AAE-7762D29B0A48}"
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{2A0FBDE0-D870-4567-85BD-F7793A5862C6}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
Debug|x64 = Debug|x64
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
Debug|x86 = Debug|x86
|
||||
Release|Any CPU = Release|Any CPU
|
||||
MinSizeRel|x64 = MinSizeRel|x64
|
||||
MinSizeRel|Any CPU = MinSizeRel|Any CPU
|
||||
MinSizeRel|x86 = MinSizeRel|x86
|
||||
Release|x64 = Release|x64
|
||||
Release|Any CPU = Release|Any CPU
|
||||
Release|x86 = Release|x86
|
||||
RelWithDebInfo|x64 = RelWithDebInfo|x64
|
||||
RelWithDebInfo|Any CPU = RelWithDebInfo|Any CPU
|
||||
RelWithDebInfo|x86 = RelWithDebInfo|x86
|
||||
EndGlobalSection
|
||||
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
||||
{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|Any CPU.ActiveCfg = Debug|x64
|
||||
{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|Any CPU.Build.0 = Debug|x64
|
||||
{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x64.Build.0 = Debug|x64
|
||||
{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x86.ActiveCfg = Debug|x64
|
||||
{684863A1-3396-4D31-A731-C0725CF07F4E}.Debug|x86.Build.0 = Debug|x64
|
||||
{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|Any CPU.ActiveCfg = Release|x64
|
||||
{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|Any CPU.Build.0 = Release|x64
|
||||
{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x64.ActiveCfg = Release|x64
|
||||
{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x64.Build.0 = Release|x64
|
||||
{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x86.ActiveCfg = Release|x64
|
||||
{684863A1-3396-4D31-A731-C0725CF07F4E}.Release|x86.Build.0 = Release|x64
|
||||
{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|Any CPU.ActiveCfg = Debug|x64
|
||||
{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|Any CPU.Build.0 = Debug|x64
|
||||
{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x64.Build.0 = Debug|x64
|
||||
{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x86.ActiveCfg = Debug|x64
|
||||
{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Debug|x86.Build.0 = Debug|x64
|
||||
{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|Any CPU.ActiveCfg = Release|x64
|
||||
{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|Any CPU.Build.0 = Release|x64
|
||||
{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x64.ActiveCfg = Release|x64
|
||||
{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x64.Build.0 = Release|x64
|
||||
{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x86.ActiveCfg = Release|x64
|
||||
{45292808-4EF3-4D4A-B3F5-B75C42EE5870}.Release|x86.Build.0 = Release|x64
|
||||
{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x64.Build.0 = Release|Any CPU
|
||||
{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6}.Release|x86.Build.0 = Release|Any CPU
|
||||
{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x64.Build.0 = Release|Any CPU
|
||||
{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{28A26DF7-5F7A-43A2-A548-B55FEBCC3958}.Release|x86.Build.0 = Release|Any CPU
|
||||
{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x64.Build.0 = Release|Any CPU
|
||||
{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{7543386A-5EBA-4AF0-9E0F-35657B58557D}.Release|x86.Build.0 = Release|Any CPU
|
||||
{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x64.Build.0 = Release|Any CPU
|
||||
{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{C13A5ABD-F739-4BE8-A2FE-511E989A5B43}.Release|x86.Build.0 = Release|Any CPU
|
||||
{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x64.Build.0 = Release|Any CPU
|
||||
{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{F7C6F079-67BA-4899-9B43-D3F8C121D264}.Release|x86.Build.0 = Release|Any CPU
|
||||
{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x64.Build.0 = Release|Any CPU
|
||||
{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7}.Release|x86.Build.0 = Release|Any CPU
|
||||
{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x64.Build.0 = Release|Any CPU
|
||||
{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{ECBAA198-EC93-4174-8156-BDA5F5E95042}.Release|x86.Build.0 = Release|Any CPU
|
||||
{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x64.Build.0 = Release|Any CPU
|
||||
{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40}.Release|x86.Build.0 = Release|Any CPU
|
||||
{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x64.Build.0 = Release|Any CPU
|
||||
{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7}.Release|x86.Build.0 = Release|Any CPU
|
||||
{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x64.Build.0 = Release|Any CPU
|
||||
{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{1E296FB3-D185-4A95-A1EB-AA5B37B85C60}.Release|x86.Build.0 = Release|Any CPU
|
||||
{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{B6070F41-8759-448B-B56D-A7D15D489569}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{B6070F41-8759-448B-B56D-A7D15D489569}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{B6070F41-8759-448B-B56D-A7D15D489569}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x64.Build.0 = Release|Any CPU
|
||||
{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{B6070F41-8759-448B-B56D-A7D15D489569}.Release|x86.Build.0 = Release|Any CPU
|
||||
{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x64.Build.0 = Release|Any CPU
|
||||
{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{B2D721AB-920C-4CD6-8DBF-6778A115A449}.Release|x86.Build.0 = Release|Any CPU
|
||||
{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x64.Build.0 = Release|Any CPU
|
||||
{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{38EE36C6-892C-4C94-8BB0-1ACB164E1F20}.Release|x86.Build.0 = Release|Any CPU
|
||||
{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|Any CPU.ActiveCfg = Debug|x64
|
||||
{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|Any CPU.Build.0 = Debug|x64
|
||||
{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x64.Build.0 = Debug|x64
|
||||
{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x86.ActiveCfg = Debug|x64
|
||||
{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Debug|x86.Build.0 = Debug|x64
|
||||
{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|Any CPU.ActiveCfg = Release|x64
|
||||
{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|Any CPU.Build.0 = Release|x64
|
||||
{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x64.ActiveCfg = Release|x64
|
||||
{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x64.Build.0 = Release|x64
|
||||
{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x86.ActiveCfg = Release|x64
|
||||
{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3}.Release|x86.Build.0 = Release|x64
|
||||
{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|Any CPU.ActiveCfg = Debug|x64
|
||||
{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|Any CPU.Build.0 = Debug|x64
|
||||
{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x64.Build.0 = Debug|x64
|
||||
{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x86.ActiveCfg = Debug|x64
|
||||
{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Debug|x86.Build.0 = Debug|x64
|
||||
{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|Any CPU.ActiveCfg = Release|x64
|
||||
{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|Any CPU.Build.0 = Release|x64
|
||||
{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x64.ActiveCfg = Release|x64
|
||||
{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x64.Build.0 = Release|x64
|
||||
{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x86.ActiveCfg = Release|x64
|
||||
{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224}.Release|x86.Build.0 = Release|x64
|
||||
{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|Any CPU.ActiveCfg = Debug|x64
|
||||
{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|Any CPU.Build.0 = Debug|x64
|
||||
{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x64.Build.0 = Debug|x64
|
||||
{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x86.ActiveCfg = Debug|x64
|
||||
{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Debug|x86.Build.0 = Debug|x64
|
||||
{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|Any CPU.ActiveCfg = Release|x64
|
||||
{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|Any CPU.Build.0 = Release|x64
|
||||
{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x64.ActiveCfg = Release|x64
|
||||
{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x64.Build.0 = Release|x64
|
||||
{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x86.ActiveCfg = Release|x64
|
||||
{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x86.Build.0 = Release|x64
|
||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
|
||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
|
||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
|
||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
|
||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
|
||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
|
||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
|
||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
|
||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
|
||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x86.Build.0 = Release|Any CPU
|
||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
|
||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
|
||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
|
||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
|
||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64
|
||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.ActiveCfg = Release|x64
|
||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.Build.0 = Release|x64
|
||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
|
||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
|
||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
|
||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
|
||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.ActiveCfg = Release|x64
|
||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.Build.0 = Release|x64
|
||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x86.Build.0 = Release|Any CPU
|
||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
|
||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
|
||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
|
||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
|
||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.Build.0 = Debug|x64
|
||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.ActiveCfg = Debug|x64
|
||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.Build.0 = Debug|x64
|
||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
|
||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
|
||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
|
||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
|
||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.ActiveCfg = Release|x64
|
||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.Build.0 = Release|x64
|
||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x86.Build.0 = Release|Any CPU
|
||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
|
||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
|
||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
|
||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
|
||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
|
||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
|
||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
|
||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
|
||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
|
||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
|
||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
|
||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
|
||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
|
||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x86.Build.0 = Release|Any CPU
|
||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
|
||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
|
||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
|
||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
|
||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.Build.0 = Debug|x64
|
||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.ActiveCfg = Release|x64
|
||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.Build.0 = Release|x64
|
||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
|
||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
|
||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
|
||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
|
||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.ActiveCfg = Release|x64
|
||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.Build.0 = Release|x64
|
||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x86.Build.0 = Release|Any CPU
|
||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
|
||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
|
||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
|
||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
|
||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.Build.0 = Debug|x64
|
||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.ActiveCfg = Debug|x64
|
||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.Build.0 = Debug|x64
|
||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
|
||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
|
||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
|
||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
|
||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.ActiveCfg = Release|x64
|
||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.Build.0 = Release|x64
|
||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x86.Build.0 = Release|Any CPU
|
||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
|
||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
|
||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
|
||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
|
||||
{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
|
||||
{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
|
||||
{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
|
||||
{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
|
||||
{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
|
||||
{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
|
||||
{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
|
||||
{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
|
||||
{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
|
||||
{3201359D-C071-4089-9EDE-B04D759F1508}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{3201359D-C071-4089-9EDE-B04D759F1508}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x86.Build.0 = Release|Any CPU
|
||||
{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||
{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||
{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
|
||||
{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
|
||||
{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
|
||||
{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
|
||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
|
||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
|
||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
|
||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
|
||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
|
||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
|
||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
|
||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
|
||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
|
||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x86.Build.0 = Release|Any CPU
|
||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
|
||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
|
||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
|
||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
|
||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
|
||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
|
||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
|
||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
|
||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.Build.0 = Release|Any CPU
|
||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x86.Build.0 = Release|Any CPU
|
||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
|
||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
|
||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
|
||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
|
||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
|
||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
|
||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
|
||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
|
||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.Build.0 = Release|Any CPU
|
||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x86.Build.0 = Release|Any CPU
|
||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
|
||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
|
||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
|
||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
|
||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
|
||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
|
||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
|
||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
|
||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.Build.0 = Release|Any CPU
|
||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x86.Build.0 = Release|Any CPU
|
||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
|
||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
|
||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
|
||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
|
||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|Any CPU.ActiveCfg = MinSizeRel|Any CPU
|
||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|Any CPU.Build.0 = MinSizeRel|Any CPU
|
||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x86.ActiveCfg = MinSizeRel|Any CPU
|
||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x86.Build.0 = MinSizeRel|Any CPU
|
||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.Build.0 = Release|Any CPU
|
||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x86.Build.0 = Release|Any CPU
|
||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|Any CPU.ActiveCfg = RelWithDebInfo|Any CPU
|
||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|Any CPU.Build.0 = RelWithDebInfo|Any CPU
|
||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x86.ActiveCfg = RelWithDebInfo|Any CPU
|
||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x86.Build.0 = RelWithDebInfo|Any CPU
|
||||
{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||
{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||
{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
|
||||
{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
|
||||
{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.MinSizeRel|x86.Build.0 = Debug|Any CPU
|
||||
{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|x64.Build.0 = Release|Any CPU
|
||||
{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.Release|x86.Build.0 = Release|Any CPU
|
||||
{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||
{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||
{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
|
||||
{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
|
||||
{5DE3B533-DCE6-4BF4-8F7E-08C440E177B4}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
|
||||
{90A48718-F6A7-4747-97EA-89632B451008}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{90A48718-F6A7-4747-97EA-89632B451008}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{90A48718-F6A7-4747-97EA-89632B451008}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{90A48718-F6A7-4747-97EA-89632B451008}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{90A48718-F6A7-4747-97EA-89632B451008}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{90A48718-F6A7-4747-97EA-89632B451008}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||
{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||
{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
|
||||
{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
|
||||
{90A48718-F6A7-4747-97EA-89632B451008}.MinSizeRel|x86.Build.0 = Debug|Any CPU
|
||||
{90A48718-F6A7-4747-97EA-89632B451008}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{90A48718-F6A7-4747-97EA-89632B451008}.Release|x64.Build.0 = Release|Any CPU
|
||||
{90A48718-F6A7-4747-97EA-89632B451008}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{90A48718-F6A7-4747-97EA-89632B451008}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{90A48718-F6A7-4747-97EA-89632B451008}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{90A48718-F6A7-4747-97EA-89632B451008}.Release|x86.Build.0 = Release|Any CPU
|
||||
{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||
{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||
{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
|
||||
{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
|
||||
{90A48718-F6A7-4747-97EA-89632B451008}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
|
||||
{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||
{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||
{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
|
||||
{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
|
||||
{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.MinSizeRel|x86.Build.0 = Debug|Any CPU
|
||||
{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|x64.Build.0 = Release|Any CPU
|
||||
{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.Release|x86.Build.0 = Release|Any CPU
|
||||
{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||
{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||
{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
|
||||
{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
|
||||
{2A0FBDE0-D870-4567-85BD-F7793A5862C6}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
EndGlobalSection
|
||||
GlobalSection(NestedProjects) = preSolution
|
||||
{D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
|
||||
{684863A1-3396-4D31-A731-C0725CF07F4E} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
|
||||
{45292808-4EF3-4D4A-B3F5-B75C42EE5870} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
|
||||
{AA57F8D9-7BD6-4161-881B-AD2CEF9A30F6} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
|
||||
{28A26DF7-5F7A-43A2-A548-B55FEBCC3958} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
|
||||
{7543386A-5EBA-4AF0-9E0F-35657B58557D} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
|
||||
{60E0D592-FFB3-0D0A-C4AA-E88919B48F9D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
|
||||
{C13A5ABD-F739-4BE8-A2FE-511E989A5B43} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
|
||||
{F7C6F079-67BA-4899-9B43-D3F8C121D264} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
|
||||
{E60C119D-2919-4E2F-9D39-90DB1A8FF9A7} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
|
||||
{ECBAA198-EC93-4174-8156-BDA5F5E95042} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
|
||||
{F54AE5C0-2ECA-4151-82E6-0D7EFE88FF40} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
|
||||
{A898C6C3-F134-4AF0-AFB3-3834C58CC0D7} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
|
||||
{1E296FB3-D185-4A95-A1EB-AA5B37B85C60} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
|
||||
{B6070F41-8759-448B-B56D-A7D15D489569} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
|
||||
{B2D721AB-920C-4CD6-8DBF-6778A115A449} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
|
||||
{38EE36C6-892C-4C94-8BB0-1ACB164E1F20} = {60E0D592-FFB3-0D0A-C4AA-E88919B48F9D}
|
||||
{C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
|
||||
{42C014A0-00C5-0620-8552-38912E891A7F} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
|
||||
{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3} = {42C014A0-00C5-0620-8552-38912E891A7F}
|
||||
{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
|
||||
{2B8243FF-60C6-4607-8AAE-7762D29B0A48} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
|
||||
{1A5B8C5D-BADC-457E-819F-0050C44FE4D3} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
|
||||
{2A0FBDE0-D870-4567-85BD-F7793A5862C6} = {1A5B8C5D-BADC-457E-819F-0050C44FE4D3}
|
||||
EndGlobalSection
|
||||
GlobalSection(ExtensibilityGlobals) = postSolution
|
||||
SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
|
||||
EndGlobalSection
|
||||
GlobalSection(Performance) = preSolution
|
||||
HasPerformanceSessions = true
|
||||
EndGlobalSection
|
||||
EndGlobal
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"dotnet.preferCSharpExtension": true
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
"""
|
||||
Simple DSPublisher DSL websocket client.
|
||||
|
||||
Example:
|
||||
python dsl_websocket_client.py --dsl "HeikinAshiStream(...)"
|
||||
python dsl_websocket_client.py --dsl-file query.dsl
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import websockets
|
||||
except ImportError:
|
||||
print("Installing missing dependency: websockets")
|
||||
subprocess.check_call([sys.executable, "-m", "pip", "install", "websockets"])
|
||||
import websockets
|
||||
|
||||
|
||||
DEFAULT_DSL = 'RSIUIStream("ES", "TIME", 5, 7)'
|
||||
DEFAULT_URL = "ws://localhost:19093/ws/"
|
||||
|
||||
|
||||
def load_dsl(dsl: str | None, dsl_file: str | None) -> str:
|
||||
if dsl and dsl_file:
|
||||
raise ValueError("Pass either --dsl or --dsl-file, not both.")
|
||||
|
||||
if dsl_file:
|
||||
return Path(dsl_file).read_text(encoding="utf-8").strip()
|
||||
|
||||
if dsl:
|
||||
return dsl.strip()
|
||||
|
||||
if DEFAULT_DSL.startswith("#"):
|
||||
raise ValueError("No DSL provided. Fill in DEFAULT_DSL, or pass --dsl / --dsl-file.")
|
||||
|
||||
return DEFAULT_DSL
|
||||
|
||||
|
||||
def load_config_path(config_path: str | None) -> Path | None:
|
||||
if config_path:
|
||||
return Path(config_path)
|
||||
|
||||
environment = os.environ.get("DOTNET_ENVIRONMENT")
|
||||
config_file_name = "appsettings.json" if not environment else f"appsettings.{environment}.json"
|
||||
script_dir = Path(__file__).resolve().parent
|
||||
candidate_dirs = [
|
||||
script_dir,
|
||||
script_dir.parent,
|
||||
Path.cwd(),
|
||||
]
|
||||
|
||||
for candidate_dir in candidate_dirs:
|
||||
candidate_path = candidate_dir / config_file_name
|
||||
if candidate_path.exists():
|
||||
return candidate_path
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def load_url(url: str | None, config_path: str | None) -> str:
|
||||
if url:
|
||||
return url
|
||||
|
||||
resolved_config_path = load_config_path(config_path)
|
||||
if not resolved_config_path:
|
||||
print(f"Config file not found; using default url: {DEFAULT_URL}")
|
||||
return DEFAULT_URL
|
||||
|
||||
config = json.loads(resolved_config_path.read_text(encoding="utf-8"))
|
||||
dsl_api_server_config = config.get("DSLAPIServer", {})
|
||||
ip = dsl_api_server_config.get("Ip", "localhost")
|
||||
port = int(dsl_api_server_config.get("Port", 19093))
|
||||
ws_url = f"ws://{ip}:{port}/ws/"
|
||||
print(f"Loaded DSPublisher DSL websocket url from {resolved_config_path}: {ws_url}")
|
||||
return ws_url
|
||||
|
||||
|
||||
async def send_subscription(websocket, dsl: str) -> None:
|
||||
payload = {
|
||||
"type": "subscribe",
|
||||
"dsl": dsl,
|
||||
}
|
||||
await websocket.send(json.dumps(payload))
|
||||
print("Sent subscribe request:")
|
||||
print(json.dumps(payload, indent=2))
|
||||
|
||||
|
||||
async def send_unsubscribe(websocket, subscription_id: str) -> None:
|
||||
payload = {
|
||||
"type": "unsubscribe",
|
||||
"subscriptionId": subscription_id,
|
||||
}
|
||||
await websocket.send(json.dumps(payload))
|
||||
print("Sent unsubscribe request:")
|
||||
print(json.dumps(payload, indent=2))
|
||||
|
||||
|
||||
def print_server_message(raw_message: str) -> str | None:
|
||||
try:
|
||||
message = json.loads(raw_message)
|
||||
except json.JSONDecodeError:
|
||||
print(raw_message)
|
||||
return None
|
||||
|
||||
print("Received server message:")
|
||||
print(json.dumps(message, indent=2))
|
||||
return message.get("subscriptionId")
|
||||
|
||||
|
||||
async def run_client(url: str, dsl: str) -> None:
|
||||
subscription_id: str | None = None
|
||||
|
||||
async with websockets.connect(url) as websocket:
|
||||
print(f"Connected to DSPublisher DSL websocket: {url}")
|
||||
await send_subscription(websocket, dsl)
|
||||
|
||||
try:
|
||||
async for raw_message in websocket:
|
||||
latest_subscription_id = print_server_message(raw_message)
|
||||
subscription_id = subscription_id or latest_subscription_id
|
||||
finally:
|
||||
if subscription_id:
|
||||
await send_unsubscribe(websocket, subscription_id)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Subscribe to a DSPublisher DSL websocket query and print responses."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--url",
|
||||
help="DSPublisher DSL websocket URL. Overrides appsettings DSLAPIServer config.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--config",
|
||||
help="Path to DSPublisher appsettings JSON. Defaults to appsettings[.$DOTNET_ENVIRONMENT].json near this script.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dsl",
|
||||
help="DSL expression to subscribe to.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dsl-file",
|
||||
help="Path to a text file containing the DSL expression.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
dsl = load_dsl(args.dsl, args.dsl_file)
|
||||
url = load_url(args.url, args.config)
|
||||
asyncio.run(run_client(url, dsl))
|
||||
return 0
|
||||
except KeyboardInterrupt:
|
||||
print("stopped")
|
||||
return 0
|
||||
except Exception as ex:
|
||||
print(f"error: {ex}")
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,264 @@
|
||||
namespace DSPublisher.DSLWSServer;
|
||||
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Net;
|
||||
using System.Net.WebSockets;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using BCG;
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
/// <summary>
|
||||
/// Websocket-facing API boundary for DSL clients. It opens the listener port,
|
||||
/// accepts subscribe/unsubscribe messages, and sends DataPacket JSON back to clients.
|
||||
/// Instantiated by DSPAPIServer.
|
||||
/// </summary>
|
||||
public sealed class DSLAPIWSListener : IDisposable {
|
||||
readonly HttpListener listener = new();
|
||||
readonly IDSLBookkeeper bookkeeper;
|
||||
readonly string prefix;
|
||||
readonly ConcurrentDictionary<string, ClientConnection> clients_by_id = new();
|
||||
CancellationTokenSource? cancellation_token_source;
|
||||
Task? listener_task;
|
||||
bool disposed;
|
||||
|
||||
sealed class ClientConnection {
|
||||
public required string client_id { get; init; }
|
||||
public required WebSocket web_socket { get; init; }
|
||||
public HashSet<Guid> subscription_ids { get; } = new();
|
||||
public SemaphoreSlim send_lock { get; } = new(1, 1);
|
||||
public DataPacketJsonSerializer serializer { get; } = new();
|
||||
}
|
||||
|
||||
public DSLAPIWSListener(string prefix, IDSLBookkeeper bookkeeper) {
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(prefix);
|
||||
ArgumentNullException.ThrowIfNull(bookkeeper);
|
||||
this.prefix = prefix;
|
||||
this.bookkeeper = bookkeeper;
|
||||
}
|
||||
|
||||
public Task StartAsync(CancellationToken cancellation_token = default) {
|
||||
_throw_if_disposed();
|
||||
if (listener_task is not null) {
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
cancellation_token_source = CancellationTokenSource.CreateLinkedTokenSource(cancellation_token);
|
||||
listener.Prefixes.Add(prefix);
|
||||
listener.Start();
|
||||
listener_task = Task.Run(() => _listen_loop(cancellation_token_source.Token), CancellationToken.None);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public async Task StopAsync() {
|
||||
if (cancellation_token_source is null) {
|
||||
return;
|
||||
}
|
||||
|
||||
cancellation_token_source.Cancel();
|
||||
if (listener.IsListening) {
|
||||
listener.Stop();
|
||||
}
|
||||
|
||||
foreach (var client_connection in clients_by_id.Values) {
|
||||
await _cleanup_client(client_connection);
|
||||
}
|
||||
|
||||
if (listener_task is not null) {
|
||||
await listener_task;
|
||||
listener_task = null;
|
||||
}
|
||||
|
||||
cancellation_token_source.Dispose();
|
||||
cancellation_token_source = null;
|
||||
}
|
||||
|
||||
public void Dispose() {
|
||||
if (disposed) {
|
||||
return;
|
||||
}
|
||||
|
||||
StopAsync().GetAwaiter().GetResult();
|
||||
listener.Close();
|
||||
disposed = true;
|
||||
}
|
||||
|
||||
async Task _listen_loop(CancellationToken cancellation_token) {
|
||||
while (!cancellation_token.IsCancellationRequested) {
|
||||
HttpListenerContext context;
|
||||
try {
|
||||
context = await listener.GetContextAsync();
|
||||
} catch (HttpListenerException) when (cancellation_token.IsCancellationRequested || !listener.IsListening) {
|
||||
break;
|
||||
} catch (ObjectDisposedException) {
|
||||
break;
|
||||
}
|
||||
|
||||
_ = Task.Run(() => _handle_request(context, cancellation_token), cancellation_token);
|
||||
}
|
||||
}
|
||||
|
||||
async Task _handle_request(HttpListenerContext context, CancellationToken cancellation_token) {
|
||||
if (!context.Request.IsWebSocketRequest) {
|
||||
context.Response.StatusCode = (int)HttpStatusCode.BadRequest;
|
||||
context.Response.Close();
|
||||
return;
|
||||
}
|
||||
|
||||
HttpListenerWebSocketContext web_socket_context = await context.AcceptWebSocketAsync(null);
|
||||
var client_connection = new ClientConnection {
|
||||
client_id = Guid.NewGuid().ToString("N"),
|
||||
web_socket = web_socket_context.WebSocket,
|
||||
};
|
||||
|
||||
clients_by_id[client_connection.client_id] = client_connection;
|
||||
try {
|
||||
await _read_messages(client_connection, cancellation_token);
|
||||
} finally {
|
||||
await _cleanup_client(client_connection);
|
||||
}
|
||||
}
|
||||
|
||||
async Task _read_messages(ClientConnection client_connection, CancellationToken cancellation_token) {
|
||||
var buffer = new byte[64 * 1024];
|
||||
while (client_connection.web_socket.State == WebSocketState.Open && !cancellation_token.IsCancellationRequested) {
|
||||
var result = await client_connection.web_socket.ReceiveAsync(buffer, cancellation_token);
|
||||
if (result.MessageType == WebSocketMessageType.Close) {
|
||||
await client_connection.web_socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "ok", cancellation_token);
|
||||
break;
|
||||
}
|
||||
|
||||
var json = Encoding.UTF8.GetString(buffer, 0, result.Count);
|
||||
await _handle_message(client_connection, json, cancellation_token);
|
||||
}
|
||||
}
|
||||
|
||||
async Task _handle_message(ClientConnection client_connection, string json, CancellationToken cancellation_token) {
|
||||
DSLWebSocketRequest? request;
|
||||
try {
|
||||
request = JsonSerializer.Deserialize<DSLWebSocketRequest>(json);
|
||||
} catch (JsonException ex) {
|
||||
await _send_error(client_connection, $"Invalid request JSON: {ex.Message}", cancellation_token);
|
||||
return;
|
||||
}
|
||||
|
||||
if (request?.type is null) {
|
||||
await _send_error(client_connection, "Request type is required.", cancellation_token);
|
||||
return;
|
||||
}
|
||||
|
||||
if (string.Equals(request.type, "subscribe", StringComparison.OrdinalIgnoreCase)) {
|
||||
await _handle_subscribe(client_connection, request, cancellation_token);
|
||||
return;
|
||||
}
|
||||
|
||||
if (string.Equals(request.type, "unsubscribe", StringComparison.OrdinalIgnoreCase)) {
|
||||
await _handle_unsubscribe(client_connection, request, cancellation_token);
|
||||
return;
|
||||
}
|
||||
|
||||
await _send_error(client_connection, $"Unknown request type: {request.type}", cancellation_token);
|
||||
}
|
||||
|
||||
async Task _handle_subscribe(ClientConnection client_connection, DSLWebSocketRequest request, CancellationToken cancellation_token) {
|
||||
if (string.IsNullOrWhiteSpace(request.dsl)) {
|
||||
await _send_error(client_connection, "Subscribe request requires dsl.", cancellation_token);
|
||||
return;
|
||||
}
|
||||
|
||||
Guid subscription_id = Guid.Empty;
|
||||
subscription_id = bookkeeper.Subscribe(request.dsl, packet => {
|
||||
if (subscription_id == Guid.Empty) {
|
||||
return;
|
||||
}
|
||||
_ = _send_packet(client_connection, subscription_id, packet, CancellationToken.None);
|
||||
}, client_connection.client_id);
|
||||
|
||||
client_connection.subscription_ids.Add(subscription_id);
|
||||
await _send_response(client_connection, new DSLWebSocketResponse {
|
||||
type = "subscribed",
|
||||
subscription_id = subscription_id,
|
||||
}, cancellation_token);
|
||||
}
|
||||
|
||||
async Task _handle_unsubscribe(ClientConnection client_connection, DSLWebSocketRequest request, CancellationToken cancellation_token) {
|
||||
if (request.subscription_id is null) {
|
||||
await _send_error(client_connection, "Unsubscribe request requires subscriptionId.", cancellation_token);
|
||||
return;
|
||||
}
|
||||
|
||||
var subscription_id = request.subscription_id.Value;
|
||||
if (!client_connection.subscription_ids.Remove(subscription_id)) {
|
||||
await _send_error(client_connection, $"Subscription does not belong to this client: {subscription_id}", cancellation_token);
|
||||
return;
|
||||
}
|
||||
|
||||
bookkeeper.Unsubscribe(subscription_id);
|
||||
await _send_response(client_connection, new DSLWebSocketResponse {
|
||||
type = "unsubscribed",
|
||||
subscription_id = subscription_id,
|
||||
}, cancellation_token);
|
||||
}
|
||||
|
||||
async Task _send_packet(ClientConnection client_connection, Guid subscription_id, DataPacket packet, CancellationToken cancellation_token) {
|
||||
var packet_json = client_connection.serializer.Serialize(packet);
|
||||
using var packet_document = JsonDocument.Parse(packet_json ?? "{}");
|
||||
await _send_response(client_connection, new DSLWebSocketResponse {
|
||||
type = "data",
|
||||
subscription_id = subscription_id,
|
||||
packet = packet_document.RootElement.Clone(),
|
||||
}, cancellation_token);
|
||||
}
|
||||
|
||||
Task _send_error(ClientConnection client_connection, string message, CancellationToken cancellation_token) =>
|
||||
_send_response(client_connection, new DSLWebSocketResponse {
|
||||
type = "error",
|
||||
message = message,
|
||||
}, cancellation_token);
|
||||
|
||||
async Task _send_response(ClientConnection client_connection, DSLWebSocketResponse response, CancellationToken cancellation_token) {
|
||||
if (client_connection.web_socket.State != WebSocketState.Open) {
|
||||
bookkeeper.DisposeClient(client_connection.client_id);
|
||||
return;
|
||||
}
|
||||
|
||||
var json = JsonSerializer.Serialize(response);
|
||||
var bytes = Encoding.UTF8.GetBytes(json);
|
||||
await client_connection.send_lock.WaitAsync(cancellation_token);
|
||||
try {
|
||||
await client_connection.web_socket.SendAsync(bytes, WebSocketMessageType.Text, true, cancellation_token);
|
||||
} catch {
|
||||
bookkeeper.DisposeClient(client_connection.client_id);
|
||||
throw;
|
||||
} finally {
|
||||
client_connection.send_lock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
async Task _cleanup_client(ClientConnection client_connection) {
|
||||
if (clients_by_id.TryRemove(client_connection.client_id, out _)) {
|
||||
bookkeeper.DisposeClient(client_connection.client_id);
|
||||
}
|
||||
|
||||
if (client_connection.web_socket.State == WebSocketState.Open || client_connection.web_socket.State == WebSocketState.CloseReceived) {
|
||||
try {
|
||||
await client_connection.web_socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "cleanup", CancellationToken.None);
|
||||
} catch {
|
||||
}
|
||||
}
|
||||
|
||||
client_connection.send_lock.Dispose();
|
||||
client_connection.web_socket.Dispose();
|
||||
}
|
||||
|
||||
void _throw_if_disposed() {
|
||||
if (disposed) {
|
||||
throw new ObjectDisposedException(nameof(DSLAPIWSListener));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
namespace DSPublisher.DSLWSServer;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
using BCG;
|
||||
using Bts.DataStreamCore;
|
||||
using bts.BlueOrb;
|
||||
using bts;
|
||||
|
||||
/// <summary>
|
||||
/// Tracks active DSL subscriptions and shared DSLStream instances. Multiple clients
|
||||
/// subscribing to the same DSL reuse one stream, and DoWork dispatches queued packets.
|
||||
/// Instantiated by DSPAPIServer.
|
||||
/// </summary>
|
||||
public sealed class DSLBookkeeper : IDSLBookkeeper {
|
||||
sealed class QueryState {
|
||||
public required string dsl { get; init; }
|
||||
public required DSLStream stream { get; init; }
|
||||
public DataStream.ChangeDelegate? stream_handler { get; set; }
|
||||
public HashSet<Guid> subscription_ids { get; } = new();
|
||||
public Queue<DataPacket> pending_packets { get; } = new();
|
||||
}
|
||||
|
||||
sealed class SubscriptionState {
|
||||
public required Guid subscription_id { get; init; }
|
||||
public required string client_id { get; init; }
|
||||
public required string dsl_key { get; init; }
|
||||
public required Action<DataPacket> on_data { get; init; }
|
||||
}
|
||||
|
||||
readonly object lock_obj = new();
|
||||
readonly Dictionary<string, QueryState> queries_by_dsl = new(StringComparer.Ordinal);
|
||||
readonly Dictionary<Guid, SubscriptionState> subscriptions_by_id = new();
|
||||
readonly Dictionary<string, HashSet<Guid>> subscriptions_by_client = new(StringComparer.Ordinal);
|
||||
bool disposed;
|
||||
|
||||
public Guid Subscribe(string dsl, Action<DataPacket> on_data, string client_id) {
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(dsl);
|
||||
ArgumentNullException.ThrowIfNull(on_data);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(client_id);
|
||||
|
||||
lock (lock_obj) {
|
||||
_throw_if_disposed();
|
||||
|
||||
var dsl_key = _normalize_dsl(dsl);
|
||||
if (!queries_by_dsl.TryGetValue(dsl_key, out var query_state)) {
|
||||
_log_lifecycle($"DSLBookkeeper.Subscribe client_id: {client_id} creating new DSLStream dsl: {dsl_key}");
|
||||
query_state = _create_query_state(dsl_key);
|
||||
queries_by_dsl.Add(dsl_key, query_state);
|
||||
} else {
|
||||
_log_lifecycle($"DSLBookkeeper.Subscribe client_id: {client_id} reusing existing DSLStream subscription_count: {query_state.subscription_ids.Count} dsl: {dsl_key}");
|
||||
}
|
||||
|
||||
var subscription_id = Guid.NewGuid();
|
||||
var subscription_state = new SubscriptionState {
|
||||
subscription_id = subscription_id,
|
||||
client_id = client_id,
|
||||
dsl_key = dsl_key,
|
||||
on_data = on_data,
|
||||
};
|
||||
|
||||
subscriptions_by_id.Add(subscription_id, subscription_state);
|
||||
query_state.subscription_ids.Add(subscription_id);
|
||||
|
||||
if (!subscriptions_by_client.TryGetValue(client_id, out var client_subscriptions)) {
|
||||
client_subscriptions = new();
|
||||
subscriptions_by_client.Add(client_id, client_subscriptions);
|
||||
}
|
||||
client_subscriptions.Add(subscription_id);
|
||||
|
||||
_log_lifecycle($"DSLBookkeeper.Subscribe client_id: {client_id} subscription_id: {subscription_id} active_subscriptions_for_dsl: {query_state.subscription_ids.Count} dsl: {dsl_key}");
|
||||
return subscription_id;
|
||||
}
|
||||
}
|
||||
|
||||
public void Unsubscribe(Guid subscription_id) {
|
||||
lock (lock_obj) {
|
||||
if (disposed) {
|
||||
return;
|
||||
}
|
||||
_unsubscribe_locked(subscription_id);
|
||||
}
|
||||
}
|
||||
|
||||
public void DisposeClient(string client_id) {
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(client_id);
|
||||
|
||||
lock (lock_obj) {
|
||||
if (disposed) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!subscriptions_by_client.TryGetValue(client_id, out var subscription_ids)) {
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var subscription_id in subscription_ids.ToArray()) {
|
||||
_unsubscribe_locked(subscription_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void DoWork() {
|
||||
List<(Action<DataPacket> on_data, DataPacket packet)> callbacks = new();
|
||||
|
||||
lock (lock_obj) {
|
||||
if (disposed) {
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var query_state in queries_by_dsl.Values) {
|
||||
while (query_state.pending_packets.TryDequeue(out var packet)) {
|
||||
foreach (var subscription_id in query_state.subscription_ids) {
|
||||
if (subscriptions_by_id.TryGetValue(subscription_id, out var subscription_state)) {
|
||||
callbacks.Add((subscription_state.on_data, packet));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var callback in callbacks) {
|
||||
callback.on_data(callback.packet);
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose() {
|
||||
lock (lock_obj) {
|
||||
if (disposed) {
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var query_state in queries_by_dsl.Values) {
|
||||
_dispose_query_state(query_state);
|
||||
}
|
||||
|
||||
queries_by_dsl.Clear();
|
||||
subscriptions_by_id.Clear();
|
||||
subscriptions_by_client.Clear();
|
||||
disposed = true;
|
||||
}
|
||||
}
|
||||
|
||||
QueryState _create_query_state(string dsl_key) {
|
||||
_log_lifecycle($"DSLBookkeeper._create_query_state creating DSLStream dsl: {dsl_key}");
|
||||
var stream = new DSLStream(dsl_key);
|
||||
var query_state = new QueryState {
|
||||
dsl = dsl_key,
|
||||
stream = stream,
|
||||
};
|
||||
|
||||
DataStream.ChangeDelegate stream_handler = new_value => {
|
||||
if (new_value is not DataPacket packet) {
|
||||
return;
|
||||
}
|
||||
|
||||
lock (lock_obj) {
|
||||
if (!disposed) {
|
||||
query_state.pending_packets.Enqueue(packet);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
query_state.stream_handler = stream_handler;
|
||||
stream.onChange += stream_handler;
|
||||
return query_state;
|
||||
}
|
||||
|
||||
void _unsubscribe_locked(Guid subscription_id) {
|
||||
if (!subscriptions_by_id.Remove(subscription_id, out var subscription_state)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (subscriptions_by_client.TryGetValue(subscription_state.client_id, out var client_subscriptions)) {
|
||||
client_subscriptions.Remove(subscription_id);
|
||||
if (client_subscriptions.Count == 0) {
|
||||
subscriptions_by_client.Remove(subscription_state.client_id);
|
||||
}
|
||||
}
|
||||
|
||||
if (queries_by_dsl.TryGetValue(subscription_state.dsl_key, out var query_state)) {
|
||||
query_state.subscription_ids.Remove(subscription_id);
|
||||
_log_lifecycle($"DSLBookkeeper.Unsubscribe subscription_id: {subscription_id} client_id: {subscription_state.client_id} remaining_subscriptions_for_dsl: {query_state.subscription_ids.Count} dsl: {subscription_state.dsl_key}");
|
||||
if (query_state.subscription_ids.Count == 0) {
|
||||
_log_lifecycle($"DSLBookkeeper.Unsubscribe garbage collecting DSLStream; last subscription removed dsl: {subscription_state.dsl_key}");
|
||||
_dispose_query_state(query_state);
|
||||
queries_by_dsl.Remove(subscription_state.dsl_key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static string _normalize_dsl(string dsl) =>
|
||||
dsl.Trim();
|
||||
|
||||
static void _dispose_query_state(QueryState query_state) {
|
||||
_log_lifecycle($"DSLBookkeeper._dispose_query_state disposing DSLStream dsl: {query_state.dsl} pending_packets: {query_state.pending_packets.Count} subscription_count: {query_state.subscription_ids.Count}");
|
||||
if (query_state.stream_handler is not null) {
|
||||
query_state.stream.onChange -= query_state.stream_handler;
|
||||
query_state.stream_handler = null;
|
||||
}
|
||||
query_state.subscription_ids.Clear();
|
||||
query_state.pending_packets.Clear();
|
||||
}
|
||||
|
||||
static void _log_lifecycle(string message) {
|
||||
Console.WriteLine(message);
|
||||
Log.Gen.info(message);
|
||||
}
|
||||
|
||||
void _throw_if_disposed() {
|
||||
if (disposed) {
|
||||
throw new ObjectDisposedException(nameof(DSLBookkeeper));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
namespace DSPublisher.DSLWSServer;
|
||||
|
||||
using System;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
/// <summary>
|
||||
/// Client-to-server websocket protocol message for subscribing and unsubscribing.
|
||||
/// Instantiated by System.Text.Json when DSLAPIWSListener deserializes client messages.
|
||||
/// </summary>
|
||||
public sealed class DSLWebSocketRequest {
|
||||
[JsonPropertyName("type")]
|
||||
public string? type { get; set; }
|
||||
|
||||
[JsonPropertyName("dsl")]
|
||||
public string? dsl { get; set; }
|
||||
|
||||
[JsonPropertyName("subscriptionId")]
|
||||
public Guid? subscription_id { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Server-to-client websocket protocol message. Data responses carry serialized
|
||||
/// DataPacket JSON in the packet field.
|
||||
/// Instantiated by DSLAPIWSListener before serializing responses to clients.
|
||||
/// </summary>
|
||||
public sealed class DSLWebSocketResponse {
|
||||
[JsonPropertyName("type")]
|
||||
public string type { get; set; } = "";
|
||||
|
||||
[JsonPropertyName("subscriptionId")]
|
||||
public Guid? subscription_id { get; set; }
|
||||
|
||||
[JsonPropertyName("message")]
|
||||
public string? message { get; set; }
|
||||
|
||||
[JsonPropertyName("packet")]
|
||||
public JsonElement? packet { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
namespace DSPublisher.DSLWSServer;
|
||||
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
/// <summary>
|
||||
/// Top-level DSPublisher API server. Owns the websocket listener and the DSL
|
||||
/// subscription bookkeeper, and exposes the small lifecycle surface Program.cs uses.
|
||||
/// Instantiated by Program.cs.
|
||||
/// </summary>
|
||||
public sealed class DSPAPIServer : IDisposable {
|
||||
readonly DSLBookkeeper bookkeeper;
|
||||
readonly DSLAPIWSListener web_socket_listener;
|
||||
|
||||
public DSPAPIServer(global::DSLAPIServerConfig config) {
|
||||
ArgumentNullException.ThrowIfNull(config);
|
||||
bookkeeper = new DSLBookkeeper();
|
||||
var web_socket_prefix = $"http://{config.Ip}:{config.Port}/ws/";
|
||||
web_socket_listener = new DSLAPIWSListener(web_socket_prefix, bookkeeper);
|
||||
Console.WriteLine($"DSPAPIServer listening for websocket DSL API requests on {web_socket_prefix}");
|
||||
}
|
||||
|
||||
public Task StartAsync(CancellationToken cancellation_token = default) =>
|
||||
web_socket_listener.StartAsync(cancellation_token);
|
||||
|
||||
public Task StopAsync() =>
|
||||
web_socket_listener.StopAsync();
|
||||
|
||||
public void DoWork() =>
|
||||
bookkeeper.DoWork();
|
||||
|
||||
public void Dispose() {
|
||||
web_socket_listener.Dispose();
|
||||
bookkeeper.Dispose();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
namespace DSPublisher.DSLWSServer;
|
||||
|
||||
using System;
|
||||
|
||||
using BCG;
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
/// <summary>
|
||||
/// Internal contract between the websocket listener and the DSL subscription owner.
|
||||
/// The listener handles transport; the bookkeeper owns stream lifetime and callbacks.
|
||||
/// Implemented by DSLBookkeeper and consumed by DSLAPIWSListener.
|
||||
/// </summary>
|
||||
public interface IDSLBookkeeper : IDisposable {
|
||||
Guid Subscribe(string dsl, Action<DataPacket> on_data, string client_id);
|
||||
void Unsubscribe(Guid subscription_id);
|
||||
void DisposeClient(string client_id);
|
||||
void DoWork();
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\DataStream\DataStream.csproj" />
|
||||
<ProjectReference Include="..\PubSub\PubSub.csproj" />
|
||||
<ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ConfigurationProfile Include="appsettings*.json" />
|
||||
|
||||
<None Update="appsettings*.json">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
<TargetPath>DSPublisher.%(Filename)%(Extension)</TargetPath>
|
||||
</None>
|
||||
<None Update="WSPublishers\**\*.py">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</None>
|
||||
<None Update="DSLWSClient\**\*.py">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</None>
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,21 @@
|
||||
using System.Collections.Generic;
|
||||
|
||||
public sealed class DSPublisherConfig {
|
||||
public List<StreamPublishingNode> StreamPublishingNodes { get; init; } = new();
|
||||
public List<PythonPublishingNode> PythonPublishingNodes { get; init; } = new();
|
||||
}
|
||||
|
||||
public sealed class DSLAPIServerConfig {
|
||||
public string Ip { get; init; } = "localhost";
|
||||
public int Port { get; init; } = 19093;
|
||||
}
|
||||
|
||||
public sealed class StreamPublishingNode {
|
||||
public string Name { get; init; } = string.Empty;
|
||||
public string Expression { get; init; } = string.Empty;
|
||||
}
|
||||
|
||||
public sealed class PythonPublishingNode {
|
||||
public string PythonScriptLocation { get; init; } = string.Empty;
|
||||
public string Args { get; init; } = string.Empty;
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
|
||||
using BCG;
|
||||
using bts;
|
||||
using Bts.PubSub;
|
||||
using bts.BlueOrb;
|
||||
using DSPublisher.DSLWSServer;
|
||||
|
||||
Console.WriteLine("Starting DSPublisher");
|
||||
|
||||
var configPath = load_config_path();
|
||||
Console.WriteLine($"Loading config from: {configPath}");
|
||||
if (!File.Exists(configPath)) {
|
||||
Console.WriteLine($"Config file not found: {configPath}");
|
||||
return;
|
||||
}
|
||||
|
||||
var json = File.ReadAllText(configPath);
|
||||
using var doc = JsonDocument.Parse(json);
|
||||
|
||||
if (!doc.RootElement.TryGetProperty("DSPublisher", out var dspPublisherElement)) {
|
||||
Console.WriteLine("DSPublisher config section not found");
|
||||
return;
|
||||
}
|
||||
|
||||
var json_options = new JsonSerializerOptions {
|
||||
PropertyNameCaseInsensitive = true,
|
||||
};
|
||||
|
||||
var config = dspPublisherElement.Deserialize<DSPublisherConfig>(json_options) ?? new DSPublisherConfig();
|
||||
var dsl_api_server_config = load_dsl_api_server_config(doc, json_options);
|
||||
|
||||
Console.WriteLine("StreamPublishingNodes:");
|
||||
foreach (var node in config.StreamPublishingNodes) {
|
||||
Console.WriteLine($" Name: {node.Name}");
|
||||
Console.WriteLine($" Expression: {node.Expression}");
|
||||
}
|
||||
|
||||
Console.WriteLine("PythonPublishingNodes:");
|
||||
foreach (var node in config.PythonPublishingNodes) {
|
||||
Console.WriteLine($" PythonScriptLocation: {node.PythonScriptLocation}");
|
||||
Console.WriteLine($" Args: {node.Args}");
|
||||
}
|
||||
|
||||
var pubsub_config = load_pubsub_config(doc, json_options);
|
||||
PubSubConfig.SetGlobalConfig(pubsub_config);
|
||||
using var view_stream_database = new ViewStreamDatabase(new SubscriberCoreFactory());
|
||||
ViewStreamDatabase.SetGlobalInstance(view_stream_database);
|
||||
|
||||
//var pub_table_broker = new PubTableBroker();
|
||||
//var stream_publishers = _process_publishing_nodes(config.StreamPublishingNodes, pub_table_broker);
|
||||
|
||||
using var dsp_api_server = new DSPAPIServer(dsl_api_server_config);
|
||||
await dsp_api_server.StartAsync();
|
||||
|
||||
while (true) {
|
||||
view_stream_database.doWork();
|
||||
dsp_api_server.DoWork();
|
||||
Thread.Sleep(TimeSpan.FromMilliseconds(10));
|
||||
}
|
||||
|
||||
string load_config_path() {
|
||||
var environment = Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT");
|
||||
var baseFileName = string.IsNullOrWhiteSpace(environment)
|
||||
? "appsettings.json"
|
||||
: $"appsettings.{environment}.json";
|
||||
var dspPublisherFileName = $"DSPublisher.{baseFileName}";
|
||||
var dspPublisherConfigPath = Path.Combine(AppContext.BaseDirectory, dspPublisherFileName);
|
||||
if (File.Exists(dspPublisherConfigPath)) {
|
||||
return dspPublisherConfigPath;
|
||||
}
|
||||
|
||||
return Path.Combine(AppContext.BaseDirectory, baseFileName);
|
||||
}
|
||||
|
||||
PubSubConfig load_pubsub_config(JsonDocument doc, JsonSerializerOptions json_options) {
|
||||
if (!doc.RootElement.TryGetProperty("PubSub", out var pubsub_element)) {
|
||||
Log.Gen.warn("PubSub config section not found; using PubSubConfig.Default().");
|
||||
return PubSubConfig.Default();
|
||||
}
|
||||
|
||||
var pubsub_config = pubsub_element.Deserialize<PubSubConfig>(json_options);
|
||||
if (pubsub_config is null) {
|
||||
Log.Gen.error("PubSub config section could not be deserialized; using PubSubConfig.Default().");
|
||||
return PubSubConfig.Default();
|
||||
}
|
||||
|
||||
Log.Gen.info("Loaded PubSub config from DSPublisher config file.");
|
||||
return pubsub_config;
|
||||
}
|
||||
|
||||
DSLAPIServerConfig load_dsl_api_server_config(JsonDocument doc, JsonSerializerOptions json_options) {
|
||||
if (!doc.RootElement.TryGetProperty("DSLAPIServer", out var dsl_api_server_element)) {
|
||||
Log.Gen.warn("DSLAPIServer config section not found; using DSLAPIServerConfig defaults.");
|
||||
return new DSLAPIServerConfig();
|
||||
}
|
||||
|
||||
var dsl_api_server_config = dsl_api_server_element.Deserialize<DSLAPIServerConfig>(json_options);
|
||||
if (dsl_api_server_config is null) {
|
||||
Log.Gen.error("DSLAPIServer config section could not be deserialized; using DSLAPIServerConfig defaults.");
|
||||
return new DSLAPIServerConfig();
|
||||
}
|
||||
|
||||
Log.Gen.info($"Loaded DSLAPIServer config ip: {dsl_api_server_config.Ip} port: {dsl_api_server_config.Port}");
|
||||
return dsl_api_server_config;
|
||||
}
|
||||
|
||||
List<IDisposable> _process_publishing_nodes(IEnumerable<StreamPublishingNode> publishing_nodes, PubTableBroker pub_table_broker) {
|
||||
var stream_publishers = new List<IDisposable>();
|
||||
foreach (var node in publishing_nodes) {
|
||||
if (string.IsNullOrWhiteSpace(node.Name) || string.IsNullOrWhiteSpace(node.Expression)) {
|
||||
Log.Gen.error("Skipping StreamPublishingNode because name or expression is empty.");
|
||||
continue;
|
||||
}
|
||||
|
||||
Log.Gen.info($"Creating ViewPublisher name: {node.Name} expression: {node.Expression}");
|
||||
var stream = new DSLStream(node.Expression);
|
||||
stream_publishers.Add(new ViewPublisher(node.Name, pub_table_broker, stream));
|
||||
}
|
||||
|
||||
return stream_publishers;
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
"""
|
||||
Requires:
|
||||
pip install -U databento
|
||||
pip install websockets
|
||||
|
||||
Sources:
|
||||
https://databento.com/docs/quickstart/set-up
|
||||
https://databento.com/docs/schemas-and-data-formats/bbo
|
||||
"""
|
||||
|
||||
import datetime as dt
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
try:
|
||||
import databento as db
|
||||
except ImportError:
|
||||
print("Installing missing dependency: databento")
|
||||
subprocess.check_call([sys.executable, "-m", "pip", "install", "-U", "databento"])
|
||||
import databento as db
|
||||
|
||||
from databento.common.error import BentoError
|
||||
|
||||
try:
|
||||
import websockets
|
||||
except ImportError:
|
||||
print("Installing missing dependency: websockets")
|
||||
subprocess.check_call([sys.executable, "-m", "pip", "install", "websockets"])
|
||||
import websockets
|
||||
|
||||
from databento_transformer import DatabentoTransformer
|
||||
from pubsub_client import PubSubClient
|
||||
|
||||
|
||||
API_KEY = "db-VksTJvhMmbLjbJu9Kg6GPS99B8FDN"
|
||||
DATASET = "GLBX.MDP3"
|
||||
SYMBOL = "ES.FUT"
|
||||
PUBSUB_WS_URL = "ws://localhost:19092/ws/"
|
||||
BBO_TABLE_NAME = "livebbo_glbx_future_es_continuouscontracts"
|
||||
TRADES_TABLE_NAME = "livetrades_glbx_future_es_continuouscontracts"
|
||||
DEFINITIONS_TABLE_NAME = "livecontracts_glbx_future_es"
|
||||
|
||||
_pubsub_client = PubSubClient(PUBSUB_WS_URL)
|
||||
_transformer = DatabentoTransformer(
|
||||
symbol=SYMBOL,
|
||||
bbo_table_name=BBO_TABLE_NAME,
|
||||
trades_table_name=TRADES_TABLE_NAME,
|
||||
definitions_table_name=DEFINITIONS_TABLE_NAME,
|
||||
)
|
||||
|
||||
|
||||
def _publish_contract_definitions() -> None:
|
||||
historical_client = db.Historical(API_KEY)
|
||||
dataset_range = historical_client.metadata.get_dataset_range(DATASET)
|
||||
schema_range = dataset_range.get("schema", {}).get("definition", {})
|
||||
available_end = schema_range.get("end", dataset_range["end"])
|
||||
|
||||
if isinstance(available_end, str):
|
||||
available_end_dt = dt.datetime.fromisoformat(available_end.replace("Z", "+00:00"))
|
||||
else:
|
||||
available_end_dt = available_end
|
||||
|
||||
# Databento's range end is exclusive. Back off slightly so the request stays
|
||||
# strictly inside the available window.
|
||||
end_dt = available_end_dt - dt.timedelta(seconds=1)
|
||||
start_dt = max(end_dt - dt.timedelta(days=1), end_dt.replace(hour=0, minute=0, second=0, microsecond=0))
|
||||
start = start_dt.isoformat()
|
||||
end = end_dt.isoformat()
|
||||
|
||||
definition_data = historical_client.timeseries.get_range(
|
||||
dataset=DATASET,
|
||||
schema="definition",
|
||||
stype_in="parent",
|
||||
symbols=SYMBOL,
|
||||
start=start,
|
||||
end=end,
|
||||
)
|
||||
|
||||
df = definition_data.to_df()
|
||||
payload = _transformer.build_definitions_payload(df)
|
||||
if payload is None:
|
||||
print("No Databento contract definitions returned")
|
||||
return
|
||||
|
||||
print(f"Publishing {len(payload['rows'])} Databento contract definition rows")
|
||||
_pubsub_client.publish(payload)
|
||||
|
||||
|
||||
def on_record(record) -> None:
|
||||
# Print every callback first so we can see whether Databento is flowing
|
||||
# even if our row extraction logic misses a field shape.
|
||||
print(record)
|
||||
|
||||
payloads = _transformer.transform_record(record)
|
||||
if payloads:
|
||||
for payload in payloads:
|
||||
_pubsub_client.publish(payload)
|
||||
return
|
||||
|
||||
print("Skipping record with no BBO/trade fields")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
_pubsub_client.start()
|
||||
print(f"Starting Databento live client for {SYMBOL} on {DATASET}")
|
||||
try:
|
||||
_publish_contract_definitions()
|
||||
except Exception as ex:
|
||||
print(f"Contract definition publish failed: {ex}", flush=True)
|
||||
|
||||
while True:
|
||||
try:
|
||||
client = db.Live(key=API_KEY)
|
||||
|
||||
client.subscribe(
|
||||
dataset=DATASET,
|
||||
schema="bbo-1s",
|
||||
stype_in="parent",
|
||||
symbols=SYMBOL,
|
||||
)
|
||||
client.subscribe(
|
||||
dataset=DATASET,
|
||||
schema="trades",
|
||||
stype_in="parent",
|
||||
symbols=SYMBOL,
|
||||
)
|
||||
|
||||
client.add_callback(on_record)
|
||||
client.start()
|
||||
print("Databento client started, waiting for BBO and trade records...", flush=True)
|
||||
client.block_for_close()
|
||||
print("Databento client closed cleanly", flush=True)
|
||||
break
|
||||
except BentoError as ex:
|
||||
print(f"Databento live connection error: {ex}", flush=True)
|
||||
print("Retrying Databento live connection in 5 seconds...", flush=True)
|
||||
time.sleep(5)
|
||||
except Exception as ex:
|
||||
print(f"Databento publisher unexpected error: {ex}", flush=True)
|
||||
print("Retrying Databento live connection in 5 seconds...", flush=True)
|
||||
time.sleep(5)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,244 @@
|
||||
import datetime as dt
|
||||
|
||||
|
||||
class DatabentoTransformer:
|
||||
def __init__(
|
||||
self,
|
||||
symbol: str,
|
||||
bbo_table_name: str,
|
||||
trades_table_name: str,
|
||||
definitions_table_name: str,
|
||||
):
|
||||
self._symbol = symbol
|
||||
self._bbo_table_name = bbo_table_name
|
||||
self._trades_table_name = trades_table_name
|
||||
self._definitions_table_name = definitions_table_name
|
||||
self._instrument_id_to_symbol: dict[int, str] = {}
|
||||
self._last_bbo_row: dict | None = None
|
||||
self._trade_row_id = 0
|
||||
self._trade_table_created = False
|
||||
|
||||
def transform_record(self, record) -> list[dict]:
|
||||
payload = self._handle_bbo_record(record)
|
||||
if payload is not None:
|
||||
return [payload]
|
||||
|
||||
payload = self._handle_trade_record(record)
|
||||
if payload is not None:
|
||||
return [payload]
|
||||
|
||||
return []
|
||||
|
||||
def build_definitions_payload(self, definition_df) -> dict | None:
|
||||
if definition_df is None or definition_df.empty:
|
||||
return None
|
||||
|
||||
df = definition_df.reset_index()
|
||||
if "instrument_class" in df.columns:
|
||||
df = df[df["instrument_class"].astype(str) == "F"]
|
||||
if "security_type" in df.columns:
|
||||
df = df[df["security_type"].astype(str) == "FUT"]
|
||||
if "raw_symbol" in df.columns:
|
||||
df = df.sort_values("ts_recv" if "ts_recv" in df.columns else "ts_event")
|
||||
df = df.drop_duplicates(subset=["raw_symbol"], keep="last")
|
||||
|
||||
rows: list[dict] = []
|
||||
for idx, row in enumerate(df.itertuples(index=False), start=1):
|
||||
instrument_id = int(getattr(row, "instrument_id", 0) or 0)
|
||||
security_name = str(getattr(row, "raw_symbol", ""))
|
||||
if instrument_id > 0 and security_name:
|
||||
self._instrument_id_to_symbol[instrument_id] = security_name
|
||||
|
||||
rows.append({
|
||||
"id": idx,
|
||||
"security_name": security_name,
|
||||
"instrument_id": instrument_id,
|
||||
"instrument_class": str(getattr(row, "instrument_class", "")),
|
||||
"security_type": str(getattr(row, "security_type", "")),
|
||||
"asset": str(getattr(row, "asset", "")),
|
||||
"exchange": str(getattr(row, "exchange", "")),
|
||||
"currency": str(getattr(row, "currency", "")),
|
||||
"expiration": self._ns_to_iso(getattr(row, "expiration", 0)),
|
||||
"activation": self._ns_to_iso(getattr(row, "activation", 0)),
|
||||
"min_price_increment": self._nano_to_float(getattr(row, "min_price_increment", 0)),
|
||||
"display_factor": self._nano_to_float(getattr(row, "display_factor", 0)),
|
||||
"price_display_multiplier": self._nano_to_float(getattr(row, "display_factor", 0)),
|
||||
"contract_multiplier": int(getattr(row, "contract_multiplier", 0) or 0),
|
||||
"contract_multiplier_unit": int(getattr(row, "contract_multiplier_unit", 0) or 0),
|
||||
"unit_of_measure": str(getattr(row, "unit_of_measure", "")),
|
||||
"unit_of_measure_qty": self._nano_to_float(getattr(row, "unit_of_measure_qty", 0)),
|
||||
"min_lot_size": int(getattr(row, "min_lot_size", 0) or 0),
|
||||
"min_trade_vol": int(getattr(row, "min_trade_vol", 0) or 0),
|
||||
"max_trade_vol": int(getattr(row, "max_trade_vol", 0) or 0),
|
||||
"maturity_year": int(getattr(row, "maturity_year", 0) or 0),
|
||||
"maturity_month": int(getattr(row, "maturity_month", 0) or 0),
|
||||
"maturity_day": int(getattr(row, "maturity_day", 0) or 0),
|
||||
"underlying": str(getattr(row, "underlying", "")),
|
||||
})
|
||||
|
||||
if not rows:
|
||||
return None
|
||||
|
||||
return {
|
||||
"op": "create_table",
|
||||
"table_name": self._definitions_table_name,
|
||||
"rows": rows,
|
||||
}
|
||||
|
||||
def _handle_bbo_record(self, record) -> dict | None:
|
||||
row = self._to_bbo_row(record)
|
||||
if row is None:
|
||||
return None
|
||||
|
||||
if self._last_bbo_row is None:
|
||||
payload = {
|
||||
"op": "create_table",
|
||||
"table_name": self._bbo_table_name,
|
||||
"rows": [row],
|
||||
}
|
||||
else:
|
||||
payload = {
|
||||
"op": "multiple_remove_add",
|
||||
"table_name": self._bbo_table_name,
|
||||
"removed_rows": [self._last_bbo_row],
|
||||
"added_rows": [row],
|
||||
}
|
||||
|
||||
self._last_bbo_row = row
|
||||
return payload
|
||||
|
||||
def _handle_trade_record(self, record) -> dict | None:
|
||||
row = self._to_trade_row(record)
|
||||
if row is None:
|
||||
return None
|
||||
|
||||
row["id"] = self._trade_row_id
|
||||
self._trade_row_id += 1
|
||||
|
||||
if not self._trade_table_created:
|
||||
payload = {
|
||||
"op": "create_table",
|
||||
"table_name": self._trades_table_name,
|
||||
"rows": [row],
|
||||
}
|
||||
self._trade_table_created = True
|
||||
return payload
|
||||
|
||||
return {
|
||||
"op": "multiple_remove_add",
|
||||
"table_name": self._trades_table_name,
|
||||
"removed_rows": [],
|
||||
"added_rows": [row],
|
||||
}
|
||||
|
||||
def _to_bbo_row(self, record) -> dict | None:
|
||||
bid_px = self._get_level_attr(record, "bid_px", default=self._get_attr(record, "bid_px_00"))
|
||||
ask_px = self._get_level_attr(record, "ask_px", default=self._get_attr(record, "ask_px_00"))
|
||||
if bid_px is None and ask_px is None:
|
||||
return None
|
||||
|
||||
return {
|
||||
"id": 0,
|
||||
"security_name": self._get_security_name(record),
|
||||
"timestamp": self._get_timestamp(record),
|
||||
"bid_px": float(bid_px or 0.0),
|
||||
"ask_px": float(ask_px or 0.0),
|
||||
"bid_sz": int(self._get_level_attr(record, "bid_sz", default=self._get_attr(record, "bid_sz_00", default=0)) or 0),
|
||||
"ask_sz": int(self._get_level_attr(record, "ask_sz", default=self._get_attr(record, "ask_sz_00", default=0)) or 0),
|
||||
}
|
||||
|
||||
def _to_trade_row(self, record) -> dict | None:
|
||||
price = self._get_attr(record, "price", "px")
|
||||
size = self._get_attr(record, "size")
|
||||
if price is None and size is None:
|
||||
return None
|
||||
|
||||
side = self._get_attr(record, "side", "aggressor_side", default="")
|
||||
|
||||
return {
|
||||
"id": 0,
|
||||
"security_name": self._get_security_name(record),
|
||||
"timestamp": self._get_timestamp(record),
|
||||
"price": float(price or 0.0),
|
||||
"size": int(size or 0),
|
||||
"side": str(side),
|
||||
}
|
||||
|
||||
def _get_security_name(self, record) -> str:
|
||||
instrument_id = self._get_attr(record, "instrument_id", default=0)
|
||||
try:
|
||||
instrument_id_int = int(instrument_id or 0)
|
||||
except (TypeError, ValueError):
|
||||
instrument_id_int = 0
|
||||
|
||||
if instrument_id_int > 0 and instrument_id_int in self._instrument_id_to_symbol:
|
||||
return self._instrument_id_to_symbol[instrument_id_int]
|
||||
|
||||
value = self._get_attr(
|
||||
record,
|
||||
"symbol",
|
||||
"raw_symbol",
|
||||
"stype_out_symbol",
|
||||
"stype_in_symbol",
|
||||
default=self._symbol,
|
||||
)
|
||||
return str(value)
|
||||
|
||||
def _get_timestamp(self, record) -> int:
|
||||
value = self._get_attr(record, "ts_event", "ts_recv", default=0)
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
def _get_attr(self, record, *names, default=None):
|
||||
for name in names:
|
||||
if hasattr(record, name):
|
||||
value = getattr(record, name)
|
||||
if value is not None:
|
||||
return value
|
||||
return default
|
||||
|
||||
def _get_level_attr(self, record, name, default=None):
|
||||
levels = self._get_attr(record, "levels", default=None)
|
||||
if levels:
|
||||
level0 = levels[0]
|
||||
if hasattr(level0, name):
|
||||
value = getattr(level0, name)
|
||||
if value is not None:
|
||||
return value
|
||||
return default
|
||||
|
||||
def _ns_to_iso(self, timestamp_ns) -> str:
|
||||
if timestamp_ns is None:
|
||||
return ""
|
||||
|
||||
if hasattr(timestamp_ns, "isoformat"):
|
||||
try:
|
||||
if hasattr(timestamp_ns, "to_pydatetime"):
|
||||
timestamp_dt = timestamp_ns.to_pydatetime()
|
||||
else:
|
||||
timestamp_dt = timestamp_ns
|
||||
|
||||
if timestamp_dt is None:
|
||||
return ""
|
||||
|
||||
if getattr(timestamp_dt, "tzinfo", None) is None:
|
||||
timestamp_dt = timestamp_dt.replace(tzinfo=dt.timezone.utc)
|
||||
return timestamp_dt.isoformat()
|
||||
except (TypeError, ValueError, AttributeError):
|
||||
pass
|
||||
|
||||
try:
|
||||
ts_int = int(timestamp_ns)
|
||||
if ts_int <= 0:
|
||||
return ""
|
||||
return dt.datetime.fromtimestamp(ts_int / 1_000_000_000, tz=dt.timezone.utc).isoformat()
|
||||
except (TypeError, ValueError, OSError, OverflowError):
|
||||
return ""
|
||||
|
||||
def _nano_to_float(self, value) -> float:
|
||||
try:
|
||||
return float(value) / 1_000_000_000.0
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
@@ -0,0 +1,58 @@
|
||||
import asyncio
|
||||
import json
|
||||
import queue
|
||||
import threading
|
||||
|
||||
import websockets
|
||||
|
||||
|
||||
class PubSubClient:
|
||||
def __init__(self, ws_url: str):
|
||||
self._ws_url = ws_url
|
||||
self._outgoing_queue: "queue.Queue[dict]" = queue.Queue()
|
||||
self._sender_thread: threading.Thread | None = None
|
||||
|
||||
def start(self) -> threading.Thread:
|
||||
if self._sender_thread is None or not self._sender_thread.is_alive():
|
||||
self._sender_thread = self._start_sender_thread()
|
||||
return self._sender_thread
|
||||
|
||||
def publish(self, payload: dict) -> None:
|
||||
self._outgoing_queue.put(payload)
|
||||
|
||||
def publish_create_table(self, table_name: str, rows: list[dict]) -> None:
|
||||
self.publish({
|
||||
"op": "create_table",
|
||||
"table_name": table_name,
|
||||
"rows": rows,
|
||||
})
|
||||
|
||||
def publish_multiple_remove_add(
|
||||
self,
|
||||
table_name: str,
|
||||
removed_rows: list[dict],
|
||||
added_rows: list[dict],
|
||||
) -> None:
|
||||
self.publish({
|
||||
"op": "multiple_remove_add",
|
||||
"table_name": table_name,
|
||||
"removed_rows": removed_rows,
|
||||
"added_rows": added_rows,
|
||||
})
|
||||
|
||||
def _start_sender_thread(self) -> threading.Thread:
|
||||
thread = threading.Thread(target=self._run_send_loop, daemon=True)
|
||||
thread.start()
|
||||
return thread
|
||||
|
||||
def _run_send_loop(self) -> None:
|
||||
asyncio.run(self._ws_send_loop())
|
||||
|
||||
async def _ws_send_loop(self) -> None:
|
||||
async with websockets.connect(self._ws_url) as ws:
|
||||
print(f"Connected to PubSubServer websocket: {self._ws_url}")
|
||||
while True:
|
||||
payload = await asyncio.to_thread(self._outgoing_queue.get)
|
||||
body = json.dumps(payload)
|
||||
print(body)
|
||||
await ws.send(body)
|
||||
@@ -0,0 +1,69 @@
|
||||
{
|
||||
"PubSub": {
|
||||
"HeartbeatInterval": 15000,
|
||||
"MissingHeartbeatsBeforeGc": 50,
|
||||
"DisposedConnectionCheckPeriod": 1000,
|
||||
"ServerPort": 22019,
|
||||
"Routing": {
|
||||
"PublisherRoutes": [
|
||||
{
|
||||
"Name": "default",
|
||||
"ServerAddress": "bts-cloud://127.0.0.1:22019",
|
||||
"TagRegExp": ".*"
|
||||
}
|
||||
],
|
||||
"SubscriberRoutes": [
|
||||
{
|
||||
"Name": "default",
|
||||
"ServerAddress": "bts-cloud://127.0.0.1:22019",
|
||||
"TagRegExp": ".*"
|
||||
}
|
||||
]
|
||||
},
|
||||
"TagTypeToAssemblyMap": {
|
||||
"Entries": [
|
||||
{
|
||||
"HandlerClass": "BCG.PubSubServerTagHandler",
|
||||
"Assembly": "DataStreamCore.dll"
|
||||
}
|
||||
]
|
||||
},
|
||||
"TagHandlerFactory": {
|
||||
"Handlers": [
|
||||
{
|
||||
"TagType": "DataStream",
|
||||
"Assembly": "DataStreamCore",
|
||||
"Class": "bts.PubsubServerStreamTagHandler"
|
||||
},
|
||||
{
|
||||
"TagType": "DS_.*",
|
||||
"Assembly": "DataStreamCore",
|
||||
"Class": "bts.PubsubServerStreamTagHandler"
|
||||
},
|
||||
{
|
||||
"TagType": ".*",
|
||||
"Assembly": "DataStreamCore",
|
||||
"Class": "bts.PubsubServerGenericDataHandler"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"DSLAPIServer": {
|
||||
"Ip": "localhost",
|
||||
"Port": 19093
|
||||
},
|
||||
"DSPublisher": {
|
||||
"StreamPublishingNodes": [
|
||||
{
|
||||
"Name": "sample_stream",
|
||||
"Expression": "MillisecondStream(1000,5)"
|
||||
}
|
||||
],
|
||||
"PythonPublishingNodes": [
|
||||
{
|
||||
"PythonScriptLocation": "WSPublishers/DatabentoPublisher.py",
|
||||
"Args": ""
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
{
|
||||
"PubSub": {
|
||||
"HeartbeatInterval": 15000,
|
||||
"MissingHeartbeatsBeforeGc": 50,
|
||||
"DisposedConnectionCheckPeriod": 1000,
|
||||
"ServerPort": 22019,
|
||||
"Routing": {
|
||||
"PublisherRoutes": [
|
||||
{
|
||||
"Name": "default",
|
||||
"ServerAddress": "bts-cloud://127.0.0.1:22019",
|
||||
"TagRegExp": ".*"
|
||||
}
|
||||
],
|
||||
"SubscriberRoutes": [
|
||||
{
|
||||
"Name": "default",
|
||||
"ServerAddress": "bts-cloud://127.0.0.1:22019",
|
||||
"TagRegExp": ".*"
|
||||
}
|
||||
]
|
||||
},
|
||||
"TagTypeToAssemblyMap": {
|
||||
"Entries": [
|
||||
{
|
||||
"HandlerClass": "BCG.PubSubServerTagHandler",
|
||||
"Assembly": "DataStreamCore.dll"
|
||||
}
|
||||
]
|
||||
},
|
||||
"TagHandlerFactory": {
|
||||
"Handlers": [
|
||||
{
|
||||
"TagType": "DataStream",
|
||||
"Assembly": "DataStreamCore",
|
||||
"Class": "bts.PubsubServerStreamTagHandler"
|
||||
},
|
||||
{
|
||||
"TagType": "DS_.*",
|
||||
"Assembly": "DataStreamCore",
|
||||
"Class": "bts.PubsubServerStreamTagHandler"
|
||||
},
|
||||
{
|
||||
"TagType": ".*",
|
||||
"Assembly": "DataStreamCore",
|
||||
"Class": "bts.PubsubServerGenericDataHandler"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"DSLAPIServer": {
|
||||
"Ip": "localhost",
|
||||
"Port": 19093
|
||||
},
|
||||
"DSPublisher": {
|
||||
"StreamPublishingNodes": [
|
||||
{
|
||||
"Name": "sample_stream",
|
||||
"Expression": "MillisecondStream(1000,5)"
|
||||
}
|
||||
],
|
||||
"PythonPublishingNodes": [
|
||||
{
|
||||
"PythonScriptLocation": "WSPublishers/DatabentoPublisher.py",
|
||||
"Args": ""
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,8 @@
|
||||
using BCG;
|
||||
|
||||
[DataStreamUserRespresentation("AddColumnsStream")]
|
||||
public class AddColumnsStream: ViewStream {
|
||||
// example: AddColumnsStream(l,"jc1:001")
|
||||
public class AddColumnsStream : ViewStream {
|
||||
public AddColumnsStream(ViewStream parent_str, string add_col_expr) {
|
||||
ViewStream rs = new BCG.AddColumnsStream(parent_str, add_col_expr);
|
||||
mySetRealSource(rs);
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
namespace BlueOrb.Operators.Core {
|
||||
using BCG;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
|
||||
[DataStreamUserRespresentation("HideColumnsStream")]
|
||||
public class HideColumnsStream : ViewStream {
|
||||
// Wrapper row to hide the columns
|
||||
public class MyRow : IBlueRow {
|
||||
#region IBlueRow members
|
||||
private IBlueTableSchema _schema;
|
||||
private IBlueRow _row;
|
||||
public MyRow(IBlueTableSchema schema, IBlueRow row) {
|
||||
_schema = schema;
|
||||
_row = row;
|
||||
}
|
||||
public IBlueTableSchema schema { get { return _schema; } }
|
||||
#endregion
|
||||
|
||||
#region ICollection members
|
||||
public int Count { get { return schema.columnNames.Length; } }
|
||||
|
||||
public bool IsSynchronized { get { return false; } }
|
||||
|
||||
public object SyncRoot { get { return this; } }
|
||||
|
||||
public void CopyTo(Array array, int index) {
|
||||
int i = index;
|
||||
foreach (object o in this as IEnumerable)
|
||||
array.SetValue(o, i++);
|
||||
}
|
||||
#endregion
|
||||
|
||||
#region IDictionary members
|
||||
public bool IsFixedSize { get { return true; } }
|
||||
|
||||
public bool IsReadOnly { get { return true; } }
|
||||
|
||||
public object this[object key] {
|
||||
get {
|
||||
if (key is string)
|
||||
return this[(string)key];
|
||||
return this[(int)key];
|
||||
}
|
||||
set { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||
}
|
||||
|
||||
public object this[int index] {
|
||||
get {
|
||||
return this[schema.columnInfo[index].colName];
|
||||
}
|
||||
}
|
||||
|
||||
public object this[string col_name] {
|
||||
get {
|
||||
if (_schema.contains_column(col_name))
|
||||
return _row[col_name];
|
||||
else
|
||||
throw new ApplicationException($"{col_name} not found");
|
||||
}
|
||||
}
|
||||
|
||||
public ICollection Keys { get { return schema.columnNames; } }
|
||||
|
||||
public ICollection Values {
|
||||
get {
|
||||
object[] retVal = new object[schema.columnNames.Length];
|
||||
CopyTo(retVal, 0);
|
||||
return retVal;
|
||||
}
|
||||
}
|
||||
|
||||
public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||
|
||||
public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||
|
||||
public bool Contains(object key) { return schema[(string)key] != null; }
|
||||
|
||||
public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
|
||||
|
||||
IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
|
||||
|
||||
public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
|
||||
#endregion
|
||||
|
||||
}
|
||||
|
||||
private readonly ViewStream _parent_str;
|
||||
private readonly HashSet<string> _removed_cols;
|
||||
private BlueTable _state;
|
||||
public HideColumnsStream(ViewStream parent_str, string hide_cols_expr) {
|
||||
_parent_str = parent_str;
|
||||
_removed_cols = new HashSet<string>(hide_cols_expr.Split(',', ' '));
|
||||
if (_removed_cols.Contains("id"))
|
||||
_removed_cols.Remove("id");
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
_parent_str.onChange += _handle_parent_str_cb;
|
||||
}
|
||||
|
||||
private void _handle_parent_str_cb(object dp_obj) {
|
||||
// this could be made incremental and hence faster
|
||||
DataPacket dp_in = dp_obj as DataPacket;
|
||||
if (_state == null) {
|
||||
List<ColumnInfo> cis = new List<ColumnInfo>();
|
||||
foreach (ColumnInfo ci in dp_in.schema.columnInfo) {
|
||||
if (!_removed_cols.Contains(ci.colName))
|
||||
cis.Add(new ColumnInfo(ci.colName, cis.Count, ci.colType));
|
||||
}
|
||||
_state = new BlueTable(new BlueTableSchema(dp_in.schema.tableName, cis.ToArray()));
|
||||
}
|
||||
_state.clear();
|
||||
foreach (IBlueRow r in dp_in.newRows) {
|
||||
_state.addRow(new MyRow(_state.schema, r));
|
||||
}
|
||||
var dp_out = BlueDataUtility.build_dp_inc(_state,
|
||||
myLastValue == null ? new BlueTable(_state.schema) : (myLastValue as DataPacket).newRows);
|
||||
myFireEvent(dp_out);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
namespace bts.BlueOrb {
|
||||
using BCG;
|
||||
|
||||
[DataStreamUserRespresentation("CartesianProductStream")]
|
||||
public class CartesianProductStream : ViewStream {
|
||||
public CartesianProductStream(ViewStream ls, ViewStream rs) {
|
||||
ls = new AddColumnsStream(ls, "ljc: 001");
|
||||
rs = new AddColumnsStream(rs, "rjc: 001");
|
||||
ViewStream res = new SimpleJoinStream(ls, "ljc", rs, "rjc");
|
||||
res = new global::BlueOrb.Operators.Core.HideColumnsStream(res, "ljc, rjc");
|
||||
mySetRealSource(res);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
namespace BlueOrb.Operators.Core {
|
||||
using System;
|
||||
|
||||
using BCG;
|
||||
|
||||
[DataStreamUserRespresentation("FullJoinStream")]
|
||||
// JoinStream is technically a LeftJoin. FullJoin only results in rows that have bpth left and right side
|
||||
public class FullJoinStream : ViewStream {
|
||||
public FullJoinStream(ViewStream l_str, string ljcs, ViewStream r_str, string rjcs) {
|
||||
ViewStream js = new global::bts.BlueOrb.SimpleJoinStream(l_str, ljcs, r_str, rjcs);
|
||||
ViewStream rs = new bts.BlueOrb.FilterStream(js, r => !DBNull.Value.Equals(r["right_id"]));
|
||||
mySetRealSource(rs);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
namespace bts.BlueOrb {
|
||||
namespace bts.BlueOrb
|
||||
{
|
||||
using BCG;
|
||||
|
||||
using Bts.DataStream;
|
||||
@@ -13,8 +14,10 @@
|
||||
using RowHash = System.Int32;
|
||||
|
||||
[DataStreamUserRespresentation("JoinStream")]
|
||||
public class SimpleJoinStream : ViewStream {
|
||||
public class JoinBookkeepingData {
|
||||
public class SimpleJoinStream : ViewStream
|
||||
{
|
||||
public class JoinBookkeepingData
|
||||
{
|
||||
// initialized in cons
|
||||
readonly string[] _join_col_names;
|
||||
private readonly BlueRowHasher _hasher;
|
||||
@@ -23,7 +26,8 @@
|
||||
int[]? _join_cols;
|
||||
private IDictionary<RowHash, List<IBlueRow>> _hash_row_map = new Dictionary<RowHash, List<IBlueRow>>();
|
||||
|
||||
public JoinBookkeepingData(ViewStream vs, List<string> join_cols, BlueRowHasher row_hasher) {
|
||||
public JoinBookkeepingData(ViewStream vs, List<string> join_cols, BlueRowHasher row_hasher)
|
||||
{
|
||||
_join_col_names = join_cols.Where(c => !string.IsNullOrEmpty(c)).ToArray();
|
||||
_hasher = row_hasher;
|
||||
}
|
||||
@@ -40,22 +44,28 @@
|
||||
public List<IBlueRow>? get_matching_rows(int hash) =>
|
||||
_hash_row_map.GetValueOrDefault(hash);
|
||||
|
||||
private RowHash _get_hash(IBlueRow r) {
|
||||
if (_join_cols == null || _join_cols.Length == 0) {
|
||||
private RowHash _get_hash(IBlueRow r)
|
||||
{
|
||||
if (_join_cols == null || _join_cols.Length == 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
return _hasher.get_hash(r, _join_cols);
|
||||
}
|
||||
|
||||
public void update_data(IDataPacket dp) {
|
||||
if (!ready) {
|
||||
public void update_data(IDataPacket dp)
|
||||
{
|
||||
if (!ready)
|
||||
{
|
||||
_schema = dp.newRows.schema;
|
||||
_join_cols = _schema.ResolveIndices(_join_col_names);
|
||||
}
|
||||
_hash_row_map.Clear();
|
||||
foreach (IBlueRow r in dp.newRows) {
|
||||
foreach (IBlueRow r in dp.newRows)
|
||||
{
|
||||
int hash = _get_hash(r);
|
||||
if (!_hash_row_map.TryGetValue(hash, out var hash_rows)) {
|
||||
if (!_hash_row_map.TryGetValue(hash, out var hash_rows))
|
||||
{
|
||||
hash_rows = new List<IBlueRow>();
|
||||
_hash_row_map.Add(hash, hash_rows);
|
||||
}
|
||||
@@ -64,8 +74,10 @@
|
||||
}
|
||||
|
||||
[MemberNotNullWhen(true, nameof(_schema))]
|
||||
public bool ready {
|
||||
get {
|
||||
public bool ready
|
||||
{
|
||||
get
|
||||
{
|
||||
return _schema != null;
|
||||
}
|
||||
}
|
||||
@@ -77,9 +89,11 @@
|
||||
IBlueTableSchema? _schema;
|
||||
|
||||
public SimpleJoinStream(ViewStream ls, string ljc, ViewStream rs, string rjc) :
|
||||
this(ViewStreamDatabase.instance, ls, ljc, rs, rjc) { }
|
||||
this(ViewStreamDatabase.instance, ls, ljc, rs, rjc)
|
||||
{ }
|
||||
|
||||
public SimpleJoinStream(ViewStreamDatabase vsd, ViewStream ls, string ljc, ViewStream rs, string rjc) {
|
||||
public SimpleJoinStream(ViewStreamDatabase vsd, ViewStream ls, string ljc, ViewStream rs, string rjc)
|
||||
{
|
||||
_ls = ls;
|
||||
_rs = rs;
|
||||
List<string> lj_col_list = ljc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
|
||||
@@ -91,30 +105,35 @@
|
||||
_rdata = new JoinBookkeepingData(rs, rj_col_list, hasher);
|
||||
}
|
||||
|
||||
protected override void myOnFirstSubscriber() {
|
||||
protected override void myOnFirstSubscriber()
|
||||
{
|
||||
_ls.onChange += _ls_onchange;
|
||||
_rs.onChange += _rs_onchange;
|
||||
}
|
||||
|
||||
private void _ls_onchange(object dp_obj) {
|
||||
private void _ls_onchange(object dp_obj)
|
||||
{
|
||||
var dp = (IDataPacket)dp_obj;
|
||||
_ldata.update_data(dp);
|
||||
_form_and_fire();
|
||||
}
|
||||
|
||||
private void _rs_onchange(object dp_obj) {
|
||||
private void _rs_onchange(object dp_obj)
|
||||
{
|
||||
var dp = (IDataPacket)dp_obj;
|
||||
_rdata.update_data(dp);
|
||||
_form_and_fire();
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber() {
|
||||
protected override void myOnLastSubscriber()
|
||||
{
|
||||
_ls.onChange -= _ls_onchange;
|
||||
_rs.onChange -= _rs_onchange;
|
||||
}
|
||||
|
||||
[MemberNotNull(nameof(_schema))]
|
||||
private void _build_schema() {
|
||||
private void _build_schema()
|
||||
{
|
||||
if (_schema != null)
|
||||
return;
|
||||
var lschema = Log.notnull(_ldata.schema);
|
||||
@@ -122,52 +141,64 @@
|
||||
var cis = new List<ColumnInfo> {
|
||||
new("id", 0, TypeCode.Int32)
|
||||
};
|
||||
foreach (ColumnInfo ci in lschema.columnInfo) {
|
||||
foreach (ColumnInfo ci in lschema.columnInfo)
|
||||
{
|
||||
string col_name = rschema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
|
||||
cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
|
||||
}
|
||||
foreach (ColumnInfo ci in rschema.columnInfo) {
|
||||
foreach (ColumnInfo ci in rschema.columnInfo)
|
||||
{
|
||||
string col_name = lschema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
|
||||
cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
|
||||
}
|
||||
_schema = new BlueTableSchema($"{lschema.tableName}_join_{rschema.tableName}", cis.ToArray());
|
||||
}
|
||||
|
||||
private void _form_and_fire() {
|
||||
private void _form_and_fire()
|
||||
{
|
||||
if (!_ldata.ready || !_rdata.ready)
|
||||
return;
|
||||
_build_schema();
|
||||
List<object[]> res_rows = new List<object[]>();
|
||||
|
||||
// this is where the join happens
|
||||
foreach (var kv in _ldata.rows) {
|
||||
foreach (var kv in _ldata.rows)
|
||||
{
|
||||
RowHash hash = kv.Key;
|
||||
List<IBlueRow> l_rows = kv.Value;
|
||||
List<IBlueRow>? r_rows = _rdata.get_matching_rows(hash);
|
||||
|
||||
foreach (IBlueRow lr in l_rows) {
|
||||
if (r_rows != null) {
|
||||
foreach (IBlueRow r in r_rows) {
|
||||
foreach (IBlueRow lr in l_rows)
|
||||
{
|
||||
if (r_rows != null)
|
||||
{
|
||||
foreach (IBlueRow r in r_rows)
|
||||
{
|
||||
object[] rowvals = build_row_array(res_rows.Count, lr);
|
||||
copy_row(rowvals, r, false);
|
||||
res_rows.Add(rowvals);
|
||||
}
|
||||
} else {
|
||||
}
|
||||
else
|
||||
{
|
||||
object[] rowvals = build_row_array(res_rows.Count, lr);
|
||||
res_rows.Add(rowvals);
|
||||
}
|
||||
}
|
||||
}
|
||||
BlueTable rv = new BlueTable(_schema);
|
||||
foreach (object[] rval in res_rows) {
|
||||
foreach (object[] rval in res_rows)
|
||||
{
|
||||
rv.addRow(new BlueRow(_schema, rval));
|
||||
}
|
||||
var dp = BlueDataUtility.build_dp(rv, myLastValue as IDataPacket);
|
||||
myFireEvent(dp);
|
||||
|
||||
object[] build_row_array(int id, IBlueRow l_row) {
|
||||
object[] build_row_array(int id, IBlueRow l_row)
|
||||
{
|
||||
object[] rv = new object[_schema.columnNames.Length];
|
||||
for (int i = 0; i < _schema.columnNames.Length; i++) {
|
||||
for (int i = 0; i < _schema.columnNames.Length; i++)
|
||||
{
|
||||
rv[i] = DBNull.Value;
|
||||
}
|
||||
rv[0] = id;
|
||||
@@ -175,30 +206,26 @@
|
||||
return rv;
|
||||
}
|
||||
|
||||
void copy_row(object?[] rval, IBlueRow r, bool is_left) {
|
||||
void copy_row(object?[] rval, IBlueRow r, bool is_left)
|
||||
{
|
||||
int offset = 1;
|
||||
JoinBookkeepingData sdata = _ldata;
|
||||
if (is_left) {
|
||||
if (is_left)
|
||||
{
|
||||
offset = 1;
|
||||
sdata = _ldata;
|
||||
} else {
|
||||
}
|
||||
else
|
||||
{
|
||||
sdata = _rdata;
|
||||
offset = 1 + _ldata.RequireSchema().columnInfo.Length;
|
||||
}
|
||||
for (int i = 0; i < sdata.RequireSchema().columnInfo.Length; i++) {
|
||||
for (int i = 0; i < sdata.RequireSchema().columnInfo.Length; i++)
|
||||
{
|
||||
rval[offset + i] = r[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[DataStreamUserRespresentation("FullJoinStream")]
|
||||
public class FullJoinStream : ViewStream {
|
||||
public FullJoinStream(ViewStream l_str, string ljcs, ViewStream r_str, string rjcs) {
|
||||
ViewStream js = new SimpleJoinStream(l_str, ljcs, r_str, rjcs);
|
||||
ViewStream rs = new bts.BlueOrb.FilterStream(js, r => !DBNull.Value.Equals(r["right_id"]));
|
||||
mySetRealSource(rs);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -29,22 +29,26 @@ namespace bts.BlueOrb {
|
||||
_stream.onChange -= new ChangeDelegate(_onstream_change);
|
||||
}
|
||||
|
||||
|
||||
private bool _orig_schema_has_timestamp = false;
|
||||
private void _onstream_change(object o) {
|
||||
DataPacket dp = o as DataPacket;
|
||||
DateTime now = DateTime.Now;
|
||||
if (_table == null) {
|
||||
IBlueTableSchema schema = new BlueTableSchema(dp.newRows.schema.columnInfo, dp.newRows.schema.tableName);
|
||||
schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
|
||||
if (!schema.contains_column("timestamp")) {
|
||||
_orig_schema_has_timestamp = true;
|
||||
schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
|
||||
}
|
||||
_table = new CircularBufferTable(schema, _size);
|
||||
}
|
||||
foreach (IBlueRow r in dp.addedRows) {
|
||||
MutableBlueRow ar = new MutableBlueRow(_table.schema, r);
|
||||
ar.setValue("timestamp", now);
|
||||
if (!_orig_schema_has_timestamp)
|
||||
ar.setValue("timestamp", now);
|
||||
ar.setValue("id", _id++);
|
||||
_table.add_row(ar);
|
||||
}
|
||||
var dp_out = BlueDataUtility.build_dp(_table, myLastValue as DataPacket);
|
||||
var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
|
||||
myFireEvent(dp_out);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,16 +23,16 @@ public static class DataStreamExtensions {
|
||||
public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source, string? name = null, ILogger? logger = null) where T : IBlueRow =>
|
||||
source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)), name, logger);
|
||||
|
||||
public static IObservable<IDataPacket> AsObservable(this ViewStream source) {
|
||||
return source as IObservable<IDataPacket> ?? Observable.Create<IDataPacket>(Subscribe);
|
||||
public static IObservable<DataPacket> AsObservable(this ViewStream source) {
|
||||
return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
|
||||
|
||||
IDisposable Subscribe(IObserver<IDataPacket> observer) {
|
||||
IDisposable Subscribe(IObserver<DataPacket> observer) {
|
||||
source.onChange += OnChange;
|
||||
return Disposable.Create(() => source.onChange -= OnChange);
|
||||
|
||||
void OnChange(object? newValue) =>
|
||||
observer.OnNext(newValue as IDataPacket ?? throw new ArgumentException(
|
||||
$"Expected IDataPacket, got {newValue?.GetType()}", nameof(newValue)));
|
||||
observer.OnNext(newValue as DataPacket ?? throw new ArgumentException(
|
||||
$"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -88,7 +88,8 @@ namespace BCG {
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
void _on_str(IDataPacket dp) {
|
||||
private void _on_str(DataPacket dp)
|
||||
{
|
||||
foreach (IBlueRow r in dp.newRows)
|
||||
{
|
||||
string name = r.get<string>("name");
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
// using System;
|
||||
// using BCG;
|
||||
// using BCG.Utilities;
|
||||
|
||||
// namespace BCG
|
||||
// {
|
||||
// public class CartesianProductStream : ViewStream
|
||||
// {
|
||||
// public CartesianProductStream(ViewStream v1, ViewStream v2)
|
||||
// {
|
||||
// mySetRealSource(
|
||||
// new JoinStream (new AddColumnsStream (v1,
|
||||
// "hiddenCartesianColumn=1"),
|
||||
// "hiddenCartesianColumn",
|
||||
// "!id hiddenCartesianColumn dateTime:dateBoundary",
|
||||
// new AddColumnsStream (v2,
|
||||
// "hiddenCartesianColumn=1"),
|
||||
// "hiddenCartesianColumn",
|
||||
// "!id hiddenCartesianColumn"));
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
@@ -14,8 +14,10 @@ namespace BCG
|
||||
public DistinctRowEvaluator(string colMapExpression, string[] columnNames)
|
||||
: base (colMapExpression, null, columnNames) {}
|
||||
|
||||
public override IBlueRow? evaluate(IDataPacket packet) {
|
||||
myOutputTable ??= new GroupByOutputTable(packet.Schema);
|
||||
|
||||
public override IBlueRow evaluate (DataPacket packet)
|
||||
{
|
||||
myOutputTable = myOutputTable == null ? new GroupByOutputTable (packet.schema) : myOutputTable;
|
||||
string hashString = GroupByStream.getRepString (packet, myGroupByCols);
|
||||
if (packet.newRows.Count == 0)
|
||||
//if there are no rows for this group we remove the group from the output table
|
||||
|
||||
@@ -42,7 +42,7 @@ namespace BCG
|
||||
return retVal;
|
||||
}
|
||||
|
||||
public abstract IBlueRow? evaluate(IDataPacket dp);
|
||||
public abstract IBlueRow evaluate (DataPacket dp) ;
|
||||
#endregion
|
||||
|
||||
#region Helper methods
|
||||
|
||||
@@ -8,13 +8,12 @@ namespace BCG {
|
||||
internal class GroupByStream : MethodDrivenStream {
|
||||
private class MyFuncGroupByEvaluator : IGroupByEvaluator {
|
||||
private string[] _groupby_cols;
|
||||
readonly Func<IDataPacket, IBlueRow> _groupby_func;
|
||||
public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<IDataPacket, IBlueRow> groupby_func) {
|
||||
private Func<DataPacket, IBlueRow> _groupby_func;
|
||||
public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func) {
|
||||
_groupby_cols = groupby_cols.ToArray();
|
||||
_groupby_func = groupby_func;
|
||||
}
|
||||
public IBlueRow evaluate(IDataPacket packet) =>
|
||||
_groupby_func(packet);
|
||||
public IBlueRow evaluate(DataPacket packet) { return _groupby_func(packet); }
|
||||
|
||||
public string[] groupByColumns { get { return _groupby_cols; } }
|
||||
|
||||
@@ -33,7 +32,7 @@ namespace BCG {
|
||||
private IBlueTableSchema myOutputSchema;
|
||||
private int myIdGenerator = 0;
|
||||
|
||||
public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<IDataPacket, IBlueRow> groupby_func)
|
||||
public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
|
||||
: this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func)) { }
|
||||
|
||||
public GroupByStream(ViewStream stream, IGroupByEvaluator groupByEvaluator)
|
||||
@@ -188,11 +187,11 @@ namespace BCG {
|
||||
}
|
||||
}
|
||||
|
||||
public static string getRepString(IDataPacket dp, string[] colNames) {
|
||||
public static string getRepString(DataPacket dp, string[] colNames) {
|
||||
return BlueDataUtility.rowToString(getRepRow(dp), colNames);
|
||||
}
|
||||
|
||||
public static IBlueRow getRepRow(IDataPacket dp) {
|
||||
public static IBlueRow getRepRow(DataPacket dp) {
|
||||
if (dp.newRows.Count > 0)
|
||||
return BlueDataUtility.extractRow(dp.newRows, 0);
|
||||
else if (dp.addedRows.Count > 0)
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
using System;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
/// <summary>
|
||||
@@ -12,7 +14,7 @@ namespace BCG
|
||||
/// deletetion) and the added and deletedRows being the increments. If a group has
|
||||
/// to be totally removed then a null should be returned
|
||||
/// </summary>
|
||||
IBlueRow? evaluate(IDataPacket packet);
|
||||
IBlueRow evaluate (DataPacket packet);
|
||||
|
||||
/// <summary>
|
||||
/// Returns the columns (belonging to the original schema) on which this IGroupByEvaluator
|
||||
|
||||
@@ -14,8 +14,9 @@ namespace BCG {
|
||||
public MaxGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
|
||||
: base(colMapEpression, outputCol, groupByCols) { }
|
||||
|
||||
public override IBlueRow? evaluate(IDataPacket dp) {
|
||||
myOutputTable ??= new GroupByOutputTable(dp.Schema);
|
||||
public override IBlueRow evaluate(DataPacket dp) {
|
||||
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) :
|
||||
myOutputTable;
|
||||
string hashString = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
|
||||
IBlueRow currMaxRow = myOutputTable.getRow(hashString);
|
||||
IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
using System;
|
||||
|
||||
using BCG.Utilities;
|
||||
|
||||
namespace BCG {
|
||||
public class MinGroupByEvaluator : MaxGroupByEvaluator {
|
||||
public MinGroupByEvaluator(string outputCol, string[] groupByCols)
|
||||
@@ -6,8 +10,8 @@ namespace BCG {
|
||||
public MinGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) :
|
||||
base(colMapEpression, outputCol, groupByCols) { }
|
||||
|
||||
public override IBlueRow? evaluate(IDataPacket dp) {
|
||||
myOutputTable ??= new GroupByOutputTable(dp.Schema);
|
||||
public override IBlueRow evaluate(DataPacket dp) {
|
||||
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
|
||||
string stringRep = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
|
||||
IBlueRow currMinRow = myOutputTable.getRow(stringRep);
|
||||
IBlueRow minRow = BlueDataUtility.computeMinRow(dp.deletedRows, myOutputColumn);
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
using System;
|
||||
|
||||
namespace BCG {
|
||||
/// <summary>
|
||||
/// Computes the sum for a given column
|
||||
@@ -8,8 +10,8 @@ namespace BCG {
|
||||
public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
|
||||
: base(colMapEpression, outputCol, groupByCols) { }
|
||||
|
||||
public override IBlueRow evaluate(IDataPacket dp) {
|
||||
myOutputTable ??= new GroupByOutputTable(dp.Schema);
|
||||
public override IBlueRow evaluate(DataPacket dp) {
|
||||
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
|
||||
|
||||
IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
|
||||
return sumRow;
|
||||
@@ -31,7 +33,7 @@ namespace BCG {
|
||||
|
||||
#region IGroupByEvaluator Members
|
||||
|
||||
public IBlueRow evaluate(IDataPacket packet) => BlueDataUtility.computeSum(packet.newRows, myOutputCols);
|
||||
public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum(packet.newRows, myOutputCols); }
|
||||
|
||||
public string[] groupByColumns { get { return myGroupByCols; } }
|
||||
|
||||
@@ -49,8 +51,9 @@ namespace BCG {
|
||||
public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
|
||||
: base(colMapEpression, outputCol, groupByCols) { }
|
||||
|
||||
public override IBlueRow evaluate(IDataPacket dp) {
|
||||
myOutputTable ??= new GroupByOutputTable(dp.Schema);
|
||||
public override IBlueRow evaluate(DataPacket dp) {
|
||||
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
|
||||
|
||||
IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
|
||||
return sumRow;
|
||||
}
|
||||
@@ -65,7 +68,7 @@ namespace BCG {
|
||||
|
||||
#region IGroupByEvaluator Members
|
||||
|
||||
public IBlueRow evaluate(IDataPacket packet) => BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols);
|
||||
public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols); }
|
||||
|
||||
public string[] groupByColumns { get { return myGroupByCols; } }
|
||||
|
||||
|
||||
@@ -2,8 +2,6 @@ using BCG;
|
||||
|
||||
using MongoDB.Driver.Core.WireProtocol.Messages;
|
||||
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
@@ -39,7 +37,7 @@ namespace bts {
|
||||
}
|
||||
|
||||
void _handle_stream_update(object dp_obj) {
|
||||
var dp = ExceptionAssert.Argument.Is<IDataPacket>(dp_obj);
|
||||
DataPacket dp = dp_obj as DataPacket;
|
||||
IDictionary<int, List<IBlueRow>> groups = !String.IsNullOrEmpty(_group_by_col) ?
|
||||
BlueDataUtility.group_together(dp.newRows, new string[] { _group_by_col })
|
||||
:
|
||||
|
||||
@@ -8,48 +8,34 @@ using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using bts;
|
||||
using BCG;
|
||||
|
||||
namespace bts
|
||||
{
|
||||
namespace BlueOrb.Pubsub.Diagnostics {
|
||||
// This publishes data to the pubsub server, subscribes, gets it back and records the time it took for the round trip
|
||||
[DataStreamUserRespresentation("pss_echo")]
|
||||
public class PubsubEchoTestStream : ViewStream, IWorker
|
||||
{
|
||||
public class PayloadMessage
|
||||
{
|
||||
public class PubsubEchoTestStream : ViewStream, IWorker {
|
||||
public class PayloadMessage {
|
||||
public readonly string? msg_id;
|
||||
public readonly Int64 timestamp = 0;
|
||||
public readonly byte[]? payload = null;
|
||||
|
||||
public static byte[] create_data(int payload_size, byte payload_byte)
|
||||
{
|
||||
public static byte[] create_data(int payload_size, byte payload_byte) {
|
||||
var rv = new byte[payload_size];
|
||||
for (int i = 0; i < payload_size; i++)
|
||||
{
|
||||
for (int i = 0; i < payload_size; i++) {
|
||||
rv[i] = payload_byte;
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
|
||||
// public PayloadMessage(string guid_, Int64 timestamp_, int payload_size, byte payload_byte)
|
||||
// {
|
||||
// guid = guid_;
|
||||
// timestamp = timestamp_;
|
||||
// payload = new byte[payload_size];
|
||||
// for (int i = 0; i < payload_size; i++)
|
||||
// {
|
||||
// payload[i] = payload_byte;
|
||||
// }
|
||||
// }
|
||||
public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_)
|
||||
{
|
||||
public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_) {
|
||||
msg_id = guid_;
|
||||
timestamp = timestamp_;
|
||||
payload = payload_;
|
||||
}
|
||||
|
||||
private PayloadMessage() { }
|
||||
public byte[] serialize()
|
||||
{
|
||||
public byte[] serialize() {
|
||||
var str = new MemoryStream();
|
||||
BinaryWriter bw = new BinaryWriter(str);
|
||||
bw.Write(msg_id ?? string.Empty);
|
||||
@@ -61,8 +47,7 @@ namespace bts
|
||||
return bytes;
|
||||
}
|
||||
|
||||
public static PayloadMessage deserialize(byte[] ser_bytes)
|
||||
{
|
||||
public static PayloadMessage deserialize(byte[] ser_bytes) {
|
||||
var str = new MemoryStream(ser_bytes) {
|
||||
Position = 0
|
||||
};
|
||||
@@ -87,8 +72,7 @@ namespace bts
|
||||
private readonly string _tag_name;
|
||||
private readonly int _payload_sz;
|
||||
private readonly IPubSubRouter _router;
|
||||
private enum IOState
|
||||
{
|
||||
private enum IOState {
|
||||
SEND = 0,
|
||||
RECV = 1
|
||||
};
|
||||
@@ -123,8 +107,7 @@ namespace bts
|
||||
public PubsubEchoTestStream(string server_url, int publish_period_msecs, int payload_size, string tag_name)
|
||||
: this(server_url, TimeSpan.FromMilliseconds(publish_period_msecs), payload_size, tag_name, ViewStreamDatabase.instance) { }
|
||||
|
||||
protected override void myOnFirstSubscriber()
|
||||
{
|
||||
protected override void myOnFirstSubscriber() {
|
||||
_pub_core = PublisherCore.Create(_router);
|
||||
_sub_core = new SubscriberCore(_router);
|
||||
_sub_core.subscribe(_tag_name);
|
||||
@@ -132,8 +115,7 @@ namespace bts
|
||||
|
||||
}
|
||||
|
||||
protected override void myOnLastSubscriber()
|
||||
{
|
||||
protected override void myOnLastSubscriber() {
|
||||
_vsd.removeWorker(this);
|
||||
Log.notnull(_sub_core).unsubscribe(_tag_name);
|
||||
}
|
||||
@@ -144,8 +126,7 @@ namespace bts
|
||||
SubscriberRoutes = new() { new(serverUrl, ".*") },
|
||||
});
|
||||
|
||||
private void _fire_dp_after_send()
|
||||
{
|
||||
private void _fire_dp_after_send() {
|
||||
Log.notnull(_last_sent_msg);
|
||||
// as soon as the data is published, its reflected in the stream state
|
||||
BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value,
|
||||
@@ -155,8 +136,7 @@ namespace bts
|
||||
myFireEvent(dp_out);
|
||||
}
|
||||
|
||||
private void _fire_dp_after_recv()
|
||||
{
|
||||
private void _fire_dp_after_recv() {
|
||||
Log.notnull(_last_recvd_msg);
|
||||
Log.notnull(_last_sent_msg);
|
||||
// update the last publish row
|
||||
@@ -173,13 +153,11 @@ namespace bts
|
||||
}
|
||||
}
|
||||
|
||||
private string _get_next_id()
|
||||
{
|
||||
private string _get_next_id() {
|
||||
return _table.Count.ToString();
|
||||
}
|
||||
|
||||
private void _publish_next_packet()
|
||||
{
|
||||
private void _publish_next_packet() {
|
||||
Log.notnull(_pub_core);
|
||||
var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte);
|
||||
_last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes);
|
||||
@@ -188,13 +166,10 @@ namespace bts
|
||||
_last_publish_time = DateTime.Now;
|
||||
}
|
||||
|
||||
public void doWork()
|
||||
{
|
||||
switch (_state)
|
||||
{
|
||||
public void doWork() {
|
||||
switch (_state) {
|
||||
case IOState.SEND:
|
||||
if (DateTime.Now - _last_publish_time >= _publish_period)
|
||||
{
|
||||
if (DateTime.Now - _last_publish_time >= _publish_period) {
|
||||
_publish_next_packet();
|
||||
_last_publish_time = DateTime.Now;
|
||||
_fire_dp_after_send();
|
||||
@@ -206,8 +181,7 @@ namespace bts
|
||||
var updates = Log.notnull(_sub_core).get_updates(TimeSpan.Zero);
|
||||
updates = updates.Where(u => u.tag == _tag_name);
|
||||
var recv_msg = find_packet(updates);
|
||||
if (recv_msg != null)
|
||||
{
|
||||
if (recv_msg != null) {
|
||||
_last_recvd_msg = recv_msg;
|
||||
_fire_dp_after_recv();
|
||||
_state = IOState.SEND;
|
||||
@@ -230,8 +204,7 @@ namespace bts
|
||||
}
|
||||
}
|
||||
|
||||
public static long epoch_now()
|
||||
{
|
||||
public static long epoch_now() {
|
||||
return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
|
||||
}
|
||||
|
||||
|
||||
@@ -1,12 +1,17 @@
|
||||
namespace bts {
|
||||
using BCG;
|
||||
|
||||
using bts.collections;
|
||||
|
||||
using IronPython.Runtime.Operations;
|
||||
|
||||
using ProtoBuf;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.Intrinsics.Arm;
|
||||
|
||||
public class ProjectData {
|
||||
// schema from the base stream
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
namespace BCG {
|
||||
namespace BlueOrb.PubSub {
|
||||
using BCG;
|
||||
// this for testing purposes mainly, its wraps a ViewPublisher around a stream,
|
||||
// and hence epublishes it out to the PS
|
||||
public class PublisherStream : ViewStream {
|
||||
|
||||
@@ -15,6 +15,7 @@ using mbts;
|
||||
using BCG.Extensions;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using Newtonsoft.Json;
|
||||
using BCG;
|
||||
using System.Runtime.CompilerServices;
|
||||
using bts;
|
||||
@@ -359,11 +360,11 @@ namespace BCG {
|
||||
}
|
||||
|
||||
public static Dictionary<string, string> convert_json_to_dictionary(string json_string) {
|
||||
Dictionary<string, string> dictionary = JsonSerializer.Deserialize<Dictionary<string, string>>(json_string);
|
||||
Dictionary<string, string> dictionary = JsonConvert.DeserializeObject<Dictionary<string, string>>(json_string);
|
||||
return dictionary;
|
||||
}
|
||||
public static IBlueRow convert_json_to_row(string json_string, int id = 0) {
|
||||
Dictionary<string, object> dictionary = JsonSerializer.Deserialize<Dictionary<string, object>>(json_string);
|
||||
Dictionary<string, object> dictionary = JsonConvert.DeserializeObject<Dictionary<string, object>>(json_string);
|
||||
if (!dictionary.ContainsKey("id")) {
|
||||
dictionary.Add("id", id);
|
||||
}
|
||||
@@ -1846,7 +1847,7 @@ namespace BCG {
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
public static string id_string(IBlueRowCollection table) {
|
||||
public static string rows_id_string(this IBlueRowCollection table) {
|
||||
string rv = String.Empty;
|
||||
foreach (IBlueRow r in table)
|
||||
rv += $"{r["id"]} ";
|
||||
@@ -2245,7 +2246,7 @@ namespace BCG {
|
||||
|
||||
public static T get_row_value<T>(IBlueRow r, string col_name, T error_val) {
|
||||
try {
|
||||
object rv = (r != null && r.Contains(col_name)) ? r[col_name] : DBNull.Value;
|
||||
object rv = (r != null && r.schema.contains_column(col_name)) ? r[col_name] : DBNull.Value;
|
||||
if (rv == null || rv == DBNull.Value)
|
||||
return error_val;
|
||||
else
|
||||
|
||||
@@ -1,6 +1,4 @@
|
||||
#nullable enable
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using System;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
@@ -9,7 +7,6 @@ namespace BCG {
|
||||
/// Resprsents information about a column schema, ie its ordinal, the type, the name and constant value or expression. where applicable
|
||||
/// </summary>
|
||||
[Serializable]
|
||||
[JsonConverter(typeof(ColumnInfoJsonConverter))]
|
||||
public class ColumnInfo : IComparable<ColumnInfo>, IComparable, IEquatable<ColumnInfo> {
|
||||
|
||||
//the default column info does not cach values in a lookup table, therefore it is in group 0
|
||||
@@ -70,12 +67,16 @@ namespace BCG {
|
||||
private Type myColType;
|
||||
private object? myColValue;
|
||||
|
||||
[JsonPropertyName("colNo")]
|
||||
public int colNo { get { return myColNo; } }
|
||||
|
||||
[JsonPropertyName("colName")]
|
||||
public string colName { get { return myColName; } }
|
||||
|
||||
[JsonPropertyName("colType")]
|
||||
public Type colType { get { return myColType; } }
|
||||
|
||||
[JsonIgnore]
|
||||
public object? colValue {get{return myColValue;}}
|
||||
}
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
namespace Bts.DataStreamCore;
|
||||
|
||||
using BCG;
|
||||
|
||||
using Spark.Core.Json;
|
||||
|
||||
using System;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
public class ColumnInfoJsonConverter : JsonConverter<ColumnInfo> {
|
||||
public override ColumnInfo Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
|
||||
var jobj = JsonSerializer.Deserialize<JsonObject>(ref reader);
|
||||
return new(
|
||||
jobj[nameof(ColumnInfo.colName)].GetValue<string>(),
|
||||
jobj[nameof(ColumnInfo.colNo)].GetValue<int>(),
|
||||
Type.GetType(jobj[nameof(ColumnInfo.colType)].GetValue<string>()));
|
||||
}
|
||||
|
||||
public override void Write(Utf8JsonWriter writer, ColumnInfo value, JsonSerializerOptions options) {
|
||||
using var _ = writer.WriteObject();
|
||||
writer.WriteString(nameof(ColumnInfo.colName), value.colName);
|
||||
writer.WriteNumber(nameof(ColumnInfo.colNo), value.colNo);
|
||||
writer.WriteString(nameof(ColumnInfo.colType), value.colType.ToString());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
namespace BlueOrb.Data {
|
||||
using BCG;
|
||||
|
||||
using bts;
|
||||
|
||||
|
||||
using System;
|
||||
using System.Linq;
|
||||
|
||||
public static class Comparisons {
|
||||
public static int compare_row(this IBlueRow row, IBlueRow other_row, params string[] col_names) {
|
||||
int comp_res = 0;
|
||||
foreach (string col_name in col_names) {
|
||||
object val1 = row[col_name];
|
||||
object val2 = other_row[col_name];
|
||||
comp_res = compare_row_values(val1, val2);
|
||||
if (comp_res != 0)
|
||||
break;
|
||||
}
|
||||
return comp_res;
|
||||
}
|
||||
|
||||
public static void compare_tables_id(this IBlueRowCollection table1,
|
||||
IBlueRowCollection table2,
|
||||
out IBlueRowCollection rows_only_in_table1,
|
||||
out IBlueRowCollection rows_only_in_table2) {
|
||||
Log.assert(table1 != null);
|
||||
table2 = table2 ?? new BlueTable(table1.schema);
|
||||
var schema = get_output_schema(table1, table2);
|
||||
var table1_only = new BlueTable(schema);
|
||||
var table2_only = new BlueTable(schema);
|
||||
|
||||
if (table1 is not null) {
|
||||
foreach (IBlueRow row in table1)
|
||||
if (table2 is null || !table2.containsID(row.get<int>("id")))
|
||||
table1_only.addRow(row);
|
||||
}
|
||||
|
||||
if (table2 is not null) {
|
||||
foreach (IBlueRow row in table2)
|
||||
if (table1 is null || !table1.containsID(row.get<int>("id")))
|
||||
table2_only.addRow(row);
|
||||
}
|
||||
|
||||
rows_only_in_table1 = table1_only;
|
||||
rows_only_in_table2 = table2_only;
|
||||
}
|
||||
|
||||
public static void compare_tables_deep(this IBlueRowCollection table1,
|
||||
IBlueRowCollection table2,
|
||||
out IBlueRowCollection rows_only_in_table1,
|
||||
out IBlueRowCollection rows_only_in_table2) {
|
||||
Log.assert(table1 != null);
|
||||
table2 = table2 ?? new BlueTable(table1.schema);
|
||||
var schema = get_output_schema(table1, table2);
|
||||
validate_matching_schemas(table1, table2);
|
||||
|
||||
var table1_only = new BlueTable(schema);
|
||||
var table2_only = new BlueTable(schema);
|
||||
var compare_cols = schema.columnNames.Where(c => c != "id").ToArray();
|
||||
|
||||
if (table1 is not null) {
|
||||
foreach (IBlueRow row in table1) {
|
||||
if (table2 is null || !table2.containsID(row.get<int>("id"))) {
|
||||
table1_only.addRow(row);
|
||||
continue;
|
||||
}
|
||||
|
||||
var other_row = table2[row.get<int>("id")];
|
||||
if (row.compare_row(other_row, compare_cols) != 0) {
|
||||
table1_only.addRow(row);
|
||||
table2_only.addRow(other_row);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (table2 is not null) {
|
||||
foreach (IBlueRow row in table2)
|
||||
if (table1 is null || !table1.containsID(row.get<int>("id")))
|
||||
table2_only.addRow(row);
|
||||
}
|
||||
|
||||
rows_only_in_table1 = table1_only;
|
||||
rows_only_in_table2 = table2_only;
|
||||
}
|
||||
|
||||
static int compare_row_values(object v1, object v2) {
|
||||
if (is_null(v1))
|
||||
return is_null(v2) ? 0 : -1;
|
||||
if (is_null(v2))
|
||||
return 1;
|
||||
try {
|
||||
var comparable1 = (IComparable)v1;
|
||||
return comparable1.CompareTo(v2);
|
||||
} catch (InvalidCastException) {
|
||||
if (v1 is byte[] bytes1 && v2 is byte[] bytes2)
|
||||
return compare_bytes(bytes1, bytes2) ? 0 : 1;
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
static bool compare_bytes(byte[] v1, byte[] v2) {
|
||||
if (v1.Length != v2.Length)
|
||||
return false;
|
||||
|
||||
for (int i = 0; i < v1.Length; i++)
|
||||
if (v1[i] != v2[i])
|
||||
return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool is_null(object o) {
|
||||
return o == null || o == DBNull.Value;
|
||||
}
|
||||
|
||||
static IBlueTableSchema get_output_schema(IBlueRowCollection table1, IBlueRowCollection table2) {
|
||||
if (table1 is not null)
|
||||
return table1.schema;
|
||||
if (table2 is not null)
|
||||
return table2.schema;
|
||||
throw new ArgumentNullException(nameof(table1), "At least one table must be non-null.");
|
||||
}
|
||||
|
||||
static void validate_matching_schemas(IBlueRowCollection table1, IBlueRowCollection table2) {
|
||||
if (table1 is null || table2 is null)
|
||||
return;
|
||||
|
||||
var schema1 = table1.schema;
|
||||
var schema2 = table2.schema;
|
||||
if (schema1.columnInfo.Length != schema2.columnInfo.Length)
|
||||
throw new ArgumentException("Tables must have matching schemas.");
|
||||
|
||||
for (int i = 0; i < schema1.columnInfo.Length; i++) {
|
||||
var c1 = schema1.columnInfo[i];
|
||||
var c2 = schema2.columnInfo[i];
|
||||
if (c1.colName != c2.colName || c1.colType != c2.colType)
|
||||
throw new ArgumentException("Tables must have matching schemas.");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
namespace BlueOrb.Data {
|
||||
using BCG;
|
||||
|
||||
using System;
|
||||
using System.Text;
|
||||
|
||||
public static class ToStrings {
|
||||
public static string ToDebugString(this IBlueRow row) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
foreach (ColumnInfo c in row.schema.columnInfo) {
|
||||
if (sb.Length != 0)
|
||||
sb.Append(", ");
|
||||
sb.Append(c.colName);
|
||||
sb.Append(": ");
|
||||
object val = row[c.colName];
|
||||
sb.Append(is_null(val) ? "NULL" : val.ToString());
|
||||
}
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
public static string ToDebugString(this IBlueRowCollection rows) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
foreach (IBlueRow row in rows) {
|
||||
if (sb.Length != 0)
|
||||
sb.AppendLine();
|
||||
sb.Append(row.ToDebugString());
|
||||
}
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
static bool is_null(object o) {
|
||||
return o == null || o == DBNull.Value;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -41,7 +41,7 @@ namespace BCG {
|
||||
}
|
||||
|
||||
public IBlueRowCollection newRows, addedRows, deletedRows;
|
||||
public Dictionary<string, object> meta_data = null;
|
||||
public IDictionary<string, object> meta_data = null;
|
||||
public DataPacket() { }
|
||||
|
||||
public DataPacket(IBlueTableSchema schema)
|
||||
@@ -53,7 +53,7 @@ namespace BCG {
|
||||
this.deletedRows = deletedRows;
|
||||
}
|
||||
public DataPacket(IBlueRowCollection newRows, IBlueRowCollection addedRows, IBlueRowCollection deletedRows,
|
||||
Dictionary<string, object> meta_data) {
|
||||
IDictionary<string, object> meta_data) {
|
||||
this.newRows = newRows;
|
||||
this.addedRows = addedRows;
|
||||
this.deletedRows = deletedRows;
|
||||
@@ -63,7 +63,6 @@ namespace BCG {
|
||||
IBlueRowCollection IDataPacket.newRows => newRows;
|
||||
IBlueRowCollection IDataPacket.addedRows => addedRows;
|
||||
IBlueRowCollection IDataPacket.deletedRows => deletedRows;
|
||||
IReadOnlyDictionary<string, object> IDataPacket.Metadata => meta_data;
|
||||
|
||||
public void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows) {
|
||||
new_rows = newRows;
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Google.Protobuf" Version="3.24.3" />
|
||||
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
|
||||
<PackageReference Include="OpenAI" Version="2.0.0" />
|
||||
<PackageReference Include="SimMetrics.Net" Version="1.0.5" />
|
||||
</ItemGroup>
|
||||
|
||||
@@ -6,89 +6,89 @@ namespace BCG {
|
||||
using System.Reflection;
|
||||
using System.IO;
|
||||
using Bts.DataStreamCore;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
public class PrimitiveManager {
|
||||
static MarketDataStreamFactory mdStreamFactory;
|
||||
private IDictionary myTypeLookupTable = new Hashtable();
|
||||
private static PrimitiveManager mySelf;
|
||||
private ArrayList myAssemblyNames = new ArrayList();
|
||||
private Dictionary<string, Type> _dslopname_to_type_map = new Dictionary<string, Type>();
|
||||
private static PrimitiveManager _self;
|
||||
private HashSet<string> _loaded_assemblies = new HashSet<string>();
|
||||
private static string[] _allowed_namespaces_matches = new string[] {
|
||||
"blueorb"
|
||||
};
|
||||
|
||||
private PrimitiveManager() {
|
||||
myLoadAllTypes();
|
||||
_load_all_streams();
|
||||
}
|
||||
|
||||
private void myLoadAllTypes() {
|
||||
AppDomain currentDomain = AppDomain.CurrentDomain;
|
||||
currentDomain.AssemblyResolve += new ResolveEventHandler(myAssemblyResolver);
|
||||
loadAssembly(this.GetType().Assembly.FullName);
|
||||
var assemblies = new string[] { "DataStream.dll", "SharedStreams.dll" };
|
||||
foreach (string assemblyName in assemblies) {
|
||||
bts.Log.Gen.info($"PrimitiveManager.myLoadAllTypes assemblyName: {assemblyName}");
|
||||
string resolvedAssemblyPath = assemblyName;
|
||||
if (!System.IO.Path.IsPathRooted(assemblyName)) {
|
||||
resolvedAssemblyPath = Path.GetFullPath(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, assemblyName));
|
||||
private static bool _is_loadable_stream(Type type) {
|
||||
Func<bool> namespace_match = () => {
|
||||
foreach (string ns in _allowed_namespaces_matches) {
|
||||
if (type.Namespace.ToLower().Contains(ns.ToLower()))
|
||||
return true;
|
||||
}
|
||||
loadAssembly(resolvedAssemblyPath);
|
||||
return false;
|
||||
};
|
||||
bool rv = type.IsSubclassOf(typeof(ViewStream))
|
||||
&& type.IsPublic;
|
||||
//rv = rv && (!type.IsInterface && !type.IsAbstract);
|
||||
rv = rv && namespace_match();
|
||||
return rv;
|
||||
}
|
||||
|
||||
private void _load_all_streams() {
|
||||
load_streams_from_assembly(this.GetType().Assembly.FullName);
|
||||
var assemblies = new string[] { "DataStream.dll", "SharedStreams.dll" };
|
||||
foreach (string assembly_name in assemblies) {
|
||||
bts.Log.Gen.info($"PrimitiveManager._load_all_streams assemblyName: {assembly_name}");
|
||||
string resolved_path_name = assembly_name;
|
||||
if (!System.IO.Path.IsPathRooted(assembly_name)) {
|
||||
resolved_path_name = Path.GetFullPath(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, assembly_name));
|
||||
}
|
||||
load_streams_from_assembly(resolved_path_name);
|
||||
}
|
||||
}
|
||||
|
||||
private static Assembly myAssemblyResolver(object sender, ResolveEventArgs args) {
|
||||
foreach (Assembly a in AppDomain.CurrentDomain.GetAssemblies())
|
||||
if (args.Name.Equals(a.FullName))
|
||||
return a;
|
||||
return null;
|
||||
public Type get_dsl_op_type(string dsl_op_name) {
|
||||
Type rv;
|
||||
if (!_dslopname_to_type_map.TryGetValue(dsl_op_name, out rv))
|
||||
return null;
|
||||
return rv;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Loads all the Stream types associated with the assembly
|
||||
/// </summary>
|
||||
/// <param name="assemblyName"></param>
|
||||
public void loadAssembly(string assemblyName) {
|
||||
bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assemblyName}");
|
||||
if (myAssemblyNames.Contains(assemblyName))
|
||||
public void load_streams_from_assembly(string assembly_name) {
|
||||
bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assembly_name}");
|
||||
if (_loaded_assemblies.Contains(assembly_name))
|
||||
return;
|
||||
myAssemblyNames.Add(assemblyName);
|
||||
Assembly loadedAssembly = GetType().Assembly.FullName.Equals(assemblyName) ?
|
||||
_loaded_assemblies.Add(assembly_name);
|
||||
Assembly loaded_assembly = GetType().Assembly.FullName.Equals(assembly_name) ?
|
||||
GetType().Assembly :
|
||||
Assembly.LoadFrom(assemblyName);
|
||||
Assembly.LoadFrom(assembly_name);
|
||||
|
||||
foreach (Type type in loadedAssembly.GetTypes())
|
||||
if (!type.IsInterface
|
||||
&& !type.IsAbstract
|
||||
&& type.IsSubclassOf(typeof(ViewStream))
|
||||
&& type.IsPublic) {
|
||||
bool isVisible = true;
|
||||
string userRep = createDefaultName(type.Name);
|
||||
foreach (Type type in loaded_assembly.GetTypes())
|
||||
if (_is_loadable_stream(type)) {
|
||||
bool is_visible = true;
|
||||
string dsl_name = type.Name; //createDefaultName(type.Name);
|
||||
foreach (Attribute att in type.GetCustomAttributes(false))
|
||||
if (att is DataStreamUserRespresentation) {
|
||||
DataStreamUserRespresentation dsatt = att as DataStreamUserRespresentation;
|
||||
isVisible = dsatt.isVisible;
|
||||
userRep = isVisible ?
|
||||
dsatt.friendlyName == null ? userRep : dsatt.friendlyName
|
||||
is_visible = dsatt.isVisible;
|
||||
dsl_name = is_visible ?
|
||||
dsatt.friendlyName == null ? dsl_name : dsatt.friendlyName
|
||||
: null;
|
||||
break;
|
||||
}
|
||||
if (isVisible) {
|
||||
// Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName);
|
||||
myTypeLookupTable[userRep] = type;
|
||||
if (is_visible) {
|
||||
// Log.assert_d(myTypeLookupTable[userRep] == null, type.FullName);
|
||||
_dslopname_to_type_map[dsl_name] = type;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static string createDefaultName(string typeName) {
|
||||
if (!typeName.EndsWith("Stream"))
|
||||
return typeName;
|
||||
string retVal = typeName.Remove(typeName.Length - "Stream".Length, "Stream".Length);
|
||||
string startString = new String(new char[] { retVal[0] }).ToLower();
|
||||
retVal = retVal.Remove(0, 1);
|
||||
retVal = retVal.Insert(0, startString);
|
||||
return retVal;
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static PrimitiveManager instance {
|
||||
get { return mySelf = mySelf == null ? new PrimitiveManager() : mySelf; }
|
||||
get { return _self = _self == null ? new PrimitiveManager() : _self; }
|
||||
}
|
||||
|
||||
public static void SetMarketDataClient(object value) {
|
||||
@@ -96,31 +96,20 @@ namespace BCG {
|
||||
mdStreamFactory = new(value);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the Type name corresponding to the primitive name
|
||||
/// </summary>
|
||||
/// <param name="friendlyName"></param>
|
||||
/// <returns></returns>
|
||||
public string find(string friendlyName) {
|
||||
if (myTypeLookupTable[friendlyName] == null)
|
||||
throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types");
|
||||
return (myTypeLookupTable[friendlyName] as Type).Name;
|
||||
public string dsl_to_type_name(string dsl_op_name) {
|
||||
Type rv;
|
||||
if (!_dslopname_to_type_map.TryGetValue(dsl_op_name, out rv))
|
||||
throw new ApplicationException(dsl_op_name + " is not a loaded dsl op");
|
||||
return rv.Name;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Constructs and returns a ViewStream of type corresponding to the friendly name
|
||||
/// </summary>
|
||||
/// <param name="friendlyName">friendly primitive name </param>
|
||||
/// <param name="args">args can be null or an empty ICollection if the paramterless constructor
|
||||
/// is the desired one</param>
|
||||
/// <returns></returns>
|
||||
public ViewStream build(string friendlyName, ICollection args) {
|
||||
object[] constructorArgs = null;
|
||||
if (args != null && args.Count > 0) {
|
||||
constructorArgs = new object[args.Count];
|
||||
args.CopyTo(constructorArgs, 0);
|
||||
}
|
||||
var type = myTypeLookupTable[friendlyName] as Type
|
||||
var type = _dslopname_to_type_map[friendlyName] as Type
|
||||
?? throw new ApplicationException(friendlyName + " is not the friendly name of any of the loaded types");
|
||||
if (type.HasCustomAttribute<LiveMarketDataStreamAttribute>()) {
|
||||
if (mdStreamFactory is null) {
|
||||
@@ -135,30 +124,14 @@ namespace BCG {
|
||||
Activator.CreateInstance(type, constructorArgs) as ViewStream;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the name of all the primitives
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
public string[] getPrimitiveNames() {
|
||||
string[] retVal = new string[myTypeLookupTable.Count];
|
||||
myTypeLookupTable.Keys.CopyTo(retVal, 0);
|
||||
return retVal;
|
||||
public string[] get_dsl_op_names() {
|
||||
return _dslopname_to_type_map.Keys.ToArray();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Return the types of all the streams
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
public Type[] getStreamTypes() {
|
||||
Type[] retVal = new Type[myTypeLookupTable.Count];
|
||||
myTypeLookupTable.Values.CopyTo(retVal, 0);
|
||||
return retVal;
|
||||
}
|
||||
|
||||
public ICollection getConstructorSigns(string primitiveName) {
|
||||
public ICollection get_dsl_op_cons_args(string dsl_op_name) {
|
||||
ArrayList retVal = new ArrayList();
|
||||
Log.assert(myTypeLookupTable[primitiveName] != null);
|
||||
Type t = myTypeLookupTable[primitiveName] as Type;
|
||||
Type t = get_dsl_op_type(dsl_op_name);
|
||||
foreach (ConstructorInfo ci in t.GetConstructors()) {
|
||||
string sign = "";
|
||||
sign = t.Name + " (";
|
||||
@@ -172,9 +145,18 @@ namespace BCG {
|
||||
return retVal;
|
||||
}
|
||||
|
||||
public string getAssemblyName(string primitiveName) {
|
||||
Log.assert(myTypeLookupTable[primitiveName] != null);
|
||||
return (myTypeLookupTable[primitiveName] as Type).Assembly.GetName().Name;
|
||||
public string get_dslop_assembly_name(string dsl_op_name) {
|
||||
Type rv;
|
||||
if (!_dslopname_to_type_map.TryGetValue(dsl_op_name, out rv))
|
||||
throw new ApplicationException(dsl_op_name + " is not a loaded dsl op");
|
||||
return rv.Assembly.FullName;
|
||||
}
|
||||
|
||||
public string get_qualified_op_name(string dsl_op_name) {
|
||||
Type dsl_op_type = get_dsl_op_type(dsl_op_name);
|
||||
Log.assert(dsl_op_type != null);
|
||||
return $"{dsl_op_type.Namespace}.{dsl_op_name}";
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -13,7 +13,6 @@ namespace BCG {
|
||||
IBlueRowCollection newRows { get; }
|
||||
IBlueRowCollection addedRows { get; }
|
||||
IBlueRowCollection deletedRows { get; }
|
||||
IReadOnlyDictionary<string, object> Metadata { get; }
|
||||
//this is an ICollection of IDictionary
|
||||
void get_new_added_deleted(out ICollection new_rows, out ICollection added_rows, out ICollection deleted_rows);
|
||||
}
|
||||
|
||||
@@ -3,23 +3,23 @@ using System.IO;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
using BlueOrb.Data;
|
||||
using bts;
|
||||
using System.Threading;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
// This class is responsible for mantaining the state fo a table. It gets the
|
||||
// bytes from the pubsubserver via the handle_updates call (called by VSD), deserializes
|
||||
namespace BCG {
|
||||
// This class is responsible for mantaining the state for a table. It gets the
|
||||
// bytes from the pubsubserver via the handle_updates call
|
||||
// (called by VSD, - VSD is polling the SubscriberCOre for udpates), deserializes,
|
||||
// updates the state of the table, and fires the TableUpdateDelegate.
|
||||
public class TableAccumulator2 : ITableAccumulator
|
||||
{
|
||||
public class TableAccumulator2 : ITableAccumulator {
|
||||
private readonly string _tag;
|
||||
private readonly ISubscriberCore _subscriber_core;
|
||||
IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
|
||||
private BlueTable _table;
|
||||
private TableUpdateDelegate _on_update_delegate;
|
||||
|
||||
public TableAccumulator2(string tag, ISubscriberCore sub_core)
|
||||
{
|
||||
public TableAccumulator2(string tag, ISubscriberCore sub_core) {
|
||||
_tag = tag;
|
||||
_subscriber_core = sub_core;
|
||||
}
|
||||
@@ -28,61 +28,54 @@ namespace BCG
|
||||
|
||||
//public bool is_active() { return _on_update_delegate != null; }
|
||||
|
||||
public event TableUpdateDelegate on_update
|
||||
{
|
||||
add
|
||||
{
|
||||
public event TableUpdateDelegate on_update {
|
||||
add {
|
||||
Log.Gen.info("TableAccumulator2.on_update.add {_tag} subscribed");
|
||||
if (_on_update_delegate == null)
|
||||
{
|
||||
if (_on_update_delegate == null) {
|
||||
//ie first subscription
|
||||
Log.Gen.info("TableAccumulator2.on_update {_tag} subscribed for the first time");
|
||||
_subscriber_core.subscribe(_tag);
|
||||
}
|
||||
_on_update_delegate =
|
||||
Delegate.Combine(_on_update_delegate, value) as TableUpdateDelegate;
|
||||
// a new subsriber should get a full callback immediatly (new rows, 0 deleted, all added)
|
||||
if (_table != null)
|
||||
value(_table, _table, new BlueTable(_table.schema));
|
||||
}
|
||||
remove
|
||||
{
|
||||
remove {
|
||||
Log.Gen.info("TableAccumulator2.on_update.remove {_tag} unsubscribed");
|
||||
if (_on_update_delegate != null)
|
||||
{
|
||||
if (_on_update_delegate != null) {
|
||||
_on_update_delegate =
|
||||
Delegate.Remove(_on_update_delegate, value) as TableUpdateDelegate;
|
||||
int n_subs = _on_update_delegate == null ? 0 : _on_update_delegate.GetInvocationList().Length;
|
||||
Log.Gen.info("TableAccumulator2.on_update.remove {_tag} unsubscribed n_subs: {n_subs}");
|
||||
if (_on_update_delegate == null)
|
||||
{
|
||||
if (_on_update_delegate == null) {
|
||||
_subscriber_core.unsubscribe(_tag);
|
||||
// TODO
|
||||
//_table = null;
|
||||
_table = null;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
private int _update_no = 0;
|
||||
public void handle_updates(IEnumerable<byte[]> msgs)
|
||||
{
|
||||
public void handle_updates(IEnumerable<byte[]> msgs) {
|
||||
if (msgs == null || msgs.Count() == 0)
|
||||
return;
|
||||
Log.Gen.info($"TableAccumulator2.handle_updates _update_no: {_update_no++} msgs: {msgs.Count()} msg_0: {msgs.ElementAt(0).Length}");
|
||||
IBlueRowCollection table_0 = _table == null ? null : new BlueTable(_table);
|
||||
foreach (var msg in msgs)
|
||||
{
|
||||
bts.StreamIOMessage str_msg = _deser.deserialize(msg);
|
||||
int i = 0;
|
||||
foreach (var msg in msgs) {
|
||||
bts.StreamIOMessage new_msg_txt = _deser.deserialize(msg);
|
||||
var state_txt = _table == null ? "NULL" : _table.Count.ToString();// BlueDataUtility.id_string(_table);
|
||||
Log.Gen.info($"TableAccumulator2.handle_update tag: {_tag} sz: {msg.Length} str_msg: {str_msg} _state: {state_txt}");
|
||||
switch (str_msg.op)
|
||||
{
|
||||
Log.Gen.info($"TableAccumulator2.handle_update tag: {_tag} object_hash: {GetHashCode()} thread_id: {Thread.CurrentThread.ManagedThreadId} msg: {i++ / msgs.Count()} msg_sz: {msg.Length} new_msg_txt: {new_msg_txt} pre_update_count: {state_txt}");
|
||||
|
||||
switch (new_msg_txt.op) {
|
||||
case bts.StreamIOMessage.StreamOp.CREATE:
|
||||
_table = new BlueTable(str_msg.create_message.rows);
|
||||
_table = new BlueTable(new_msg_txt.create_message.rows);
|
||||
break;
|
||||
case bts.StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD:
|
||||
_table.deleteRows(str_msg.multiple_remove_add_message.removed_rows);
|
||||
_table.addRows(str_msg.multiple_remove_add_message.added_rows);
|
||||
_table.deleteRows(new_msg_txt.multiple_remove_add_message.removed_rows);
|
||||
_table.addRows(new_msg_txt.multiple_remove_add_message.added_rows);
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -91,11 +84,16 @@ namespace BCG
|
||||
IBlueRowCollection d_rows, a_rows;
|
||||
//BlueDataUtility.compute_differences_id_based(_table, table_0, out d_rows, out a_rows);
|
||||
BlueDataUtility.compute_differences_fast(_table, table_0, out d_rows, out a_rows);
|
||||
Log.Gen.info($"TableAccumulator2.handle_updates firing update, _table: {_table.rows_id_string()} d_rows: {d_rows.rows_id_string()} a_rows: {a_rows.rows_id_string()}");
|
||||
if (a_rows.Count == 0 && d_rows.Count == 0 && table_0 != null) {
|
||||
Log.Gen.info($"TableAccmulator2.handle_updates ex! _table: {_table.ToDebugString()} table_0: {table_0.ToDebugString()} a_rows: {a_rows.ToDebugString()} d_rows: {d_rows.ToDebugString()}");
|
||||
//same table was pubslished (maybe surpress it on the publisher or pubsubs)
|
||||
return;
|
||||
}
|
||||
_on_update_delegate?.Invoke(_table, a_rows, d_rows);
|
||||
}
|
||||
|
||||
public void update(Stream[] streams)
|
||||
{
|
||||
public void update(Stream[] streams) {
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
+1
-1
@@ -30,7 +30,7 @@ namespace bts {
|
||||
}
|
||||
}
|
||||
|
||||
// This class is created by the SingleTagData for stream tag_type. Every pub
|
||||
// This class is created by the SingleTagData for stream tag_type. Every DS tag has a PubsubServerSingleStreamTagHandler
|
||||
|
||||
public class PubsubServerSingleStreamTagHandler : IPubSubServerTagHandler {
|
||||
// this is kept around to deserielize the incoming messages and keep the state of the table
|
||||
|
||||
@@ -20,7 +20,7 @@ public class PrimitiveExpression : StatementExpression {
|
||||
}
|
||||
|
||||
public override string print(int startCol, IList<string> activeBindings) {
|
||||
string prelude = "new " + PrimitiveManager.instance.find(myName) + " (";
|
||||
string prelude = "new " + PrimitiveManager.instance.dsl_to_type_name(myName) + " (";
|
||||
|
||||
return prelude + myPrintArgs(startCol + prelude.Length, activeBindings) + ")";
|
||||
}
|
||||
|
||||
@@ -8,11 +8,11 @@ using Bts.DataStreamCore;
|
||||
using Bts.DataStreamExpressions;
|
||||
using Bts.PubSub;
|
||||
|
||||
using global::Spark.Core.Disposables;
|
||||
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
using Spark.Core.Disposables;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
"""
|
||||
Requires:
|
||||
pip install -U databento
|
||||
pip install websockets
|
||||
|
||||
Sources:
|
||||
https://databento.com/docs/quickstart/set-up
|
||||
https://databento.com/docs/schemas-and-data-formats/bbo
|
||||
"""
|
||||
|
||||
import datetime as dt
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
try:
|
||||
import databento as db
|
||||
except ImportError:
|
||||
print("Installing missing dependency: databento")
|
||||
subprocess.check_call([sys.executable, "-m", "pip", "install", "-U", "databento"])
|
||||
import databento as db
|
||||
|
||||
from databento.common.error import BentoError
|
||||
|
||||
try:
|
||||
import websockets
|
||||
except ImportError:
|
||||
print("Installing missing dependency: websockets")
|
||||
subprocess.check_call([sys.executable, "-m", "pip", "install", "websockets"])
|
||||
import websockets
|
||||
|
||||
from databento_transformer import DatabentoTransformer
|
||||
from pubsub_client import PubSubClient
|
||||
|
||||
|
||||
API_KEY = "db-VksTJvhMmbLjbJu9Kg6GPS99B8FDN"
|
||||
DATASET = "GLBX.MDP3"
|
||||
SYMBOL = "ES.FUT"
|
||||
PUBSUB_WS_URL = "ws://localhost:19092/ws/"
|
||||
BBO_TABLE_NAME = "livebbo_glbx_future_es_continuouscontracts"
|
||||
TRADES_TABLE_NAME = "livetrades_glbx_future_es_continuouscontracts"
|
||||
DEFINITIONS_TABLE_NAME = "livecontracts_glbx_future_es"
|
||||
|
||||
_pubsub_client = PubSubClient(PUBSUB_WS_URL)
|
||||
_transformer = DatabentoTransformer(
|
||||
symbol=SYMBOL,
|
||||
bbo_table_name=BBO_TABLE_NAME,
|
||||
trades_table_name=TRADES_TABLE_NAME,
|
||||
definitions_table_name=DEFINITIONS_TABLE_NAME,
|
||||
)
|
||||
|
||||
|
||||
def _publish_contract_definitions() -> None:
|
||||
historical_client = db.Historical(API_KEY)
|
||||
dataset_range = historical_client.metadata.get_dataset_range(DATASET)
|
||||
schema_range = dataset_range.get("schema", {}).get("definition", {})
|
||||
available_end = schema_range.get("end", dataset_range["end"])
|
||||
|
||||
if isinstance(available_end, str):
|
||||
available_end_dt = dt.datetime.fromisoformat(available_end.replace("Z", "+00:00"))
|
||||
else:
|
||||
available_end_dt = available_end
|
||||
|
||||
# Databento's range end is exclusive. Back off slightly so the request stays
|
||||
# strictly inside the available window.
|
||||
end_dt = available_end_dt - dt.timedelta(seconds=1)
|
||||
start_dt = max(end_dt - dt.timedelta(days=1), end_dt.replace(hour=0, minute=0, second=0, microsecond=0))
|
||||
start = start_dt.isoformat()
|
||||
end = end_dt.isoformat()
|
||||
|
||||
definition_data = historical_client.timeseries.get_range(
|
||||
dataset=DATASET,
|
||||
schema="definition",
|
||||
stype_in="parent",
|
||||
symbols=SYMBOL,
|
||||
start=start,
|
||||
end=end,
|
||||
)
|
||||
|
||||
df = definition_data.to_df()
|
||||
payload = _transformer.build_definitions_payload(df)
|
||||
if payload is None:
|
||||
print("No Databento contract definitions returned")
|
||||
return
|
||||
|
||||
print(f"Publishing {len(payload['rows'])} Databento contract definition rows")
|
||||
_pubsub_client.publish(payload)
|
||||
|
||||
|
||||
def on_record(record) -> None:
|
||||
# Print every callback first so we can see whether Databento is flowing
|
||||
# even if our row extraction logic misses a field shape.
|
||||
print(record)
|
||||
|
||||
payloads = _transformer.transform_record(record)
|
||||
if payloads:
|
||||
for payload in payloads:
|
||||
_pubsub_client.publish(payload)
|
||||
return
|
||||
|
||||
print("Skipping record with no BBO/trade fields")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
_pubsub_client.start()
|
||||
print(f"Starting Databento live client for {SYMBOL} on {DATASET}")
|
||||
try:
|
||||
_publish_contract_definitions()
|
||||
except Exception as ex:
|
||||
print(f"Contract definition publish failed: {ex}", flush=True)
|
||||
|
||||
while True:
|
||||
try:
|
||||
client = db.Live(key=API_KEY)
|
||||
|
||||
client.subscribe(
|
||||
dataset=DATASET,
|
||||
schema="bbo-1s",
|
||||
stype_in="parent",
|
||||
symbols=SYMBOL,
|
||||
)
|
||||
client.subscribe(
|
||||
dataset=DATASET,
|
||||
schema="trades",
|
||||
stype_in="parent",
|
||||
symbols=SYMBOL,
|
||||
)
|
||||
|
||||
client.add_callback(on_record)
|
||||
client.start()
|
||||
print("Databento client started, waiting for BBO and trade records...", flush=True)
|
||||
client.block_for_close()
|
||||
print("Databento client closed cleanly", flush=True)
|
||||
break
|
||||
except BentoError as ex:
|
||||
print(f"Databento live connection error: {ex}", flush=True)
|
||||
print("Retrying Databento live connection in 5 seconds...", flush=True)
|
||||
time.sleep(5)
|
||||
except Exception as ex:
|
||||
print(f"Databento publisher unexpected error: {ex}", flush=True)
|
||||
print("Retrying Databento live connection in 5 seconds...", flush=True)
|
||||
time.sleep(5)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,244 @@
|
||||
import datetime as dt
|
||||
|
||||
|
||||
class DatabentoTransformer:
|
||||
def __init__(
|
||||
self,
|
||||
symbol: str,
|
||||
bbo_table_name: str,
|
||||
trades_table_name: str,
|
||||
definitions_table_name: str,
|
||||
):
|
||||
self._symbol = symbol
|
||||
self._bbo_table_name = bbo_table_name
|
||||
self._trades_table_name = trades_table_name
|
||||
self._definitions_table_name = definitions_table_name
|
||||
self._instrument_id_to_symbol: dict[int, str] = {}
|
||||
self._last_bbo_row: dict | None = None
|
||||
self._trade_row_id = 0
|
||||
self._trade_table_created = False
|
||||
|
||||
def transform_record(self, record) -> list[dict]:
|
||||
payload = self._handle_bbo_record(record)
|
||||
if payload is not None:
|
||||
return [payload]
|
||||
|
||||
payload = self._handle_trade_record(record)
|
||||
if payload is not None:
|
||||
return [payload]
|
||||
|
||||
return []
|
||||
|
||||
def build_definitions_payload(self, definition_df) -> dict | None:
|
||||
if definition_df is None or definition_df.empty:
|
||||
return None
|
||||
|
||||
df = definition_df.reset_index()
|
||||
if "instrument_class" in df.columns:
|
||||
df = df[df["instrument_class"].astype(str) == "F"]
|
||||
if "security_type" in df.columns:
|
||||
df = df[df["security_type"].astype(str) == "FUT"]
|
||||
if "raw_symbol" in df.columns:
|
||||
df = df.sort_values("ts_recv" if "ts_recv" in df.columns else "ts_event")
|
||||
df = df.drop_duplicates(subset=["raw_symbol"], keep="last")
|
||||
|
||||
rows: list[dict] = []
|
||||
for idx, row in enumerate(df.itertuples(index=False), start=1):
|
||||
instrument_id = int(getattr(row, "instrument_id", 0) or 0)
|
||||
security_name = str(getattr(row, "raw_symbol", ""))
|
||||
if instrument_id > 0 and security_name:
|
||||
self._instrument_id_to_symbol[instrument_id] = security_name
|
||||
|
||||
rows.append({
|
||||
"id": idx,
|
||||
"security_name": security_name,
|
||||
"instrument_id": instrument_id,
|
||||
"instrument_class": str(getattr(row, "instrument_class", "")),
|
||||
"security_type": str(getattr(row, "security_type", "")),
|
||||
"asset": str(getattr(row, "asset", "")),
|
||||
"exchange": str(getattr(row, "exchange", "")),
|
||||
"currency": str(getattr(row, "currency", "")),
|
||||
"expiration": self._ns_to_iso(getattr(row, "expiration", 0)),
|
||||
"activation": self._ns_to_iso(getattr(row, "activation", 0)),
|
||||
"min_price_increment": self._nano_to_float(getattr(row, "min_price_increment", 0)),
|
||||
"display_factor": self._nano_to_float(getattr(row, "display_factor", 0)),
|
||||
"price_display_multiplier": self._nano_to_float(getattr(row, "display_factor", 0)),
|
||||
"contract_multiplier": int(getattr(row, "contract_multiplier", 0) or 0),
|
||||
"contract_multiplier_unit": int(getattr(row, "contract_multiplier_unit", 0) or 0),
|
||||
"unit_of_measure": str(getattr(row, "unit_of_measure", "")),
|
||||
"unit_of_measure_qty": self._nano_to_float(getattr(row, "unit_of_measure_qty", 0)),
|
||||
"min_lot_size": int(getattr(row, "min_lot_size", 0) or 0),
|
||||
"min_trade_vol": int(getattr(row, "min_trade_vol", 0) or 0),
|
||||
"max_trade_vol": int(getattr(row, "max_trade_vol", 0) or 0),
|
||||
"maturity_year": int(getattr(row, "maturity_year", 0) or 0),
|
||||
"maturity_month": int(getattr(row, "maturity_month", 0) or 0),
|
||||
"maturity_day": int(getattr(row, "maturity_day", 0) or 0),
|
||||
"underlying": str(getattr(row, "underlying", "")),
|
||||
})
|
||||
|
||||
if not rows:
|
||||
return None
|
||||
|
||||
return {
|
||||
"op": "create_table",
|
||||
"table_name": self._definitions_table_name,
|
||||
"rows": rows,
|
||||
}
|
||||
|
||||
def _handle_bbo_record(self, record) -> dict | None:
|
||||
row = self._to_bbo_row(record)
|
||||
if row is None:
|
||||
return None
|
||||
|
||||
if self._last_bbo_row is None:
|
||||
payload = {
|
||||
"op": "create_table",
|
||||
"table_name": self._bbo_table_name,
|
||||
"rows": [row],
|
||||
}
|
||||
else:
|
||||
payload = {
|
||||
"op": "multiple_remove_add",
|
||||
"table_name": self._bbo_table_name,
|
||||
"removed_rows": [self._last_bbo_row],
|
||||
"added_rows": [row],
|
||||
}
|
||||
|
||||
self._last_bbo_row = row
|
||||
return payload
|
||||
|
||||
def _handle_trade_record(self, record) -> dict | None:
|
||||
row = self._to_trade_row(record)
|
||||
if row is None:
|
||||
return None
|
||||
|
||||
row["id"] = self._trade_row_id
|
||||
self._trade_row_id += 1
|
||||
|
||||
if not self._trade_table_created:
|
||||
payload = {
|
||||
"op": "create_table",
|
||||
"table_name": self._trades_table_name,
|
||||
"rows": [row],
|
||||
}
|
||||
self._trade_table_created = True
|
||||
return payload
|
||||
|
||||
return {
|
||||
"op": "multiple_remove_add",
|
||||
"table_name": self._trades_table_name,
|
||||
"removed_rows": [],
|
||||
"added_rows": [row],
|
||||
}
|
||||
|
||||
def _to_bbo_row(self, record) -> dict | None:
|
||||
bid_px = self._get_level_attr(record, "bid_px", default=self._get_attr(record, "bid_px_00"))
|
||||
ask_px = self._get_level_attr(record, "ask_px", default=self._get_attr(record, "ask_px_00"))
|
||||
if bid_px is None and ask_px is None:
|
||||
return None
|
||||
|
||||
return {
|
||||
"id": 0,
|
||||
"security_name": self._get_security_name(record),
|
||||
"timestamp": self._get_timestamp(record),
|
||||
"bid_px": float(bid_px or 0.0),
|
||||
"ask_px": float(ask_px or 0.0),
|
||||
"bid_sz": int(self._get_level_attr(record, "bid_sz", default=self._get_attr(record, "bid_sz_00", default=0)) or 0),
|
||||
"ask_sz": int(self._get_level_attr(record, "ask_sz", default=self._get_attr(record, "ask_sz_00", default=0)) or 0),
|
||||
}
|
||||
|
||||
def _to_trade_row(self, record) -> dict | None:
|
||||
price = self._get_attr(record, "price", "px")
|
||||
size = self._get_attr(record, "size")
|
||||
if price is None and size is None:
|
||||
return None
|
||||
|
||||
side = self._get_attr(record, "side", "aggressor_side", default="")
|
||||
|
||||
return {
|
||||
"id": 0,
|
||||
"security_name": self._get_security_name(record),
|
||||
"timestamp": self._get_timestamp(record),
|
||||
"price": float(price or 0.0),
|
||||
"size": int(size or 0),
|
||||
"side": str(side),
|
||||
}
|
||||
|
||||
def _get_security_name(self, record) -> str:
|
||||
instrument_id = self._get_attr(record, "instrument_id", default=0)
|
||||
try:
|
||||
instrument_id_int = int(instrument_id or 0)
|
||||
except (TypeError, ValueError):
|
||||
instrument_id_int = 0
|
||||
|
||||
if instrument_id_int > 0 and instrument_id_int in self._instrument_id_to_symbol:
|
||||
return self._instrument_id_to_symbol[instrument_id_int]
|
||||
|
||||
value = self._get_attr(
|
||||
record,
|
||||
"symbol",
|
||||
"raw_symbol",
|
||||
"stype_out_symbol",
|
||||
"stype_in_symbol",
|
||||
default=self._symbol,
|
||||
)
|
||||
return str(value)
|
||||
|
||||
def _get_timestamp(self, record) -> int:
|
||||
value = self._get_attr(record, "ts_event", "ts_recv", default=0)
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
def _get_attr(self, record, *names, default=None):
|
||||
for name in names:
|
||||
if hasattr(record, name):
|
||||
value = getattr(record, name)
|
||||
if value is not None:
|
||||
return value
|
||||
return default
|
||||
|
||||
def _get_level_attr(self, record, name, default=None):
|
||||
levels = self._get_attr(record, "levels", default=None)
|
||||
if levels:
|
||||
level0 = levels[0]
|
||||
if hasattr(level0, name):
|
||||
value = getattr(level0, name)
|
||||
if value is not None:
|
||||
return value
|
||||
return default
|
||||
|
||||
def _ns_to_iso(self, timestamp_ns) -> str:
|
||||
if timestamp_ns is None:
|
||||
return ""
|
||||
|
||||
if hasattr(timestamp_ns, "isoformat"):
|
||||
try:
|
||||
if hasattr(timestamp_ns, "to_pydatetime"):
|
||||
timestamp_dt = timestamp_ns.to_pydatetime()
|
||||
else:
|
||||
timestamp_dt = timestamp_ns
|
||||
|
||||
if timestamp_dt is None:
|
||||
return ""
|
||||
|
||||
if getattr(timestamp_dt, "tzinfo", None) is None:
|
||||
timestamp_dt = timestamp_dt.replace(tzinfo=dt.timezone.utc)
|
||||
return timestamp_dt.isoformat()
|
||||
except (TypeError, ValueError, AttributeError):
|
||||
pass
|
||||
|
||||
try:
|
||||
ts_int = int(timestamp_ns)
|
||||
if ts_int <= 0:
|
||||
return ""
|
||||
return dt.datetime.fromtimestamp(ts_int / 1_000_000_000, tz=dt.timezone.utc).isoformat()
|
||||
except (TypeError, ValueError, OSError, OverflowError):
|
||||
return ""
|
||||
|
||||
def _nano_to_float(self, value) -> float:
|
||||
try:
|
||||
return float(value) / 1_000_000_000.0
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
@@ -0,0 +1,58 @@
|
||||
import asyncio
|
||||
import json
|
||||
import queue
|
||||
import threading
|
||||
|
||||
import websockets
|
||||
|
||||
|
||||
class PubSubClient:
|
||||
def __init__(self, ws_url: str):
|
||||
self._ws_url = ws_url
|
||||
self._outgoing_queue: "queue.Queue[dict]" = queue.Queue()
|
||||
self._sender_thread: threading.Thread | None = None
|
||||
|
||||
def start(self) -> threading.Thread:
|
||||
if self._sender_thread is None or not self._sender_thread.is_alive():
|
||||
self._sender_thread = self._start_sender_thread()
|
||||
return self._sender_thread
|
||||
|
||||
def publish(self, payload: dict) -> None:
|
||||
self._outgoing_queue.put(payload)
|
||||
|
||||
def publish_create_table(self, table_name: str, rows: list[dict]) -> None:
|
||||
self.publish({
|
||||
"op": "create_table",
|
||||
"table_name": table_name,
|
||||
"rows": rows,
|
||||
})
|
||||
|
||||
def publish_multiple_remove_add(
|
||||
self,
|
||||
table_name: str,
|
||||
removed_rows: list[dict],
|
||||
added_rows: list[dict],
|
||||
) -> None:
|
||||
self.publish({
|
||||
"op": "multiple_remove_add",
|
||||
"table_name": table_name,
|
||||
"removed_rows": removed_rows,
|
||||
"added_rows": added_rows,
|
||||
})
|
||||
|
||||
def _start_sender_thread(self) -> threading.Thread:
|
||||
thread = threading.Thread(target=self._run_send_loop, daemon=True)
|
||||
thread.start()
|
||||
return thread
|
||||
|
||||
def _run_send_loop(self) -> None:
|
||||
asyncio.run(self._ws_send_loop())
|
||||
|
||||
async def _ws_send_loop(self) -> None:
|
||||
async with websockets.connect(self._ws_url) as ws:
|
||||
print(f"Connected to PubSubServer websocket: {self._ws_url}")
|
||||
while True:
|
||||
payload = await asyncio.to_thread(self._outgoing_queue.get)
|
||||
body = json.dumps(payload)
|
||||
print(body)
|
||||
await ws.send(body)
|
||||
@@ -59,14 +59,9 @@
|
||||
"Class": "bts.PubsubServerStreamTagHandler"
|
||||
},
|
||||
{
|
||||
"TagType": "System_Sub",
|
||||
"Assembly": "PubSub",
|
||||
"Class": "BCG.SystemSubscriptionMapPublisher"
|
||||
},
|
||||
{
|
||||
"TagType": "System_Stats",
|
||||
"Assembly": "PubSub",
|
||||
"Class": "BCG.SystemStatsPublisher"
|
||||
"TagType": "DS_.*",
|
||||
"Assembly": "DataStreamCore",
|
||||
"Class": "bts.PubsubServerStreamTagHandler"
|
||||
},
|
||||
{
|
||||
"TagType": ".*",
|
||||
@@ -90,4 +85,4 @@
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -21,11 +21,55 @@ using System.Reactive.Subjects;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading;
|
||||
using System.Windows.Forms;
|
||||
using BlueOrb.Pubsub.Diagnostics;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
public partial class MainForm : System.Windows.Forms.Form
|
||||
{
|
||||
private const string LastQueryRegistryKey = "LastQuery";
|
||||
private const string RecentSavedFilesRegistryKey = "RecentSavedFiles";
|
||||
private const string DateTimeDisplayFormat = "M/d/yyyy h:mm:ss.fff tt";
|
||||
private const int MaxRecentSavedFiles = 10;
|
||||
private static readonly Color QueryEditorDefaultColor = Color.White;
|
||||
private static readonly Color QueryEditorCommentColor = Color.FromArgb(106, 115, 125);
|
||||
private static readonly Color QueryEditorOperatorColor = Color.FromArgb(86, 156, 214);
|
||||
private static readonly Color QueryEditorAssignmentNameColor = Color.FromArgb(78, 201, 176);
|
||||
private static readonly Color GridBackgroundColor = Color.FromArgb(37, 37, 38);
|
||||
private static readonly Color GridAltRowColor = Color.FromArgb(43, 43, 48);
|
||||
private static readonly Color GridHeaderColor = Color.FromArgb(62, 62, 66);
|
||||
private static readonly Color GridGridLineColor = Color.FromArgb(58, 58, 61);
|
||||
private static readonly Color GridSelectionColor = Color.FromArgb(9, 71, 113);
|
||||
private static readonly Color GridSelectionForeColor = Color.White;
|
||||
private static readonly Color GridTimestampColor = Color.FromArgb(206, 215, 222);
|
||||
private static readonly Color GridIdentifierColor = Color.FromArgb(156, 163, 175);
|
||||
private static readonly Color GridIndicatorColor = Color.FromArgb(156, 220, 254);
|
||||
|
||||
class PrimitiveTreeNodeComparer : IComparer
|
||||
{
|
||||
public int Compare(object? x, object? y)
|
||||
{
|
||||
var left = x as TreeNode;
|
||||
var right = y as TreeNode;
|
||||
if (left is null || right is null)
|
||||
return 0;
|
||||
|
||||
string left_key = _sort_key(left);
|
||||
string right_key = _sort_key(right);
|
||||
return StringComparer.OrdinalIgnoreCase.Compare(left_key, right_key);
|
||||
}
|
||||
|
||||
static string _sort_key(TreeNode node)
|
||||
{
|
||||
if (node.Parent is null)
|
||||
return node.Text;
|
||||
|
||||
if (node.Tag is string primitive_name)
|
||||
return primitive_name;
|
||||
|
||||
return node.Text;
|
||||
}
|
||||
}
|
||||
// static async Task _run() {
|
||||
// var login = "bts";
|
||||
// var passwd = String.Empty;
|
||||
@@ -40,6 +84,8 @@ namespace BCG
|
||||
private string? myLastFileName;
|
||||
private ViewStream? myCurrentStream;
|
||||
private List<ToolStripMenuItem> mySiteItems = new List<ToolStripMenuItem>();
|
||||
private readonly List<string> _recent_saved_files = new List<string>();
|
||||
private ToolStripSeparator? _recent_saved_files_separator;
|
||||
// private string mySite = SitesCore.getMySiteName ();
|
||||
private string mySite = "TODO";
|
||||
private string myLastClassName = "MyView";
|
||||
@@ -49,11 +95,20 @@ namespace BCG
|
||||
private TextWriter? myConsoleOutput;
|
||||
IDataPacket? pendingData;
|
||||
IDisposable? titleSub;
|
||||
readonly ToolTip _primitive_tool_tip = new ToolTip();
|
||||
readonly Dictionary<string, string?> _primitive_source_path_cache = new Dictionary<string, string?>();
|
||||
Dictionary<string, string?>? _source_file_index;
|
||||
string? _repo_root_path;
|
||||
char? _last_primitive_search_char;
|
||||
TreeNode? _last_primitive_search_assembly_node;
|
||||
bool _is_applying_query_syntax_highlighting;
|
||||
HashSet<string>? _dsl_operator_names;
|
||||
|
||||
public MainForm()
|
||||
{
|
||||
//Task.Run(async () => await _run());
|
||||
InitializeComponent();
|
||||
_apply_grid_theme();
|
||||
// myKQ = ConfigurationRequester.getApplicationQualifier();
|
||||
myKQ = "TODO";
|
||||
var handleCreatedSubject = new Subject<bool>();
|
||||
@@ -72,13 +127,12 @@ namespace BCG
|
||||
myWindowState = new PersistWindowState();
|
||||
myWindowState.parent = this;
|
||||
myWindowState.saveState = false;
|
||||
myWindowState.LoadState += myLoadPersistedState;
|
||||
myWindowState.SaveState += mySavePersistedState;
|
||||
|
||||
this.Load += new EventHandler(myLoad);
|
||||
this.Closing += new CancelEventHandler(myClosing);
|
||||
myQueryTextBox.TextChanged += (o, e) =>
|
||||
{
|
||||
myQueryTextBox.ForeColor = Color.White;
|
||||
};
|
||||
myQueryTextBox.TextChanged += (o, e) => _apply_query_syntax_highlighting();
|
||||
}
|
||||
|
||||
event Action? titleInvalidated;
|
||||
@@ -86,14 +140,20 @@ namespace BCG
|
||||
public string query_text
|
||||
{
|
||||
get { return myQueryTextBox.Text; }
|
||||
set { myQueryTextBox.Text = value; }
|
||||
set
|
||||
{
|
||||
myQueryTextBox.Text = value;
|
||||
_apply_query_syntax_highlighting();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Clean up any resources being used.
|
||||
/// </summary>
|
||||
protected override void Dispose(bool disposing) {
|
||||
if (disposing) {
|
||||
protected override void Dispose(bool disposing)
|
||||
{
|
||||
if (disposing)
|
||||
{
|
||||
Disposable.Dispose(ref titleSub);
|
||||
components?.Dispose();
|
||||
}
|
||||
@@ -104,7 +164,7 @@ namespace BCG
|
||||
{
|
||||
// TODO: Port PublishedTagsStream from edge (then we can uncomment impl below)
|
||||
//ViewStream vs = new PubSubServerStateStream();
|
||||
ViewStream vs = new PubSubServerStateStream(new SubscriberCoreFactory(), ViewStreamDatabase.instance);
|
||||
ViewStream vs = new PubSubServerStateStream(ViewStreamDatabase.instance);
|
||||
return new FilterStream(vs, "tag~='DS_'");
|
||||
//return new LiteralStream("tag/String", "DS_table1;DS_table2");
|
||||
}
|
||||
@@ -114,7 +174,8 @@ namespace BCG
|
||||
titleInvalidated?.Invoke();
|
||||
}
|
||||
|
||||
void refreshTitle(IBlueRowCollection rows) {
|
||||
void refreshTitle(IBlueRowCollection rows)
|
||||
{
|
||||
string route_txt = String.Empty;
|
||||
foreach (IBlueRow r in rows)
|
||||
{
|
||||
@@ -135,11 +196,11 @@ namespace BCG
|
||||
private IDictionary myGetLibraryData()
|
||||
{
|
||||
SortedList retVal = new SortedList();
|
||||
string[] primitiveNames = PrimitiveManager.instance.getPrimitiveNames();
|
||||
string[] primitiveNames = PrimitiveManager.instance.get_dsl_op_names();
|
||||
foreach (string pName in primitiveNames)
|
||||
{
|
||||
string aName = PrimitiveManager.instance.getAssemblyName(pName);
|
||||
ICollection cNames = PrimitiveManager.instance.getConstructorSigns(pName);
|
||||
string aName = PrimitiveManager.instance.get_dslop_assembly_name(pName);
|
||||
ICollection cNames = PrimitiveManager.instance.get_dsl_op_cons_args(pName);
|
||||
if (!retVal.Contains(aName))
|
||||
retVal.Add(aName, new SortedList());
|
||||
(retVal[aName] as IDictionary)[pName] = new ArrayList(cNames);
|
||||
@@ -151,6 +212,65 @@ namespace BCG
|
||||
return retVal;
|
||||
}
|
||||
|
||||
HashSet<string> _get_dsl_operator_names()
|
||||
{
|
||||
if (_dsl_operator_names != null)
|
||||
return _dsl_operator_names;
|
||||
|
||||
_dsl_operator_names = new HashSet<string>(
|
||||
PrimitiveManager.instance.get_dsl_op_names().Cast<string>(),
|
||||
StringComparer.OrdinalIgnoreCase);
|
||||
return _dsl_operator_names;
|
||||
}
|
||||
|
||||
void _apply_query_syntax_highlighting()
|
||||
{
|
||||
if (_is_applying_query_syntax_highlighting || myQueryTextBox.IsDisposed)
|
||||
return;
|
||||
|
||||
_is_applying_query_syntax_highlighting = true;
|
||||
try
|
||||
{
|
||||
int selection_start = myQueryTextBox.SelectionStart;
|
||||
int selection_length = myQueryTextBox.SelectionLength;
|
||||
|
||||
myQueryTextBox.SuspendLayout();
|
||||
myQueryTextBox.SelectAll();
|
||||
myQueryTextBox.SelectionColor = QueryEditorDefaultColor;
|
||||
|
||||
foreach (Match match in Regex.Matches(myQueryTextBox.Text, @"^\s*([A-Za-z_]\w*)\s*=", RegexOptions.Multiline))
|
||||
{
|
||||
Group group = match.Groups[1];
|
||||
myQueryTextBox.Select(group.Index, group.Length);
|
||||
myQueryTextBox.SelectionColor = QueryEditorAssignmentNameColor;
|
||||
}
|
||||
|
||||
foreach (Match match in Regex.Matches(myQueryTextBox.Text, @"\b([A-Za-z_]\w*)\s*\("))
|
||||
{
|
||||
Group group = match.Groups[1];
|
||||
if (!_get_dsl_operator_names().Contains(group.Value))
|
||||
continue;
|
||||
myQueryTextBox.Select(group.Index, group.Length);
|
||||
myQueryTextBox.SelectionColor = QueryEditorOperatorColor;
|
||||
}
|
||||
|
||||
foreach (Match match in Regex.Matches(myQueryTextBox.Text, @"//.*$", RegexOptions.Multiline))
|
||||
{
|
||||
myQueryTextBox.Select(match.Index, match.Length);
|
||||
myQueryTextBox.SelectionColor = QueryEditorCommentColor;
|
||||
}
|
||||
|
||||
myQueryTextBox.Select(selection_start, selection_length);
|
||||
if (selection_length == 0)
|
||||
myQueryTextBox.SelectionColor = QueryEditorDefaultColor;
|
||||
myQueryTextBox.ResumeLayout();
|
||||
}
|
||||
finally
|
||||
{
|
||||
_is_applying_query_syntax_highlighting = false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void myLoadAssemblies()
|
||||
{
|
||||
@@ -161,13 +281,18 @@ namespace BCG
|
||||
TreeNode assemblyNode = new TreeNode(de.Key as String);
|
||||
foreach (DictionaryEntry innerDe in (de.Value as IDictionary))
|
||||
{
|
||||
TreeNode primitiveNode = new TreeNode(innerDe.Key as string);
|
||||
string primitive_name = innerDe.Key as string;
|
||||
TreeNode primitiveNode = new TreeNode(primitive_name)
|
||||
{
|
||||
Tag = primitive_name
|
||||
};
|
||||
foreach (string con in (innerDe.Value as ICollection))
|
||||
primitiveNode.Nodes.Add(con);
|
||||
assemblyNode.Nodes.Add(primitiveNode);
|
||||
}
|
||||
myTreeView.Nodes.Add(assemblyNode);
|
||||
}
|
||||
myTreeView.Sort();
|
||||
/*
|
||||
myPrimitivesTextBox.Clear ();
|
||||
ArrayList al = new ArrayList(PrimitiveManager.instance.getPrimitiveNames());
|
||||
@@ -182,7 +307,8 @@ namespace BCG
|
||||
|
||||
private void myHandleTableListChange(object o)
|
||||
{
|
||||
BeginInvoke(() => {
|
||||
BeginInvoke(() =>
|
||||
{
|
||||
tagRows = ((IDataPacket)o).newRows;
|
||||
myUpdateTableList();
|
||||
});
|
||||
@@ -195,6 +321,19 @@ namespace BCG
|
||||
get { return myQueryTextBox.Height; }
|
||||
set { myQueryTextBox.Height = value; }
|
||||
}
|
||||
|
||||
private void myLoadPersistedState(object sender, BlueRegistry blueReg)
|
||||
{
|
||||
myQueryTextBox.Text = blueReg.getValue(RegistryRoot.HKCU, LastQueryRegistryKey, string.Empty)?.ToString() ?? string.Empty;
|
||||
_load_recent_saved_files(blueReg.getValue(RegistryRoot.HKCU, RecentSavedFilesRegistryKey, string.Empty)?.ToString());
|
||||
_refresh_recent_saved_files_menu();
|
||||
}
|
||||
|
||||
private void mySavePersistedState(object sender, BlueRegistry blueReg)
|
||||
{
|
||||
blueReg.setValue(RegistryRoot.HKCU, LastQueryRegistryKey, myQueryTextBox.Text ?? string.Empty);
|
||||
blueReg.setValue(RegistryRoot.HKCU, RecentSavedFilesRegistryKey, string.Join("|", _recent_saved_files));
|
||||
}
|
||||
#endregion
|
||||
|
||||
private Stopwatch _stopwatch = new Stopwatch();
|
||||
@@ -287,6 +426,7 @@ namespace BCG
|
||||
}
|
||||
myLastFileName = fileName;
|
||||
this.Text = fileName;
|
||||
_track_recent_saved_file(fileName);
|
||||
}
|
||||
catch (Exception exc)
|
||||
{
|
||||
@@ -371,6 +511,122 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
|
||||
private void myApplyDateTimeFormatting(IBlueTableSchema schema)
|
||||
{
|
||||
var dateTimeColumns = new HashSet<string>(
|
||||
schema.columnInfo
|
||||
.Where(ci => ci.colType == typeof(DateTime))
|
||||
.Select(ci => ci.colName));
|
||||
|
||||
foreach (DataGridViewColumn column in myDataGrid.Columns)
|
||||
{
|
||||
if (!dateTimeColumns.Contains(column.DataPropertyName) && !dateTimeColumns.Contains(column.Name))
|
||||
continue;
|
||||
|
||||
column.DefaultCellStyle.Format = DateTimeDisplayFormat;
|
||||
column.Width = Math.Max(column.Width, 220);
|
||||
}
|
||||
}
|
||||
|
||||
void _apply_grid_theme()
|
||||
{
|
||||
myDataGrid.BackgroundColor = GridBackgroundColor;
|
||||
myDataGrid.BorderStyle = BorderStyle.None;
|
||||
myDataGrid.EnableHeadersVisualStyles = false;
|
||||
myDataGrid.GridColor = GridGridLineColor;
|
||||
myDataGrid.CellBorderStyle = DataGridViewCellBorderStyle.SingleHorizontal;
|
||||
myDataGrid.SelectionMode = DataGridViewSelectionMode.FullRowSelect;
|
||||
myDataGrid.MultiSelect = false;
|
||||
myDataGrid.RowHeadersVisible = false;
|
||||
myDataGrid.ColumnHeadersBorderStyle = DataGridViewHeaderBorderStyle.Single;
|
||||
myDataGrid.RowHeadersBorderStyle = DataGridViewHeaderBorderStyle.None;
|
||||
myDataGrid.ColumnHeadersHeight = 36;
|
||||
myDataGrid.RowTemplate.Height = 24;
|
||||
myDataGrid.Font = new Font("Consolas", 9F, FontStyle.Regular, GraphicsUnit.Point);
|
||||
|
||||
myDataGrid.DefaultCellStyle.BackColor = GridBackgroundColor;
|
||||
myDataGrid.DefaultCellStyle.ForeColor = Color.White;
|
||||
myDataGrid.DefaultCellStyle.SelectionBackColor = GridSelectionColor;
|
||||
myDataGrid.DefaultCellStyle.SelectionForeColor = GridSelectionForeColor;
|
||||
myDataGrid.DefaultCellStyle.Padding = new Padding(4, 1, 4, 1);
|
||||
|
||||
myDataGrid.AlternatingRowsDefaultCellStyle.BackColor = GridAltRowColor;
|
||||
myDataGrid.AlternatingRowsDefaultCellStyle.ForeColor = Color.White;
|
||||
myDataGrid.AlternatingRowsDefaultCellStyle.SelectionBackColor = GridSelectionColor;
|
||||
myDataGrid.AlternatingRowsDefaultCellStyle.SelectionForeColor = GridSelectionForeColor;
|
||||
|
||||
myDataGrid.ColumnHeadersDefaultCellStyle.BackColor = GridHeaderColor;
|
||||
myDataGrid.ColumnHeadersDefaultCellStyle.ForeColor = Color.White;
|
||||
myDataGrid.ColumnHeadersDefaultCellStyle.SelectionBackColor = GridHeaderColor;
|
||||
myDataGrid.ColumnHeadersDefaultCellStyle.SelectionForeColor = Color.White;
|
||||
myDataGrid.ColumnHeadersDefaultCellStyle.Font = new Font("Consolas", 9F, FontStyle.Bold, GraphicsUnit.Point);
|
||||
myDataGrid.ColumnHeadersDefaultCellStyle.Alignment = DataGridViewContentAlignment.MiddleLeft;
|
||||
myDataGrid.ColumnHeadersDefaultCellStyle.Padding = new Padding(6, 2, 6, 2);
|
||||
}
|
||||
|
||||
void _apply_grid_column_styling(IBlueTableSchema schema)
|
||||
{
|
||||
var column_types = schema.columnInfo.ToDictionary(ci => ci.colName, ci => ci.colType, StringComparer.OrdinalIgnoreCase);
|
||||
foreach (DataGridViewColumn column in myDataGrid.Columns)
|
||||
{
|
||||
if (!column_types.TryGetValue(column.DataPropertyName, out var col_type) &&
|
||||
!column_types.TryGetValue(column.Name, out col_type))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
column.SortMode = DataGridViewColumnSortMode.NotSortable;
|
||||
|
||||
if (_is_numeric_type(col_type))
|
||||
{
|
||||
column.DefaultCellStyle.Alignment = DataGridViewContentAlignment.MiddleRight;
|
||||
}
|
||||
else
|
||||
{
|
||||
column.DefaultCellStyle.Alignment = DataGridViewContentAlignment.MiddleLeft;
|
||||
}
|
||||
|
||||
if (col_type == typeof(DateTime))
|
||||
{
|
||||
column.DefaultCellStyle.ForeColor = GridTimestampColor;
|
||||
}
|
||||
else if (string.Equals(column.HeaderText, "id", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
column.DefaultCellStyle.ForeColor = GridIdentifierColor;
|
||||
}
|
||||
else if (_looks_like_indicator(column.HeaderText))
|
||||
{
|
||||
column.DefaultCellStyle.ForeColor = GridIndicatorColor;
|
||||
}
|
||||
else
|
||||
{
|
||||
column.DefaultCellStyle.ForeColor = Color.White;
|
||||
}
|
||||
}
|
||||
|
||||
static bool _is_numeric_type(Type type)
|
||||
{
|
||||
return type == typeof(int) ||
|
||||
type == typeof(long) ||
|
||||
type == typeof(float) ||
|
||||
type == typeof(double) ||
|
||||
type == typeof(decimal) ||
|
||||
type == typeof(short) ||
|
||||
type == typeof(uint) ||
|
||||
type == typeof(ulong);
|
||||
}
|
||||
|
||||
static bool _looks_like_indicator(string column_name)
|
||||
{
|
||||
return column_name.Contains("rsi", StringComparison.OrdinalIgnoreCase) ||
|
||||
column_name.Contains("macd", StringComparison.OrdinalIgnoreCase) ||
|
||||
column_name.Contains("atr", StringComparison.OrdinalIgnoreCase) ||
|
||||
column_name.Contains("moving_average", StringComparison.OrdinalIgnoreCase) ||
|
||||
column_name.Contains("signal", StringComparison.OrdinalIgnoreCase) ||
|
||||
column_name.Contains("histogram", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
}
|
||||
|
||||
// this is the stream callback
|
||||
private void myCallback(object o)
|
||||
{
|
||||
@@ -386,7 +642,8 @@ namespace BCG
|
||||
private void myTimer_Tick(object sender, System.EventArgs e)
|
||||
{
|
||||
var dp = Interlocked.Exchange(ref pendingData, null);
|
||||
if (dp is null) {
|
||||
if (dp is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
_stopwatch.Stop();
|
||||
@@ -403,6 +660,10 @@ namespace BCG
|
||||
int hscrollval = myChangeStyle ? 0 : myDataGrid.HorizScrollBarValue;
|
||||
|
||||
myDataGrid.DataSource = new SimpleBindableCollection(dp.newRows.schema, dp.newRows);
|
||||
_apply_grid_theme();
|
||||
myApplyDateTimeFormatting(dp.newRows.schema);
|
||||
_apply_grid_column_styling(dp.newRows.schema);
|
||||
myDataGrid.ClearSelection();
|
||||
var force = false;
|
||||
if (myChangeStyle && force)
|
||||
{
|
||||
@@ -475,7 +736,7 @@ namespace BCG
|
||||
private void myMenuItemOpenClick(object sender, System.EventArgs e)
|
||||
{
|
||||
OpenFileDialog od = new OpenFileDialog();
|
||||
od.Filter = "C# class (*.cs)|*.cs";
|
||||
od.Filter = "All files (*.*)|*.*|C# class (*.cs)|*.cs";
|
||||
|
||||
if (od.ShowDialog() == DialogResult.OK)
|
||||
loadFile(od.FileName);
|
||||
@@ -487,6 +748,7 @@ namespace BCG
|
||||
w.Write(myCSharpText.Text);
|
||||
myLastFileName = fileName;
|
||||
this.Text = fileName;
|
||||
_track_recent_saved_file(fileName);
|
||||
}
|
||||
|
||||
private void myMenuItemSaveClick(object sender, System.EventArgs e)
|
||||
@@ -509,6 +771,75 @@ namespace BCG
|
||||
mySave(sd.FileName);
|
||||
}
|
||||
|
||||
private void _load_recent_saved_files(string? serialized_paths)
|
||||
{
|
||||
_recent_saved_files.Clear();
|
||||
if (string.IsNullOrWhiteSpace(serialized_paths))
|
||||
return;
|
||||
|
||||
foreach (string path in serialized_paths.Split('|'))
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(path))
|
||||
continue;
|
||||
if (_recent_saved_files.Contains(path, StringComparer.OrdinalIgnoreCase))
|
||||
continue;
|
||||
_recent_saved_files.Add(path);
|
||||
if (_recent_saved_files.Count >= MaxRecentSavedFiles)
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private void _track_recent_saved_file(string file_name)
|
||||
{
|
||||
_recent_saved_files.RemoveAll(path => string.Equals(path, file_name, StringComparison.OrdinalIgnoreCase));
|
||||
_recent_saved_files.Insert(0, file_name);
|
||||
if (_recent_saved_files.Count > MaxRecentSavedFiles)
|
||||
_recent_saved_files.RemoveRange(MaxRecentSavedFiles, _recent_saved_files.Count - MaxRecentSavedFiles);
|
||||
_refresh_recent_saved_files_menu();
|
||||
}
|
||||
|
||||
private void _refresh_recent_saved_files_menu()
|
||||
{
|
||||
var file_menu = myToolStripMenuItemFile.DropDownItems;
|
||||
for (int i = file_menu.Count - 1; i >= 0; i--)
|
||||
{
|
||||
if (file_menu[i] is ToolStripMenuItem menu_item && Equals(menu_item.Tag, "recent-saved-file"))
|
||||
file_menu.RemoveAt(i);
|
||||
}
|
||||
|
||||
if (_recent_saved_files_separator != null)
|
||||
{
|
||||
file_menu.Remove(_recent_saved_files_separator);
|
||||
_recent_saved_files_separator.Dispose();
|
||||
_recent_saved_files_separator = null;
|
||||
}
|
||||
|
||||
if (_recent_saved_files.Count == 0)
|
||||
return;
|
||||
|
||||
int insert_index = file_menu.IndexOf(myToolStripMenuItemExit);
|
||||
if (insert_index < 0)
|
||||
insert_index = file_menu.Count;
|
||||
|
||||
_recent_saved_files_separator = new ToolStripSeparator();
|
||||
file_menu.Insert(insert_index, _recent_saved_files_separator);
|
||||
insert_index++;
|
||||
|
||||
foreach (string path in _recent_saved_files)
|
||||
{
|
||||
var menu_item = new ToolStripMenuItem(Path.GetFileName(path))
|
||||
{
|
||||
Tag = "recent-saved-file",
|
||||
ToolTipText = path,
|
||||
BackColor = Color.FromArgb(36, 36, 49),
|
||||
ForeColor = Color.Lime
|
||||
};
|
||||
menu_item.Click += (_, __) => loadFile(path);
|
||||
file_menu.Insert(insert_index, menu_item);
|
||||
insert_index++;
|
||||
}
|
||||
}
|
||||
|
||||
private void myTableListBox_DoubleClick(object sender, System.EventArgs e)
|
||||
{
|
||||
if (myTableListBox.SelectedItem != null)
|
||||
@@ -599,6 +930,7 @@ namespace BCG
|
||||
}
|
||||
Console.WriteLine("Redirecting console ouput to this window...");
|
||||
|
||||
myTreeView.TreeViewNodeSorter = new PrimitiveTreeNodeComparer();
|
||||
myLoadAssemblies();
|
||||
|
||||
// ViewStreamDatabase.instance = new ViewStreamDatabase();
|
||||
@@ -606,6 +938,9 @@ namespace BCG
|
||||
myAllTagsStream.onChange += new BCG.DataStream.ChangeDelegate(myHandleTableListChange);
|
||||
myTabControl.SelectedIndex = 3;
|
||||
myTreeView.AfterSelect += new TreeViewEventHandler(myHandleTreeNodeSelect);
|
||||
myTreeView.KeyPress += myHandleTreeViewKeyPress;
|
||||
myTreeView.NodeMouseHover += myHandleTreeNodeMouseHover;
|
||||
myTreeView.MouseLeave += myHandleTreeViewMouseLeave;
|
||||
//do this lazily so we don't have to have bps, etc up to run dsqa
|
||||
//_library = new LibraryForm(this);
|
||||
}
|
||||
@@ -646,12 +981,138 @@ namespace BCG
|
||||
{
|
||||
if (myTreeView.SelectedNode.LastNode == null)
|
||||
{
|
||||
string sn = myTreeView.SelectedNode.Parent.Text + "()";
|
||||
string primitive_name = myTreeView.SelectedNode.Parent.Tag as string ?? myTreeView.SelectedNode.Parent.Text;
|
||||
string sn = primitive_name + "()";
|
||||
myQueryTextBox.Text = myQueryTextBox.Text == "" ? sn : myQueryTextBox.Text + "\n" + sn;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void myHandleTreeViewKeyPress(object? sender, KeyPressEventArgs e)
|
||||
{
|
||||
if (!Char.IsLetterOrDigit(e.KeyChar))
|
||||
return;
|
||||
|
||||
char search_char = Char.ToLowerInvariant(e.KeyChar);
|
||||
TreeNode? assembly_node = _get_selected_assembly_node();
|
||||
if (assembly_node is null)
|
||||
return;
|
||||
|
||||
int start_idx = 0;
|
||||
if (_last_primitive_search_char == search_char &&
|
||||
ReferenceEquals(_last_primitive_search_assembly_node, assembly_node) &&
|
||||
myTreeView.SelectedNode?.Parent == assembly_node)
|
||||
{
|
||||
start_idx = myTreeView.SelectedNode.Index + 1;
|
||||
}
|
||||
|
||||
TreeNode? match_node = _find_matching_primitive_node(assembly_node, search_char, start_idx) ??
|
||||
_find_matching_primitive_node(assembly_node, search_char, 0);
|
||||
|
||||
if (match_node is not null)
|
||||
{
|
||||
myTreeView.SelectedNode = match_node;
|
||||
match_node.EnsureVisible();
|
||||
_last_primitive_search_char = search_char;
|
||||
_last_primitive_search_assembly_node = assembly_node;
|
||||
e.Handled = true;
|
||||
}
|
||||
}
|
||||
|
||||
private TreeNode? _get_selected_assembly_node()
|
||||
{
|
||||
if (myTreeView.SelectedNode is null)
|
||||
return myTreeView.Nodes.Count > 0 ? myTreeView.Nodes[0] : null;
|
||||
|
||||
return myTreeView.SelectedNode.Parent ?? myTreeView.SelectedNode;
|
||||
}
|
||||
|
||||
private TreeNode? _find_matching_primitive_node(TreeNode assembly_node, char search_char, int start_idx)
|
||||
{
|
||||
for (int i = start_idx; i < assembly_node.Nodes.Count; i++)
|
||||
{
|
||||
TreeNode primitive_node = assembly_node.Nodes[i];
|
||||
string primitive_name = primitive_node.Tag as string ?? primitive_node.Text;
|
||||
if (primitive_name.Length > 0 && Char.ToLowerInvariant(primitive_name[0]) == search_char)
|
||||
return primitive_node;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private void myHandleTreeNodeMouseHover(object? sender, TreeNodeMouseHoverEventArgs e)
|
||||
{
|
||||
if (e.Node?.Tag is not string primitive_name)
|
||||
{
|
||||
_primitive_tool_tip.Hide(myTreeView);
|
||||
return;
|
||||
}
|
||||
|
||||
Type? primitive_type = PrimitiveManager.instance.get_dsl_op_type(primitive_name);
|
||||
if (primitive_type is null)
|
||||
{
|
||||
_primitive_tool_tip.Hide(myTreeView);
|
||||
return;
|
||||
}
|
||||
|
||||
string tooltip_text = $"Namespace: {primitive_type.Namespace ?? "(none)"}";
|
||||
string? source_path = _get_source_file_path(primitive_type);
|
||||
if (!String.IsNullOrWhiteSpace(source_path))
|
||||
tooltip_text += $"\nSource: {source_path}";
|
||||
|
||||
Point mouse_point = myTreeView.PointToClient(Cursor.Position);
|
||||
_primitive_tool_tip.Show(tooltip_text, myTreeView, mouse_point.X + 12, mouse_point.Y + 12, 8000);
|
||||
}
|
||||
|
||||
private void myHandleTreeViewMouseLeave(object? sender, EventArgs e)
|
||||
{
|
||||
_primitive_tool_tip.Hide(myTreeView);
|
||||
}
|
||||
|
||||
private string? _get_source_file_path(Type primitive_type)
|
||||
{
|
||||
string cache_key = primitive_type.FullName ?? primitive_type.Name;
|
||||
if (_primitive_source_path_cache.TryGetValue(cache_key, out string? cached_path))
|
||||
return cached_path;
|
||||
|
||||
Dictionary<string, string?> source_file_index = _source_file_index ??= _build_source_file_index();
|
||||
if (!source_file_index.TryGetValue(primitive_type.Name, out string? source_path))
|
||||
return null;
|
||||
_primitive_source_path_cache[cache_key] = source_path;
|
||||
return source_path;
|
||||
}
|
||||
|
||||
private Dictionary<string, string?> _build_source_file_index()
|
||||
{
|
||||
Dictionary<string, string?> source_file_index = new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
|
||||
string? repo_root = _repo_root_path ??= _find_repo_root();
|
||||
if (String.IsNullOrWhiteSpace(repo_root))
|
||||
return source_file_index;
|
||||
|
||||
string source_root = Path.Combine(repo_root, "src", "DataStreamSuite");
|
||||
if (!Directory.Exists(source_root))
|
||||
return source_file_index;
|
||||
|
||||
foreach (string file_path in Directory.EnumerateFiles(source_root, "*.cs", SearchOption.AllDirectories))
|
||||
{
|
||||
string type_name = Path.GetFileNameWithoutExtension(file_path);
|
||||
if (!source_file_index.ContainsKey(type_name))
|
||||
source_file_index[type_name] = file_path;
|
||||
}
|
||||
return source_file_index;
|
||||
}
|
||||
|
||||
private string? _find_repo_root()
|
||||
{
|
||||
string? curr_path = AppDomain.CurrentDomain.BaseDirectory;
|
||||
while (!String.IsNullOrWhiteSpace(curr_path))
|
||||
{
|
||||
if (Directory.Exists(Path.Combine(curr_path, "src", "DataStreamSuite")))
|
||||
return curr_path;
|
||||
curr_path = Directory.GetParent(curr_path)?.FullName;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private void myToolStripMenuItemExit_Click(object sender, EventArgs e)
|
||||
{
|
||||
Close();
|
||||
@@ -681,7 +1142,7 @@ namespace BCG
|
||||
DialogResult dr = ofd.ShowDialog(this);
|
||||
if (ofd.CheckFileExists && ofd.CheckPathExists && dr == DialogResult.OK)
|
||||
{
|
||||
PrimitiveManager.instance.loadAssembly(ofd.FileName);
|
||||
PrimitiveManager.instance.load_streams_from_assembly(ofd.FileName);
|
||||
myLoadAssemblies();
|
||||
}
|
||||
}
|
||||
@@ -768,7 +1229,8 @@ namespace BCG
|
||||
s = new MutableAppendStream(s, s);
|
||||
s.onChange += OnChange;
|
||||
|
||||
void OnChange(object? obj) {
|
||||
void OnChange(object? obj)
|
||||
{
|
||||
s.onChange -= OnChange;
|
||||
//s = new MutableAppendStream(s, s);
|
||||
var rowCollection = ((IDataPacket)obj!).newRows;
|
||||
|
||||
@@ -8,7 +8,6 @@ using Bts.PubSub.Services;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
|
||||
using Spark.Api.MarketData;
|
||||
using Spark.Api.Services;
|
||||
using Spark.Core.Services;
|
||||
|
||||
@@ -26,10 +25,6 @@ static class Program {
|
||||
host = HostBuilderExtensions.CreateBtsBuilder(args)
|
||||
.ConfigureServices(ConfigureServices)
|
||||
.Build();
|
||||
var mdClient = host.Services.GetService<IMarketDataClient>();
|
||||
if (mdClient is not null) {
|
||||
PrimitiveManager.SetMarketDataClient(mdClient);
|
||||
}
|
||||
await host.StartAsync();
|
||||
var winFormsThread = new Thread(RunWinFormsApp);
|
||||
winFormsThread.SetApartmentState(ApartmentState.STA);
|
||||
|
||||
@@ -5,8 +5,7 @@
|
||||
"ViewStreamDatabase": {
|
||||
"Interval": "00:00:00.1"
|
||||
},
|
||||
"Spark": {
|
||||
},
|
||||
"Spark": {},
|
||||
"SparkApi": {
|
||||
"IsEnabled": false,
|
||||
"Credentials": {
|
||||
@@ -53,21 +52,16 @@
|
||||
"Class": "bts.PubsubServerStreamTagHandler"
|
||||
},
|
||||
{
|
||||
"TagType": "System_Sub",
|
||||
"Assembly": "PubSub",
|
||||
"Class": "BCG.SystemSubscriptionMapPublisher"
|
||||
},
|
||||
{
|
||||
"TagType": "System_Stats",
|
||||
"Assembly": "PubSub",
|
||||
"Class": "BCG.SystemStatsPublisher"
|
||||
"TagType": "DS_.*",
|
||||
"Assembly": "DataStreamCore",
|
||||
"Class": "bts.PubsubServerStreamTagHandler"
|
||||
},
|
||||
{
|
||||
"TagType": ".*",
|
||||
"Assembly": "DataStreamCore",
|
||||
"Class": "bts.PubsubServerGenericDataHandler"
|
||||
"Class": "bts.PubsubServerGenericDataHandler"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "PubSub\PubSub.csp
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSubServer", "PubSubServer\PubSubServer.csproj", "{0DBD814A-E17F-4836-8EEE-6E91B00C578F}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PersistenceServer", "PersistenceServer\PersistenceServer.csproj", "{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "BooleanExpressions\BooleanExpressions.csproj", "{EE53687E-BB62-45C6-A1B0-9B8F16B4E2C1}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Core", "..\cs\Bts.Spark.Core\Bts.Spark.Core.csproj", "{88E70136-91AE-4F56-B243-8F4642B46D6A}"
|
||||
@@ -43,6 +45,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "..\cs\Bts.Net\Bt
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "..\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DSPublisher", "DSPublisher\DSPublisher.csproj", "{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "APIBridgeServer", "APIBridgeServer\APIBridgeServer.csproj", "{3B14C417-531A-4E39-80C1-A4C14BD000D5}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
@@ -113,6 +119,18 @@ Global
|
||||
{0DBD814A-E17F-4836-8EEE-6E91B00C578F}.Release|x64.Build.0 = Release|Any CPU
|
||||
{0DBD814A-E17F-4836-8EEE-6E91B00C578F}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{0DBD814A-E17F-4836-8EEE-6E91B00C578F}.Release|x86.Build.0 = Release|Any CPU
|
||||
{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|x64.Build.0 = Release|Any CPU
|
||||
{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{915A604A-C4B0-4C2B-A70D-D5E0C7AA2B80}.Release|x86.Build.0 = Release|Any CPU
|
||||
{EE53687E-BB62-45C6-A1B0-9B8F16B4E2C1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{EE53687E-BB62-45C6-A1B0-9B8F16B4E2C1}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{EE53687E-BB62-45C6-A1B0-9B8F16B4E2C1}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
@@ -293,6 +311,30 @@ Global
|
||||
{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x64.Build.0 = Release|Any CPU
|
||||
{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x86.Build.0 = Release|Any CPU
|
||||
{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|x64.Build.0 = Release|Any CPU
|
||||
{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{72A4C31D-1680-4E8F-AB5E-A33B3D095DD3}.Release|x86.Build.0 = Release|Any CPU
|
||||
{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|x64.Build.0 = Release|Any CPU
|
||||
{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{3B14C417-531A-4E39-80C1-A4C14BD000D5}.Release|x86.Build.0 = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
|
||||
@@ -27,6 +27,7 @@
|
||||
<ProjectReference Include="..\PubSub\PubSub.csproj" />
|
||||
<ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
|
||||
<ProjectReference Include="..\DataStream\DataStream.csproj" />
|
||||
<ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
namespace Bts.DataStreamTests;
|
||||
|
||||
using BCG;
|
||||
|
||||
using BlueOrb.Operators.Signal;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
[TestFixture]
|
||||
public class HeikinAshiStreamTests {
|
||||
readonly BlueTableSchema schema = new(
|
||||
new ColumnInfo[] {
|
||||
new("id", 0, TypeCode.Int32),
|
||||
new("ts_event", 1, TypeCode.DateTime),
|
||||
new("open", 2, TypeCode.Double),
|
||||
new("high", 3, TypeCode.Double),
|
||||
new("low", 4, TypeCode.Double),
|
||||
new("close", 5, TypeCode.Double),
|
||||
new("volume", 6, TypeCode.Double),
|
||||
}, "bars");
|
||||
|
||||
[Test]
|
||||
public void AppendsHistoryAndCalculatesRecursiveHeikinAshiValues() {
|
||||
var driver = new TestDriverStream(schema, null);
|
||||
var heikinAshi = new HeikinAshiStream(driver);
|
||||
var packets = new List<IDataPacket>();
|
||||
heikinAshi.onChange += nv => packets.Add((IDataPacket)nv);
|
||||
|
||||
driver.Reset(
|
||||
Row(1, new DateTime(2026, 1, 1, 9, 30, 0, DateTimeKind.Utc), 100.0, 110.0, 90.0, 105.0, 10.0),
|
||||
Row(2, new DateTime(2026, 1, 1, 9, 31, 0, DateTimeKind.Utc), 106.0, 112.0, 104.0, 110.0, 20.0)
|
||||
);
|
||||
|
||||
Assert.That(packets, Has.Count.EqualTo(2));
|
||||
|
||||
Assert.That(packets[0].newRows.Count, Is.EqualTo(1));
|
||||
Assert.That(packets[0].addedRows.Count, Is.EqualTo(1));
|
||||
Assert.That(packets[0].deletedRows.Count, Is.EqualTo(0));
|
||||
|
||||
var first = packets[0].newRows[0];
|
||||
Assert.That(first.get<double>("open"), Is.EqualTo(100.0).Within(1e-9));
|
||||
Assert.That(first.get<double>("high"), Is.EqualTo(110.0).Within(1e-9));
|
||||
Assert.That(first.get<double>("low"), Is.EqualTo(90.0).Within(1e-9));
|
||||
Assert.That(first.get<double>("close"), Is.EqualTo(105.0).Within(1e-9));
|
||||
Assert.That(first.get<double>("ha_open"), Is.EqualTo(102.5).Within(1e-9));
|
||||
Assert.That(first.get<double>("ha_high"), Is.EqualTo(110.0).Within(1e-9));
|
||||
Assert.That(first.get<double>("ha_low"), Is.EqualTo(90.0).Within(1e-9));
|
||||
Assert.That(first.get<double>("ha_close"), Is.EqualTo(101.25).Within(1e-9));
|
||||
|
||||
Assert.That(packets[1].newRows.Count, Is.EqualTo(2));
|
||||
Assert.That(packets[1].addedRows.Count, Is.EqualTo(1));
|
||||
Assert.That(packets[1].deletedRows.Count, Is.EqualTo(0));
|
||||
|
||||
var second = packets[1].newRows[1];
|
||||
Assert.That(second.get<double>("open"), Is.EqualTo(106.0).Within(1e-9));
|
||||
Assert.That(second.get<double>("high"), Is.EqualTo(112.0).Within(1e-9));
|
||||
Assert.That(second.get<double>("low"), Is.EqualTo(104.0).Within(1e-9));
|
||||
Assert.That(second.get<double>("close"), Is.EqualTo(110.0).Within(1e-9));
|
||||
Assert.That(second.get<double>("ha_open"), Is.EqualTo(101.875).Within(1e-9));
|
||||
Assert.That(second.get<double>("ha_high"), Is.EqualTo(112.0).Within(1e-9));
|
||||
Assert.That(second.get<double>("ha_low"), Is.EqualTo(101.875).Within(1e-9));
|
||||
Assert.That(second.get<double>("ha_close"), Is.EqualTo(108.0).Within(1e-9));
|
||||
Assert.That(second.get<double>("volume"), Is.EqualTo(20.0).Within(1e-9));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RebuildsHistoryWhenSourceSnapshotChanges() {
|
||||
var driver = new TestDriverStream(schema, null);
|
||||
var heikinAshi = new HeikinAshiStream(driver);
|
||||
var packets = new List<IDataPacket>();
|
||||
heikinAshi.onChange += nv => packets.Add((IDataPacket)nv);
|
||||
|
||||
driver.Reset(
|
||||
Row(1, new DateTime(2026, 1, 1, 9, 30, 0, DateTimeKind.Utc), 100.0, 110.0, 90.0, 105.0, 10.0),
|
||||
Row(2, new DateTime(2026, 1, 1, 9, 31, 0, DateTimeKind.Utc), 106.0, 112.0, 104.0, 110.0, 20.0)
|
||||
);
|
||||
|
||||
driver.Reset(
|
||||
Row(1, new DateTime(2026, 1, 1, 9, 30, 0, DateTimeKind.Utc), 100.0, 110.0, 90.0, 105.0, 10.0),
|
||||
Row(2, new DateTime(2026, 1, 1, 9, 31, 0, DateTimeKind.Utc), 106.0, 114.0, 103.0, 111.0, 21.0)
|
||||
);
|
||||
|
||||
Assert.That(packets, Has.Count.EqualTo(3));
|
||||
|
||||
Assert.That(packets[2].newRows.Count, Is.EqualTo(2));
|
||||
Assert.That(packets[2].addedRows.Count, Is.EqualTo(1));
|
||||
Assert.That(packets[2].deletedRows.Count, Is.EqualTo(1));
|
||||
|
||||
var updated = packets[2].newRows[1];
|
||||
Assert.That(updated.get<double>("open"), Is.EqualTo(106.0).Within(1e-9));
|
||||
Assert.That(updated.get<double>("high"), Is.EqualTo(114.0).Within(1e-9));
|
||||
Assert.That(updated.get<double>("low"), Is.EqualTo(103.0).Within(1e-9));
|
||||
Assert.That(updated.get<double>("close"), Is.EqualTo(111.0).Within(1e-9));
|
||||
Assert.That(updated.get<double>("ha_open"), Is.EqualTo(101.875).Within(1e-9));
|
||||
Assert.That(updated.get<double>("ha_high"), Is.EqualTo(114.0).Within(1e-9));
|
||||
Assert.That(updated.get<double>("ha_low"), Is.EqualTo(101.875).Within(1e-9));
|
||||
Assert.That(updated.get<double>("ha_close"), Is.EqualTo(108.5).Within(1e-9));
|
||||
Assert.That(updated.get<double>("volume"), Is.EqualTo(21.0).Within(1e-9));
|
||||
}
|
||||
|
||||
static IBlueRow Row(int id, DateTime tsEvent, double open, double high, double low, double close, double volume) {
|
||||
var row = new MutableBlueRow(
|
||||
new BlueTableSchema(new[] {
|
||||
new ColumnInfo("id", 0, TypeCode.Int32),
|
||||
new ColumnInfo("ts_event", 1, TypeCode.DateTime),
|
||||
new ColumnInfo("open", 2, TypeCode.Double),
|
||||
new ColumnInfo("high", 3, TypeCode.Double),
|
||||
new ColumnInfo("low", 4, TypeCode.Double),
|
||||
new ColumnInfo("close", 5, TypeCode.Double),
|
||||
new ColumnInfo("volume", 6, TypeCode.Double),
|
||||
}, "bars"));
|
||||
row.setValue("id", id);
|
||||
row.setValue("ts_event", tsEvent);
|
||||
row.setValue("open", open);
|
||||
row.setValue("high", high);
|
||||
row.setValue("low", low);
|
||||
row.setValue("close", close);
|
||||
row.setValue("volume", volume);
|
||||
return row;
|
||||
}
|
||||
}
|
||||
@@ -5,7 +5,7 @@ using BCG.Utilities.General;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using global::Spark.Core.Collections;
|
||||
using Spark.Core.Collections;
|
||||
|
||||
public class TestDriverStream : ViewStream, IWorker {
|
||||
readonly IBlueTableSchema schema;
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
namespace Bts.DataStreamTests;
|
||||
|
||||
using BCG;
|
||||
|
||||
using BlueOrb.Operators.Signal;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
[TestFixture]
|
||||
public class VWAPStreamTests {
|
||||
readonly BlueTableSchema schema = new(
|
||||
new ColumnInfo[] {
|
||||
new("id", 0, TypeCode.Int32),
|
||||
new("ts_event", 1, TypeCode.DateTime),
|
||||
new("close", 2, TypeCode.Double),
|
||||
new("volume", 3, TypeCode.Double),
|
||||
}, "bars");
|
||||
|
||||
[Test]
|
||||
public void EmitsRowsAndCalculatesRollingVWAP() {
|
||||
var driver = new TestDriverStream(schema, null);
|
||||
var vwap = new VWAPStream(driver, "close", "volume", 2);
|
||||
var packets = new List<IDataPacket>();
|
||||
vwap.onChange += nv => packets.Add((IDataPacket)nv);
|
||||
|
||||
driver.Reset(
|
||||
Row(1, new DateTime(2026, 1, 1, 9, 30, 0, DateTimeKind.Utc), 100.0, 10.0),
|
||||
Row(2, new DateTime(2026, 1, 1, 9, 30, 5, DateTimeKind.Utc), 110.0, 20.0)
|
||||
);
|
||||
|
||||
Assert.That(packets, Has.Count.EqualTo(1));
|
||||
Assert.That(packets[0].newRows.Count, Is.EqualTo(2));
|
||||
|
||||
var rows = packets[0].newRows.to_list().OrderBy(r => r.get<int>("id")).ToList();
|
||||
Assert.That(rows[0].get<double>("vwap"), Is.EqualTo(100.0).Within(1e-9));
|
||||
Assert.That(rows[1].get<double>("vwap"), Is.EqualTo((100.0 * 10.0 + 110.0 * 20.0) / 30.0).Within(1e-9));
|
||||
}
|
||||
|
||||
static IBlueRow Row(int id, DateTime tsEvent, double close, double volume) {
|
||||
var row = new MutableBlueRow(
|
||||
new BlueTableSchema(new[] {
|
||||
new ColumnInfo("id", 0, TypeCode.Int32),
|
||||
new ColumnInfo("ts_event", 1, TypeCode.DateTime),
|
||||
new ColumnInfo("close", 2, TypeCode.Double),
|
||||
new ColumnInfo("volume", 3, TypeCode.Double),
|
||||
}, "bars"));
|
||||
row.setValue("id", id);
|
||||
row.setValue("ts_event", tsEvent);
|
||||
row.setValue("close", close);
|
||||
row.setValue("volume", volume);
|
||||
return row;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Text.Json;
|
||||
|
||||
public static class ConfigLoader {
|
||||
public static PersistenceServerConfig load() {
|
||||
var config = new PersistenceServerConfig();
|
||||
merge_from_file(config, Path.Combine(AppContext.BaseDirectory, "appsettings.json"));
|
||||
merge_from_file(config, Path.Combine(AppContext.BaseDirectory, "appsettings.Development.json"));
|
||||
return config;
|
||||
}
|
||||
|
||||
private static void merge_from_file(PersistenceServerConfig target, string path) {
|
||||
if (!File.Exists(path)) {
|
||||
return;
|
||||
}
|
||||
|
||||
var json = File.ReadAllText(path);
|
||||
var loaded = JsonSerializer.Deserialize<PersistenceServerConfig>(json, new JsonSerializerOptions {
|
||||
PropertyNameCaseInsensitive = true
|
||||
});
|
||||
|
||||
if (loaded?.pub_sub is null) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (target.pub_sub is null) {
|
||||
target.pub_sub = loaded.pub_sub;
|
||||
} else {
|
||||
target.pub_sub.MergeWith(loaded.pub_sub);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Data.SQLite;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
|
||||
public class PeristenceHelper {
|
||||
private readonly string _connectionString;
|
||||
public string database_path { get; }
|
||||
|
||||
public PeristenceHelper(string? databasePath = null) {
|
||||
var resolvedPath = databasePath ?? Path.Combine(AppContext.BaseDirectory, "PersistenceServer.sqlite");
|
||||
database_path = resolvedPath;
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(resolvedPath)!);
|
||||
_connectionString = $"Data Source={resolvedPath};Version=3";
|
||||
}
|
||||
|
||||
public void InsertRow(string tableName, IReadOnlyDictionary<string, string> row) {
|
||||
ValidateRow(tableName, row);
|
||||
using var connection = OpenConnection();
|
||||
EnsureTable(connection, tableName, row.Keys);
|
||||
|
||||
var columns = row.Keys.Select(QuoteIdentifier).ToArray();
|
||||
var parameterNames = row.Keys.Select(CreateParameterName).ToArray();
|
||||
var commandText =
|
||||
$"INSERT INTO {QuoteIdentifier(tableName)} ({string.Join(", ", columns)}) VALUES ({string.Join(", ", parameterNames)})";
|
||||
|
||||
using var command = new SQLiteCommand(commandText, connection);
|
||||
AddParameters(command, row);
|
||||
command.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
public void UpdateRow(string tableName, IReadOnlyDictionary<string, string> row) {
|
||||
ValidateRow(tableName, row);
|
||||
using var connection = OpenConnection();
|
||||
EnsureTable(connection, tableName, row.Keys);
|
||||
|
||||
var keyColumns = GetKeyColumns(row);
|
||||
var valueColumns = row.Keys.Where(column => !keyColumns.Contains(column, StringComparer.OrdinalIgnoreCase)).ToArray();
|
||||
if (valueColumns.Length == 0) {
|
||||
throw new InvalidOperationException("UpdateRow requires at least one non-key column to update.");
|
||||
}
|
||||
|
||||
var setClause = string.Join(", ", valueColumns.Select(column => $"{QuoteIdentifier(column)} = {CreateParameterName(column)}"));
|
||||
var whereClause = string.Join(" AND ", keyColumns.Select(column => $"{QuoteIdentifier(column)} = {CreateWhereParameterName(column)}"));
|
||||
var commandText = $"UPDATE {QuoteIdentifier(tableName)} SET {setClause} WHERE {whereClause}";
|
||||
|
||||
using var command = new SQLiteCommand(commandText, connection);
|
||||
foreach (var column in valueColumns) {
|
||||
command.Parameters.AddWithValue(CreateParameterName(column), row[column]);
|
||||
}
|
||||
|
||||
foreach (var column in keyColumns) {
|
||||
command.Parameters.AddWithValue(CreateWhereParameterName(column), row[column]);
|
||||
}
|
||||
|
||||
command.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
public void RemoveRow(string tableName, IReadOnlyDictionary<string, string> row) {
|
||||
ValidateRow(tableName, row);
|
||||
using var connection = OpenConnection();
|
||||
EnsureTable(connection, tableName, row.Keys);
|
||||
|
||||
var keyColumns = GetKeyColumns(row);
|
||||
var whereClause = string.Join(" AND ", keyColumns.Select(column => $"{QuoteIdentifier(column)} = {CreateWhereParameterName(column)}"));
|
||||
var commandText = $"DELETE FROM {QuoteIdentifier(tableName)} WHERE {whereClause}";
|
||||
|
||||
using var command = new SQLiteCommand(commandText, connection);
|
||||
foreach (var column in keyColumns) {
|
||||
command.Parameters.AddWithValue(CreateWhereParameterName(column), row[column]);
|
||||
}
|
||||
|
||||
command.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
private SQLiteConnection OpenConnection() {
|
||||
var connection = new SQLiteConnection(_connectionString);
|
||||
connection.Open();
|
||||
return connection;
|
||||
}
|
||||
|
||||
private static void ValidateRow(string tableName, IReadOnlyDictionary<string, string> row) {
|
||||
if (string.IsNullOrWhiteSpace(tableName)) {
|
||||
throw new ArgumentException("Table name is required.", nameof(tableName));
|
||||
}
|
||||
|
||||
if (row.Count == 0) {
|
||||
throw new ArgumentException("Row must contain at least one column.", nameof(row));
|
||||
}
|
||||
}
|
||||
|
||||
private static string[] GetKeyColumns(IReadOnlyDictionary<string, string> row) {
|
||||
if (row.ContainsKey("namespace") && row.ContainsKey("key")) {
|
||||
return new[] { "namespace", "key" };
|
||||
}
|
||||
|
||||
if (row.ContainsKey("key")) {
|
||||
return new[] { "key" };
|
||||
}
|
||||
|
||||
throw new InvalidOperationException("Row must contain 'key', or both 'namespace' and 'key', for update/delete.");
|
||||
}
|
||||
|
||||
private static void EnsureTable(SQLiteConnection connection, string tableName, IEnumerable<string> columns) {
|
||||
var existingColumns = GetExistingColumns(connection, tableName);
|
||||
if (existingColumns.Count == 0) {
|
||||
var createColumns = string.Join(", ", columns.Select(column => $"{QuoteIdentifier(column)} TEXT"));
|
||||
using var createCommand =
|
||||
new SQLiteCommand($"CREATE TABLE IF NOT EXISTS {QuoteIdentifier(tableName)} ({createColumns})", connection);
|
||||
createCommand.ExecuteNonQuery();
|
||||
existingColumns = GetExistingColumns(connection, tableName);
|
||||
}
|
||||
|
||||
foreach (var column in columns) {
|
||||
if (existingColumns.Contains(column)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
using var alterCommand =
|
||||
new SQLiteCommand($"ALTER TABLE {QuoteIdentifier(tableName)} ADD COLUMN {QuoteIdentifier(column)} TEXT", connection);
|
||||
alterCommand.ExecuteNonQuery();
|
||||
}
|
||||
}
|
||||
|
||||
private static HashSet<string> GetExistingColumns(SQLiteConnection connection, string tableName) {
|
||||
var columns = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
using var command = new SQLiteCommand($"PRAGMA table_info({QuoteIdentifier(tableName)})", connection);
|
||||
using var reader = command.ExecuteReader();
|
||||
while (reader.Read()) {
|
||||
columns.Add(reader["name"].ToString()!);
|
||||
}
|
||||
|
||||
return columns;
|
||||
}
|
||||
|
||||
private static void AddParameters(SQLiteCommand command, IReadOnlyDictionary<string, string> row) {
|
||||
foreach (var pair in row) {
|
||||
command.Parameters.AddWithValue(CreateParameterName(pair.Key), pair.Value);
|
||||
}
|
||||
}
|
||||
|
||||
private static string QuoteIdentifier(string identifier) => $"\"{identifier.Replace("\"", "\"\"")}\"";
|
||||
|
||||
private static string CreateParameterName(string columnName) => $"@p_{columnName.Replace("\"", string.Empty)}";
|
||||
|
||||
private static string CreateWhereParameterName(string columnName) => $"@w_{columnName.Replace("\"", string.Empty)}";
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
|
||||
public class PersistenceCallHandler {
|
||||
private const string KeyValueTableName = "key_value_store";
|
||||
private readonly PeristenceHelper _persistenceHelper;
|
||||
private readonly PubSubSync _pub_sub_sync;
|
||||
|
||||
public PersistenceCallHandler(PeristenceHelper persistenceHelper, PubSubSync pub_sub_sync) {
|
||||
_persistenceHelper = persistenceHelper;
|
||||
_pub_sub_sync = pub_sub_sync;
|
||||
}
|
||||
|
||||
public void Handle(HttpListenerRequest request, string payload) {
|
||||
Console.WriteLine("----- Incoming REST call -----");
|
||||
Console.WriteLine($"Timestamp: {DateTimeOffset.Now:O}");
|
||||
Console.WriteLine($"Method: {request.HttpMethod}");
|
||||
Console.WriteLine($"Url: {request.Url}");
|
||||
Console.WriteLine($"RawUrl: {request.RawUrl}");
|
||||
Console.WriteLine($"RemoteEndPoint: {request.RemoteEndPoint}");
|
||||
Console.WriteLine($"UserAgent: {request.UserAgent}");
|
||||
Console.WriteLine($"ContentType: {request.ContentType}");
|
||||
Console.WriteLine($"ContentLength: {request.ContentLength64}");
|
||||
|
||||
if (request.QueryString.Count > 0) {
|
||||
Console.WriteLine("Query:");
|
||||
foreach (var key in request.QueryString.AllKeys) {
|
||||
if (key is null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
Console.WriteLine($" {key} = {request.QueryString[key]}");
|
||||
}
|
||||
}
|
||||
|
||||
if (request.Headers.Count > 0) {
|
||||
Console.WriteLine("Headers:");
|
||||
foreach (var headerKey in request.Headers.AllKeys) {
|
||||
if (headerKey is null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
Console.WriteLine($" {headerKey}: {request.Headers[headerKey]}");
|
||||
}
|
||||
}
|
||||
|
||||
Console.WriteLine("Body:");
|
||||
Console.WriteLine(string.IsNullOrWhiteSpace(payload) ? " <empty>" : payload);
|
||||
|
||||
var rpcRequest = ParseRpcRequest(payload);
|
||||
HandleRpc(rpcRequest.FunctionName, rpcRequest.Arguments);
|
||||
Console.WriteLine("------------------------------");
|
||||
}
|
||||
|
||||
private static PersistenceRpcRequest ParseRpcRequest(string payload) {
|
||||
if (string.IsNullOrWhiteSpace(payload)) {
|
||||
throw new InvalidOperationException("RPC payload is empty.");
|
||||
}
|
||||
|
||||
using var document = JsonDocument.Parse(payload);
|
||||
var root = document.RootElement;
|
||||
|
||||
if (!root.TryGetProperty("function_name", out var functionNameElement) ||
|
||||
functionNameElement.ValueKind != JsonValueKind.String) {
|
||||
throw new InvalidOperationException("RPC payload must contain a string 'function_name'.");
|
||||
}
|
||||
|
||||
if (!root.TryGetProperty("arguments", out var argumentsElement) ||
|
||||
argumentsElement.ValueKind != JsonValueKind.Array) {
|
||||
throw new InvalidOperationException("RPC payload must contain an 'arguments' array.");
|
||||
}
|
||||
|
||||
var arguments = new List<string>();
|
||||
foreach (var argument in argumentsElement.EnumerateArray()) {
|
||||
if (argument.ValueKind != JsonValueKind.String) {
|
||||
throw new InvalidOperationException("RPC arguments must all be strings.");
|
||||
}
|
||||
|
||||
arguments.Add(argument.GetString()!);
|
||||
}
|
||||
|
||||
return new PersistenceRpcRequest(functionNameElement.GetString()!, arguments.ToArray());
|
||||
}
|
||||
|
||||
private void HandleRpc(string functionName, params string[] arguments) {
|
||||
switch (functionName) {
|
||||
case "keyvalue_insert":
|
||||
RequireArgumentCount(functionName, arguments, 3);
|
||||
HandleKeyValueInsert(arguments[0], arguments[1], arguments[2]);
|
||||
break;
|
||||
case "keyvalue_update":
|
||||
RequireArgumentCount(functionName, arguments, 3);
|
||||
HandleKeyValueUpdate(arguments[0], arguments[1], arguments[2]);
|
||||
break;
|
||||
case "keyvalue_delete":
|
||||
RequireArgumentCount(functionName, arguments, 2);
|
||||
HandleKeyValueDelete(arguments[0], arguments[1]);
|
||||
break;
|
||||
default:
|
||||
throw new InvalidOperationException($"Unknown RPC function '{functionName}'.");
|
||||
}
|
||||
}
|
||||
|
||||
private static void RequireArgumentCount(string functionName, string[] arguments, int expectedCount) {
|
||||
if (arguments.Length != expectedCount) {
|
||||
throw new InvalidOperationException(
|
||||
$"RPC function '{functionName}' expected {expectedCount} arguments but received {arguments.Length}.");
|
||||
}
|
||||
}
|
||||
|
||||
private void HandleKeyValueInsert(string itemNamespace, string key, string value) {
|
||||
var row = new Dictionary<string, string> {
|
||||
["namespace"] = itemNamespace,
|
||||
["key"] = key,
|
||||
["value"] = value,
|
||||
};
|
||||
_persistenceHelper.InsertRow(KeyValueTableName, row);
|
||||
_pub_sub_sync.publish_table(KeyValueTableName);
|
||||
Console.WriteLine($"RPC: keyvalue_insert(namespace='{itemNamespace}', key='{key}', value='{value}')");
|
||||
}
|
||||
|
||||
private void HandleKeyValueUpdate(string itemNamespace, string key, string value) {
|
||||
var row = new Dictionary<string, string> {
|
||||
["namespace"] = itemNamespace,
|
||||
["key"] = key,
|
||||
["value"] = value,
|
||||
};
|
||||
_persistenceHelper.UpdateRow(KeyValueTableName, row);
|
||||
_pub_sub_sync.publish_table(KeyValueTableName);
|
||||
Console.WriteLine($"RPC: keyvalue_update(namespace='{itemNamespace}', key='{key}', value='{value}')");
|
||||
}
|
||||
|
||||
private void HandleKeyValueDelete(string itemNamespace, string key) {
|
||||
var row = new Dictionary<string, string> {
|
||||
["namespace"] = itemNamespace,
|
||||
["key"] = key,
|
||||
};
|
||||
_persistenceHelper.RemoveRow(KeyValueTableName, row);
|
||||
_pub_sub_sync.publish_table(KeyValueTableName);
|
||||
Console.WriteLine($"RPC: keyvalue_delete(namespace='{itemNamespace}', key='{key}')");
|
||||
}
|
||||
}
|
||||
|
||||
public record PersistenceRpcRequest {
|
||||
public PersistenceRpcRequest(string functionName, string[] arguments) {
|
||||
FunctionName = functionName;
|
||||
Arguments = arguments;
|
||||
Console.WriteLine($"PersistenceRpcRequest created for function '{FunctionName}' with {Arguments.Length} argument(s).");
|
||||
}
|
||||
|
||||
public string FunctionName { get; init; }
|
||||
public string[] Arguments { get; init; }
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="System.Data.SQLite" Version="1.0.118" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Content Include="appsettings.json">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</Content>
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,5 @@
|
||||
using Bts.PubSub;
|
||||
|
||||
public class PersistenceServerConfig {
|
||||
public PubSubConfig? pub_sub { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
using Bts.DataStreamCore;
|
||||
using BCG;
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Net;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
const int Port = 5079;
|
||||
var prefix = $"http://localhost:{Port}/";
|
||||
|
||||
var listener = new HttpListener();
|
||||
listener.Prefixes.Add(prefix);
|
||||
var config = ConfigLoader.load();
|
||||
var persistenceHelper = new PeristenceHelper();
|
||||
var pub_sub_sync = new PubSubSync(persistenceHelper.database_path, config.pub_sub);
|
||||
var callHandler = new PersistenceCallHandler(persistenceHelper, pub_sub_sync);
|
||||
|
||||
Console.WriteLine("PersistenceServer starting.");
|
||||
Console.WriteLine($"Listening on {prefix}");
|
||||
Console.WriteLine($"DataStreamCore loaded from: {typeof(IDataPacket).Assembly.FullName}");
|
||||
Console.WriteLine("Press Ctrl+C to stop.");
|
||||
|
||||
var shutdown = new CancellationTokenSource();
|
||||
|
||||
Console.CancelKeyPress += (_, eventArgs) => {
|
||||
eventArgs.Cancel = true;
|
||||
Console.WriteLine("Shutdown requested.");
|
||||
shutdown.Cancel();
|
||||
if (listener.IsListening) {
|
||||
listener.Stop();
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
pub_sub_sync.publish_all_tables();
|
||||
listener.Start();
|
||||
await RunAsync(listener, callHandler, shutdown.Token);
|
||||
return 0;
|
||||
} catch (HttpListenerException ex) when (shutdown.IsCancellationRequested) {
|
||||
Console.WriteLine($"Listener stopped: {ex.Message}");
|
||||
return 0;
|
||||
} catch (Exception ex) {
|
||||
Console.Error.WriteLine($"PersistenceServer crashed: {ex}");
|
||||
return 1;
|
||||
} finally {
|
||||
listener.Close();
|
||||
}
|
||||
|
||||
static async Task RunAsync(
|
||||
HttpListener listener,
|
||||
PersistenceCallHandler callHandler,
|
||||
CancellationToken cancellationToken) {
|
||||
while (!cancellationToken.IsCancellationRequested) {
|
||||
HttpListenerContext context;
|
||||
try {
|
||||
context = await listener.GetContextAsync();
|
||||
} catch (HttpListenerException) when (cancellationToken.IsCancellationRequested) {
|
||||
break;
|
||||
} catch (ObjectDisposedException) when (cancellationToken.IsCancellationRequested) {
|
||||
break;
|
||||
}
|
||||
|
||||
_ = Task.Run(() => HandleRequestAsync(context, callHandler, cancellationToken), cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
static async Task HandleRequestAsync(
|
||||
HttpListenerContext context,
|
||||
PersistenceCallHandler callHandler,
|
||||
CancellationToken cancellationToken) {
|
||||
var request = context.Request;
|
||||
var response = context.Response;
|
||||
|
||||
try {
|
||||
var body = await ReadBodyAsync(request, cancellationToken);
|
||||
callHandler.Handle(request, body);
|
||||
|
||||
var responseText =
|
||||
$$"""
|
||||
{"status":"ok","method":"{{request.HttpMethod}}","path":"{{request.RawUrl ?? "/"}}"}
|
||||
""";
|
||||
|
||||
var bytes = Encoding.UTF8.GetBytes(responseText);
|
||||
response.StatusCode = (int)HttpStatusCode.OK;
|
||||
response.ContentType = "application/json";
|
||||
response.ContentEncoding = Encoding.UTF8;
|
||||
response.ContentLength64 = bytes.LongLength;
|
||||
await response.OutputStream.WriteAsync(bytes, cancellationToken);
|
||||
} catch (Exception ex) {
|
||||
Console.Error.WriteLine($"Request handling failed: {ex}");
|
||||
response.StatusCode = (int)HttpStatusCode.InternalServerError;
|
||||
} finally {
|
||||
response.OutputStream.Close();
|
||||
response.Close();
|
||||
}
|
||||
}
|
||||
|
||||
static async Task<string> ReadBodyAsync(HttpListenerRequest request, CancellationToken cancellationToken) {
|
||||
if (!request.HasEntityBody) {
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
using var reader = new StreamReader(
|
||||
request.InputStream,
|
||||
request.ContentEncoding ?? Encoding.UTF8,
|
||||
detectEncodingFromByteOrderMarks: true,
|
||||
leaveOpen: true);
|
||||
|
||||
return await reader.ReadToEndAsync(cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Data;
|
||||
using System.Data.SQLite;
|
||||
|
||||
using BCG;
|
||||
using Bts.DataStreamCore;
|
||||
using Bts.PubSub;
|
||||
using Bts.PubSub.Routing;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
|
||||
using bts;
|
||||
|
||||
public class PubSubSync {
|
||||
private readonly string _connectionString;
|
||||
private readonly Dictionary<string, IPubTableBroker> _brokers = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly object _sync_root = new();
|
||||
private readonly IServiceProvider _serviceProvider;
|
||||
private readonly PubSubRouterConfig _router_config;
|
||||
private readonly PubSubTagHandlerFactoryConfig _tag_handler_factory_config;
|
||||
|
||||
public PubSubSync(string databasePath, PubSubConfig? pubSubConfig = null) {
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(databasePath);
|
||||
_connectionString = $"Data Source={databasePath};Version=3";
|
||||
var resolved_pub_sub_config = get_or_initialize_pub_sub_config(pubSubConfig);
|
||||
_router_config = resolved_pub_sub_config.Routing ?? throw new InvalidOperationException("PubSub routing config is required.");
|
||||
_tag_handler_factory_config = resolved_pub_sub_config.TagHandlerFactory ?? PubSubTagHandlerFactoryConfig.Default();
|
||||
_serviceProvider = new ServiceCollection().BuildServiceProvider();
|
||||
}
|
||||
|
||||
public void publish_all_tables() {
|
||||
lock (_sync_root) {
|
||||
foreach (var table_name in get_table_names()) {
|
||||
publish_table_core(table_name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void publish_table(string table_name) {
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(table_name);
|
||||
|
||||
lock (_sync_root) {
|
||||
publish_table_core(table_name);
|
||||
}
|
||||
}
|
||||
|
||||
private void publish_table_core(string table_name) {
|
||||
var rows = load_table(table_name);
|
||||
var broker = get_or_create_broker(table_name);
|
||||
broker.create_table(rows);
|
||||
Console.WriteLine($"PubSubSync published table '{table_name}' with {rows.Count} row(s).");
|
||||
}
|
||||
|
||||
private IPubTableBroker get_or_create_broker(string table_name) {
|
||||
if (_brokers.TryGetValue(table_name, out var broker)) {
|
||||
return broker;
|
||||
}
|
||||
|
||||
var publisher_core = new PublisherCore(
|
||||
new PubSubRouter(_router_config),
|
||||
new PubsubTagHandlerFactory(_serviceProvider, _tag_handler_factory_config));
|
||||
broker = new PubTableBroker(publisher_core);
|
||||
_brokers[table_name] = broker;
|
||||
return broker;
|
||||
}
|
||||
|
||||
private string[] get_table_names() {
|
||||
using var connection = open_connection();
|
||||
using var command = new SQLiteCommand(
|
||||
"SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name",
|
||||
connection);
|
||||
using var reader = command.ExecuteReader();
|
||||
|
||||
var table_names = new List<string>();
|
||||
while (reader.Read()) {
|
||||
table_names.Add(reader.GetString(0));
|
||||
}
|
||||
|
||||
return table_names.ToArray();
|
||||
}
|
||||
|
||||
private IBlueRowCollection load_table(string table_name) {
|
||||
using var connection = open_connection();
|
||||
using var adapter = new SQLiteDataAdapter($"SELECT * FROM {quote_identifier(table_name)}", connection);
|
||||
var table = new DataTable(table_name);
|
||||
adapter.Fill(table);
|
||||
table.TableName = table_name;
|
||||
return BlueDataUtility.toIBlueRowCollection(table);
|
||||
}
|
||||
|
||||
private SQLiteConnection open_connection() {
|
||||
var connection = new SQLiteConnection(_connectionString);
|
||||
connection.Open();
|
||||
return connection;
|
||||
}
|
||||
|
||||
private static PubSubConfig get_or_initialize_pub_sub_config(PubSubConfig? configured_pub_sub) {
|
||||
try {
|
||||
PubSubConfig.SetGlobalConfig(PubSubConfig.Default());
|
||||
var existing = PubSubConfig.Instance;
|
||||
if (configured_pub_sub is not null) {
|
||||
existing.MergeWith(configured_pub_sub);
|
||||
}
|
||||
return existing;
|
||||
} catch (InvalidOperationException) {
|
||||
var config = PubSubConfig.Default();
|
||||
if (configured_pub_sub is not null) {
|
||||
config.MergeWith(configured_pub_sub);
|
||||
}
|
||||
PubSubConfig.SetGlobalConfig(config);
|
||||
return config;
|
||||
}
|
||||
}
|
||||
|
||||
private static string quote_identifier(string identifier) => $"\"{identifier.Replace("\"", "\"\"")}\"";
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"PubSub": {
|
||||
"Routing": {
|
||||
"PublisherRoutes": [
|
||||
{
|
||||
"Name": "default",
|
||||
"ServerAddress": "bts-cloud://localhost:22019",
|
||||
"TagRegExp": ".*"
|
||||
}
|
||||
],
|
||||
"SubscriberRoutes": [
|
||||
{
|
||||
"Name": "default",
|
||||
"ServerAddress": "bts-cloud://localhost:22019",
|
||||
"TagRegExp": ".*"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import json
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
|
||||
SERVER_URL = "http://localhost:5079/save?source=python-client"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
payload = {
|
||||
"function_name": "keyvalue_insert",
|
||||
"arguments": [
|
||||
"example-namespace",
|
||||
"example-key",
|
||||
"hello from python",
|
||||
],
|
||||
}
|
||||
|
||||
data = json.dumps(payload).encode("utf-8")
|
||||
request = urllib.request.Request(
|
||||
SERVER_URL,
|
||||
data=data,
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"X-Client": "PersistenceServerPythonClient",
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(request) as response:
|
||||
body = response.read().decode("utf-8")
|
||||
print(f"status: {response.status}")
|
||||
print(f"body: {body}")
|
||||
return 0
|
||||
except urllib.error.URLError as error:
|
||||
print(f"request failed: {error}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+36
-36
@@ -1,44 +1,44 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
using System.Runtime.InteropServices;
|
||||
// using System;
|
||||
// using System.Collections.Generic;
|
||||
// using System.Text;
|
||||
// using System.Runtime.InteropServices;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
public class ClientToServerConnectionWrapper
|
||||
{
|
||||
[DllImport("BlutoInterop.dll")]
|
||||
public unsafe static extern void init();
|
||||
// namespace BCG
|
||||
// {
|
||||
// public class ClientToServerConnectionWrapper
|
||||
// {
|
||||
// [DllImport("BlutoInterop.dll")]
|
||||
// public unsafe static extern void init();
|
||||
|
||||
[DllImport("BlutoInterop.dll")]
|
||||
public unsafe static extern void initServerConnection(byte[] ip, int port);
|
||||
// [DllImport("BlutoInterop.dll")]
|
||||
// public unsafe static extern void initServerConnection(byte[] ip, int port);
|
||||
|
||||
[DllImport("BlutoInterop.dll")]
|
||||
public static extern void sendDataToServer(byte[] ip, int port, byte[] message, int size);
|
||||
// [DllImport("BlutoInterop.dll")]
|
||||
// public static extern void sendDataToServer(byte[] ip, int port, byte[] message, int size);
|
||||
|
||||
[DllImport("BlutoInterop.dll")]
|
||||
public unsafe static extern void getServerData(int timeout, int maxElems, BlutoTCPClientMessage*** retpts, out int npts);
|
||||
// [DllImport("BlutoInterop.dll")]
|
||||
// public unsafe static extern void getServerData(int timeout, int maxElems, BlutoTCPClientMessage*** retpts, out int npts);
|
||||
|
||||
public static byte[] stringToBytes(string s)
|
||||
{
|
||||
byte[] rv = new byte[s.Length + 1];
|
||||
for (int i = 0; i < s.Length; i++)
|
||||
rv[i] = Convert.ToByte(s[i]);
|
||||
rv[s.Length] = Convert.ToByte ('\0');
|
||||
return rv;
|
||||
}
|
||||
// public static byte[] stringToBytes(string s)
|
||||
// {
|
||||
// byte[] rv = new byte[s.Length + 1];
|
||||
// for (int i = 0; i < s.Length; i++)
|
||||
// rv[i] = Convert.ToByte(s[i]);
|
||||
// rv[s.Length] = Convert.ToByte ('\0');
|
||||
// return rv;
|
||||
// }
|
||||
|
||||
public static string intPtrToString(IntPtr iptr)
|
||||
{
|
||||
return Marshal.PtrToStringAnsi(iptr);
|
||||
}
|
||||
// public static string intPtrToString(IntPtr iptr)
|
||||
// {
|
||||
// return Marshal.PtrToStringAnsi(iptr);
|
||||
// }
|
||||
|
||||
public static byte[] intPtrToByteArray(IntPtr iptr, int sz)
|
||||
{
|
||||
byte[] data = new byte[sz];
|
||||
Marshal.Copy(iptr, data, 0, data.Length);
|
||||
return data;
|
||||
}
|
||||
// public static byte[] intPtrToByteArray(IntPtr iptr, int sz)
|
||||
// {
|
||||
// byte[] data = new byte[sz];
|
||||
// Marshal.Copy(iptr, data, 0, data.Length);
|
||||
// return data;
|
||||
// }
|
||||
|
||||
}
|
||||
}
|
||||
// }
|
||||
// }
|
||||
@@ -39,8 +39,6 @@ public static class PubSubMessageUtils {
|
||||
}
|
||||
}
|
||||
|
||||
// Subscribe //////////////////////////////////////////////////////////////
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
public class SubscribeMsg : IPubSubMessage, IEquatable<SubscribeMsg> {
|
||||
public string? connection;
|
||||
public string subscribing_app = ClientInfo.thisClient.app;
|
||||
@@ -86,8 +84,6 @@ public class SubscribeMsg : IPubSubMessage, IEquatable<SubscribeMsg> {
|
||||
$"{GetType().Name} connection: {connection} subscribing_app: {subscribing_app} tags: {string.Concat(tags.empty_if_null())}";
|
||||
}
|
||||
|
||||
// Unsubscribe ////////////////////////////////////////////////////////////
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
public class UnsubscribeMsg : SubscribeMsg, IEquatable<UnsubscribeMsg> {
|
||||
public bool Equals(UnsubscribeMsg? other) =>
|
||||
other is not null && base.Equals(other);
|
||||
@@ -99,8 +95,6 @@ public class UnsubscribeMsg : SubscribeMsg, IEquatable<UnsubscribeMsg> {
|
||||
base.GetHashCode();
|
||||
}
|
||||
|
||||
// Trasnaction /////////////////////////////////////////////////////////////
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
public class TransactionMsg : IPubSubMessage, IEquatable<TransactionMsg> {
|
||||
uint n_msgs;
|
||||
|
||||
@@ -127,15 +121,12 @@ public class TransactionMsg : IPubSubMessage, IEquatable<TransactionMsg> {
|
||||
n_msgs.GetHashCode();
|
||||
}
|
||||
|
||||
// Publish ////////////////////////////////////////////////////////////////
|
||||
/// ///////////////////////////////////////////////////////////////////////
|
||||
public class PublishMsg : IPubSubMessage, IEquatable<PublishMsg> {
|
||||
public string? connection;
|
||||
public string publishing_app = ClientInfo.thisClient.app;
|
||||
public string? tag;
|
||||
public string? tag_type;
|
||||
public bool is_compressed = false;
|
||||
// this is the size that of the msg, regarless of it being serialized or not
|
||||
public long publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
|
||||
public byte[]? data_to_publish;
|
||||
|
||||
@@ -144,8 +135,6 @@ public class PublishMsg : IPubSubMessage, IEquatable<PublishMsg> {
|
||||
Log.assert(!string.IsNullOrEmpty(tag_type));
|
||||
Log.assert(!string.IsNullOrEmpty(tag));
|
||||
Log.assert(!string.IsNullOrEmpty(connection));
|
||||
// The 'ThrowIfNullOrEmpty' calls are redundant (given the assert statements above), but
|
||||
// necessary for the compiler to recognize the strings won't be null when this method exits.
|
||||
ArgumentException.ThrowIfNullOrEmpty(tag_type);
|
||||
ArgumentException.ThrowIfNullOrEmpty(tag);
|
||||
ArgumentException.ThrowIfNullOrEmpty(connection);
|
||||
@@ -181,7 +170,7 @@ public class PublishMsg : IPubSubMessage, IEquatable<PublishMsg> {
|
||||
}
|
||||
|
||||
public bool Equals(PublishMsg? other) =>
|
||||
other is not null
|
||||
other is not null
|
||||
&& connection == other.connection
|
||||
&& publishing_app == other.publishing_app
|
||||
&& tag == other.tag
|
||||
@@ -220,7 +209,6 @@ public enum PubSubMessageType {
|
||||
|
||||
public class PubSubClientMessage : IPubSubMessage {
|
||||
public PubSubMessageType msg_type;
|
||||
// only one of these wil be non null
|
||||
public SubscribeMsg? sub_msg;
|
||||
public UnsubscribeMsg? unsub_msg;
|
||||
public TransactionMsg? transaction_msg;
|
||||
@@ -265,7 +253,7 @@ public class PubSubClientMessage : IPubSubMessage {
|
||||
ValidateMsg(pub_msg).serialize_into_str(str);
|
||||
break;
|
||||
|
||||
T ValidateMsg<T>([NotNull]T? msg, [CallerArgumentExpression(nameof(msg))]string name = "") {
|
||||
T ValidateMsg<T>([NotNull] T? msg, [CallerArgumentExpression(nameof(msg))] string name = "") {
|
||||
if (msg is null) {
|
||||
throw new InvalidOperationException($"msg type is {msg_type} but {name} is null");
|
||||
}
|
||||
@@ -383,7 +371,7 @@ public class PubSubServerMessage : IPubSubMessage, IEquatable<PubSubServerMessag
|
||||
|
||||
[MemberNotNull(nameof(tag), nameof(tag_type))]
|
||||
public void construct_deserialize(byte[] msg) =>
|
||||
construct_deserialize(new MemoryStream(msg) { Position = 0});
|
||||
construct_deserialize(new MemoryStream(msg) { Position = 0 });
|
||||
|
||||
public bool Equals(PubSubServerMessage? other) =>
|
||||
other is not null
|
||||
|
||||
@@ -48,16 +48,16 @@ public class PubSubConfig {
|
||||
HeartbeatInterval = 15000,
|
||||
MissingHeartbeatsBeforeGc = 50,
|
||||
DisposedConnectionCheckPeriod = 1000,
|
||||
ServerPort = 12345,
|
||||
ServerPort = 22019,
|
||||
TagTypeToAssemblyMap = new(),
|
||||
TagHandlerFactory = PubSubTagHandlerFactoryConfig.Default(),
|
||||
SimpleEchoServer = false,
|
||||
Routing = new() {
|
||||
PublisherRoutes = new() {
|
||||
new("bts-cloud://127.0.0.1:12345", ".*")
|
||||
new("bts-cloud://127.0.0.1:22019", ".*")
|
||||
},
|
||||
SubscriberRoutes = new() {
|
||||
new("bts-cloud://127.0.0.1:12345", ".*")
|
||||
new("bts-cloud://127.0.0.1:22019", ".*")
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -12,6 +12,7 @@ using System.Net.Sockets;
|
||||
|
||||
// opens a TCP port and starts listening to connections
|
||||
public class AsyncTCPConnectionAcceptor_APM {
|
||||
// This acceptor exists as an alternate implementation and is not the one currently used by the pubsub server.
|
||||
private readonly int _port;
|
||||
Socket? _accept_socket;
|
||||
readonly ITCPConnectionHandler _connection_handler;
|
||||
|
||||
@@ -13,6 +13,7 @@ namespace bts {
|
||||
|
||||
// opens a TCP port and starts listening to connections. Unlike AsyncTCPConnectionAcceptor_APM,
|
||||
// it uses TcpListener, instead of raw sockets.
|
||||
// This is the acceptor currently used by the pubsub server via ManagedTCPMessageHub.
|
||||
public class AsyncTCPConnectionAcceptor_AsyncWrapper {
|
||||
private readonly int _accept_port;
|
||||
// private Socket _accept_socket;
|
||||
|
||||
@@ -10,6 +10,7 @@ namespace bts {
|
||||
// Unlike AsyncTCPConnectionAcceptor_APM this class is static since the requests to connect are stand alone requests
|
||||
// whereas AsyncTCPConnectionAcceptor_APM is always listening for incoming connections which lends itself to more
|
||||
// statefullness and hence non static
|
||||
// This connector exists as an alternate implementation and is not the one currently used by pubsub clients.
|
||||
public static class AsyncTCPConnector_APM {
|
||||
|
||||
internal class SocketConnectionArgs {
|
||||
|
||||
@@ -13,17 +13,18 @@ using System.Diagnostics.CodeAnalysis;
|
||||
|
||||
|
||||
namespace bts {
|
||||
// Unlike AsyncTCPConnectionAcceptor_APM this class is static since the requests to connect are stand alone requests
|
||||
// whereas AsyncTCPConnectionAcceptor_APM is always listening for incoming connections which lends itself to more
|
||||
// Unlike AsyncTCPConnectionAcceptor_AsyncWrappe this class is static since the requests to connect are stand alone requests
|
||||
// whereas AsyncTCPConnectionAcceptor_AsyncWrappe is always listening for incoming connections which lends itself to more
|
||||
// statefullness and hence non static
|
||||
// This is the connector currently used by pubsub clients via PubSubServerManagedTCPConnection.
|
||||
public static class AsyncTCPConnector_AsyncWrapper {
|
||||
public static void connect_to(RemoteConnectionInfo server_address, ITCPConnectionHandler handler) {
|
||||
bts.Log.Gen.info($"AsyncTCPConnector_AsyncWrapper.connect_to server: {server_address}");
|
||||
//Task.Run(() => _connect_to(server_address, handler));
|
||||
ThreadPool.QueueUserWorkItem(_ => _keep_trying_blocking_function(
|
||||
Connect, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5)));
|
||||
_connect, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5)));
|
||||
|
||||
bool Connect() {
|
||||
bool _connect() {
|
||||
try {
|
||||
return _connect_to(server_address, handler);
|
||||
} catch (Exception ex) {
|
||||
@@ -49,7 +50,7 @@ namespace bts {
|
||||
handler.onConnection(conn);
|
||||
return true;
|
||||
|
||||
bool CreateTcpClient([NotNullWhen(true)]out TcpClient? tcpClient) {
|
||||
bool CreateTcpClient([NotNullWhen(true)] out TcpClient? tcpClient) {
|
||||
try {
|
||||
tcpClient = new(server_rci.ip, server_rci.port);
|
||||
return true;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
using System;
|
||||
using System;
|
||||
using System.Text;
|
||||
|
||||
namespace BCG
|
||||
@@ -24,8 +24,8 @@ namespace BCG
|
||||
{
|
||||
get { return myApp; }
|
||||
set { myApp = value; }
|
||||
|
||||
}
|
||||
|
||||
private string myKeyQualifier;
|
||||
public string keyQualifier
|
||||
{
|
||||
@@ -38,8 +38,8 @@ namespace BCG
|
||||
{
|
||||
get { return myStartTime; }
|
||||
set { myStartTime = value; }
|
||||
|
||||
}
|
||||
|
||||
string? myProcessString;
|
||||
public string processString
|
||||
{
|
||||
@@ -49,10 +49,8 @@ namespace BCG
|
||||
myProcessString = getProcessString(";");
|
||||
return myProcessString;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
public ClientInfo(string theMachineName, int thePid, string theProcessName, string theKeyQualifier, DateTime theStartTime)
|
||||
{
|
||||
myMachineName = theMachineName;
|
||||
@@ -68,10 +66,8 @@ namespace BCG
|
||||
myMachineName = Environment.MachineName;
|
||||
myPid = System.Diagnostics.Process.GetCurrentProcess().Id;
|
||||
myApp = System.Diagnostics.Process.GetCurrentProcess().ProcessName.ToUpper();
|
||||
string[] args = System.Environment.GetCommandLineArgs();
|
||||
string[] args = Environment.GetCommandLineArgs();
|
||||
|
||||
//the key qualifier is the last argument given. If they didn't give any arguments
|
||||
//but the program itself, then the KQ is empty.
|
||||
if (args.Length > 1)
|
||||
myKeyQualifier = args[args.Length - 1];
|
||||
else
|
||||
@@ -86,13 +82,13 @@ namespace BCG
|
||||
|
||||
public static ClientInfo thisClient
|
||||
{
|
||||
get
|
||||
get
|
||||
{
|
||||
if (myLocalClient == null)
|
||||
{
|
||||
lock(myLock)
|
||||
lock (myLock)
|
||||
{
|
||||
if(myLocalClient == null)
|
||||
if (myLocalClient == null)
|
||||
myLocalClient = new ClientInfo();
|
||||
}
|
||||
}
|
||||
@@ -102,7 +98,6 @@ namespace BCG
|
||||
|
||||
public static ClientInfo fromProcessString(string procString)
|
||||
{
|
||||
|
||||
ClientInfo outputGuy = new ClientInfo();
|
||||
|
||||
string[] info = procString.Split(';');
|
||||
@@ -112,9 +107,7 @@ namespace BCG
|
||||
outputGuy.myKeyQualifier = info[2];
|
||||
outputGuy.myPid = Int32.Parse(info[3]);
|
||||
|
||||
|
||||
return outputGuy;
|
||||
|
||||
}
|
||||
|
||||
private string getProcessString(string seperator)
|
||||
|
||||
@@ -3,7 +3,7 @@ using System;
|
||||
namespace BCG.Utilities.General
|
||||
{
|
||||
public interface IWorker
|
||||
{
|
||||
{
|
||||
void doWork();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -79,5 +79,4 @@ namespace mbts
|
||||
return ToString().GetHashCode();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -9,7 +9,7 @@ namespace BCG {
|
||||
/// <summary>
|
||||
/// Collection of utility methods used throughout
|
||||
/// </summary>
|
||||
public static class StreamUtilityMethods {
|
||||
public static class StreamUtilityMethods {
|
||||
|
||||
// extension method to write a string C++ style (len prefixed)
|
||||
public static void write_string_cpp(this BinaryWriter bw, string? str) {
|
||||
@@ -98,7 +98,7 @@ namespace BCG {
|
||||
byte[] secondStreamBytes = new byte[stream.Length - splitLocation];
|
||||
for (int i = 0; i < stream.Length; ++i) {
|
||||
//for some resaon, the 'getBytesFromStream' function prepends the length of the byte array to the bytes. This means
|
||||
//we have to ignore those bytes here.
|
||||
//we have to ignore those bytes here.
|
||||
if (i < splitLocation)
|
||||
firstStreamBytes[i] = streamBytes[i + 4];
|
||||
else
|
||||
@@ -107,8 +107,6 @@ namespace BCG {
|
||||
}
|
||||
|
||||
return new Stream[] { new MemoryStream(firstStreamBytes), new MemoryStream(secondStreamBytes) };
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,8 @@ using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace bts {
|
||||
// This caches all the published data. This class is created and held by
|
||||
// This caches all the published data, on the pubsliher (client) side.
|
||||
// This class is created and held by
|
||||
// PublisherCore. When data is publsihed , PublisherCore passes it to this
|
||||
// class via the publish call. On a reconnection to the PubSubServer,
|
||||
// PublisherCore calls get_content to get the data to re-publish, to
|
||||
|
||||
@@ -43,20 +43,20 @@ namespace BCG {
|
||||
|
||||
#region constructors
|
||||
|
||||
public PublisherCore(IPubSubRouter router, IPubSubTagHandlerFactory tagHandlerFactory) {
|
||||
public PublisherCore(IPubSubRouter router, IPubSubTagHandlerFactory tag_handler_factory) {
|
||||
_router = router;
|
||||
_state = new(tagHandlerFactory);
|
||||
_state = new(tag_handler_factory);
|
||||
Thread pump_thread = new Thread(new ThreadStart(_message_pump));
|
||||
pump_thread.IsBackground = true;
|
||||
pump_thread.Start();
|
||||
}
|
||||
|
||||
public static PublisherCore Create(IPubSubRouter? router = null, IPubSubTagHandlerFactory? tagHandlerFactory = null, IServiceProvider? serviceProvider = null) {
|
||||
public static PublisherCore Create(IPubSubRouter? router = null, IPubSubTagHandlerFactory? tag_handler_factory = null, IServiceProvider? service_provider = null) {
|
||||
router ??= new PubSubRouter();
|
||||
tagHandlerFactory ??= new PubsubTagHandlerFactory(
|
||||
serviceProvider ?? PubSubServices.ServiceProvider,
|
||||
tag_handler_factory ??= new PubsubTagHandlerFactory(
|
||||
service_provider ?? PubSubServices.ServiceProvider,
|
||||
PubSubConfig.Instance.TagHandlerFactory ?? PubSubTagHandlerFactoryConfig.Default());
|
||||
return new(router, tagHandlerFactory);
|
||||
return new(router, tag_handler_factory);
|
||||
}
|
||||
|
||||
public void Dispose() { }
|
||||
@@ -119,7 +119,7 @@ namespace BCG {
|
||||
#endregion
|
||||
|
||||
// static void setGlobalInstance() {
|
||||
|
||||
|
||||
// }
|
||||
|
||||
#region Message Pump Thread - this is where messages are dequeued and the action performed
|
||||
|
||||
@@ -2,6 +2,6 @@ namespace Bts.PubSub.Server;
|
||||
|
||||
using bts;
|
||||
|
||||
interface IInternalPublisher {
|
||||
public interface IInternalPublisher {
|
||||
void Publish(PublishMsg message);
|
||||
}
|
||||
@@ -10,7 +10,7 @@ using System.Collections.Generic;
|
||||
/// messages to be published. It is also passed to the <see cref="BCG.PubSubMessageReactor"/>, so it
|
||||
/// can dequeue these messages and send them over the wire.
|
||||
/// </summary>
|
||||
interface IInternalPublisherQueue {
|
||||
public interface IInternalPublisherQueue {
|
||||
void Enqueue(PublishMsg message);
|
||||
List<PublishMsg> DequeueMultiple(TimeSpan ts, int maxElements);
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
namespace BlueOrb.PubSub.Server;
|
||||
|
||||
using BCG.Utilities.General;
|
||||
|
||||
using System;
|
||||
|
||||
public interface IPubSubInternalPublisher : IWorker {
|
||||
event Action<string, string, byte[]>? onNewData;
|
||||
}
|
||||
@@ -10,6 +10,16 @@ public delegate void PubSubStateSingleTagChangeDelegate(PubSubSingleTagStateEven
|
||||
public delegate void PubSubStateChangeDelegate(string[] tags);
|
||||
public delegate void PubSubStatsDelegate(string tag, int size);
|
||||
|
||||
public class PubSubServerTagStats {
|
||||
public int n_updates { get; set; }
|
||||
public long total_bytes { get; set; }
|
||||
public int min_size { get; set; }
|
||||
public int max_size { get; set; }
|
||||
public int last_update_siz { get; set; }
|
||||
public int average_size { get; set; }
|
||||
public long last_update_time { get; set; }
|
||||
}
|
||||
|
||||
public interface IPubSubState {
|
||||
event PubSubStateChangeDelegate? onTagMapChanged;
|
||||
event PubSubStatsDelegate? onPublishStats;
|
||||
@@ -18,6 +28,7 @@ public interface IPubSubState {
|
||||
|
||||
byte[] get_content(string tag);
|
||||
string getTagType(string tag);
|
||||
PubSubServerTagStats get_stats(string tag);
|
||||
void get_all_connections(
|
||||
string tag,
|
||||
out Dictionary<string, string> subscribers,
|
||||
|
||||
@@ -7,7 +7,7 @@ using bts;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
class InternalPublisherQueue : IInternalPublisherQueue {
|
||||
public class InternalPublisherQueue : IInternalPublisherQueue {
|
||||
readonly SyncQueue<PublishMsg> queue = new();
|
||||
readonly object lockObj = new();
|
||||
|
||||
|
||||
@@ -1,72 +1,76 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
using BCG.Utilities.General;
|
||||
using System.IO;
|
||||
using bts;
|
||||
using Bts.PubSub;
|
||||
using Bts.PubSub.Server;
|
||||
using Spark.Core;
|
||||
// using System;
|
||||
// using BCG.Utilities.General;
|
||||
// using System.IO;
|
||||
// using bts;
|
||||
// using Bts.PubSub;
|
||||
// using Bts.PubSub.Server;
|
||||
// using Spark.Core;
|
||||
|
||||
namespace BCG {
|
||||
class Pubsubheartbeat {
|
||||
public Int64 hb_time_epoch;
|
||||
public byte[] serialize() {
|
||||
MemoryStream str = new MemoryStream();
|
||||
BinaryWriter bw = new BinaryWriter(str);
|
||||
bw.Write(hb_time_epoch);
|
||||
str.Position = 0;
|
||||
return str.ToArray();
|
||||
}
|
||||
// namespace BCG {
|
||||
// class Pubsubheartbeat {
|
||||
// public Int64 hb_time_epoch;
|
||||
// public byte[] serialize() {
|
||||
// MemoryStream str = new MemoryStream();
|
||||
// BinaryWriter bw = new BinaryWriter(str);
|
||||
// bw.Write(hb_time_epoch);
|
||||
// str.Position = 0;
|
||||
// return str.ToArray();
|
||||
// }
|
||||
|
||||
public Pubsubheartbeat() {
|
||||
hb_time_epoch = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
|
||||
}
|
||||
// public Pubsubheartbeat() {
|
||||
// hb_time_epoch = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
|
||||
// }
|
||||
|
||||
public static Pubsubheartbeat deserialize(byte[] data) {
|
||||
long hb_time = BitConverter.ToInt64(data);
|
||||
return new Pubsubheartbeat() {
|
||||
hb_time_epoch = hb_time,
|
||||
};
|
||||
}
|
||||
}
|
||||
/// <summary>
|
||||
/// Periodicaly publishes a heartbeat, under the tag name System_Heartbeat.
|
||||
/// A subscriber can subscribe to it, and find out of a PubSub is alive or
|
||||
/// not. Infact more so, if the subsriptions are working or not.
|
||||
/// </summary>
|
||||
internal class SystemHeartbeatPublisher : VirtualSingleton<SystemHeartbeatPublisher>, IWorker, IParameterizedWorker {
|
||||
public static string tag_type = "System_Heartbeat";
|
||||
public static string tag_name = "System_Heartbeat";
|
||||
readonly IInternalPublisher internalPublisher;
|
||||
readonly IWorker workerImpl;
|
||||
// public static Pubsubheartbeat deserialize(byte[] data) {
|
||||
// long hb_time = BitConverter.ToInt64(data);
|
||||
// return new Pubsubheartbeat() {
|
||||
// hb_time_epoch = hb_time,
|
||||
// };
|
||||
// }
|
||||
// }
|
||||
// /// <summary>
|
||||
// /// Periodicaly publishes a heartbeat, under the tag name System_Heartbeat.
|
||||
// /// A subscriber can subscribe to it, and find out if a PubSubServer is alive
|
||||
// /// or not. Infact more so, if the subsriptions are working or not.
|
||||
// /// </summary>
|
||||
// internal class SystemHeartbeatPublisher : VirtualSingleton<SystemHeartbeatPublisher>, IWorker, IParameterizedWorker {
|
||||
// public static string tag_type = "System_Heartbeat";
|
||||
// public static string tag_name = "System_Heartbeat";
|
||||
|
||||
public SystemHeartbeatPublisher(IInternalPublisher internalPublisher) {
|
||||
this.internalPublisher = internalPublisher;
|
||||
workerImpl = new SingleThreadedWorker(new AnonymousWorker(doWorkImpl));
|
||||
}
|
||||
// readonly IInternalPublisher internalPublisher;
|
||||
// readonly IWorker workerImpl;
|
||||
|
||||
TimeSpan IParameterizedWorker.Interval => TimeSpan.FromMilliseconds(500);
|
||||
// public SystemHeartbeatPublisher(IInternalPublisher internalPublisher) {
|
||||
// this.internalPublisher = internalPublisher;
|
||||
// workerImpl = new SingleThreadedWorker(new AnonymousWorker(doWorkImpl));
|
||||
// }
|
||||
|
||||
#region IWorker Members
|
||||
// TimeSpan IParameterizedWorker.Interval => TimeSpan.FromMilliseconds(500);
|
||||
|
||||
public void doWork() =>
|
||||
workerImpl.doWork();
|
||||
// #region IWorker Members
|
||||
|
||||
void doWorkImpl() {
|
||||
Pubsubheartbeat hb = new Pubsubheartbeat();
|
||||
PublishMsg msg = new PublishMsg() {
|
||||
connection = "Internal",
|
||||
publishing_app = ClientInfo.thisClient.app,
|
||||
tag = "pss_hb",
|
||||
tag_type = "Internal",
|
||||
is_compressed = false,
|
||||
publish_time_epoch_ms = hb.hb_time_epoch,
|
||||
data_to_publish = hb.serialize()
|
||||
};
|
||||
internalPublisher.Publish(msg);
|
||||
}
|
||||
// public void doWork() =>
|
||||
// workerImpl.doWork();
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
// void doWorkImpl() {
|
||||
// // why not publish a table
|
||||
// Pubsubheartbeat hb = new Pubsubheartbeat();
|
||||
// PublishMsg msg = new PublishMsg() {
|
||||
// connection = "Internal",
|
||||
// publishing_app = ClientInfo.thisClient.app,
|
||||
// tag = "pss_hb",
|
||||
// tag_type = "Internal",
|
||||
// is_compressed = false,
|
||||
// publish_time_epoch_ms = hb.hb_time_epoch,
|
||||
// data_to_publish = hb.serialize()
|
||||
// };
|
||||
// internalPublisher.Publish(msg);
|
||||
// }
|
||||
|
||||
// #endregion
|
||||
|
||||
// private void _publish_hb() {
|
||||
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
Loaded 100 of 591 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user