Compare commits

..
Author SHA1 Message Date
John Colvin 19fef8492d Update externals to remove useless nugets with filenames that are too long
Too long for Windows to clone externals (at least for the Windows Server
box running jenkins)
2026-08-12 14:05:07 -04:00
arham 7521b01022 bug fixes for api stream 2026-08-09 22:13:09 +00:00
arham d0faa95d7a add: fee adjusted stream 2026-08-09 22:12:56 +00:00
arham 36aa2caa2a fractional quantities spreader working 2026-07-30 05:12:58 +00:00
arham 97e7343110 fix: add add raydium and orca to crypto in spreader 2026-07-26 19:28:21 +00:00
arham 5dccc2e9c3 fix: synth name 2026-07-26 19:27:30 +00:00
arham 365012102e basic create synthetic working 2026-07-22 10:25:41 +00:00
arham c3d14c8e31 api server works in default docker 2026-07-02 04:50:44 +00:00
arham 5cc80db09d config change 2026-07-02 04:50:01 +00:00
arham 12c2deb833 market orders for hedge order in spreader 2026-07-02 04:49:44 +00:00
arham 2222fb8e3e add okx to conf 2026-06-30 23:37:26 +00:00
arham 266fdf93e5 fix null check 2026-06-30 23:36:59 +00:00
arham fd121bd85b emnative order hadles limit orders now 2026-06-19 03:13:13 +00:00
arham 1ffaf7c05b Add: rakurai dex execution 2026-06-19 03:12:52 +00:00
arham 5a87288aaa quantity fix 2026-06-15 18:33:37 +00:00
arham a1d62e552c Add:orca raydoum to risk ui 2026-06-11 13:40:10 +00:00
arham 80656a0890 Fix: orca and raydium show up in the api server 2026-06-08 04:10:49 +00:00
arham 188daf2b84 fix: error in cpp 2026-05-31 23:39:38 +00:00
arham 5908adcb09 add: cpp to dex handler 2026-05-15 13:36:09 +00:00
arham 36cf021b70 Add: libs for rakurai to work 2026-05-15 13:35:32 +00:00
arham d45e0e08a2 fix: compiler error 2026-05-15 13:00:25 +00:00
arham 888e751243 raydium orca working 2026-04-24 06:35:50 +00:00
arham bbe3350b4e Add: raydium and orca to frond end 2026-04-24 02:33:24 -04:00
arham 80ea6ad16e add orca nad radium to benue options 2026-04-23 23:46:14 +00:00
arham 0bc0e821f3 basic rakurai hookup done 2026-04-23 17:07:51 +00:00
arham 1b82d07a0d fix: fixed api rounding issues 2026-04-23 17:07:51 +00:00
arham a83404f20f Add:: added orca and raydium 2026-04-23 17:07:51 +00:00
97 changed files with 5663 additions and 5467 deletions

No files matched your search

+9 -9
View File
@@ -183,10 +183,11 @@
"windows": {
"program": "PubSubServer.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
],
"cwd": "build/Debug/bin/net8.0",
"env": {
"DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
},
"stopAtEntry": false,
"console": "integratedTerminal"
},
@@ -448,15 +449,14 @@
"default": "Local",
},
{
"id": "pubSubServerEnvironment",
"id": "pubSubServerConfig",
"type": "pickString",
"description": "PubSubServer Configuration",
"description": "PubSub Configuration",
"options": [
"Development",
"EchoServer",
"Local",
"PubSubServer",
"SimpleEchoClient",
],
"default": "Development",
"default": "PubSubServer",
},
{
"id": "sdkDemoEnvironment",
+6
View File
@@ -19,6 +19,9 @@ services:
- "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
- "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
- "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
- 8008:8008
- 8080:8080
- 9443:9443
entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
deploy:
resources:
@@ -72,6 +75,9 @@ services:
- "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
- "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
- "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
- 8008:8008
- 8080:8080
- 9443:9443
entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
deploy:
restart_policy:
@@ -19,6 +19,13 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
RUN apt-get update \
&& apt-get install -y dotnet8
RUN apt-get update && apt-get install -y libpq-dev
RUN apt-get update && apt-get install -y libsodium-dev
RUN apt-get update && apt-get install -y fish
RUN apt-get update && apt-get install -y less
RUN apt-get update && apt-get install -y vim
RUN locale-gen en_US.UTF-8
ARG docker_user="bts"
@@ -98,7 +98,7 @@ RUN apt-get update && apt-get install -y libpcap0.8-dev
RUN apt-get update && apt-get -y install libzstd-dev
# Rakurai
ARG RAKURAI_DEB_URL="https://api.bitbucket.org/2.0/repositories/bluetradesys/spark/downloads/rakurai-dex-connector_0.0.5_amd64.deb"
ARG RAKURAI_DEB_URL="https://api.bitbucket.org/2.0/repositories/bluetradesys/spark/downloads/rakurai-dex-connector_0.0.8_amd64.deb"
RUN --mount=type=secret,id=bitbucket_access_token \
set -eux \
&& apt-get update \
Submodule docker/linux deleted from 6537812262.
Submodule docker/windows deleted from de7d40c09d.
@@ -1 +0,0 @@
r=TimeSeriesStream(ReplayFutureLiveBBOStream("ES"))
@@ -1,3 +0,0 @@
x,y,z,id
123,1.001,abc,1
123,1.001,abc,0
1 x y z id
2 123 1.001 abc 1
3 123 1.001 abc 0
@@ -1,3 +0,0 @@
x,y,z,id
123,1.001,abc,1
123,1.001,abc,0
1 x y z id
2 123 1.001 abc 1
3 123 1.001 abc 0
@@ -1,78 +0,0 @@
id,ts_event,contract,bid_price,bid_size,ask_price,ask_size,timestamp
76,2026-03-13T05:57:25.177965500Z,ESU5,6363.75,51,6364,63,3/13/2026 5:57:25 AM
75,2026-03-13T05:57:25.068062900Z,ESU5,6363.75,52,6364,63,3/13/2026 5:57:25 AM
74,2026-03-13T05:57:24.963428500Z,ESU5,6363.75,56,6364,63,3/13/2026 5:57:24 AM
73,2026-03-13T05:57:24.781215700Z,ESU5,6363.75,57,6364,63,3/13/2026 5:57:24 AM
72,2026-03-13T05:57:24.676290100Z,ESU5,6363.75,57,6364,63,3/13/2026 5:57:24 AM
71,2026-03-13T05:57:24.566898400Z,ESU5,6363.75,56,6364,63,3/13/2026 5:57:24 AM
70,2026-03-13T05:57:24.430800400Z,ESU5,6363.75,57,6364,63,3/13/2026 5:57:24 AM
69,2026-03-13T05:57:24.273683400Z,ESU5,6363.75,56,6364,63,3/13/2026 5:57:24 AM
68,2026-03-13T05:57:24.167656100Z,ESU5,6363.75,56,6364,64,3/13/2026 5:57:24 AM
67,2026-03-13T05:57:23.995720100Z,ESU5,6363.75,56,6364,65,3/13/2026 5:57:23 AM
66,2026-03-13T05:57:23.894029100Z,ESU5,6363.75,56,6364,68,3/13/2026 5:57:23 AM
65,2026-03-13T05:57:23.728897300Z,ESU5,6363.75,56,6364,68,3/13/2026 5:57:23 AM
64,2026-03-13T05:57:23.614316800Z,ESU5,6363.75,52,6364,68,3/13/2026 5:57:23 AM
63,2026-03-13T05:57:23.503752700Z,ESU5,6363.75,51,6364,68,3/13/2026 5:57:23 AM
62,2026-03-13T05:57:23.289108200Z,ESU5,6363.75,50,6364,68,3/13/2026 5:57:23 AM
61,2026-03-13T05:57:23.155673400Z,ESU5,6363.75,49,6364,68,3/13/2026 5:57:23 AM
60,2026-03-13T05:57:23.028898400Z,ESU5,6363.75,48,6364,68,3/13/2026 5:57:23 AM
59,2026-03-13T05:57:22.887132700Z,ESU5,6363.75,46,6364,68,3/13/2026 5:57:22 AM
58,2026-03-13T05:57:22.676916300Z,ESU5,6363.75,44,6364,68,3/13/2026 5:57:22 AM
57,2026-03-13T05:57:22.520328000Z,ESU5,6363.75,43,6364,68,3/13/2026 5:57:22 AM
56,2026-03-13T05:57:22.405244600Z,ESU5,6363.75,43,6364,69,3/13/2026 5:57:22 AM
55,2026-03-13T05:57:22.294674100Z,ESU5,6363.75,41,6364,69,3/13/2026 5:57:22 AM
54,2026-03-13T05:57:22.163467900Z,ESU5,6363.75,38,6364,69,3/13/2026 5:57:22 AM
53,2026-03-13T05:57:21.992443000Z,ESU5,6363.75,38,6364,71,3/13/2026 5:57:21 AM
52,2026-03-13T05:57:21.872338400Z,ESU5,6363.75,38,6364,71,3/13/2026 5:57:21 AM
51,2026-03-13T05:57:21.742801800Z,ESU5,6363.75,37,6364,71,3/13/2026 5:57:21 AM
50,2026-03-13T05:57:21.557711100Z,ESU5,6363.75,37,6364,72,3/13/2026 5:57:21 AM
49,2026-03-13T05:57:21.434977000Z,ESU5,6363.75,37,6364,73,3/13/2026 5:57:21 AM
48,2026-03-13T05:57:21.285751600Z,ESU5,6363.75,37,6364,72,3/13/2026 5:57:21 AM
47,2026-03-13T05:57:21.184392000Z,ESU5,6363.75,37,6364,73,3/13/2026 5:57:21 AM
46,2026-03-13T05:57:21.076843800Z,ESU5,6363.75,34,6364,73,3/13/2026 5:57:21 AM
45,2026-03-13T05:57:20.894131800Z,ESU5,6363.75,32,6364,73,3/13/2026 5:57:20 AM
44,2026-03-13T05:57:20.759550000Z,ESU5,6363.75,30,6364,73,3/13/2026 5:57:20 AM
43,2026-03-13T05:57:20.641547900Z,ESU5,6363.75,28,6364,73,3/13/2026 5:57:20 AM
42,2026-03-13T05:57:20.540293200Z,ESU5,6363.75,27,6364,73,3/13/2026 5:57:20 AM
41,2026-03-13T05:57:20.439034600Z,ESU5,6363.75,27,6364,74,3/13/2026 5:57:20 AM
40,2026-03-13T05:57:20.337795000Z,ESU5,6363.75,27,6364,73,3/13/2026 5:57:20 AM
39,2026-03-13T05:57:20.236606600Z,ESU5,6363.75,27,6364,72,3/13/2026 5:57:20 AM
38,2026-03-13T05:57:20.135417700Z,ESU5,6363.75,30,6364,72,3/13/2026 5:57:20 AM
37,2026-03-13T05:57:20.034272700Z,ESU5,6363.75,32,6364,72,3/13/2026 5:57:20 AM
36,2026-03-13T05:57:19.933096800Z,ESU5,6363.75,34,6364,72,3/13/2026 5:57:19 AM
35,2026-03-13T05:57:19.832012600Z,ESU5,6363.75,36,6364,72,3/13/2026 5:57:19 AM
34,2026-03-13T05:57:19.730878500Z,ESU5,6363.75,38,6364,72,3/13/2026 5:57:19 AM
33,2026-03-13T05:57:19.629787400Z,ESU5,6363.75,38,6364,71,3/13/2026 5:57:19 AM
32,2026-03-13T05:57:19.528697000Z,ESU5,6363.75,38,6364,70,3/13/2026 5:57:19 AM
31,2026-03-13T05:57:19.427077000Z,ESU5,6363.75,38,6364,71,3/13/2026 5:57:19 AM
30,2026-03-13T05:57:19.326450300Z,ESU5,6363.75,39,6364,71,3/13/2026 5:57:19 AM
29,2026-03-13T05:57:19.225427900Z,ESU5,6363.75,39,6364,70,3/13/2026 5:57:19 AM
28,2026-03-13T05:57:19.123339800Z,ESU5,6363.75,40,6364,70,3/13/2026 5:57:19 AM
27,2026-03-13T05:57:19.021936500Z,ESU5,6363.75,41,6364,70,3/13/2026 5:57:19 AM
26,2026-03-13T05:57:18.920859200Z,ESU5,6363.75,45,6364,70,3/13/2026 5:57:18 AM
25,2026-03-13T05:57:18.820148400Z,ESU5,6363.75,46,6364,70,3/13/2026 5:57:18 AM
24,2026-03-13T05:57:18.718988100Z,ESU5,6363.75,46,6364,69,3/13/2026 5:57:18 AM
23,2026-03-13T05:57:18.618299700Z,ESU5,6363.75,49,6364,69,3/13/2026 5:57:18 AM
22,2026-03-13T05:57:18.517138100Z,ESU5,6363.75,52,6364,69,3/13/2026 5:57:18 AM
21,2026-03-13T05:57:18.416335200Z,ESU5,6363.75,53,6364,69,3/13/2026 5:57:18 AM
20,2026-03-13T05:57:18.315335900Z,ESU5,6363.75,55,6364,69,3/13/2026 5:57:18 AM
19,2026-03-13T05:57:18.214368600Z,ESU5,6363.75,57,6364,69,3/13/2026 5:57:18 AM
18,2026-03-13T05:57:18.113754700Z,ESU5,6363.75,57,6364,65,3/13/2026 5:57:18 AM
17,2026-03-13T05:57:18.012706500Z,ESU5,6363.75,56,6364,65,3/13/2026 5:57:18 AM
16,2026-03-13T05:57:17.911672600Z,ESU5,6363.75,54,6364,65,3/13/2026 5:57:17 AM
15,2026-03-13T05:57:17.810879900Z,ESU5,6363.75,52,6364,65,3/13/2026 5:57:17 AM
14,2026-03-13T05:57:17.710089700Z,ESU5,6363.75,49,6364,65,3/13/2026 5:57:17 AM
13,2026-03-13T05:57:17.609388300Z,ESU5,6363.75,49,6364,67,3/13/2026 5:57:17 AM
12,2026-03-13T05:57:17.508697400Z,ESU5,6363.75,49,6364,67,3/13/2026 5:57:17 AM
11,2026-03-13T05:57:17.408027000Z,ESU5,6363.75,53,6364,67,3/13/2026 5:57:17 AM
10,2026-03-13T05:57:17.307406200Z,ESU5,6363.75,49,6364,67,3/13/2026 5:57:17 AM
9,2026-03-13T05:57:17.206714300Z,ESU5,6363.75,49,6364,71,3/13/2026 5:57:17 AM
8,2026-03-13T05:57:17.105530300Z,ESU5,6363.75,48,6364,71,3/13/2026 5:57:17 AM
7,2026-03-13T05:57:17.004796800Z,ESU5,6363.75,47,6364,71,3/13/2026 5:57:17 AM
6,2026-03-13T05:57:16.904113900Z,ESU5,6363.75,46,6364,71,3/13/2026 5:57:16 AM
5,2026-03-13T05:57:16.803381900Z,ESU5,6363.75,43,6364,71,3/13/2026 5:57:16 AM
4,2026-03-13T05:57:16.701969600Z,ESU5,6363.75,43,6364,72,3/13/2026 5:57:16 AM
3,2026-03-13T05:57:16.600765600Z,ESU5,6363.75,43,6364,73,3/13/2026 5:57:16 AM
2,2026-03-13T05:57:16.499986300Z,ESU5,6363.75,43,6364,74,3/13/2026 5:57:16 AM
1,2026-03-13T05:57:16.399241700Z,ESU5,6363.75,43,6364,75,3/13/2026 5:57:16 AM
0,2026-03-13T05:57:16.264038000Z,ESU5,6363.75,43,6364,76,3/13/2026 5:57:16 AM
1 id ts_event contract bid_price bid_size ask_price ask_size timestamp
2 76 2026-03-13T05:57:25.177965500Z ESU5 6363.75 51 6364 63 3/13/2026 5:57:25 AM
3 75 2026-03-13T05:57:25.068062900Z ESU5 6363.75 52 6364 63 3/13/2026 5:57:25 AM
4 74 2026-03-13T05:57:24.963428500Z ESU5 6363.75 56 6364 63 3/13/2026 5:57:24 AM
5 73 2026-03-13T05:57:24.781215700Z ESU5 6363.75 57 6364 63 3/13/2026 5:57:24 AM
6 72 2026-03-13T05:57:24.676290100Z ESU5 6363.75 57 6364 63 3/13/2026 5:57:24 AM
7 71 2026-03-13T05:57:24.566898400Z ESU5 6363.75 56 6364 63 3/13/2026 5:57:24 AM
8 70 2026-03-13T05:57:24.430800400Z ESU5 6363.75 57 6364 63 3/13/2026 5:57:24 AM
9 69 2026-03-13T05:57:24.273683400Z ESU5 6363.75 56 6364 63 3/13/2026 5:57:24 AM
10 68 2026-03-13T05:57:24.167656100Z ESU5 6363.75 56 6364 64 3/13/2026 5:57:24 AM
11 67 2026-03-13T05:57:23.995720100Z ESU5 6363.75 56 6364 65 3/13/2026 5:57:23 AM
12 66 2026-03-13T05:57:23.894029100Z ESU5 6363.75 56 6364 68 3/13/2026 5:57:23 AM
13 65 2026-03-13T05:57:23.728897300Z ESU5 6363.75 56 6364 68 3/13/2026 5:57:23 AM
14 64 2026-03-13T05:57:23.614316800Z ESU5 6363.75 52 6364 68 3/13/2026 5:57:23 AM
15 63 2026-03-13T05:57:23.503752700Z ESU5 6363.75 51 6364 68 3/13/2026 5:57:23 AM
16 62 2026-03-13T05:57:23.289108200Z ESU5 6363.75 50 6364 68 3/13/2026 5:57:23 AM
17 61 2026-03-13T05:57:23.155673400Z ESU5 6363.75 49 6364 68 3/13/2026 5:57:23 AM
18 60 2026-03-13T05:57:23.028898400Z ESU5 6363.75 48 6364 68 3/13/2026 5:57:23 AM
19 59 2026-03-13T05:57:22.887132700Z ESU5 6363.75 46 6364 68 3/13/2026 5:57:22 AM
20 58 2026-03-13T05:57:22.676916300Z ESU5 6363.75 44 6364 68 3/13/2026 5:57:22 AM
21 57 2026-03-13T05:57:22.520328000Z ESU5 6363.75 43 6364 68 3/13/2026 5:57:22 AM
22 56 2026-03-13T05:57:22.405244600Z ESU5 6363.75 43 6364 69 3/13/2026 5:57:22 AM
23 55 2026-03-13T05:57:22.294674100Z ESU5 6363.75 41 6364 69 3/13/2026 5:57:22 AM
24 54 2026-03-13T05:57:22.163467900Z ESU5 6363.75 38 6364 69 3/13/2026 5:57:22 AM
25 53 2026-03-13T05:57:21.992443000Z ESU5 6363.75 38 6364 71 3/13/2026 5:57:21 AM
26 52 2026-03-13T05:57:21.872338400Z ESU5 6363.75 38 6364 71 3/13/2026 5:57:21 AM
27 51 2026-03-13T05:57:21.742801800Z ESU5 6363.75 37 6364 71 3/13/2026 5:57:21 AM
28 50 2026-03-13T05:57:21.557711100Z ESU5 6363.75 37 6364 72 3/13/2026 5:57:21 AM
29 49 2026-03-13T05:57:21.434977000Z ESU5 6363.75 37 6364 73 3/13/2026 5:57:21 AM
30 48 2026-03-13T05:57:21.285751600Z ESU5 6363.75 37 6364 72 3/13/2026 5:57:21 AM
31 47 2026-03-13T05:57:21.184392000Z ESU5 6363.75 37 6364 73 3/13/2026 5:57:21 AM
32 46 2026-03-13T05:57:21.076843800Z ESU5 6363.75 34 6364 73 3/13/2026 5:57:21 AM
33 45 2026-03-13T05:57:20.894131800Z ESU5 6363.75 32 6364 73 3/13/2026 5:57:20 AM
34 44 2026-03-13T05:57:20.759550000Z ESU5 6363.75 30 6364 73 3/13/2026 5:57:20 AM
35 43 2026-03-13T05:57:20.641547900Z ESU5 6363.75 28 6364 73 3/13/2026 5:57:20 AM
36 42 2026-03-13T05:57:20.540293200Z ESU5 6363.75 27 6364 73 3/13/2026 5:57:20 AM
37 41 2026-03-13T05:57:20.439034600Z ESU5 6363.75 27 6364 74 3/13/2026 5:57:20 AM
38 40 2026-03-13T05:57:20.337795000Z ESU5 6363.75 27 6364 73 3/13/2026 5:57:20 AM
39 39 2026-03-13T05:57:20.236606600Z ESU5 6363.75 27 6364 72 3/13/2026 5:57:20 AM
40 38 2026-03-13T05:57:20.135417700Z ESU5 6363.75 30 6364 72 3/13/2026 5:57:20 AM
41 37 2026-03-13T05:57:20.034272700Z ESU5 6363.75 32 6364 72 3/13/2026 5:57:20 AM
42 36 2026-03-13T05:57:19.933096800Z ESU5 6363.75 34 6364 72 3/13/2026 5:57:19 AM
43 35 2026-03-13T05:57:19.832012600Z ESU5 6363.75 36 6364 72 3/13/2026 5:57:19 AM
44 34 2026-03-13T05:57:19.730878500Z ESU5 6363.75 38 6364 72 3/13/2026 5:57:19 AM
45 33 2026-03-13T05:57:19.629787400Z ESU5 6363.75 38 6364 71 3/13/2026 5:57:19 AM
46 32 2026-03-13T05:57:19.528697000Z ESU5 6363.75 38 6364 70 3/13/2026 5:57:19 AM
47 31 2026-03-13T05:57:19.427077000Z ESU5 6363.75 38 6364 71 3/13/2026 5:57:19 AM
48 30 2026-03-13T05:57:19.326450300Z ESU5 6363.75 39 6364 71 3/13/2026 5:57:19 AM
49 29 2026-03-13T05:57:19.225427900Z ESU5 6363.75 39 6364 70 3/13/2026 5:57:19 AM
50 28 2026-03-13T05:57:19.123339800Z ESU5 6363.75 40 6364 70 3/13/2026 5:57:19 AM
51 27 2026-03-13T05:57:19.021936500Z ESU5 6363.75 41 6364 70 3/13/2026 5:57:19 AM
52 26 2026-03-13T05:57:18.920859200Z ESU5 6363.75 45 6364 70 3/13/2026 5:57:18 AM
53 25 2026-03-13T05:57:18.820148400Z ESU5 6363.75 46 6364 70 3/13/2026 5:57:18 AM
54 24 2026-03-13T05:57:18.718988100Z ESU5 6363.75 46 6364 69 3/13/2026 5:57:18 AM
55 23 2026-03-13T05:57:18.618299700Z ESU5 6363.75 49 6364 69 3/13/2026 5:57:18 AM
56 22 2026-03-13T05:57:18.517138100Z ESU5 6363.75 52 6364 69 3/13/2026 5:57:18 AM
57 21 2026-03-13T05:57:18.416335200Z ESU5 6363.75 53 6364 69 3/13/2026 5:57:18 AM
58 20 2026-03-13T05:57:18.315335900Z ESU5 6363.75 55 6364 69 3/13/2026 5:57:18 AM
59 19 2026-03-13T05:57:18.214368600Z ESU5 6363.75 57 6364 69 3/13/2026 5:57:18 AM
60 18 2026-03-13T05:57:18.113754700Z ESU5 6363.75 57 6364 65 3/13/2026 5:57:18 AM
61 17 2026-03-13T05:57:18.012706500Z ESU5 6363.75 56 6364 65 3/13/2026 5:57:18 AM
62 16 2026-03-13T05:57:17.911672600Z ESU5 6363.75 54 6364 65 3/13/2026 5:57:17 AM
63 15 2026-03-13T05:57:17.810879900Z ESU5 6363.75 52 6364 65 3/13/2026 5:57:17 AM
64 14 2026-03-13T05:57:17.710089700Z ESU5 6363.75 49 6364 65 3/13/2026 5:57:17 AM
65 13 2026-03-13T05:57:17.609388300Z ESU5 6363.75 49 6364 67 3/13/2026 5:57:17 AM
66 12 2026-03-13T05:57:17.508697400Z ESU5 6363.75 49 6364 67 3/13/2026 5:57:17 AM
67 11 2026-03-13T05:57:17.408027000Z ESU5 6363.75 53 6364 67 3/13/2026 5:57:17 AM
68 10 2026-03-13T05:57:17.307406200Z ESU5 6363.75 49 6364 67 3/13/2026 5:57:17 AM
69 9 2026-03-13T05:57:17.206714300Z ESU5 6363.75 49 6364 71 3/13/2026 5:57:17 AM
70 8 2026-03-13T05:57:17.105530300Z ESU5 6363.75 48 6364 71 3/13/2026 5:57:17 AM
71 7 2026-03-13T05:57:17.004796800Z ESU5 6363.75 47 6364 71 3/13/2026 5:57:17 AM
72 6 2026-03-13T05:57:16.904113900Z ESU5 6363.75 46 6364 71 3/13/2026 5:57:16 AM
73 5 2026-03-13T05:57:16.803381900Z ESU5 6363.75 43 6364 71 3/13/2026 5:57:16 AM
74 4 2026-03-13T05:57:16.701969600Z ESU5 6363.75 43 6364 72 3/13/2026 5:57:16 AM
75 3 2026-03-13T05:57:16.600765600Z ESU5 6363.75 43 6364 73 3/13/2026 5:57:16 AM
76 2 2026-03-13T05:57:16.499986300Z ESU5 6363.75 43 6364 74 3/13/2026 5:57:16 AM
77 1 2026-03-13T05:57:16.399241700Z ESU5 6363.75 43 6364 75 3/13/2026 5:57:16 AM
78 0 2026-03-13T05:57:16.264038000Z ESU5 6363.75 43 6364 76 3/13/2026 5:57:16 AM
@@ -1,91 +0,0 @@
id,ts_event,contract,bid_price,bid_size,ask_price,ask_size,timestamp
89,2026-03-13T05:57:26.904401600Z,ESU5,6363.75,43,6364,67,3/13/2026 5:57:26 AM
88,2026-03-13T05:57:26.778037200Z,ESU5,6363.75,43,6364,69,3/13/2026 5:57:26 AM
87,2026-03-13T05:57:26.597038500Z,ESU5,6363.75,43,6364,69,3/13/2026 5:57:26 AM
86,2026-03-13T05:57:26.438768300Z,ESU5,6363.75,43,6364,68,3/13/2026 5:57:26 AM
85,2026-03-13T05:57:26.328259100Z,ESU5,6363.75,43,6364,67,3/13/2026 5:57:26 AM
84,2026-03-13T05:57:26.216211200Z,ESU5,6363.75,44,6364,67,3/13/2026 5:57:26 AM
83,2026-03-13T05:57:26.099281200Z,ESU5,6363.75,46,6364,67,3/13/2026 5:57:26 AM
82,2026-03-13T05:57:25.979814900Z,ESU5,6363.75,48,6364,67,3/13/2026 5:57:25 AM
81,2026-03-13T05:57:25.852317700Z,ESU5,6363.75,48,6364,66,3/13/2026 5:57:25 AM
80,2026-03-13T05:57:25.684323100Z,ESU5,6363.75,48,6364,65,3/13/2026 5:57:25 AM
79,2026-03-13T05:57:25.539436900Z,ESU5,6363.75,48,6364,64,3/13/2026 5:57:25 AM
78,2026-03-13T05:57:25.425439200Z,ESU5,6363.75,49,6364,64,3/13/2026 5:57:25 AM
77,2026-03-13T05:57:25.298054100Z,ESU5,6363.75,51,6364,64,3/13/2026 5:57:25 AM
76,2026-03-13T05:57:25.177965500Z,ESU5,6363.75,51,6364,63,3/13/2026 5:57:25 AM
75,2026-03-13T05:57:25.068062900Z,ESU5,6363.75,52,6364,63,3/13/2026 5:57:25 AM
74,2026-03-13T05:57:24.963428500Z,ESU5,6363.75,56,6364,63,3/13/2026 5:57:24 AM
73,2026-03-13T05:57:24.781215700Z,ESU5,6363.75,57,6364,63,3/13/2026 5:57:24 AM
72,2026-03-13T05:57:24.676290100Z,ESU5,6363.75,57,6364,63,3/13/2026 5:57:24 AM
71,2026-03-13T05:57:24.566898400Z,ESU5,6363.75,56,6364,63,3/13/2026 5:57:24 AM
70,2026-03-13T05:57:24.430800400Z,ESU5,6363.75,57,6364,63,3/13/2026 5:57:24 AM
69,2026-03-13T05:57:24.273683400Z,ESU5,6363.75,56,6364,63,3/13/2026 5:57:24 AM
68,2026-03-13T05:57:24.167656100Z,ESU5,6363.75,56,6364,64,3/13/2026 5:57:24 AM
67,2026-03-13T05:57:23.995720100Z,ESU5,6363.75,56,6364,65,3/13/2026 5:57:23 AM
66,2026-03-13T05:57:23.894029100Z,ESU5,6363.75,56,6364,68,3/13/2026 5:57:23 AM
65,2026-03-13T05:57:23.728897300Z,ESU5,6363.75,56,6364,68,3/13/2026 5:57:23 AM
64,2026-03-13T05:57:23.614316800Z,ESU5,6363.75,52,6364,68,3/13/2026 5:57:23 AM
63,2026-03-13T05:57:23.503752700Z,ESU5,6363.75,51,6364,68,3/13/2026 5:57:23 AM
62,2026-03-13T05:57:23.289108200Z,ESU5,6363.75,50,6364,68,3/13/2026 5:57:23 AM
61,2026-03-13T05:57:23.155673400Z,ESU5,6363.75,49,6364,68,3/13/2026 5:57:23 AM
60,2026-03-13T05:57:23.028898400Z,ESU5,6363.75,48,6364,68,3/13/2026 5:57:23 AM
59,2026-03-13T05:57:22.887132700Z,ESU5,6363.75,46,6364,68,3/13/2026 5:57:22 AM
58,2026-03-13T05:57:22.676916300Z,ESU5,6363.75,44,6364,68,3/13/2026 5:57:22 AM
57,2026-03-13T05:57:22.520328000Z,ESU5,6363.75,43,6364,68,3/13/2026 5:57:22 AM
56,2026-03-13T05:57:22.405244600Z,ESU5,6363.75,43,6364,69,3/13/2026 5:57:22 AM
55,2026-03-13T05:57:22.294674100Z,ESU5,6363.75,41,6364,69,3/13/2026 5:57:22 AM
54,2026-03-13T05:57:22.163467900Z,ESU5,6363.75,38,6364,69,3/13/2026 5:57:22 AM
53,2026-03-13T05:57:21.992443000Z,ESU5,6363.75,38,6364,71,3/13/2026 5:57:21 AM
52,2026-03-13T05:57:21.872338400Z,ESU5,6363.75,38,6364,71,3/13/2026 5:57:21 AM
51,2026-03-13T05:57:21.742801800Z,ESU5,6363.75,37,6364,71,3/13/2026 5:57:21 AM
50,2026-03-13T05:57:21.557711100Z,ESU5,6363.75,37,6364,72,3/13/2026 5:57:21 AM
49,2026-03-13T05:57:21.434977000Z,ESU5,6363.75,37,6364,73,3/13/2026 5:57:21 AM
48,2026-03-13T05:57:21.285751600Z,ESU5,6363.75,37,6364,72,3/13/2026 5:57:21 AM
47,2026-03-13T05:57:21.184392000Z,ESU5,6363.75,37,6364,73,3/13/2026 5:57:21 AM
46,2026-03-13T05:57:21.076843800Z,ESU5,6363.75,34,6364,73,3/13/2026 5:57:21 AM
45,2026-03-13T05:57:20.894131800Z,ESU5,6363.75,32,6364,73,3/13/2026 5:57:20 AM
44,2026-03-13T05:57:20.759550000Z,ESU5,6363.75,30,6364,73,3/13/2026 5:57:20 AM
43,2026-03-13T05:57:20.641547900Z,ESU5,6363.75,28,6364,73,3/13/2026 5:57:20 AM
42,2026-03-13T05:57:20.540293200Z,ESU5,6363.75,27,6364,73,3/13/2026 5:57:20 AM
41,2026-03-13T05:57:20.439034600Z,ESU5,6363.75,27,6364,74,3/13/2026 5:57:20 AM
40,2026-03-13T05:57:20.337795000Z,ESU5,6363.75,27,6364,73,3/13/2026 5:57:20 AM
39,2026-03-13T05:57:20.236606600Z,ESU5,6363.75,27,6364,72,3/13/2026 5:57:20 AM
38,2026-03-13T05:57:20.135417700Z,ESU5,6363.75,30,6364,72,3/13/2026 5:57:20 AM
37,2026-03-13T05:57:20.034272700Z,ESU5,6363.75,32,6364,72,3/13/2026 5:57:20 AM
36,2026-03-13T05:57:19.933096800Z,ESU5,6363.75,34,6364,72,3/13/2026 5:57:19 AM
35,2026-03-13T05:57:19.832012600Z,ESU5,6363.75,36,6364,72,3/13/2026 5:57:19 AM
34,2026-03-13T05:57:19.730878500Z,ESU5,6363.75,38,6364,72,3/13/2026 5:57:19 AM
33,2026-03-13T05:57:19.629787400Z,ESU5,6363.75,38,6364,71,3/13/2026 5:57:19 AM
32,2026-03-13T05:57:19.528697000Z,ESU5,6363.75,38,6364,70,3/13/2026 5:57:19 AM
31,2026-03-13T05:57:19.427077000Z,ESU5,6363.75,38,6364,71,3/13/2026 5:57:19 AM
30,2026-03-13T05:57:19.326450300Z,ESU5,6363.75,39,6364,71,3/13/2026 5:57:19 AM
29,2026-03-13T05:57:19.225427900Z,ESU5,6363.75,39,6364,70,3/13/2026 5:57:19 AM
28,2026-03-13T05:57:19.123339800Z,ESU5,6363.75,40,6364,70,3/13/2026 5:57:19 AM
27,2026-03-13T05:57:19.021936500Z,ESU5,6363.75,41,6364,70,3/13/2026 5:57:19 AM
26,2026-03-13T05:57:18.920859200Z,ESU5,6363.75,45,6364,70,3/13/2026 5:57:18 AM
25,2026-03-13T05:57:18.820148400Z,ESU5,6363.75,46,6364,70,3/13/2026 5:57:18 AM
24,2026-03-13T05:57:18.718988100Z,ESU5,6363.75,46,6364,69,3/13/2026 5:57:18 AM
23,2026-03-13T05:57:18.618299700Z,ESU5,6363.75,49,6364,69,3/13/2026 5:57:18 AM
22,2026-03-13T05:57:18.517138100Z,ESU5,6363.75,52,6364,69,3/13/2026 5:57:18 AM
21,2026-03-13T05:57:18.416335200Z,ESU5,6363.75,53,6364,69,3/13/2026 5:57:18 AM
20,2026-03-13T05:57:18.315335900Z,ESU5,6363.75,55,6364,69,3/13/2026 5:57:18 AM
19,2026-03-13T05:57:18.214368600Z,ESU5,6363.75,57,6364,69,3/13/2026 5:57:18 AM
18,2026-03-13T05:57:18.113754700Z,ESU5,6363.75,57,6364,65,3/13/2026 5:57:18 AM
17,2026-03-13T05:57:18.012706500Z,ESU5,6363.75,56,6364,65,3/13/2026 5:57:18 AM
16,2026-03-13T05:57:17.911672600Z,ESU5,6363.75,54,6364,65,3/13/2026 5:57:17 AM
15,2026-03-13T05:57:17.810879900Z,ESU5,6363.75,52,6364,65,3/13/2026 5:57:17 AM
14,2026-03-13T05:57:17.710089700Z,ESU5,6363.75,49,6364,65,3/13/2026 5:57:17 AM
13,2026-03-13T05:57:17.609388300Z,ESU5,6363.75,49,6364,67,3/13/2026 5:57:17 AM
12,2026-03-13T05:57:17.508697400Z,ESU5,6363.75,49,6364,67,3/13/2026 5:57:17 AM
11,2026-03-13T05:57:17.408027000Z,ESU5,6363.75,53,6364,67,3/13/2026 5:57:17 AM
10,2026-03-13T05:57:17.307406200Z,ESU5,6363.75,49,6364,67,3/13/2026 5:57:17 AM
9,2026-03-13T05:57:17.206714300Z,ESU5,6363.75,49,6364,71,3/13/2026 5:57:17 AM
8,2026-03-13T05:57:17.105530300Z,ESU5,6363.75,48,6364,71,3/13/2026 5:57:17 AM
7,2026-03-13T05:57:17.004796800Z,ESU5,6363.75,47,6364,71,3/13/2026 5:57:17 AM
6,2026-03-13T05:57:16.904113900Z,ESU5,6363.75,46,6364,71,3/13/2026 5:57:16 AM
5,2026-03-13T05:57:16.803381900Z,ESU5,6363.75,43,6364,71,3/13/2026 5:57:16 AM
4,2026-03-13T05:57:16.701969600Z,ESU5,6363.75,43,6364,72,3/13/2026 5:57:16 AM
3,2026-03-13T05:57:16.600765600Z,ESU5,6363.75,43,6364,73,3/13/2026 5:57:16 AM
2,2026-03-13T05:57:16.499986300Z,ESU5,6363.75,43,6364,74,3/13/2026 5:57:16 AM
1,2026-03-13T05:57:16.399241700Z,ESU5,6363.75,43,6364,75,3/13/2026 5:57:16 AM
0,2026-03-13T05:57:16.264038000Z,ESU5,6363.75,43,6364,76,3/13/2026 5:57:16 AM
1 id ts_event contract bid_price bid_size ask_price ask_size timestamp
2 89 2026-03-13T05:57:26.904401600Z ESU5 6363.75 43 6364 67 3/13/2026 5:57:26 AM
3 88 2026-03-13T05:57:26.778037200Z ESU5 6363.75 43 6364 69 3/13/2026 5:57:26 AM
4 87 2026-03-13T05:57:26.597038500Z ESU5 6363.75 43 6364 69 3/13/2026 5:57:26 AM
5 86 2026-03-13T05:57:26.438768300Z ESU5 6363.75 43 6364 68 3/13/2026 5:57:26 AM
6 85 2026-03-13T05:57:26.328259100Z ESU5 6363.75 43 6364 67 3/13/2026 5:57:26 AM
7 84 2026-03-13T05:57:26.216211200Z ESU5 6363.75 44 6364 67 3/13/2026 5:57:26 AM
8 83 2026-03-13T05:57:26.099281200Z ESU5 6363.75 46 6364 67 3/13/2026 5:57:26 AM
9 82 2026-03-13T05:57:25.979814900Z ESU5 6363.75 48 6364 67 3/13/2026 5:57:25 AM
10 81 2026-03-13T05:57:25.852317700Z ESU5 6363.75 48 6364 66 3/13/2026 5:57:25 AM
11 80 2026-03-13T05:57:25.684323100Z ESU5 6363.75 48 6364 65 3/13/2026 5:57:25 AM
12 79 2026-03-13T05:57:25.539436900Z ESU5 6363.75 48 6364 64 3/13/2026 5:57:25 AM
13 78 2026-03-13T05:57:25.425439200Z ESU5 6363.75 49 6364 64 3/13/2026 5:57:25 AM
14 77 2026-03-13T05:57:25.298054100Z ESU5 6363.75 51 6364 64 3/13/2026 5:57:25 AM
15 76 2026-03-13T05:57:25.177965500Z ESU5 6363.75 51 6364 63 3/13/2026 5:57:25 AM
16 75 2026-03-13T05:57:25.068062900Z ESU5 6363.75 52 6364 63 3/13/2026 5:57:25 AM
17 74 2026-03-13T05:57:24.963428500Z ESU5 6363.75 56 6364 63 3/13/2026 5:57:24 AM
18 73 2026-03-13T05:57:24.781215700Z ESU5 6363.75 57 6364 63 3/13/2026 5:57:24 AM
19 72 2026-03-13T05:57:24.676290100Z ESU5 6363.75 57 6364 63 3/13/2026 5:57:24 AM
20 71 2026-03-13T05:57:24.566898400Z ESU5 6363.75 56 6364 63 3/13/2026 5:57:24 AM
21 70 2026-03-13T05:57:24.430800400Z ESU5 6363.75 57 6364 63 3/13/2026 5:57:24 AM
22 69 2026-03-13T05:57:24.273683400Z ESU5 6363.75 56 6364 63 3/13/2026 5:57:24 AM
23 68 2026-03-13T05:57:24.167656100Z ESU5 6363.75 56 6364 64 3/13/2026 5:57:24 AM
24 67 2026-03-13T05:57:23.995720100Z ESU5 6363.75 56 6364 65 3/13/2026 5:57:23 AM
25 66 2026-03-13T05:57:23.894029100Z ESU5 6363.75 56 6364 68 3/13/2026 5:57:23 AM
26 65 2026-03-13T05:57:23.728897300Z ESU5 6363.75 56 6364 68 3/13/2026 5:57:23 AM
27 64 2026-03-13T05:57:23.614316800Z ESU5 6363.75 52 6364 68 3/13/2026 5:57:23 AM
28 63 2026-03-13T05:57:23.503752700Z ESU5 6363.75 51 6364 68 3/13/2026 5:57:23 AM
29 62 2026-03-13T05:57:23.289108200Z ESU5 6363.75 50 6364 68 3/13/2026 5:57:23 AM
30 61 2026-03-13T05:57:23.155673400Z ESU5 6363.75 49 6364 68 3/13/2026 5:57:23 AM
31 60 2026-03-13T05:57:23.028898400Z ESU5 6363.75 48 6364 68 3/13/2026 5:57:23 AM
32 59 2026-03-13T05:57:22.887132700Z ESU5 6363.75 46 6364 68 3/13/2026 5:57:22 AM
33 58 2026-03-13T05:57:22.676916300Z ESU5 6363.75 44 6364 68 3/13/2026 5:57:22 AM
34 57 2026-03-13T05:57:22.520328000Z ESU5 6363.75 43 6364 68 3/13/2026 5:57:22 AM
35 56 2026-03-13T05:57:22.405244600Z ESU5 6363.75 43 6364 69 3/13/2026 5:57:22 AM
36 55 2026-03-13T05:57:22.294674100Z ESU5 6363.75 41 6364 69 3/13/2026 5:57:22 AM
37 54 2026-03-13T05:57:22.163467900Z ESU5 6363.75 38 6364 69 3/13/2026 5:57:22 AM
38 53 2026-03-13T05:57:21.992443000Z ESU5 6363.75 38 6364 71 3/13/2026 5:57:21 AM
39 52 2026-03-13T05:57:21.872338400Z ESU5 6363.75 38 6364 71 3/13/2026 5:57:21 AM
40 51 2026-03-13T05:57:21.742801800Z ESU5 6363.75 37 6364 71 3/13/2026 5:57:21 AM
41 50 2026-03-13T05:57:21.557711100Z ESU5 6363.75 37 6364 72 3/13/2026 5:57:21 AM
42 49 2026-03-13T05:57:21.434977000Z ESU5 6363.75 37 6364 73 3/13/2026 5:57:21 AM
43 48 2026-03-13T05:57:21.285751600Z ESU5 6363.75 37 6364 72 3/13/2026 5:57:21 AM
44 47 2026-03-13T05:57:21.184392000Z ESU5 6363.75 37 6364 73 3/13/2026 5:57:21 AM
45 46 2026-03-13T05:57:21.076843800Z ESU5 6363.75 34 6364 73 3/13/2026 5:57:21 AM
46 45 2026-03-13T05:57:20.894131800Z ESU5 6363.75 32 6364 73 3/13/2026 5:57:20 AM
47 44 2026-03-13T05:57:20.759550000Z ESU5 6363.75 30 6364 73 3/13/2026 5:57:20 AM
48 43 2026-03-13T05:57:20.641547900Z ESU5 6363.75 28 6364 73 3/13/2026 5:57:20 AM
49 42 2026-03-13T05:57:20.540293200Z ESU5 6363.75 27 6364 73 3/13/2026 5:57:20 AM
50 41 2026-03-13T05:57:20.439034600Z ESU5 6363.75 27 6364 74 3/13/2026 5:57:20 AM
51 40 2026-03-13T05:57:20.337795000Z ESU5 6363.75 27 6364 73 3/13/2026 5:57:20 AM
52 39 2026-03-13T05:57:20.236606600Z ESU5 6363.75 27 6364 72 3/13/2026 5:57:20 AM
53 38 2026-03-13T05:57:20.135417700Z ESU5 6363.75 30 6364 72 3/13/2026 5:57:20 AM
54 37 2026-03-13T05:57:20.034272700Z ESU5 6363.75 32 6364 72 3/13/2026 5:57:20 AM
55 36 2026-03-13T05:57:19.933096800Z ESU5 6363.75 34 6364 72 3/13/2026 5:57:19 AM
56 35 2026-03-13T05:57:19.832012600Z ESU5 6363.75 36 6364 72 3/13/2026 5:57:19 AM
57 34 2026-03-13T05:57:19.730878500Z ESU5 6363.75 38 6364 72 3/13/2026 5:57:19 AM
58 33 2026-03-13T05:57:19.629787400Z ESU5 6363.75 38 6364 71 3/13/2026 5:57:19 AM
59 32 2026-03-13T05:57:19.528697000Z ESU5 6363.75 38 6364 70 3/13/2026 5:57:19 AM
60 31 2026-03-13T05:57:19.427077000Z ESU5 6363.75 38 6364 71 3/13/2026 5:57:19 AM
61 30 2026-03-13T05:57:19.326450300Z ESU5 6363.75 39 6364 71 3/13/2026 5:57:19 AM
62 29 2026-03-13T05:57:19.225427900Z ESU5 6363.75 39 6364 70 3/13/2026 5:57:19 AM
63 28 2026-03-13T05:57:19.123339800Z ESU5 6363.75 40 6364 70 3/13/2026 5:57:19 AM
64 27 2026-03-13T05:57:19.021936500Z ESU5 6363.75 41 6364 70 3/13/2026 5:57:19 AM
65 26 2026-03-13T05:57:18.920859200Z ESU5 6363.75 45 6364 70 3/13/2026 5:57:18 AM
66 25 2026-03-13T05:57:18.820148400Z ESU5 6363.75 46 6364 70 3/13/2026 5:57:18 AM
67 24 2026-03-13T05:57:18.718988100Z ESU5 6363.75 46 6364 69 3/13/2026 5:57:18 AM
68 23 2026-03-13T05:57:18.618299700Z ESU5 6363.75 49 6364 69 3/13/2026 5:57:18 AM
69 22 2026-03-13T05:57:18.517138100Z ESU5 6363.75 52 6364 69 3/13/2026 5:57:18 AM
70 21 2026-03-13T05:57:18.416335200Z ESU5 6363.75 53 6364 69 3/13/2026 5:57:18 AM
71 20 2026-03-13T05:57:18.315335900Z ESU5 6363.75 55 6364 69 3/13/2026 5:57:18 AM
72 19 2026-03-13T05:57:18.214368600Z ESU5 6363.75 57 6364 69 3/13/2026 5:57:18 AM
73 18 2026-03-13T05:57:18.113754700Z ESU5 6363.75 57 6364 65 3/13/2026 5:57:18 AM
74 17 2026-03-13T05:57:18.012706500Z ESU5 6363.75 56 6364 65 3/13/2026 5:57:18 AM
75 16 2026-03-13T05:57:17.911672600Z ESU5 6363.75 54 6364 65 3/13/2026 5:57:17 AM
76 15 2026-03-13T05:57:17.810879900Z ESU5 6363.75 52 6364 65 3/13/2026 5:57:17 AM
77 14 2026-03-13T05:57:17.710089700Z ESU5 6363.75 49 6364 65 3/13/2026 5:57:17 AM
78 13 2026-03-13T05:57:17.609388300Z ESU5 6363.75 49 6364 67 3/13/2026 5:57:17 AM
79 12 2026-03-13T05:57:17.508697400Z ESU5 6363.75 49 6364 67 3/13/2026 5:57:17 AM
80 11 2026-03-13T05:57:17.408027000Z ESU5 6363.75 53 6364 67 3/13/2026 5:57:17 AM
81 10 2026-03-13T05:57:17.307406200Z ESU5 6363.75 49 6364 67 3/13/2026 5:57:17 AM
82 9 2026-03-13T05:57:17.206714300Z ESU5 6363.75 49 6364 71 3/13/2026 5:57:17 AM
83 8 2026-03-13T05:57:17.105530300Z ESU5 6363.75 48 6364 71 3/13/2026 5:57:17 AM
84 7 2026-03-13T05:57:17.004796800Z ESU5 6363.75 47 6364 71 3/13/2026 5:57:17 AM
85 6 2026-03-13T05:57:16.904113900Z ESU5 6363.75 46 6364 71 3/13/2026 5:57:16 AM
86 5 2026-03-13T05:57:16.803381900Z ESU5 6363.75 43 6364 71 3/13/2026 5:57:16 AM
87 4 2026-03-13T05:57:16.701969600Z ESU5 6363.75 43 6364 72 3/13/2026 5:57:16 AM
88 3 2026-03-13T05:57:16.600765600Z ESU5 6363.75 43 6364 73 3/13/2026 5:57:16 AM
89 2 2026-03-13T05:57:16.499986300Z ESU5 6363.75 43 6364 74 3/13/2026 5:57:16 AM
90 1 2026-03-13T05:57:16.399241700Z ESU5 6363.75 43 6364 75 3/13/2026 5:57:16 AM
91 0 2026-03-13T05:57:16.264038000Z ESU5 6363.75 43 6364 76 3/13/2026 5:57:16 AM
-187
View File
@@ -1,187 +0,0 @@
from __future__ import annotations
from dataclasses import dataclass
import json
import os
import subprocess
import threading
from typing import Any, Callable, Dict, List, Optional
@dataclass
class DataPacket:
state: List[Any]
deleted_ids: List[int]
added_ids: List[int]
@dataclass
class DslResponse:
data_packet: DataPacket | None
req_id: str | None
error: str | None
def _resolve_interop_exe() -> str:
env_path = os.environ.get("BLUEORB_DSL_INTEROP")
if env_path and os.path.isfile(env_path):
return env_path
candidates = [
"/workspaces/spark/build/Debug/bin/net8.0/BlueOrbDSLInteropServer",
"/workspaces/spark/build/Release/bin/net8.0/BlueOrbDSLInteropServer",
]
for path in candidates:
if os.path.isfile(path):
return path
raise FileNotFoundError(
"BlueOrbDSLInterop executable not found. "
"Set BLUEORB_DSL_INTEROP to the full path."
)
_interop_proc: Optional[subprocess.Popen] = None
_reader_thread: Optional[threading.Thread] = None
_callbacks: Dict[str, Callable[[DslResponse], None]] = {}
_first_response_events: Dict[str, threading.Event] = {}
_first_responses: Dict[str, DslResponse] = {}
_state_lock = threading.Lock()
def start_interop() -> None:
"""
Start the BlueOrbDSLInteropServer process if it's not already running.
"""
global _interop_proc
if _interop_proc is not None and _interop_proc.poll() is None:
return
exe = _resolve_interop_exe()
_interop_proc = subprocess.Popen(
[exe],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
_start_reader_thread()
def stop_interop() -> None:
"""
Stop the BlueOrbDSLInteropServer process if it's running.
"""
global _interop_proc
if _interop_proc is None:
return
if _interop_proc.poll() is None and _interop_proc.stdin is not None:
_interop_proc.stdin.write(":exit\n")
_interop_proc.stdin.flush()
_interop_proc.stdin.close()
_interop_proc.wait(timeout=5)
global _reader_thread
if _reader_thread is not None:
_reader_thread.join(timeout=1)
_reader_thread = None
with _state_lock:
_callbacks.clear()
_first_response_events.clear()
_first_responses.clear()
_interop_proc = None
def get_interop_status() -> tuple[bool, int | None]:
"""
Returns current interop process status as (is_running, exit_code).
exit_code is None when process was never started or is still running.
"""
proc = _interop_proc
if proc is None:
return (False, None)
rc = proc.poll()
return (rc is None, rc)
def get_dsl(dsl: str, req_id: str, callback: Callable[[DslResponse], None]) -> DslResponse:
"""
Sends the DSL to BlueOrbDSLInteropServer and returns the first response for this req_id.
Subsequent streaming updates for the same req_id are delivered via callback.
Requires start_interop() to be called first.
"""
assert _interop_proc is not None and _interop_proc.poll() is None, (
"Interop process is not running. Call start_interop() first."
)
assert _interop_proc.stdin is not None
payload = {"dsl": dsl, "req_id": req_id}
first_event = threading.Event()
with _state_lock:
_callbacks[req_id] = callback
_first_response_events[req_id] = first_event
_first_responses.pop(req_id, None)
_interop_proc.stdin.write(json.dumps(payload) + "\n")
_interop_proc.stdin.flush()
if not first_event.wait(timeout=10):
with _state_lock:
_first_response_events.pop(req_id, None)
_callbacks.pop(req_id, None)
raise TimeoutError(f"No response received for req_id '{req_id}' from BlueOrbDSLInterop.")
with _state_lock:
response = _first_responses.pop(req_id, None)
_first_response_events.pop(req_id, None)
if response is None:
raise RuntimeError(f"Missing first response for req_id '{req_id}'.")
return response
def _start_reader_thread() -> None:
global _reader_thread
if _reader_thread is not None and _reader_thread.is_alive():
return
_reader_thread = threading.Thread(target=_reader_loop, daemon=True)
_reader_thread.start()
def _reader_loop() -> None:
while True:
proc = _interop_proc
if proc is None or proc.poll() is not None:
return
assert proc.stdout is not None
raw = proc.stdout.readline()
if raw == "":
return
line = raw.strip()
if not line:
continue
try:
payload = json.loads(line)
except json.JSONDecodeError:
continue
response = DslResponse(
data_packet=DataPacket(state=[payload.get("data_packet")], deleted_ids=[], added_ids=[]),
req_id=payload.get("req_id"),
error=payload.get("error"),
)
req_id = response.req_id
if not req_id:
continue
callback: Optional[Callable[[DslResponse], None]]
event: Optional[threading.Event]
with _state_lock:
callback = _callbacks.get(req_id)
event = _first_response_events.get(req_id)
if event is not None and req_id not in _first_responses:
_first_responses[req_id] = response
event.set()
if callback is not None:
try:
callback(response)
except Exception:
# Keep the reader alive if callback logic fails.
pass
@@ -1,33 +0,0 @@
import os
import sys
import uuid
_here = os.path.dirname(os.path.abspath(__file__))
_parent = os.path.dirname(_here)
if _parent not in sys.path:
sys.path.insert(0, _parent)
from BlueOrbDSL import get_dsl, start_interop, stop_interop, DslResponse
def on_packet(response: DslResponse) -> None:
print("callback from DSL Server, DslResponse:", response)
def main() -> None:
start_interop()
try:
while True:
dsl = input("Enter DSL (or 'quit'): ").strip()
if dsl.lower() == "quit":
break
if not dsl:
continue
response = get_dsl(dsl, str(uuid.uuid4()), on_packet)
print("return:", response)
finally:
stop_interop()
if __name__ == "__main__":
main()
-844
View File
@@ -1,844 +0,0 @@
from __future__ import annotations
import csv
import time
import os
import sys
import threading
import uuid
from collections import deque
from datetime import datetime
from typing import Any
from rich.text import Text
from rich.style import Style
from textual import events
from textual.app import App, ComposeResult
from textual.binding import Binding
from textual.containers import Container, Horizontal, Vertical
from textual.widgets.text_area import TextAreaTheme
from textual.widgets import Button, DataTable, Input, Static, TabbedContent, TabPane, TextArea, Tree
_here = os.path.dirname(os.path.abspath(__file__))
_parent = os.path.dirname(_here)
if _parent not in sys.path:
sys.path.insert(0, _parent)
from BlueOrbDSL import DslResponse, get_dsl, get_interop_status, start_interop, stop_interop
class DSQALiveConsole(App):
DEFAULT_QUERY = (
'FROM market.ticks\n'
'WHERE symbol IN ("BTCUSD", "ETHUSD")\n'
"SELECT symbol, price, change, time"
)
LAST_QUERY_PATH = os.path.join(_here, "DSQA_LAST_EXECUTED_QUERY.txt")
DSL_HIGHLIGHT_QUERY = r"""
; comments
(comment) @comment
; assignment vars: lhs in x = SomeStream(...)
(assignment_expression
left: (identifier) @variable)
; stream operators (function names ending with Stream)
(call_expression
function: (identifier) @function
(#match? @function "^[A-Za-z_][A-Za-z0-9_]*Stream$"))
; normal function calls
(call_expression
function: (identifier) @function.method)
; arguments and literals
(arguments (identifier) @parameter)
(string) @string
(number) @number
(true) @constant.builtin
(false) @constant.builtin
(null) @constant.builtin
"""
BINDINGS = [
Binding("ctrl+r", "execute", "Execute"),
Binding("f11", "execute", show=False),
Binding("alt+enter", "execute", show=False),
# Override Textual's default Ctrl+Q quit to avoid accidental VS Code close.
Binding("ctrl+q", "noop", show=False),
Binding("ctrl+x", "quit", "Quit"),
]
CSS = """
Screen {
background: #060a1c;
color: #d8dff4;
}
#window_close_button {
width: 3;
min-width: 3;
height: 1;
margin: 0;
border: none;
background: #3a1015;
color: #ff9ca8;
text-style: bold;
}
#window_close_button:hover,
#window_close_button:focus {
background: #5a1620;
color: #ffd6db;
}
#root {
padding: 0 1 0 1;
height: 1fr;
}
#query_panel {
background: #070d27;
height: 33%;
margin-top: 1;
}
#query_header_row {
height: 1;
border-bottom: solid #596285;
}
#query_header {
width: auto;
color: #9eb0d4;
content-align: left middle;
}
#query_header_spacer {
width: 1fr;
}
#query_input {
border: solid #697597;
background: #050b22;
color: #cfd7eb;
height: 1fr;
padding-left: 1;
}
#results_panel {
background: #070d27;
height: 1fr;
margin-top: 1;
}
#results_toolbar {
height: 3;
padding: 0 1;
}
#results_left {
width: auto;
height: 3;
margin-top: 0;
}
#stream_toggle_button {
width: auto;
min-width: 11;
margin-top: 0;
margin-left: 1;
border: none;
background: #070d27;
color: #79afe7;
}
#stream_toggle_button:hover,
#stream_toggle_button:focus {
background: #111a3f;
color: #9bc9f6;
}
#execute_button {
width: 5;
min-width: 5;
margin-top: 0;
margin-left: 1;
border: none;
background: #10341f;
color: #7ef1ac;
text-style: bold;
}
#execute_button:hover,
#execute_button:focus {
background: #1a4f30;
color: #b3ffd0;
}
#results_actions {
width: auto;
height: 3;
margin-top: 0;
}
#results_spacer {
width: 1fr;
}
#results_search {
width: 30;
min-width: 24;
margin-right: 1;
border: none;
background: #060d28;
color: #d5def3;
padding: 0 1;
}
#search_button,
#copy_button,
#export_button {
width: auto;
min-width: 11;
margin-left: 1;
border: none;
background: #070d27;
color: #79afe7;
}
#search_button:hover,
#copy_button:hover,
#export_button:hover,
#search_button:focus,
#copy_button:focus,
#export_button:focus {
background: #111a3f;
color: #9bc9f6;
}
#results_table {
border: solid #697597;
background: #050b22;
height: 1fr;
}
#results_table .datatable--header,
#results_table .datatable--header-hover,
#results_table .datatable--header-cursor {
background: #0c1535;
color: #6ea6df;
text-style: bold;
}
#results_table .datatable--even-row {
background: #050b22;
}
#results_table .datatable--odd-row {
background: #07102a;
}
#results_table .datatable--cursor {
background: #19335b;
color: #dbe8ff;
}
#results_table .datatable--fixed-cursor {
background: #132949;
color: #c8dbff;
}
#results_table.error .datatable--odd-row,
#results_table.error .datatable--even-row,
#results_table.error .datatable--cursor,
#results_table.error .datatable--fixed-cursor {
color: #f17486;
text-style: bold;
}
#results_panel.paused {
background: #150f1f;
}
#results_table.paused {
background: #130f18;
}
#results_table.paused .datatable--header,
#results_table.paused .datatable--header-hover,
#results_table.paused .datatable--header-cursor {
background: #262338;
color: #8b90a1;
text-style: bold;
}
#results_tabs {
height: 1fr;
background: #070d27;
}
#data_tab,
#tree_tab {
padding: 0;
}
#ops_tree {
border: solid #697597;
background: #050b22;
color: #cfd7eb;
height: 1fr;
}
#bottom_status_bar {
border-top: solid #596285;
background: #070d27;
height: 2;
margin: 0;
padding: 0 1;
}
#status_right_wrap {
width: 1fr;
}
#status_row {
height: 1;
margin-top: 0;
width: 1fr;
}
#live_status {
width: 1fr;
color: #b3bed9;
content-align: right middle;
}
"""
def compose(self) -> ComposeResult:
with Container(id="root"):
with Vertical(id="query_panel"):
with Horizontal(id="query_header_row"):
yield Static("DSL Query", id="query_header")
yield Static("", id="query_header_spacer")
yield Button("✕", id="window_close_button")
yield TextArea(
text=self._load_last_query(),
id="query_input",
language="javascript",
theme="vscode_dark",
show_line_numbers=True,
)
with Vertical(id="results_panel"):
with Horizontal(id="results_toolbar"):
with Horizontal(id="results_left"):
yield Button("▶", id="execute_button")
yield Button("⏸ Pause", id="stream_toggle_button")
yield Static("", id="results_spacer")
with Horizontal(id="results_actions"):
yield Input(placeholder="Search: col:value", id="results_search")
yield Button("⌕ Search", id="search_button")
yield Button("⧉ Copy", id="copy_button")
yield Button("⇩ Export", id="export_button")
with TabbedContent(initial="data_tab", id="results_tabs"):
with TabPane("Data", id="data_tab"):
table = DataTable(id="results_table")
table.cursor_type = "cell"
table.show_cursor = True
table.zebra_stripes = True
yield table
with TabPane("Operators", id="tree_tab"):
tree = Tree("Assemblies", id="ops_tree")
tree.show_root = False
yield tree
with Horizontal(id="bottom_status_bar"):
with Horizontal(id="status_right_wrap"):
with Horizontal(id="status_row"):
yield Static(
"Rows: 0 | Rate: 0.0/s | Total Updates: 0 | Uptime: 00:00:00 | Last Update: --:--:--",
id="live_status",
)
def on_mount(self) -> None:
self._update_timestamps: deque[float] = deque()
self._last_update_at: datetime | None = None
self._stream_started_at: float | None = None
self._total_updates = 0
self._last_row_count = 0
self._current_col_names: list[str] = []
self._current_rows: list[dict[str, Any]] = []
self._column_width_cache: dict[str, int] = {}
self._ops_catalog_rows: list[dict[str, Any]] = []
self._ops_catalog_error: str | None = None
self._ops_catalog_loaded = False
self._ops_catalog_loading = False
self._interop_is_running = False
self._interop_exit_code: int | None = None
self._grid_updates_enabled = True
self._paused_latest_response: DslResponse | None = None
self._ensure_interop_started()
self._refresh_ops_tree()
self._refresh_ops_tree_async()
self._configure_query_highlighting()
self._set_pause_visuals()
self.set_interval(0.5, self._refresh_status)
def _configure_query_highlighting(self) -> None:
query_input = self.query_one("#query_input", TextArea)
query_input.language = "javascript"
dsl_theme = TextAreaTheme(
"dsqa_ide",
base_style=Style.parse("#d7deee on #171a22"),
gutter_style=Style.parse("#8a93a8 on #1c212b"),
cursor_style=Style.parse("#ffffff on #2b3242"),
cursor_line_style=Style.parse("on #202534"),
cursor_line_gutter_style=Style.parse("#a6afc4 on #202534"),
selection_style=Style.parse("on #2a3f67"),
bracket_matching_style=Style.parse("bold #7fd9ff"),
syntax_styles={
"comment": Style.parse("italic #6f778a"),
"variable": Style.parse("bold #4ec9b0"),
"function": Style.parse("bold #4fc1ff"),
"function.method": Style.parse("bold #70d4ff"),
"parameter": Style.parse("#c6d4f8"),
"string": Style.parse("#d7ba7d"),
"number": Style.parse("#b5cea8"),
"constant.builtin": Style.parse("bold #569cd6"),
},
)
query_input.register_theme(dsl_theme)
query_input.theme = "dsqa_ide"
try:
query_input.update_highlight_query(self.DSL_HIGHLIGHT_QUERY)
except Exception:
pass
def on_shutdown(self) -> None:
stop_interop()
@staticmethod
def _ensure_interop_started() -> None:
try:
start_interop()
except Exception:
pass
def action_execute(self) -> None:
tabs = self.query_one("#results_tabs", TabbedContent)
tabs.active = "data_tab"
self._grid_updates_enabled = True
self._set_stream_toggle_label()
self._set_pause_visuals()
self._execute_query_async()
def action_noop(self) -> None:
pass
def on_button_pressed(self, event: Button.Pressed) -> None:
button_id = event.button.id
if button_id == "search_button":
self._search_results()
elif button_id == "copy_button":
self._copy_selected_cell()
elif button_id == "export_button":
self._export_results_csv()
elif button_id == "execute_button":
self.action_execute()
elif button_id == "window_close_button":
self.exit()
elif button_id == "stream_toggle_button":
self._toggle_stream_painting()
def on_input_submitted(self, event: Input.Submitted) -> None:
if event.input.id == "results_search":
self._search_results()
def on_click(self, event: events.Click) -> None:
if getattr(event.widget, "id", None) != "ops_tree":
return
line = event.style.meta.get("line")
if line is None:
return
tree = self.query_one("#ops_tree", Tree)
node = tree.get_node_at_line(line)
if node is None:
return
node_data = node.data
if not isinstance(node_data, dict) or node_data.get("kind") != "constructor":
return
self._append_operator_to_query(self._stringify(node_data.get("op", "")).strip())
def _append_operator_to_query(self, op_name: str) -> None:
if not op_name:
return
query_input = self.query_one("#query_input", TextArea)
existing = query_input.text
if existing.strip():
separator = "" if existing.endswith("\n") else "\n"
query_input.text = f"{existing}{separator}{op_name}"
else:
query_input.text = op_name
query_input.focus()
def _execute_query_async(self) -> None:
self._ensure_interop_started()
query = self.query_one(TextArea).text.strip()
if not query:
return
self._column_width_cache.clear()
self._save_last_query(query)
self._update_timestamps.clear()
self._last_update_at = None
self._stream_started_at = time.monotonic()
self._total_updates = 0
self._last_row_count = 0
self._paused_latest_response = None
self._refresh_status()
req_id = str(uuid.uuid4())
def _run() -> None:
try:
def _on_response(resp: DslResponse) -> None:
self.call_from_thread(self._render_response, resp)
get_dsl(query, req_id, _on_response)
except Exception as exc:
error_resp = DslResponse(data_packet=None, req_id=req_id, error=str(exc))
self.call_from_thread(self._render_response, error_resp)
threading.Thread(target=_run, daemon=True).start()
def _render_response(self, response: DslResponse) -> None:
if not self._grid_updates_enabled:
self._paused_latest_response = response
return
self._render_response_internal(response)
def _render_response_internal(self, response: DslResponse) -> None:
self._record_update()
table = self.query_one(DataTable)
table.remove_class("error")
prev_scroll_x = table.scroll_offset.x
prev_scroll_y = table.scroll_offset.y
table.clear(columns=True)
if response.error:
table.add_class("error")
error_text = self._stringify(response.error)
error_lines = error_text.splitlines() or [error_text]
self._current_col_names = []
self._current_rows = []
self._last_row_count = len(error_lines)
self._refresh_ops_tree()
table.add_columns("error")
for line in error_lines:
table.add_row(Text(line, style="bold #f17486"))
return
if response.data_packet is None or not response.data_packet.state:
self._current_col_names = []
self._current_rows = []
self._last_row_count = 0
self._refresh_ops_tree()
table.add_columns("empty")
table.add_row("no data")
return
data_packet = response.data_packet.state[0]
columns = data_packet.get("columns", [])
rows = data_packet.get("all", [])
if not isinstance(rows, list):
rows = []
col_names = [c.get("name", "") for c in columns]
if not col_names:
col_names = list(rows[0].keys()) if rows else ["empty"]
self._current_col_names = col_names
self._current_rows = rows
self._last_row_count = len(rows)
self._refresh_ops_tree()
for col_name in col_names:
width = self._column_width_cache.get(col_name, len(col_name))
for row in rows:
if not isinstance(row, dict):
continue
cell_text = self._stringify(row.get(col_name, ""))
width = max(width, len(cell_text))
self._column_width_cache[col_name] = width
table.add_column(col_name, width=width)
for row in rows:
table.add_row(*[self._format_cell(c, row.get(c, "")) for c in col_names])
# Preserve user's viewport during streaming refreshes.
table.scroll_to(
x=min(prev_scroll_x, table.max_scroll_x),
y=min(prev_scroll_y, table.max_scroll_y),
animate=False,
)
@staticmethod
def _stringify(value: Any) -> str:
return "" if value is None else str(value)
@staticmethod
def _format_cell(column_name: str, value: Any) -> str | Text:
text = DSQALiveConsole._stringify(value)
lowered = column_name.lower()
if lowered == "id":
return Text(text, style="bold #9eb1d8")
if lowered == "symbol":
return Text(text, style="bold #dce6ff")
if lowered == "price":
return Text(text, style="bold #d4ddf2")
if lowered == "time":
return Text(text, style="#aab5d0")
if lowered == "change":
cleaned = text.strip()
if cleaned.startswith("-"):
return Text(text, style="bold #f17486")
if cleaned.startswith("+"):
return Text(text, style="bold #92d67e")
return Text(text, style="bold #c7d3ef")
if lowered == "bid":
return Text(text, style="bold #8cf3c0 on #123526")
if lowered == "ask":
return Text(text, style="bold #ffd5d9 on #3b1b25")
return text
def _record_update(self) -> None:
now = time.monotonic()
self._update_timestamps.append(now)
self._total_updates += 1
if self._stream_started_at is None:
self._stream_started_at = now
self._last_update_at = datetime.now()
self._refresh_status()
def _refresh_status(self) -> None:
self._refresh_interop_heartbeat()
now = time.monotonic()
cutoff = now - 1.0
while self._update_timestamps and self._update_timestamps[0] < cutoff:
self._update_timestamps.popleft()
rate = float(len(self._update_timestamps))
if self._stream_started_at is not None:
uptime_seconds = max(0, int(now - self._stream_started_at))
uptime = self._format_duration(uptime_seconds)
else:
uptime = "--:--:--"
last_update = self._last_update_at.strftime("%H:%M:%S") if self._last_update_at else "--:--:--"
status = Text()
if self._interop_is_running:
status.append("Interop: ● UP", style="bold #92d67e")
elif self._interop_exit_code is not None:
status.append(f"Interop: ● DOWN({self._interop_exit_code})", style="bold #f17486")
else:
status.append("Interop: ● DOWN", style="bold #8b90a1")
status.append(" | ")
status.append(f"Rows: {self._last_row_count} | Rate: {rate:.1f}/s | ")
status.append(f"Total Updates: {self._total_updates}", style="bold #79afe7")
status.append(f" | Uptime: {uptime} | ")
status.append(f"Last Update: {last_update}", style="bold #79afe7")
self.query_one("#live_status", Static).update(status)
def _refresh_interop_heartbeat(self) -> None:
is_running, exit_code = get_interop_status()
self._interop_is_running = is_running
self._interop_exit_code = exit_code
@staticmethod
def _format_duration(total_seconds: int) -> str:
hours = total_seconds // 3600
minutes = (total_seconds % 3600) // 60
seconds = total_seconds % 60
return f"{hours:02d}:{minutes:02d}:{seconds:02d}"
def _search_results(self) -> None:
raw = self.query_one("#results_search", Input).value.strip()
if ":" not in raw:
return
col_part, value_part = raw.split(":", 1)
target_col = col_part.strip()
target_value = value_part.strip()
if not target_col:
return
lookup = {name.lower(): idx for idx, name in enumerate(self._current_col_names)}
col_idx = lookup.get(target_col.lower())
if col_idx is None:
return
col_name = self._current_col_names[col_idx]
needle = target_value.lower()
for row_idx, row in enumerate(self._current_rows):
if not isinstance(row, dict):
continue
cell_value = self._stringify(row.get(col_name, ""))
if needle in cell_value.lower():
table = self.query_one("#results_table", DataTable)
table.move_cursor(row=row_idx, column=col_idx)
table.focus()
return
def _copy_selected_cell(self) -> None:
table = self.query_one("#results_table", DataTable)
if table.row_count == 0 or not self._current_col_names:
return
coordinate = table.cursor_coordinate
try:
cell = table.get_cell_at(coordinate)
except Exception:
return
if isinstance(cell, Text):
value = cell.plain
else:
value = self._stringify(cell)
self.copy_to_clipboard(value)
def _export_results_csv(self) -> None:
if not self._current_col_names:
return
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
export_path = os.path.join(_here, f"DSQA_RESULTS_EXPORT_{timestamp}.csv")
with open(export_path, "w", newline="", encoding="utf-8") as csv_file:
writer = csv.writer(csv_file)
writer.writerow(self._current_col_names)
for row in self._current_rows:
if isinstance(row, dict):
writer.writerow([self._stringify(row.get(col, "")) for col in self._current_col_names])
def _refresh_ops_tree(self) -> None:
tree = self.query_one("#ops_tree", Tree)
root = tree.root
root.remove_children()
if self._ops_catalog_error:
root.add(Text(self._ops_catalog_error, style="bold #f17486"))
root.expand()
return
grouped: dict[str, dict[str, dict[str, set[str]]]] = {}
for row in self._ops_catalog_rows:
if not isinstance(row, dict):
continue
row_lookup = {self._stringify(k).lower(): v for k, v in row.items()}
assembly_name = self._stringify(row_lookup.get("assembly_name", "")).strip() or "(unknown assembly)"
namespace = self._stringify(row_lookup.get("namespace", "")).strip() or "(global)"
op = self._stringify(row_lookup.get("op", "")).strip() or "(unknown op)"
constructor = self._stringify(row_lookup.get("constructor", "")).strip() or "(none)"
grouped.setdefault(assembly_name, {}).setdefault(namespace, {}).setdefault(op, set()).add(constructor)
if not grouped:
root.add(Text("No operator metadata available", style="#8ea0c6"))
root.expand()
return
for assembly_name in sorted(grouped.keys(), key=str.lower):
assembly_node = root.add(Text(assembly_name, style="bold #79afe7"))
for namespace in sorted(grouped[assembly_name].keys(), key=str.lower):
namespace_node = assembly_node.add(Text(namespace, style="#cfd7eb"))
for op in sorted(grouped[assembly_name][namespace].keys(), key=str.lower):
op_node = namespace_node.add(
Text(op, style="#b8c8e7"),
data={"kind": "op", "op": op},
)
for constructor in sorted(grouped[assembly_name][namespace][op], key=str.lower):
op_node.add(
Text(constructor, style="#9eb0d4"),
data={"kind": "constructor", "op": op, "constructor": constructor},
)
op_node.expand()
namespace_node.expand()
assembly_node.expand()
root.expand()
def _refresh_ops_tree_async(self) -> None:
if self._ops_catalog_loaded or self._ops_catalog_loading:
return
self._ensure_interop_started()
self._ops_catalog_loading = True
req_id = str(uuid.uuid4())
def _run() -> None:
try:
def _on_response(resp: DslResponse) -> None:
self.call_from_thread(self._update_ops_catalog, resp)
get_dsl("OpsStream()", req_id, _on_response)
except Exception as exc:
error_resp = DslResponse(data_packet=None, req_id=req_id, error=str(exc))
self.call_from_thread(self._update_ops_catalog, error_resp)
threading.Thread(target=_run, daemon=True).start()
def _update_ops_catalog(self, response: DslResponse) -> None:
self._ops_catalog_loading = False
if response.error:
self._ops_catalog_rows = []
self._ops_catalog_error = self._stringify(response.error)
self._refresh_ops_tree()
return
if response.data_packet is None or not response.data_packet.state:
self._ops_catalog_rows = []
self._ops_catalog_error = "OpsStream() returned no data"
self._refresh_ops_tree()
return
data_packet = response.data_packet.state[0]
rows = data_packet.get("all", [])
if not isinstance(rows, list):
rows = []
normalized_rows: list[dict[str, Any]] = []
for row in rows:
if isinstance(row, dict):
normalized_rows.append(row)
self._ops_catalog_rows = normalized_rows
self._ops_catalog_error = None
self._ops_catalog_loaded = True
self._refresh_ops_tree()
def _toggle_stream_painting(self) -> None:
self._grid_updates_enabled = not self._grid_updates_enabled
self._set_stream_toggle_label()
self._set_pause_visuals()
if self._grid_updates_enabled and self._paused_latest_response is not None:
latest = self._paused_latest_response
self._paused_latest_response = None
self._render_response_internal(latest)
def _set_stream_toggle_label(self) -> None:
button = self.query_one("#stream_toggle_button", Button)
button.label = "⏸ Pause" if self._grid_updates_enabled else "▶ Play"
def _set_pause_visuals(self) -> None:
table = self.query_one("#results_table", DataTable)
panel = self.query_one("#results_panel", Vertical)
if self._grid_updates_enabled:
table.remove_class("paused")
panel.remove_class("paused")
else:
table.add_class("paused")
panel.add_class("paused")
def _load_last_query(self) -> str:
try:
with open(self.LAST_QUERY_PATH, "r", encoding="utf-8") as file:
text = file.read().strip()
return text if text else self.DEFAULT_QUERY
except FileNotFoundError:
return self.DEFAULT_QUERY
except OSError:
return self.DEFAULT_QUERY
def _save_last_query(self, query: str) -> None:
try:
with open(self.LAST_QUERY_PATH, "w", encoding="utf-8") as file:
file.write(query)
except OSError:
pass
if __name__ == "__main__":
DSQALiveConsole().run()
@@ -1,31 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<AssemblyName>BlueOrbDSLInteropServer</AssemblyName>
<RootNamespace>BlueOrbDSLInteropServer</RootNamespace>
</PropertyGroup>
<Choose>
<When Condition="'$(UseProjectRefs)' == 'true'">
<ItemGroup>
<ProjectReference Include="..\..\cs\Bts.Spark.Api\Bts.Spark.Api.csproj" />
<ProjectReference Include="..\DataStream\DataStream.csproj" />
<ProjectReference Include="..\PubSub\PubSub.csproj" />
<ProjectReference Include="..\BlueOrbOperatorStreams\BlueOrbOperatorStreams.csproj" />
</ItemGroup>
</When>
<Otherwise>
<ItemGroup>
<PackageReference Include="Bts.Spark.Api" Version="$(SparkVersion)" />
<PackageReference Include="DataStream" Version="$(SparkVersion)" />
<PackageReference Include="PubSub" Version="$(SparkVersion)" />
<PackageReference Include="BlueOrbOperatorStreams" Version="$(SparkVersion)" />
</ItemGroup>
</Otherwise>
</Choose>
</Project>
@@ -1,122 +0,0 @@
using System;
using System.Text.Json;
using System.Text.Json.Serialization;
using BCG;
using bts.BlueOrb;
using Bts.DataStream;
using Bts.DataStreamCore;
using Bts.DataStreamExpressions;
using Bts.PubSub;
using MathNet.Numerics.Statistics;
namespace BlueOrbDSLInteropServer {
// BlueOrbDSLInteropServer reads DSL lines from stdin, validates them,
// and writes one DataPacket JSON line to stdout per input.
internal static class Program {
private const string ExitSignal = ":exit";
private static SyncQueue<DslRequest> _work_queue = new SyncQueue<DslRequest>();
private static JsonSerializerOptions response_json_options = new JsonSerializerOptions {
NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals
};
private static string _current_req_id;
// Request from Python
private sealed class DslRequest {
public string? dsl { get; set; }
public string? req_id { get; set; }
}
// Response envelope for Python: includes DataPacket, request id, and optional error.
private sealed class DslResponse {
public DataPacket? data_packet { get; set; }
public string? req_id { get; set; }
public string? error { get; set; }
}
private static void Main(string[] args) {
//System.Threading.Thread.Sleep(20000);
System.Threading.Thread thread = new Thread(_worker_loop);
thread.IsBackground = true;
thread.Start();
response_json_options.Converters.Add(new DataPacketJsonConverter {
IncludeCurrent = true,
UseObjectRows = true,
});
string? line;
while ((line = Console.ReadLine()) != null) {
line = line.Trim();
if (line.Length == 0) {
continue;
}
if (string.Equals(line, ExitSignal, StringComparison.OrdinalIgnoreCase)) {
break;
}
try {
DslRequest req = JsonSerializer.Deserialize<DslRequest>(line);
_work_queue.enqueue(req);
} catch (Exception ex) {
bts.Log.assert(false, ex.ToString());
}
}
}
private static void _worker_loop() {
Dictionary<string, ViewStream> req_map = new Dictionary<string, ViewStream>();
while (true) {
System.Threading.Thread.Sleep(100);
ViewStreamDatabase.instance.doWork();
var new_reqs = _work_queue.dequeue_all();
if (new_reqs == null || new_reqs.Count == 0) {
continue;
}
// ubsubscribing to previous stream(s)
foreach (var kv in req_map) {
kv.Value.onChange -= dp_obj => _on_dp(kv.Key, dp_obj);
}
foreach (var r in new_reqs) {
var req_id = string.IsNullOrWhiteSpace(r.req_id) ? Guid.NewGuid().ToString() : r.req_id;
_current_req_id = req_id;
try {
DSLStream dsl_str = new DSLStream(r.dsl);
req_map[req_id] = dsl_str;
dsl_str.onChange += dp_obj => _on_dp(req_id, dp_obj);
} catch (Exception ex) {
_write_error(req_id, ex.StackTrace);
}
}
}
}
private static void _on_dp(string req_id, object dp_obj) {
// eventhough we unsubscribe to all the other streams except the currrent one,
// sometimes the onLastSub is not implemented and we still get updates, this masks them
if (req_id != _current_req_id)
return;
DataPacket dp = dp_obj as DataPacket;
DslResponse response = new DslResponse() {
data_packet = dp,
req_id = req_id,
error = ""
};
var json = JsonSerializer.Serialize(response, response_json_options);
Console.WriteLine(json);
}
private static void _write_error(string req_id, string error) {
DslResponse response = new DslResponse() {
data_packet = null,
req_id = req_id,
error = error
};
var json = JsonSerializer.Serialize(response, response_json_options);
Console.WriteLine(json);
}
}
}
@@ -1,514 +0,0 @@
// namespace BlueOrb.Operators.Signal;
// using BCG;
// using bts;
// using System;
// using System.Collections.Generic;
// using System.Globalization;
// using System.Linq;
// [DataStreamUserRespresentation("BarBuilderStream")]
// public class BarBuilderStream : ViewStream
// {
// enum BarType
// {
// TIME,
// TICK,
// VOLUME,
// DOLLAR,
// }
// enum PriceSource
// {
// trade_price,
// mid,
// bid,
// ask,
// }
// enum VolumeSource
// {
// trade_size,
// none,
// }
// enum SessionReset
// {
// NONE,
// DAY,
// RTH,
// ETH,
// }
// sealed class BarState
// {
// public string Contract = string.Empty;
// public DateTime StartTs;
// public DateTime EndTs;
// public double Open;
// public double High;
// public double Low;
// public double Close;
// public double Volume;
// public int TradeCount;
// public double SumPriceSize;
// public double SumSize;
// public double DollarNotional;
// public bool HasData;
// public DateTime SessionDay;
// public bool InitialSnapshotEmitted;
// }
// readonly ViewStream _priceStream;
// readonly BarType _barType;
// readonly int _barSize;
// readonly PriceSource _priceSource;
// readonly VolumeSource _volumeSource;
// readonly string _timestampCol;
// readonly bool _emitForming;
// readonly bool _emitInitialSnapshot;
// readonly SessionReset _sessionReset;
// readonly Dictionary<string, BarState> _stateByContract = new(StringComparer.Ordinal);
// readonly Dictionary<string, bool> _validCols = new(StringComparer.OrdinalIgnoreCase);
// bool _validated;
// int _nextId;
// readonly BlueTable _outTable;
// readonly IBlueTableSchema _outSchema;
// public BarBuilderStream(
// ViewStream price_stream,
// string bar_type,
// int bar_size,
// string price_source,
// string volume_source,
// string timestamp_col = "ts_event",
// int emit_forming_i = 0,
// string session_reset = "NONE",
// int emit_initial_snapshot_i = 0
// )
// {
// _priceStream = price_stream;
// _barType = ParseBarType(bar_type);
// _barSize = bar_size;
// _priceSource = ParsePriceSource(price_source);
// _volumeSource = ParseVolumeSource(volume_source);
// _timestampCol = timestamp_col;
// _emitForming = emit_forming_i == 1;
// _emitInitialSnapshot = emit_initial_snapshot_i == 1;
// _sessionReset = ParseSessionReset(session_reset);
// if (_barSize <= 0)
// {
// throw new ArgumentException("bar_size must be > 0");
// }
// _outSchema = BlueDataUtility.form_schema(
// "BarBuilder",
// Tuple.Create("id", TypeCode.Int32),
// Tuple.Create("ts_event", TypeCode.String),
// Tuple.Create("bar_start_ts", TypeCode.String),
// Tuple.Create("bar_end_ts", TypeCode.String),
// Tuple.Create("contract", TypeCode.String),
// Tuple.Create("open", TypeCode.Double),
// Tuple.Create("high", TypeCode.Double),
// Tuple.Create("low", TypeCode.Double),
// Tuple.Create("close", TypeCode.Double),
// Tuple.Create("volume", TypeCode.Double),
// Tuple.Create("trade_count", TypeCode.Int32),
// Tuple.Create("vwap", TypeCode.Double),
// Tuple.Create("is_forming", TypeCode.Boolean)
// );
// _outTable = new BlueTable(_outSchema);
// }
// protected override void myOnFirstSubscriber()
// {
// _priceStream.onChange += _on_price_change;
// }
// protected override void myOnLastSubscriber()
// {
// _priceStream.onChange -= _on_price_change;
// }
// void _on_price_change(object newValue)
// {
// var dp = newValue as IDataPacket;
// if (dp is null)
// {
// return;
// }
// if (!_validated)
// {
// ValidateInputSchema(dp.newRows.schema);
// _validated = true;
// }
// var emitted = new List<IBlueRow>();
// foreach (IBlueRow row in dp.addedRows)
// {
// ProcessEvent(row, emitted);
// }
// if (emitted.Count == 0)
// {
// return;
// }
// foreach (var r in emitted)
// {
// _outTable.addRow(r);
// }
// var outDp = BlueDataUtility.build_dp_inc(_outTable, myLastValue as IDataPacket);
// myFireEvent(outDp);
// }
// void ValidateInputSchema(IBlueTableSchema schema)
// {
// _validCols.Clear();
// foreach (var name in schema.columnNames)
// {
// _validCols[name] = true;
// }
// if (!_validCols.ContainsKey(_timestampCol))
// {
// throw new ArgumentException($"Missing required column: {_timestampCol}");
// }
// switch (_priceSource)
// {
// case PriceSource.trade_price:
// RequireCol("trade_price");
// break;
// case PriceSource.mid:
// RequireCol("bid_price");
// RequireCol("ask_price");
// break;
// case PriceSource.bid:
// RequireCol("bid_price");
// break;
// case PriceSource.ask:
// RequireCol("ask_price");
// break;
// }
// if (_barType is BarType.VOLUME or BarType.DOLLAR)
// {
// if (_volumeSource == VolumeSource.none || !_validCols.ContainsKey("trade_size"))
// {
// throw new ArgumentException("VOLUME/DOLLAR require executed trade volume; invalid for BBO-only streams.");
// }
// }
// else if (_volumeSource == VolumeSource.trade_size)
// {
// RequireCol("trade_size");
// }
// }
// void RequireCol(string name)
// {
// if (!_validCols.ContainsKey(name))
// {
// throw new ArgumentException($"Missing required column for selected source: {name}");
// }
// }
// void ProcessEvent(IBlueRow row, List<IBlueRow> emitted)
// {
// var ts = ParseTimestamp(row);
// var contract = ResolveContract(row);
// var price = ResolvePrice(row);
// var size = ResolveSize(row);
// var state = GetOrCreateState(contract);
// if (state.HasData && _sessionReset != SessionReset.NONE && ts.Date != state.SessionDay.Date)
// {
// emitted.Add(ToBarRow(state, ts_event: state.EndTs, isForming: false));
// state.HasData = false;
// }
// switch (_barType)
// {
// case BarType.TIME:
// ProcessTimeBar(state, ts, price, size, emitted);
// break;
// case BarType.TICK:
// ProcessThresholdBar(state, ts, price, size, emitted, state.TradeCount + 1 >= _barSize);
// break;
// case BarType.VOLUME:
// ProcessThresholdBar(state, ts, price, size, emitted, state.Volume + size >= _barSize);
// break;
// case BarType.DOLLAR:
// ProcessThresholdBar(state, ts, price, size, emitted, state.DollarNotional + (price * size) >= _barSize);
// break;
// }
// }
// void ProcessTimeBar(BarState state, DateTime ts, double price, double size, List<IBlueRow> emitted)
// {
// var bucketStart = AlignTimeBucket(ts, _barSize);
// var bucketEnd = bucketStart.AddSeconds(_barSize);
// if (!state.HasData)
// {
// StartBar(state, ts, bucketStart, bucketEnd, price, size);
// MaybeEmitInitialSnapshot(state, ts, emitted);
// if (_emitForming)
// {
// emitted.Add(ToBarRow(state, ts, isForming: true));
// }
// return;
// }
// if (bucketStart < state.StartTs)
// {
// Log.Gen.error(
// $"BarBuilderStream ignored out-of-order event. contract={state.Contract} ts={ToIso(ts)} " +
// $"current_bar_start={ToIso(state.StartTs)}");
// return;
// }
// if (bucketStart > state.StartTs)
// {
// emitted.Add(ToBarRow(state, state.EndTs, isForming: false));
// StartBar(state, ts, bucketStart, bucketEnd, price, size);
// if (_emitForming)
// {
// emitted.Add(ToBarRow(state, ts, isForming: true));
// }
// return;
// }
// Accumulate(state, price, size);
// state.EndTs = bucketEnd;
// if (_emitForming)
// {
// emitted.Add(ToBarRow(state, ts, isForming: true));
// }
// }
// void ProcessThresholdBar(
// BarState state,
// DateTime ts,
// double price,
// double size,
// List<IBlueRow> emitted,
// bool shouldClose
// )
// {
// if (!state.HasData)
// {
// StartBar(state, ts, ts, ts, price, size);
// MaybeEmitInitialSnapshot(state, ts, emitted);
// }
// else
// {
// Accumulate(state, price, size);
// }
// state.EndTs = ts;
// if (_emitForming)
// {
// emitted.Add(ToBarRow(state, ts, isForming: true));
// }
// if (shouldClose)
// {
// emitted.Add(ToBarRow(state, ts, isForming: false));
// state.HasData = false;
// }
// }
// static DateTime AlignTimeBucket(DateTime ts, int seconds)
// {
// var utc = ts.ToUniversalTime();
// long bucketTicks = TimeSpan.FromSeconds(seconds).Ticks;
// long aligned = (utc.Ticks / bucketTicks) * bucketTicks;
// return new DateTime(aligned, DateTimeKind.Utc);
// }
// void MaybeEmitInitialSnapshot(BarState state, DateTime ts, List<IBlueRow> emitted)
// {
// if (!_emitInitialSnapshot || state.InitialSnapshotEmitted)
// {
// return;
// }
// emitted.Add(ToBarRow(state, ts, isForming: false));
// state.InitialSnapshotEmitted = true;
// }
// void StartBar(
// BarState state,
// DateTime ts,
// DateTime barStart,
// DateTime barEnd,
// double price,
// double size
// )
// {
// state.StartTs = barStart;
// state.EndTs = barEnd;
// state.Open = price;
// state.High = price;
// state.Low = price;
// state.Close = price;
// state.Volume = 0;
// state.TradeCount = 0;
// state.SumPriceSize = 0;
// state.SumSize = 0;
// state.DollarNotional = 0;
// state.SessionDay = ts.Date;
// state.HasData = true;
// Accumulate(state, price, size);
// }
// static void Accumulate(BarState state, double price, double size)
// {
// state.High = Math.Max(state.High, price);
// state.Low = Math.Min(state.Low, price);
// state.Close = price;
// state.TradeCount++;
// if (size > 0)
// {
// state.Volume += size;
// state.SumSize += size;
// state.SumPriceSize += price * size;
// state.DollarNotional += price * size;
// }
// }
// IBlueRow ToBarRow(BarState state, DateTime ts_event, bool isForming)
// {
// double vwap = state.SumSize > 0 ? state.SumPriceSize / state.SumSize : state.Close;
// var row = new MutableBlueRow(_outSchema);
// row.setValue("id", _nextId++);
// row.setValue("ts_event", ToIso(ts_event));
// row.setValue("bar_start_ts", ToIso(state.StartTs));
// row.setValue("bar_end_ts", ToIso(state.EndTs));
// row.setValue("contract", state.Contract);
// row.setValue("open", state.Open);
// row.setValue("high", state.High);
// row.setValue("low", state.Low);
// row.setValue("close", state.Close);
// row.setValue("volume", _volumeSource == VolumeSource.none ? 0.0 : state.Volume);
// row.setValue("trade_count", state.TradeCount);
// row.setValue("vwap", vwap);
// row.setValue("is_forming", isForming);
// return row;
// }
// BarState GetOrCreateState(string contract)
// {
// if (_stateByContract.TryGetValue(contract, out var existing))
// {
// return existing;
// }
// var created = new BarState { Contract = contract };
// _stateByContract[contract] = created;
// return created;
// }
// DateTime ParseTimestamp(IBlueRow row)
// {
// object? raw = row[_timestampCol];
// if (raw is DateTime dt)
// {
// return dt;
// }
// if (raw is string s && DateTime.TryParse(s, null, DateTimeStyles.RoundtripKind, out dt))
// {
// return dt;
// }
// throw new ArgumentException($"Invalid timestamp value in column '{_timestampCol}'.");
// }
// string ResolveContract(IBlueRow row)
// {
// if (_validCols.ContainsKey("contract"))
// {
// return (row["contract"]?.ToString() ?? string.Empty).Trim();
// }
// if (_validCols.ContainsKey("symbol"))
// {
// return (row["symbol"]?.ToString() ?? string.Empty).Trim();
// }
// return string.Empty;
// }
// double ResolvePrice(IBlueRow row)
// {
// return _priceSource switch
// {
// PriceSource.trade_price => ToDouble(row["trade_price"], "trade_price"),
// PriceSource.mid => (ToDouble(row["bid_price"], "bid_price") + ToDouble(row["ask_price"], "ask_price")) / 2.0,
// PriceSource.bid => ToDouble(row["bid_price"], "bid_price"),
// PriceSource.ask => ToDouble(row["ask_price"], "ask_price"),
// _ => throw new ArgumentOutOfRangeException()
// };
// }
// double ResolveSize(IBlueRow row)
// {
// if (_volumeSource == VolumeSource.none)
// {
// return 0;
// }
// return ToDouble(row["trade_size"], "trade_size");
// }
// static double ToDouble(object? value, string colName)
// {
// var d = BlueDataUtility.GetDouble(value!);
// if (!d.HasValue)
// {
// throw new ArgumentException($"Column '{colName}' must be numeric.");
// }
// return d.Value;
// }
// static string ToIso(DateTime dt) =>
// dt.ToUniversalTime().ToString("O");
// static BarType ParseBarType(string value)
// {
// if (Enum.TryParse<BarType>(value, true, out var parsed))
// {
// return parsed;
// }
// throw new ArgumentException("bar_type must be one of: TIME | TICK | VOLUME | DOLLAR");
// }
// static PriceSource ParsePriceSource(string value)
// {
// if (Enum.TryParse<PriceSource>(value, true, out var parsed))
// {
// return parsed;
// }
// throw new ArgumentException("price_source must be one of: trade_price | mid | bid | ask");
// }
// static VolumeSource ParseVolumeSource(string value)
// {
// if (Enum.TryParse<VolumeSource>(value, true, out var parsed))
// {
// return parsed;
// }
// throw new ArgumentException("volume_source must be one of: trade_size | none");
// }
// static SessionReset ParseSessionReset(string value)
// {
// if (!Enum.TryParse<SessionReset>(value, true, out var parsed))
// {
// throw new ArgumentException("session_reset must be one of: NONE | DAY | RTH | ETH");
// }
// // TODO: differentiate RTH/ETH session boundaries; currently treated as DAY.
// if (parsed is SessionReset.RTH or SessionReset.ETH)
// {
// return SessionReset.DAY;
// }
// return parsed;
// }
// }
@@ -1,41 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="MathNet.Numerics" Version="5.0.0" />
<PackageReference Include="OpenAI" Version="2.0.0" />
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
</ItemGroup>
<Choose>
<When Condition="'$(UseProjectRefs)' == 'true'">
<ItemGroup>
<ProjectReference Include="..\..\cs\Bts.Spark.Api\Bts.Spark.Api.csproj" />
<ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
<ProjectReference Include="..\DataStream\DataStream.csproj" />
<ProjectReference Include="..\PubSub\PubSub.csproj" />
</ItemGroup>
</When>
<Otherwise>
<ItemGroup>
<PackageReference Include="Bts.Spark.Api" Version="$(SparkVersion)" />
<PackageReference Include="DataStream" Version="$(SparkVersion)" />
<PackageReference Include="PubSub" Version="$(SparkVersion)" />
</ItemGroup>
</Otherwise>
</Choose>
<!-- <ItemGroup>
<RowType Include="RowTemplates\RefTypeRow.tt" SchemaJson="RowTemplates\PubSubHeartbeatSchema.json" TypeName="PubSubHeartbeatRow" TableName="PubSubHeartbeat" />
<RowType Include="RowTemplates\RefTypeRow.tt" SchemaJson="RowTemplates\PubSubRouteSchema.json" TypeName="PubSubRouteRow" TableName="PubSubRoute" />
<RowType Include="RowTemplates\RefTypeRow.tt" SchemaJson="RowTemplates\PubSubServerStateSchema.json" TypeName="PubSubServerStateRow" TableName="PubSubServerState" />
<RowType Include="RowTemplates\RefTypeRow.tt" SchemaJson="RowTemplates\PubSubServerTagStatsSchema.json" TypeName="PubSubServerTagStatsRow" TableName="PubSubServerTagStats" />
<RowType Include="RowTemplates\RefTypeRow.tt" SchemaJson="RowTemplates\TableMetadataSchema.json" TypeName="TableMetadataRow" TableName="TableMetadata" />
</ItemGroup> -->
</Project>
@@ -1,294 +0,0 @@
namespace BlueOrb.Internal
{
using bts;
//using bts.api;
//using bts.spark;
using System;
using System.Diagnostics.CodeAnalysis;
public static class ContractUtils
{
public record FutureTradableContract(
string Symbol,
DateTime Expiration,
Exchange Exchange,
string? ContractText);
public class FutureSpreadContract
{
public List<Tuple<double, FutureTradableContract>> legs = new(2);
public string Symbol
{
get
{
return legs == null || legs.Count == 0 ? string.Empty : legs.ElementAt(0).Item2.Symbol;
}
}
public string? ContractText;
}
public record FutureContinousContract(string Symbol, Exchange Exchange);
// this is for all future conytracts belonging to a symbol
public record FutureWildcardContract(string Symbol, Exchange Exchange);
public class FutureContract
{
public FutureWildcardContract? wildcard_contract;
public FutureTradableContract? future_tradable_contract;
public FutureSpreadContract? spread_tradable_contract;
public FutureContinousContract? continous_constract;
public FutureContract(string txt)
{
wildcard_contract = parse_wildcard_contract(txt);
if (wildcard_contract != null)
return;
var fc = parse_future_contract(txt);
if (fc != null)
{
future_tradable_contract = fc;
return;
}
var sc = parse_future_spread_contract(txt);
if (sc != null)
{
spread_tradable_contract = sc;
return;
}
// this is not totally true, since ParseFutureSpreadContract does not
// parse ICS spreads
continous_constract = new(txt, Exchange.CME);
}
[MemberNotNullWhen(true, nameof(continous_constract))]
public bool is_continous_contract => continous_constract != null;
[MemberNotNullWhen(true, nameof(wildcard_contract))]
public bool is_wildcard_contract => wildcard_contract != null;
[MemberNotNullWhen(true, nameof(future_tradable_contract))]
public bool is_future_tradable_contract => future_tradable_contract != null;
[MemberNotNullWhen(true, nameof(spread_tradable_contract))]
public bool is_spread_tradable_contract => spread_tradable_contract != null;
public string symbol()
{
if (wildcard_contract != null)
return wildcard_contract.Symbol;
if (continous_constract != null)
{
return continous_constract.Symbol;
}
else if (future_tradable_contract != null)
{
return future_tradable_contract.Symbol;
}
else if (spread_tradable_contract != null)
{
return spread_tradable_contract.Symbol;
}
Log.assert(false);
return string.Empty;
}
public string? contract_text
{
get
{
if (future_tradable_contract != null)
return future_tradable_contract.ContractText;
if (spread_tradable_contract != null)
return spread_tradable_contract.ContractText;
return null;
}
}
}
#region Parsing
public static FutureWildcardContract? parse_wildcard_contract(string contract_txt, Exchange exchange = Exchange.CME)
{
if (contract_txt.EndsWith("*"))
{
contract_txt = contract_txt.Replace("*", string.Empty);
return new FutureWildcardContract(contract_txt, exchange);
}
return null;
}
public static FutureTradableContract? parse_future_contract(string contract_txt, Exchange exchange = Exchange.CME)
{
// Validate the future string length
if (!contract_txt.All(c => char.IsLetterOrDigit(c)))
return null;
if (contract_txt.Length < 4)
{
return null;
}
// Extract the symbol and the expiration month/year codes
string symbol = contract_txt.Substring(0, contract_txt.Length - 2);
char monthCode = contract_txt[contract_txt.Length - 2];
char yearCode = contract_txt[contract_txt.Length - 1];
// Map month code to month, return null if the month code is invalid
int month = monthCode switch
{
'F' => 1, // January
'G' => 2, // February
'H' => 3, // March
'J' => 4, // April
'K' => 5, // May
'M' => 6, // June
'N' => 7, // July
'Q' => 8, // August
'U' => 9, // September
'V' => 10, // October
'X' => 11, // November
'Z' => 12, // December
_ => 0 // Return 0 for invalid month code
};
if (month == 0)
{
return null; // Invalid month code
}
// Calculate the year based on the current decade
int currentYear = DateTime.Now.Year;
int decadeStart = currentYear / 10 * 10;
int year = decadeStart + (yearCode - '0');
// If the year is in the past, assume the next decade
if (year < currentYear)
{
year += 10;
}
// Create the FutureContract instance and set its properties
DateTime expiration = new DateTime(year, month, 1);
return new(symbol, expiration, exchange, contract_txt);
}
// Method to parse a futures spread, e.g. "CLX4-CLZ4"
public static FutureSpreadContract? parse_future_spread_contract(string contract_txt, Exchange exchange = Exchange.CME)
{
// Split the string by '-' to handle each leg
var spreadParts = contract_txt.Split('-');
if (spreadParts.Length < 2)
{
return null; // Not a valid spread format
}
var spreadContract = new FutureSpreadContract { legs = new List<Tuple<double, FutureTradableContract>>() };
foreach (var part in spreadParts)
{
// Split each leg by '*', assuming the format is weight*future or just future
string[] legParts = part.Split('*');
double weight = 1.0;
string futurePart;
if (legParts.Length == 2)
{
if (!double.TryParse(legParts[0], out weight))
{
return null; // Invalid weight, parsing failed
}
futurePart = legParts[1];
}
else if (legParts.Length == 1)
{
futurePart = legParts[0];
}
else
{
return null; // Invalid format, too many '*' parts
}
// Parse the future contract
var futureContract = parse_future_contract(futurePart, exchange);
if (futureContract == null)
{
return null; // Failed to parse the future part
}
// Add the leg to the spread contract
spreadContract.legs.Add(new Tuple<double, FutureTradableContract>(weight, futureContract));
}
return spreadContract;
}
// public static ContractType DetermineFutureOrSpread(string contract) {
// if (contract.Contains("-")) {
// return ContractType.FutureSpread;
// } else {
// return ContractType.Future;
// }
// }
#endregion
public static ContractType get_type(string contract_txt)
{
var fc = parse_future_contract(contract_txt);
if (fc != null)
{
return ContractType.Future;
}
var sc = parse_future_spread_contract(contract_txt);
if (sc != null)
{
return ContractType.FutureSpread;
}
// tehre are some non calendar spreads we cant parse
//Log.assert(false);
return ContractType.FutureSpread;
}
public static ContractCategory get_category(string symbol)
{
var tp = get_symbol_category(symbol);
return tp.Item1;
}
public static Tuple<ContractCategory, string> get_symbol_category(string symbol)
{
var fc = parse_future_contract(symbol);
if (fc != null)
{
return Tuple.Create(ContractCategory.FutureString, fc.Symbol);
}
var sc = parse_future_spread_contract(symbol);
if (sc != null)
{
return Tuple.Create(ContractCategory.FutureString, sc.Symbol);
}
return Tuple.Create(ContractCategory.Symbol, symbol);
}
public enum ContractType
{
Spot,
Option,
Future,
FutureSpread,
OptionSpread
}
public enum ContractCategory
{
Symbol = 0,
FutureString,
SpreadString
}
public enum CallPut
{
Call,
Put
}
public enum Exchange
{
CME
}
}
}
@@ -1,99 +0,0 @@
namespace BlueOrb.Operators.Data {
using BCG;
using bts;
using Spark.Core.Diagnostics;
using BlueOrb.Internal;
using System;
using System.Collections.Generic;
using System.Linq;
[DataStreamUserRespresentation("ReplayFutureLiveBBOStream")]
public class ReplayFutureLiveBBOStream : ViewStream, BCG.Utilities.General.IWorker {
private class LastFireTime {
public DateTime last_fireitme_file = DateTime.MinValue;
public DateTime last_firetime_clock = DateTime.MinValue;
}
private class RowsWithPointer {
public List<IBlueRow>? rows;
public int pointer;
}
readonly ViewStreamDatabase? _vsd;
readonly ContractUtils.FutureContinousContract? _fcc;
private readonly LastFireTime _last_fire_time = new LastFireTime();
private RowsWithPointer _rows = new RowsWithPointer();
public ReplayFutureLiveBBOStream(string continous_contract) {
Func<string> iso_8601_now = () => {
DateTime utc_now = DateTime.UtcNow;
long ticks = utc_now.Ticks; // 1 tick = 100 ns
long nanoseconds = ticks % TimeSpan.TicksPerSecond * 100;
string formatted = utc_now.ToString("yyyy-MM-ddTHH:mm:ss");
formatted += $".{nanoseconds:D9}Z";
return formatted;
};
ViewStream rs = new ReplayFutureLiveBBOStream(continous_contract, true);
IBlueTableSchema schema_out = BlueDataUtility.form_schema("BBO",
Tuple.Create("id", TypeCode.Int32),
Tuple.Create("ts_event", TypeCode.String),
Tuple.Create("contract", TypeCode.String),
Tuple.Create("bid_price", TypeCode.Double),
Tuple.Create("bid_size", TypeCode.Int32),
Tuple.Create("ask_price", TypeCode.Double),
Tuple.Create("ask_size", TypeCode.Int32));
rs = new MapStream(rs, s => schema_out,
r =>
new BlueRow(schema_out, r["id"], /*r.get<string>("ts_event")*/iso_8601_now(), r.get<string>("symbol"),
r.get<double>("bid_px_00"), r.get<int>("bid_sz_00"),
r.get<double>("ask_px_00"), r.get<int>("ask_sz_00")));
mySetRealSource(rs);
}
private ReplayFutureLiveBBOStream(string continous_contract, bool noop)
: this(ViewStreamDatabase.instance, new ContractUtils.FutureContract(continous_contract)) { }
public ReplayFutureLiveBBOStream(ViewStreamDatabase vsd, ContractUtils.FutureContract fc) {
var fcc = fc.continous_constract;
Log.assert(fcc != null);
_vsd = vsd;
_fcc = fcc;
}
protected override void myOnFirstSubscriber() {
Log.notnull(_fcc);
var file_path = Path.Combine(SQLiteHelper.location, $"mbp1_{_fcc.Symbol}_small.csv".ToLower());
var rows = BCG.CSVReaderStream.LoadFile(file_path);
_rows.rows = new(rows.Count);
_rows.rows.AddRange(rows.OfType<IBlueRow>().OrderBy(r => r.get<string>("ts_event")));
_rows.pointer = 0;
ExceptionAssert.NotNull(_vsd).addWorker(this);
}
protected override void myOnLastSubscriber() {
ExceptionAssert.NotNull(_vsd).removeWorker(this);
}
public void doWork() {
var rows = ExceptionAssert.NotNull(_rows.rows);
IBlueRow cand_row = rows[_rows.pointer];
string cand_ts_event = cand_row.get<string>("ts_event");
DateTime cand_ts_event_dt = DateTime.Parse(cand_ts_event, null, System.Globalization.DateTimeStyles.RoundtripKind);
TimeSpan cand_ts_span_file = cand_ts_event_dt - _last_fire_time.last_fireitme_file;
TimeSpan ts_clock = DateTime.Now - _last_fire_time.last_firetime_clock;
if (ts_clock >= cand_ts_span_file) {
_rows.pointer++;
_last_fire_time.last_firetime_clock = DateTime.Now;
_last_fire_time.last_fireitme_file = cand_ts_event_dt;
BlueTable bt_out = new BlueTable(cand_row.schema);
bt_out.addRow(cand_row);
var dp_out = BlueDataUtility.build_dp(bt_out, myLastValue as IDataPacket);
myFireEvent(dp_out);
}
}
}
}
@@ -1,111 +0,0 @@
namespace Blueorb.Operators.Data
{
using BCG;
using Spark.Core.Diagnostics;
using BlueOrb.Internal;
using bts;
using BlueOrb.Operators.Core;
using System;
using System.Collections.Generic;
using System.Linq;
[DataStreamUserRespresentation("ReplayFutureLiveTradeStream")]
public class ReplayFutureLiveTradeStream : ViewStream, BCG.Utilities.General.IWorker
{
private class LastFireTime
{
public DateTime last_fireitme_file = DateTime.MinValue;
public DateTime last_firetime_clock = DateTime.MinValue;
}
private class RowsWithPointer
{
public List<IBlueRow>? rows;
public int pointer;
}
readonly ViewStreamDatabase? _vsd;
readonly ContractUtils.FutureContinousContract? _fcc;
private readonly LastFireTime _last_fire_time = new LastFireTime();
readonly RowsWithPointer _rows = new();
public ReplayFutureLiveTradeStream(string continous_contract)
{
Func<string> iso_8601_now = () =>
{
DateTime utc_now = DateTime.UtcNow;
long ticks = utc_now.Ticks; // 1 tick = 100 ns
long nanoseconds = ticks % TimeSpan.TicksPerSecond * 100;
string formatted = utc_now.ToString("yyyy-MM-ddTHH:mm:ss");
formatted += $".{nanoseconds:D9}Z";
return formatted;
};
ViewStream rs = new ReplayFutureLiveTradeStream(continous_contract, true);
IBlueTableSchema schema_out = BlueDataUtility.form_schema("BBO",
Tuple.Create("id", TypeCode.Int32),
Tuple.Create("ts_event", TypeCode.String),
Tuple.Create("contract", TypeCode.String),
Tuple.Create("trade_size", TypeCode.Double),
Tuple.Create("trade_price", TypeCode.Int32),
Tuple.Create("aggressor_side", TypeCode.String));
rs = new MapStream(rs, s => schema_out,
r =>
new BlueRow(schema_out, r["id"], /*r.get<string>("ts_event")*/iso_8601_now(), r.get<string>("symbol"),
r.get<double>("size"), r.get<double>("price"),
r.get<string>("side").StartsWith("B") ? "BUY" : (r.get<string>("side").StartsWith("A") ? "SELL" : "NONE")));
rs = new TimeSeriesStream(rs);
mySetRealSource(rs);
}
private ReplayFutureLiveTradeStream(string continous_contract, bool noop)
: this(ViewStreamDatabase.instance, new ContractUtils.FutureContract(continous_contract)) { }
public ReplayFutureLiveTradeStream(ViewStreamDatabase vsd, ContractUtils.FutureContract fc)
{
var fcc = fc.continous_constract;
Log.assert(fcc != null);
_vsd = vsd;
_fcc = fcc;
}
protected override void myOnFirstSubscriber()
{
Log.notnull(_vsd);
Log.notnull(_fcc);
var file_path = Path.Combine(SQLiteHelper.location, $"trades_{_fcc.Symbol}.csv");
var csv_rd = new BlueOrb.Operators.Core.CSVReaderStream(file_path, false);
var rows = BCG.CSVReaderStream.LoadFile(file_path);
_rows.rows = new(rows.Count);
_rows.rows.AddRange(rows.OfType<IBlueRow>().OrderBy(r => r.get<string>("ts_event")));
_rows.pointer = 0;
ExceptionAssert.NotNull(_vsd).addWorker(this);
}
protected override void myOnLastSubscriber()
{
ExceptionAssert.NotNull(_vsd).removeWorker(this);
}
public void doWork()
{
var rows = ExceptionAssert.NotNull(_rows.rows);
IBlueRow cand_row = rows[_rows.pointer];
string cand_ts_event = cand_row.get<string>("ts_event");
DateTime cand_ts_event_dt = DateTime.Parse(cand_ts_event, null, System.Globalization.DateTimeStyles.RoundtripKind);
TimeSpan cand_ts_span_file = cand_ts_event_dt - _last_fire_time.last_fireitme_file;
TimeSpan ts_clock = DateTime.Now - _last_fire_time.last_firetime_clock;
if (ts_clock >= cand_ts_span_file)
{
_rows.pointer++;
_last_fire_time.last_firetime_clock = DateTime.Now;
_last_fire_time.last_fireitme_file = cand_ts_event_dt;
BlueTable bt_out = new BlueTable(cand_row.schema);
bt_out.addRow(cand_row);
var dp_out = BlueDataUtility.build_dp(bt_out, myLastValue as IDataPacket);
myFireEvent(dp_out);
}
}
}
}
@@ -1,144 +0,0 @@
using System;
using System.Collections.Generic;
using System.Data;
using System.Data.SQLite;
using System.IO;
using System.Runtime.InteropServices;
namespace BlueOrb.Internal
{
public static class SQLiteHelper
{
public static string location = RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
? "C:\\BlueOrb_Data\\"
: Path.Combine(Environment.GetEnvironmentVariable("HOME") ?? string.Empty, "BlueOrb_Data");
public static string db_name = "stream_data.db";
public static DataTable execute_select_query(string connection_string, string query)
{
// Create a new DataTable to store the results
DataTable dataTable = new DataTable();
// Create and open a SQLite connection using the provided connection string
using (SQLiteConnection connection = new SQLiteConnection(connection_string))
{
try
{
connection.Open();
// Create a SQLite command with the provided query
using (SQLiteCommand command = new SQLiteCommand(query, connection))
{
// Use SQLiteDataAdapter to fill the DataTable
using (SQLiteDataAdapter adapter = new SQLiteDataAdapter(command))
{
adapter.Fill(dataTable);
}
}
}
catch (Exception ex)
{
Console.WriteLine("An error occurred: " + ex.Message);
}
}
// Return the populated DataTable
return dataTable;
}
public static List<Dictionary<string, object>> ExecuteBulkRead(string connectionString, string query)
{
var result = new List<Dictionary<string, object>>();
using (var connection = new SQLiteConnection(connectionString))
{
connection.Open();
// Begin a transaction to optimize read speed
using (var transaction = connection.BeginTransaction())
using (var command = new SQLiteCommand(query, connection, transaction))
using (var reader = command.ExecuteReader())
{
while (reader.Read())
{
var row = new Dictionary<string, object>();
for (int i = 0; i < reader.FieldCount; i++)
{
string columnName = reader.GetName(i);
object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
row[columnName] = columnValue;
}
result.Add(row);
}
// Commit the transaction
transaction.Commit();
}
}
return result;
}
public static List<Dictionary<string, object>> get_table(string connectionString, string query)
{
var result = new List<Dictionary<string, object>>();
using (var connection = new SQLiteConnection(connectionString))
{
try
{
connection.Open();
using (var command = new SQLiteCommand(query, connection))
using (var reader = command.ExecuteReader())
{
while (reader.Read())
{
var row = new Dictionary<string, object>();
for (int i = 0; i < reader.FieldCount; i++)
{
string columnName = reader.GetName(i);
object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
row[columnName] = columnValue;
}
result.Add(row);
}
}
}
catch (Exception ex)
{
Console.WriteLine("An error occurred: " + ex.Message);
}
}
return result;
}
public static IEnumerable<Dictionary<string, object>> get_table_2(string connectionString, string query)
{
using (var connection = new SQLiteConnection(connectionString))
{
connection.Open();
using (var command = new SQLiteCommand(query, connection))
using (var reader = command.ExecuteReader())
{
while (reader.Read())
{
var row = new Dictionary<string, object>();
for (int i = 0; i < reader.FieldCount; i++)
{
string columnName = reader.GetName(i);
object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
row[columnName] = columnValue;
}
yield return row;
}
}
}
}
}
}
@@ -1,43 +0,0 @@
namespace BlueOrb.Internal
{
using bts;
using System;
using BCG;
using System.Collections.Generic;
using Bts.DataStreamExpressions;
using System.Xml;
// All this does is take in a dsl expression, parse it and fire the data that come out of running it.
// So for example, if an expresison is passed
[DataStreamUserRespresentation("dsl")]
public class DSLStream : ViewStream
{
private readonly string _dsl;
public DSLStream(string dsl)
{
Log.Gen.info($"DSLStream.cons dsl: {dsl}");
// Remove all colons
string sanitizedDsl = dsl.Replace(":", "COLON");
_dsl = dsl;
ViewStream rs = build_stream(dsl) ?? new LiteralStream("error", $"msg:invalid dsl");
mySetRealSource(rs);
}
static ViewStream? build_stream(string stream_expression)
{
try
{
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
return rv;
}
catch (Exception ex)
{
//send to blueorb
Log.Gen.error($"Could not build stream: {stream_expression} ex: {ex}");
return null;
}
}
}
}
@@ -1,103 +0,0 @@
using System;
using BCG.Utilities.General;
using BCG;
using bts;
namespace BlueOrb.Internal
{
[DataStreamUserRespresentation("MillisecondStream")]
public class MillisecondStream : ViewStream, IWorker
{
private DateTime _last_fire_time = DateTime.MinValue;
private readonly TimeSpan _period;
private readonly int _size;
private readonly ViewStreamDatabase _vsd;
private IBlueTableSchema _schema;
private bts.CircularBufferTable _table;
public MillisecondStream(int interval_msecs, int size)
: this(ViewStreamDatabase.instance, interval_msecs, size)
{
}
public MillisecondStream(ViewStreamDatabase vsd, int interval_msecs, int size)
{
_vsd = vsd;
_period = TimeSpan.FromMilliseconds(interval_msecs);
_size = size;
_schema = BlueDataUtility.form_schema("Milliseconds",
Tuple.Create("id", TypeCode.Int32),
Tuple.Create("time", TypeCode.DateTime),
Tuple.Create("display_time", TypeCode.String));
_table = new CircularBufferTable(_schema, _size);
}
protected override void myOnFirstSubscriber()
{
_vsd.addWorker(this);
}
protected override void myOnLastSubscriber()
{
_vsd.removeWorker(this);
}
private int _id = 0;
public void doWork()
{
if (DateTime.Now - _last_fire_time >= _period)
{
_last_fire_time = DateTime.Now;
IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
_last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
_table.add_row(nr);
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
myFireEvent(dp);
}
}
}
internal class MSecondStream : TimeUnitStream
{
public MSecondStream(int intervalInms, int size)
: base(DateTime.Now, new TimeSpan(0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) { }
protected override DateTime myRoundTime(DateTime dt) { return dt; }
}
internal class DisplayableMillisecondStream : ViewStream, IStreamMapper
{
private IBlueTableSchema mySchema = new BlueTableSchema("ms",
new ColumnInfo("id", 0, typeof(int)),
new ColumnInfo("time", 1, typeof(string)));
public DisplayableMillisecondStream(int intervalInMs, int sz)
{
mySetRealSource(new MapStream(new MSecondStream(intervalInMs, sz), this));
}
#region IStreamMapper Members
public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
{
return mySchema;
}
#endregion
#region IDynamicStreamMapper Members
public IBlueRow map(IBlueRow row)
{
DateTime time = row.get_datetime("time");
var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
return new BlueRow(mySchema, row["id"], time_txt);
}
#endregion
private string myToString(DateTime dt)
{
string sep = ":";
return dt.Hour + sep + dt.Minute + sep + dt.Second + sep + dt.Millisecond;
}
}
}
@@ -1,348 +0,0 @@
namespace BlueOrb.Operators.Signal
{
using System;
using System.Collections.Concurrent;
using System.IO.MemoryMappedFiles;
using BCG;
using System.Text.Json.Serialization;
using System.Text.Json;
using System.Data;
public enum BarType
{
TIME,
TICK,
VOLUME,
DOLLAR,
}
public class BarBuilderSchemaInfo
{
[JsonPropertyName("price_col")]
public string price_col { get; set; } = "trade_price";
[JsonPropertyName("volume_col")]
public string volume_col { get; set; } = "trade_size";
[JsonPropertyName("timestamp_col")]
public string timestamp_col { get; set; } = "ts_event";
}
public class SingleBar : IBlueRow
{
private BarBuilderStream _parent_str;
private IBlueTableSchema _schema;
private MutableBlueRow _row;
public SingleBar(BarBuilderStream parent_str)
{
_parent_str = parent_str;
_schema = BlueDataUtility.form_schema(
"bar_builder_single_bar",
Tuple.Create("id", TypeCode.Int32),
Tuple.Create("ts_event", TypeCode.DateTime),
Tuple.Create("bar_start_ts", TypeCode.DateTime),
Tuple.Create("bar_end_ts", TypeCode.DateTime),
Tuple.Create("open", TypeCode.Double),
Tuple.Create("high", TypeCode.Double),
Tuple.Create("low", TypeCode.Double),
Tuple.Create("close", TypeCode.Double),
Tuple.Create("volume", TypeCode.Double),
Tuple.Create("notional", TypeCode.Double),
Tuple.Create("trade_count", TypeCode.Int32),
Tuple.Create("n_events", TypeCode.Int32),
Tuple.Create("vwap", TypeCode.Double),
Tuple.Create("is_active", TypeCode.Boolean)
);
_row = new MutableBlueRow(_schema);
}
public IBlueTableSchema schema => _row.schema;
public object this[int index] => _row[index];
public object this[string col_name] => _row[col_name];
public IBlueRow add_event(IBlueRow r)
{
// initiaizes _row
void reset_new_bar()
{
_row = new MutableBlueRow(_schema);
_row.setValue("id", _parent_str.get_new_id());
_row.setValue("volume", 0.0);
_row.setValue("notional", 0.0);
_row.setValue("trade_count", 0);
_row.setValue("n_events", 0);
_row.setValue("is_active", true);
}
void absorb_into_row(IBlueRow input_row)
{
var ts = input_row.get_datetime(_parent_str._schema_info.timestamp_col, DateTime.UtcNow).ToUniversalTime();
var price = input_row.get<double>(_parent_str._schema_info.price_col, 0.0);
var volume = input_row.get<double>(_parent_str._schema_info.volume_col, 0.0);
var notional = price * volume;
if (_row.is_null("bar_start_ts"))
{
DateTime bar_start_ts;
DateTime bar_end_ts;
if (_parent_str._bar_type == BarType.TIME)
{
var bucket_ticks = (long)(Math.Max(1.0, _parent_str._bar_size) * TimeSpan.TicksPerSecond);
var start_ticks = (ts.Ticks / bucket_ticks) * bucket_ticks;
bar_start_ts = new DateTime(start_ticks, DateTimeKind.Utc);
bar_end_ts = bar_start_ts.AddTicks(bucket_ticks);
}
else
{
bar_start_ts = ts;
bar_end_ts = ts;
}
_row.setValue("bar_start_ts", bar_start_ts);
_row.setValue("bar_end_ts", bar_end_ts);
_row.setValue("open", price);
_row.setValue("high", price);
_row.setValue("low", price);
}
else
{
var high = _row.get<double>("high", price);
var low = _row.get<double>("low", price);
_row.setValue("high", Math.Max(high, price));
_row.setValue("low", Math.Min(low, price));
}
_row.setValue("ts_event", ts);
_row.setValue("close", price);
_row.setValue("volume", _row.get<double>("volume", 0.0) + volume);
_row.setValue("notional", _row.get<double>("notional", 0.0) + notional);
_row.setValue("trade_count", _row.get<int>("trade_count", 0) + 1);
_row.setValue("n_events", _row.get<int>("n_events", 0) + 1);
var running_volume = _row.get<double>("volume", 0.0);
var running_notional = _row.get<double>("notional", 0.0);
_row.setValue("vwap", running_volume > 0.0 ? running_notional / running_volume : price);
}
bool is_closed()
{
bool closed;
if (_parent_str._bar_type == BarType.TIME)
{
var ts = _row.get_datetime("ts_event", DateTime.MinValue);
var bar_end_ts = _row.get_datetime("bar_end_ts", DateTime.MaxValue);
closed = ts >= bar_end_ts;
}
else if (_parent_str._bar_type == BarType.TICK)
{
closed = _row.get<int>("n_events", 0) >= _parent_str._bar_size;
}
else if (_parent_str._bar_type == BarType.VOLUME)
{
closed = _row.get<double>("volume", 0.0) >= _parent_str._bar_size;
}
else
{
closed = _row.get<double>("notional", 0.0) >= _parent_str._bar_size;
}
_row.setValue("is_active", !closed);
return closed;
}
if (_row == null)
{
reset_new_bar();
}
absorb_into_row(r);
if (is_closed())
{
IBlueRow closed_row = new BlueRow(_row);
reset_new_bar();
return closed_row;
}
return null;
}
}
public class BarState : IBlueRowCollection
{
private SingleBar _curr_row;
private BlueTable _state;
public BarState(BarBuilderStream parent_str)
{
_curr_row = new SingleBar(parent_str);
_state = new BlueTable(_curr_row.schema);
}
public bool add_event(IBlueRow r)
{
var closed_bar = _curr_row.add_event(r);
if (closed_bar is not null)
{
// curr bar is closed, only then its added
// when bars are evolving we dont add it
_state.addRow(closed_bar);
return true;
}
return false;
}
public IBlueRow this[int id] => _state[id];
public IBlueTableSchema schema => _state.schema;
public bool containsID(int id) => _state.containsID(id);
public int Count => _state.Count;
public System.Collections.IEnumerator GetEnumerator() => _state.GetEnumerator();
}
[DataStreamUserRespresentation("BarBuilderStream")]
public class BarBuilderStream : ViewStream
{
readonly ViewStream _price_stream;
internal readonly BarType _bar_type;
internal readonly double _bar_size;
internal readonly bool _show_open_bars;
internal readonly BarBuilderSchemaInfo _schema_info;
private BarState _state;
public BarBuilderStream(ViewStream price_stream,
BarType bar_type,
double bar_size,
bool show_open_bars,
BarBuilderSchemaInfo schema_info)
{
_price_stream = price_stream;
_bar_type = bar_type;
_bar_size = bar_size;
_show_open_bars = show_open_bars;
_schema_info = schema_info;
_state = new BarState(this);
}
int _id = 0;
internal int get_new_id()
{
return _id++;
}
public BarBuilderStream(ViewStream price_stream,
string bar_type,
double bar_size,
string schema_info_json)
:
this(price_stream,
Enum.Parse<BarType>(bar_type),
bar_size,
true,
_parse_and_validate_schema_info_json(schema_info_json))
{
}
public BarBuilderStream(ViewStream price_stream,
string bar_type,
double bar_size,
string price_col,
string volume_col,
string timestamp_col)
:
this(price_stream,
Enum.Parse<BarType>(bar_type),
bar_size,
true,
_create_and_validate_schema_info(price_col, volume_col, timestamp_col))
{
}
protected override void myOnFirstSubscriber()
{
_price_stream.onChange += _on_price_stream_change;
}
protected override void myOnLastSubscriber()
{
_price_stream.onChange -= _on_price_stream_change;
}
void _on_price_stream_change(object dp_obj)
{
DataPacket dp = dp_obj as DataPacket;
var sz_0 = _state.Count;
foreach (IBlueRow r in dp.addedRows)
{
_state.add_event(r);
}
if (_state.Count != sz_0)
// we will check for _show_open_bars here
{
var dp_out = BlueDataUtility.build_dp_inc(_state, myLastValue as DataPacket, false);
myFireEvent(dp_out);
}
}
static BarBuilderSchemaInfo _parse_and_validate_schema_info_json(string schema_info_json)
{
if (string.IsNullOrWhiteSpace(schema_info_json))
{
throw new ArgumentException("schema_info_json cannot be null or empty.", nameof(schema_info_json));
}
var schema_info = JsonSerializer.Deserialize<BarBuilderSchemaInfo>(schema_info_json);
_validate_schema_info(schema_info);
return schema_info!;
}
static BarBuilderSchemaInfo _create_and_validate_schema_info(
string price_col,
string volume_col,
string timestamp_col)
{
var schema_info = new BarBuilderSchemaInfo
{
price_col = price_col,
volume_col = volume_col,
timestamp_col = timestamp_col,
};
_validate_schema_info(schema_info);
return schema_info;
}
static void _validate_schema_info(BarBuilderSchemaInfo? schema_info)
{
if (schema_info is null)
{
throw new ArgumentException("schema_info_json did not deserialize into BarBuilderSchemaInfo.");
}
if (string.IsNullOrWhiteSpace(schema_info.price_col))
{
throw new ArgumentException("BarBuilderSchemaInfo.PriceCol cannot be null or empty.");
}
if (string.IsNullOrWhiteSpace(schema_info.volume_col))
{
throw new ArgumentException("BarBuilderSchemaInfo.VolumeCol cannot be null or empty.");
}
if (string.IsNullOrWhiteSpace(schema_info.timestamp_col))
{
throw new ArgumentException("BarBuilderSchemaInfo.TimestampCol cannot be null or empty.");
}
if (schema_info.price_col.Equals(schema_info.volume_col, StringComparison.OrdinalIgnoreCase) ||
schema_info.price_col.Equals(schema_info.timestamp_col, StringComparison.OrdinalIgnoreCase) ||
schema_info.volume_col.Equals(schema_info.timestamp_col, StringComparison.OrdinalIgnoreCase))
{
throw new ArgumentException("BarBuilderSchemaInfo columns must be distinct.");
}
}
}
}
@@ -15,18 +15,21 @@ static class ArgumentComparer {
}
var obj1 = value1.GetValue(env);
var obj2 = value2.GetValue(env);
if (obj1 is null) {
if (obj2 is null) {
if (isNullLike(obj1)) {
if (isNullLike(obj2)) {
return 0;
}
return -1;
}
if (obj2 is null) {
if (isNullLike(obj2)) {
return 1;
}
return compareObjs(obj1, obj2);
return compareObjs(obj1!, obj2!);
}
static bool isNullLike(object? value) =>
value is null || DBNull.Value.Equals(value);
static int compareObjs(object obj1, object obj2) {
try {
var typ1 = obj1.GetType();
@@ -1,58 +0,0 @@
namespace BlueOrb.Internal;
using BCG;
using System;
using System.Linq;
[DataStreamUserRespresentation("TestStream")]
public class TestStream : ViewStream {
public TestStream() {
var rs = new BlueOrb.Operators.Core.LiteralStream("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc");
mySetRealSource(rs);
}
}
[DataStreamUserRespresentation("OpsStream")]
public class OpsStream : ViewStream {
public OpsStream() {
}
protected override void myOnFirstSubscriber() {
IBlueTableSchema schema = new BlueTableSchema("Ops");
schema = schema.add_column(System.TypeCode.Int32, "id");
schema = schema.add_column(System.TypeCode.String, "op");
schema = schema.add_column(System.TypeCode.String, "namespace");
schema = schema.add_column(System.TypeCode.String, "assembly_name");
schema = schema.add_column(System.TypeCode.String, "constructor");
var loaded_streams = PrimitiveManager.instance.get_loaded_streams();
BlueTable rv = new BlueTable(schema);
foreach (var kv in loaded_streams.OrderBy(kv => kv.Key, StringComparer.OrdinalIgnoreCase)) {
string sn = kv.Key;
Type t = kv.Value;
string ns = t.Namespace ?? string.Empty;
string assembly_name = t.Assembly.GetName().Name ?? string.Empty;
var constructors = t.GetConstructors()
.OrderBy(c => c.GetParameters().Length)
.ThenBy(c => c.ToString());
if (!constructors.Any()) {
IBlueRow r = new BlueRow(schema, rv.Count, sn, ns, assembly_name, "(no public constructor)");
rv.addRow(r);
continue;
}
foreach (var ctor in constructors) {
var parameters = ctor.GetParameters();
string constructor = $"{t.Name}({string.Join(", ", parameters.Select(p => p.ParameterType.Name))})";
IBlueRow r = new BlueRow(schema, rv.Count, sn, ns, assembly_name, constructor);
rv.addRow(r);
}
}
myFireEvent(new DataPacket(rv, rv, new BlueTable(schema)));
}
protected override void myOnLastSubscriber() {
base.myOnLastSubscriber();
}
}
@@ -15,7 +15,7 @@ namespace bts.BlueOrb {
private int _size = 0;
public TimeSeriesStream(ViewStream stream)
: this(stream, Int32.MaxValue) { }
: this(stream, 1000) { }
public TimeSeriesStream(ViewStream stream, int size) {
_stream = stream;
@@ -44,7 +44,7 @@ namespace bts.BlueOrb {
ar.setValue("id", _id++);
_table.add_row(ar);
}
var dp_out = BlueDataUtility.build_dp(_table, myLastValue as DataPacket);
var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
myFireEvent(dp_out);
}
}
@@ -117,6 +117,9 @@ namespace BCG {
addWorker(op.w);
else
removeWorker(op.w);
}
catch (InvalidOperationException ex) {
bts.Log.Gen.error($"ViewStreamDatabase worker collection was modified during enumeration: {ex}");
} finally {
_pending_worker_ops.Clear();
_is_worker_call_in_progress = false;
@@ -1,218 +0,0 @@
// namespace Bts.DataStreamTests;
// using BCG;
// using BlueOrb.Operators.Core;
// [TestFixture]
// public class BarBuilderStreamTests {
// static readonly DateTime T0 = new(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
// [Test]
// public void TimeBars_FromTrades_AcrossBoundary() {
// var schema = TradeSchema();
// var driver = new TestDriverStream(schema);
// var bars = new BarBuilderStream(driver, "TIME", 60, "trade_price", "trade_size");
// var rows = CaptureAddedRows(bars);
// var sent = new List<IBlueRow>();
// sent.Add(TradeRow(schema, 1, T0.AddSeconds(10), "BTCUSD", 100, 2));
// driver.Reset(sent);
// sent.Add(TradeRow(schema, 2, T0.AddSeconds(20), "BTCUSD", 101, 3));
// driver.Reset(sent);
// sent.Add(TradeRow(schema, 3, T0.AddMinutes(1).AddSeconds(5), "BTCUSD", 99, 1));
// driver.Reset(sent);
// sent.Add(TradeRow(schema, 4, T0.AddMinutes(2), "BTCUSD", 102, 1));
// driver.Reset(sent);
// Assert.That(rows.Count, Is.EqualTo(2));
// AssertBar(rows[0], 100, 101, 100, 101, 5, 2, 100.6, false);
// AssertBar(rows[1], 99, 99, 99, 99, 1, 1, 99, false);
// }
// [Test]
// public void TimeBars_FromBboMid_VolumeIsZero() {
// var schema = BboSchema();
// var driver = new TestDriverStream(schema);
// var bars = new BarBuilderStream(driver, "TIME", 60, "mid", "none");
// var rows = CaptureAddedRows(bars);
// var sent = new List<IBlueRow>();
// sent.Add(BboRow(schema, 1, T0.AddSeconds(0), "ETHUSD", 99, 101));
// driver.Reset(sent);
// sent.Add(BboRow(schema, 2, T0.AddSeconds(30), "ETHUSD", 101, 103));
// driver.Reset(sent);
// sent.Add(BboRow(schema, 3, T0.AddMinutes(1).AddSeconds(1), "ETHUSD", 100, 104));
// driver.Reset(sent);
// sent.Add(BboRow(schema, 4, T0.AddMinutes(2), "ETHUSD", 100, 102));
// driver.Reset(sent);
// Assert.That(rows.Count, Is.EqualTo(2));
// AssertBar(rows[0], 100, 102, 100, 102, 0, 2, 102, false);
// AssertBar(rows[1], 102, 102, 102, 102, 0, 1, 102, false);
// }
// [Test]
// public void TickBars_CloseExactlyOnNEvents() {
// var schema = TradeSchema();
// var driver = new TestDriverStream(schema);
// var bars = new BarBuilderStream(driver, "TICK", 3, "trade_price", "trade_size");
// var rows = CaptureAddedRows(bars);
// var sent = new List<IBlueRow>();
// sent.Add(TradeRow(schema, 1, T0, "NQ", 10, 1));
// driver.Reset(sent);
// sent.Add(TradeRow(schema, 2, T0.AddSeconds(1), "NQ", 11, 1));
// driver.Reset(sent);
// sent.Add(TradeRow(schema, 3, T0.AddSeconds(2), "NQ", 9, 1));
// driver.Reset(sent);
// Assert.That(rows.Count, Is.EqualTo(1));
// AssertBar(rows[0], 10, 11, 9, 9, 3, 3, 10, false);
// }
// [Test]
// public void VolumeBars_CloseOnThreshold() {
// var schema = TradeSchema();
// var driver = new TestDriverStream(schema);
// var bars = new BarBuilderStream(driver, "VOLUME", 10, "trade_price", "trade_size");
// var rows = CaptureAddedRows(bars);
// var sent = new List<IBlueRow>();
// sent.Add(TradeRow(schema, 1, T0, "ES", 100, 3));
// driver.Reset(sent);
// sent.Add(TradeRow(schema, 2, T0.AddSeconds(1), "ES", 120, 4));
// driver.Reset(sent);
// sent.Add(TradeRow(schema, 3, T0.AddSeconds(2), "ES", 110, 5));
// driver.Reset(sent);
// Assert.That(rows.Count, Is.EqualTo(1));
// AssertBar(rows[0], 100, 120, 100, 110, 12, 3, (100 * 3 + 120 * 4 + 110 * 5) / 12.0, false);
// }
// [Test]
// public void DollarBars_CloseOnThreshold() {
// var schema = TradeSchema();
// var driver = new TestDriverStream(schema);
// var bars = new BarBuilderStream(driver, "DOLLAR", 1000, "trade_price", "trade_size");
// var rows = CaptureAddedRows(bars);
// var sent = new List<IBlueRow>();
// sent.Add(TradeRow(schema, 1, T0, "CL", 100, 3));
// driver.Reset(sent);
// sent.Add(TradeRow(schema, 2, T0.AddSeconds(1), "CL", 120, 4));
// driver.Reset(sent);
// sent.Add(TradeRow(schema, 3, T0.AddSeconds(2), "CL", 110, 3));
// driver.Reset(sent);
// Assert.That(rows.Count, Is.EqualTo(1));
// AssertBar(rows[0], 100, 120, 100, 110, 10, 3, 111, false);
// }
// [Test]
// public void Validation_VolumeOnBboOnly_Throws() {
// var schema = BboSchema();
// var driver = new TestDriverStream(schema);
// var bars = new BarBuilderStream(driver, "VOLUME", 10, "mid", "trade_size");
// var row = BboRow(schema, 1, T0, "BTCUSD", 99, 101);
// Assert.Throws<ArgumentException>(() => {
// using var _ = SubscribeNoOp(bars);
// driver.Reset(row);
// });
// }
// [Test]
// public void EmitForming_EmitsFormingAndFinal() {
// var schema = TradeSchema();
// var driver = new TestDriverStream(schema);
// var bars = new BarBuilderStream(driver, "TICK", 2, "trade_price", "trade_size", emit_forming: true);
// var rows = CaptureAddedRows(bars);
// var sent = new List<IBlueRow>();
// sent.Add(TradeRow(schema, 1, T0, "MNQ", 10, 1));
// driver.Reset(sent);
// sent.Add(TradeRow(schema, 2, T0.AddSeconds(1), "MNQ", 12, 1));
// driver.Reset(sent);
// Assert.That(rows.Count, Is.EqualTo(3));
// Assert.That(rows[0].get<bool>("is_forming"), Is.True);
// Assert.That(rows[1].get<bool>("is_forming"), Is.True);
// Assert.That(rows[2].get<bool>("is_forming"), Is.False);
// AssertBar(rows[2], 10, 12, 10, 12, 2, 2, 11, false);
// }
// static IDisposable SubscribeNoOp(ViewStream stream) {
// DataStream.ChangeDelegate d = _ => { };
// stream.onChange += d;
// return new Subscription(() => stream.onChange -= d);
// }
// static List<IBlueRow> CaptureAddedRows(ViewStream stream) {
// var rows = new List<IBlueRow>();
// stream.onChange += o => {
// var dp = (IDataPacket)o;
// foreach (IBlueRow r in dp.addedRows) {
// rows.Add(new BlueRow(r));
// }
// };
// return rows;
// }
// static BlueTableSchema TradeSchema() =>
// new(
// new[] {
// new ColumnInfo("id", 0, typeof(int)),
// new ColumnInfo("ts_event", 1, typeof(string)),
// new ColumnInfo("contract", 2, typeof(string)),
// new ColumnInfo("trade_price", 3, typeof(double)),
// new ColumnInfo("trade_size", 4, typeof(double)),
// },
// "trade_events"
// );
// static BlueTableSchema BboSchema() =>
// new(
// new[] {
// new ColumnInfo("id", 0, typeof(int)),
// new ColumnInfo("ts_event", 1, typeof(string)),
// new ColumnInfo("contract", 2, typeof(string)),
// new ColumnInfo("bid_price", 3, typeof(double)),
// new ColumnInfo("ask_price", 4, typeof(double)),
// },
// "bbo_events"
// );
// static IBlueRow TradeRow(IBlueTableSchema schema, int id, DateTime ts, string contract, double px, double sz) =>
// new BlueRow(schema, id, ts.ToString("O"), contract, px, sz);
// static IBlueRow BboRow(IBlueTableSchema schema, int id, DateTime ts, string contract, double bid, double ask) =>
// new BlueRow(schema, id, ts.ToString("O"), contract, bid, ask);
// static void AssertBar(
// IBlueRow r,
// double open,
// double high,
// double low,
// double close,
// double volume,
// int tradeCount,
// double vwap,
// bool isForming
// ) {
// Assert.That(r.get<double>("open"), Is.EqualTo(open).Within(1e-9));
// Assert.That(r.get<double>("high"), Is.EqualTo(high).Within(1e-9));
// Assert.That(r.get<double>("low"), Is.EqualTo(low).Within(1e-9));
// Assert.That(r.get<double>("close"), Is.EqualTo(close).Within(1e-9));
// Assert.That(r.get<double>("volume"), Is.EqualTo(volume).Within(1e-9));
// Assert.That(r.get<int>("trade_count"), Is.EqualTo(tradeCount));
// Assert.That(r.get<double>("vwap"), Is.EqualTo(vwap).Within(1e-9));
// Assert.That(r.get<bool>("is_forming"), Is.EqualTo(isForming));
// }
// sealed class Subscription : IDisposable {
// readonly Action _dispose;
// public Subscription(Action dispose) => _dispose = dispose;
// public void Dispose() => _dispose();
// }
// }
@@ -1,5 +1,7 @@
namespace bts.BlueOrb {
using System;
using System.Collections.Generic;
using System.IO;
using BCG;
@@ -10,7 +12,14 @@ namespace bts.BlueOrb {
[LiveMarketDataStream]
public class CrossExchangeSymbolMatcherStream : ViewStream {
public CrossExchangeSymbolMatcherStream([MarketDataClientParam] IMarketDataClient mdClient, IWorkScheduler scheduler) {
const string CanonicalSymbolColumn = "canonical_symbol";
readonly CrossExchangeSymbolAliasResolver symbolAliasResolver;
public CrossExchangeSymbolMatcherStream([MarketDataClientParam] IMarketDataClient mdClient, IWorkScheduler scheduler)
: this(mdClient, scheduler, null) { }
public CrossExchangeSymbolMatcherStream([MarketDataClientParam] IMarketDataClient mdClient, IWorkScheduler scheduler, string? symbolAliasConfigFile) {
symbolAliasResolver = CrossExchangeSymbolAliasResolver.Load(ResolveSymbolAliasConfigFile(symbolAliasConfigFile));
ViewStream r;
ViewStream okx_usdc;
ViewStream okx_usdt;
@@ -24,6 +33,9 @@ namespace bts.BlueOrb {
r = new InstrumentsStream(mdClient, scheduler);
r = new DateTimeExpanstionStream(r,
"expiration");
r = new MapStream(r,
_add_canonical_symbol_schema,
_add_canonical_symbol_row);
okx_usdc = new FilterStream(r,
"venue='Okx' and sec~='USDC'");
okx_usdc = new AddColumnsStream(okx_usdc,
@@ -63,7 +75,7 @@ namespace bts.BlueOrb {
r,
"");
r = new FilterStream(r,
"left_venue!=right_venue and left_symbol=right_symbol and left_cur=right_cur and left_sec_type=right_sec_type");
"left_venue!=right_venue and left_canonical_symbol=right_canonical_symbol and left_cur=right_cur and left_sec_type=right_sec_type");
perps = new FilterStream(r,
"left_expiration_year=2100 and right_expiration_year=2100");
perps = new FilterStream(perps,
@@ -78,5 +90,127 @@ namespace bts.BlueOrb {
mySetRealSource(r);
}
static string ResolveSymbolAliasConfigFile(string? symbolAliasConfigFile) {
if (!string.IsNullOrWhiteSpace(symbolAliasConfigFile)) {
return symbolAliasConfigFile;
}
var home = Environment.GetEnvironmentVariable("HOME");
if (!string.IsNullOrWhiteSpace(home)) {
var sparkdConf = Path.Combine(home, ".local", "share", "bts", "sparkd", "conf", "cross_exchange_symbol_aliases.conf");
if (File.Exists(sparkdConf)) {
return sparkdConf;
}
}
var appData = Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData);
if (!string.IsNullOrWhiteSpace(appData)) {
var appDataConf = Path.Combine(appData, "bts", "conf", "cross_exchange_symbol_aliases.conf");
if (File.Exists(appDataConf)) {
return appDataConf;
}
}
return !string.IsNullOrWhiteSpace(home)
? Path.Combine(home, ".local", "share", "bts", "sparkd", "conf", "cross_exchange_symbol_aliases.conf")
: "cross_exchange_symbol_aliases.conf";
}
IBlueTableSchema _add_canonical_symbol_schema(IBlueTableSchema schema) =>
BlueDataUtility.add_column(schema, CanonicalSymbolColumn, typeof(string));
IBlueRow _add_canonical_symbol_row(IBlueRow row) {
var rv = new MutableBlueRow(_add_canonical_symbol_schema(row.schema), row);
rv.setValue(CanonicalSymbolColumn, symbolAliasResolver.Canonicalize(row.get<string>("symbol")));
return rv;
}
}
public sealed class CrossExchangeSymbolAliasResolver {
static readonly System.Text.RegularExpressions.Regex TokenRegex =
new(@"[A-Za-z0-9]+|[^A-Za-z0-9]+", System.Text.RegularExpressions.RegexOptions.Compiled);
readonly IReadOnlyDictionary<string, string> aliases;
CrossExchangeSymbolAliasResolver(IReadOnlyDictionary<string, string> aliases) =>
this.aliases = aliases;
public static CrossExchangeSymbolAliasResolver Load(string configFilePath) {
if (!File.Exists(configFilePath)) {
Log.Gen.warn($"CrossExchangeSymbolMatcherStream alias config not found: {configFilePath}");
return new CrossExchangeSymbolAliasResolver(new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase));
}
var aliases = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
foreach (var rawLine in File.ReadLines(configFilePath)) {
var line = StripComments(rawLine).Trim();
if (line.Length == 0) {
continue;
}
if (!TryParseAlias(line, out var alias, out var canonical)) {
Log.Gen.warn($"CrossExchangeSymbolMatcherStream skipping invalid alias line: '{rawLine}'");
continue;
}
aliases[NormalizeToken(alias)] = NormalizeToken(canonical);
}
return new CrossExchangeSymbolAliasResolver(aliases);
}
public string Canonicalize(string? symbol) {
if (string.IsNullOrWhiteSpace(symbol)) {
return string.Empty;
}
var rv = new System.Text.StringBuilder(symbol.Length);
foreach (System.Text.RegularExpressions.Match match in TokenRegex.Matches(symbol)) {
var token = match.Value;
if (IsToken(token)) {
var normalized = NormalizeToken(token);
rv.Append(aliases.TryGetValue(normalized, out var canonical) ? canonical : normalized);
} else {
rv.Append(token);
}
}
return rv.ToString();
}
static bool TryParseAlias(string line, out string alias, out string canonical) {
var match = System.Text.RegularExpressions.Regex.Match(line, @"^(?<alias>.+?)\s*(?:=|:|->)\s*(?<canonical>.+)$");
if (match.Success) {
alias = match.Groups["alias"].Value.Trim();
canonical = match.Groups["canonical"].Value.Trim();
return alias.Length > 0 && canonical.Length > 0;
}
var parts = line.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries);
if (parts.Length == 2) {
alias = parts[0];
canonical = parts[1];
return true;
}
alias = string.Empty;
canonical = string.Empty;
return false;
}
static string StripComments(string line) {
var commentIdx = line.IndexOf('#');
var slashIdx = line.IndexOf("//", StringComparison.Ordinal);
if (slashIdx >= 0 && (commentIdx < 0 || slashIdx < commentIdx)) {
commentIdx = slashIdx;
}
return commentIdx >= 0 ? line[..commentIdx] : line;
}
static bool IsToken(string token) =>
token.Length > 0 && char.IsLetterOrDigit(token[0]);
static string NormalizeToken(string token) =>
token.Trim().ToUpperInvariant();
}
}
@@ -0,0 +1,31 @@
namespace bts.BlueOrb {
using BCG;
using Bts.DataStreamCore;
using Bts.PubSub;
using Spark.Api.MarketData;
[LiveMarketDataStream]
public class FeeAdjustedCrossExchangeArbDiscoveryStream : ViewStream {
public FeeAdjustedCrossExchangeArbDiscoveryStream(
[MarketDataClientParam] IMarketDataClient mdClient,
IWorkScheduler scheduler) {
ViewStream r = new CrossExchangeArbDiscoveryStream(mdClient, scheduler);
r = new AddColumnsStream(r, "fee:0.0002");
r = new AddColumnsStream(r, "capital_available:100000");
r = new MathStream(r, """
left_leg_cost = left_bid_price*left_bid_qty*left_cpp;
right_leg_cost = right_bid_price*right_bid_qty*right_cpp;
total_cost = left_leg_cost + right_leg_cost;
total_fees = total_cost * fee;
fee_adjusted_arb = arb-total_fees;
possible_arb = fee_adjusted_arb*(Min(capital_available,total_cost)/total_cost);
""");
r = new FilterStream(r, "arb>0");
r = new TimeSeriesStream(r);
r = new ProjectStream(r,
"spread_name, left_sec_name,left_venue,right_sec_name,right_venue,arb, fee_adjusted_arb, timestamp");
mySetRealSource(r);
}
}
}
+11 -1
View File
@@ -17,7 +17,7 @@ type Tab =
| PositionManagementTab
| MarginTab
let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.BYBIT; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.GATEIO; Venue.OKX; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.TWSE; Venue.SIM; |]
let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.BYBIT; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.GATEIO; Venue.OKX; Venue.ORCA; Venue.RAYDIUM; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.TWSE; Venue.SIM; |]
let securityTypes = [| SecurityType.SPOT; SecurityType.FUTURE; SecurityType.OPTION; SecurityType.SPREAD; |]
@@ -94,6 +94,16 @@ type OkxExchangeField =
| Origin of CustomerOrFirm
| ClearingAccount of string
type OrcaExchangeField =
| CTI of CTICode
| Origin of CustomerOrFirm
| ClearingAccount of string
type RaydiumExchangeField =
| CTI of CTICode
| Origin of CustomerOrFirm
| ClearingAccount of string
type SIMExchangeField =
| CTI of CTICode
| Origin of CustomerOrFirm
@@ -801,6 +801,8 @@ let venueToString (venue:bts.api.Venue) =
| bts.api.Venue.GATEIO -> "GATEIO"
| bts.api.Venue.FTX -> "FTX"
| bts.api.Venue.OKX -> "OKX"
| bts.api.Venue.ORCA -> "ORCA"
| bts.api.Venue.RAYDIUM -> "RAYDIUM"
| bts.api.Venue.TAIFEX -> "TAIFEX"
| bts.api.Venue.TWSE -> "TWSE"
+22 -4
View File
@@ -37,6 +37,20 @@ protobuf_generate_cpp(
)
add_executable(sparkd ${${PROJECT_NAME}_SOURCES} ${${PROJECT_NAME}_PROTO_CXX_SOURCES})
find_package(PkgConfig REQUIRED)
pkg_check_modules(
emumba-deribit-connector-library
REQUIRED
IMPORTED_TARGET
emumba-deribit-connector
)
pkg_check_modules(
rakurai-dex-connector-library
REQUIRED
IMPORTED_TARGET
rakurai-dex-connector
)
# include paths
target_include_directories(
sparkd
@@ -44,11 +58,9 @@ target_include_directories(
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>/include
$<BUILD_INTERFACE:${emumba-deribit-connector-library_INCLUDE_DIRS}>
$<BUILD_INTERFACE:${rakurai-dex-connector-library_INCLUDE_DIRS}>
)
find_package(PkgConfig REQUIRED)
pkg_check_modules(emumba-deribit-connector-library REQUIRED emumba-deribit-connector)
# This dir contains rapidjsonConfig.cmake
list(APPEND CMAKE_PREFIX_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules)
find_package(rapidjson REQUIRED)
@@ -127,7 +139,8 @@ target_link_libraries(
python_interop
python_interop_module
databento::databento
${emumba-deribit-connector-library_LIBRARIES}
PkgConfig::emumba-deribit-connector-library
PkgConfig::rakurai-dex-connector-library
tbb
quickfix
benchmark
@@ -140,6 +153,11 @@ target_link_libraries(
-static-libstdc++
-pthread)
# Both vendor static connector archives currently export the same global
# `print_books` symbol. Allow that duplicate so sparkd can link until the
# upstream packages make the symbol internal or rename it.
target_link_options(${PROJECT_NAME} PRIVATE -Wl,--allow-multiple-definition)
# Onixs
target_link_libraries(
${PROJECT_NAME}
+3
View File
@@ -40,6 +40,9 @@ ARG SOURCE_DIR
ARG PYTHON_REQUIREMENTS="requirements.txt"
ARG SPARKD_ENTRYPOINT
ENV DOTNET_ENVIRONMENT=Development
RUN dotnet dev-certs https --trust
USER "root"
RUN apt-get update && apt-get install -y liblttng-ust-dev
RUN apt-get update && apt-get install -y libpcap0.8-dev
@@ -0,0 +1,3 @@
SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dex_dev
SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/dex_dev/requirements.txt
SPARKD__SHM_SIZE=2gb
@@ -0,0 +1,2 @@
* Connects to raydium and orca for dex
* Execution is SIM
@@ -0,0 +1,22 @@
{
"app_logging": {
"severity": "trace",
"file": false,
"console": true
},
"dex": [
{
"id": "orca_clmm",
"pool_ids": ["Czfq3xZZDmsdGdUyrNLtRhGc47cXcZtLG4crryfu44zE", "4HppGTweoGQ8ZZ6UcCgwJKfi5mJD9Dqwy6htCpnbfBLW"
]
}
],
"cluster_mode": false,
"wallet_pub": "HFru3Rc3Rf6UjZtrYPsBSKvmq5uyha2DWefV77rFXU64",
"wallet_pvt": [168,123,234,72,218,209,180,225,42,193,240,65,11,15,105,91,95,255,231,214,188,119,90,45,119,144,177,207,203,245,250,27,241,137,138,138,103,247,64,134,70,212,107,134,75,97,87,167,53,32,169,108,75,126,254,17,213,216,74,14,245,28,231,161],
"md_server_uri_1": "ws://185.191.117.159:8130",
"node_oms_uri_1": "ws://185.191.117.159:8133",
"md_server_uri_2": "ws://189.1.171.199:8130",
"node_oms_uri_2": "ws://189.1.171.199:8133",
"core": 1
}
@@ -0,0 +1,6 @@
{
"account": "DEX_DEV",
"operator_id": "DEX_DEV",
"json_conf_file_name": "config.json",
"default_priority_fee": 1000
}
@@ -0,0 +1,6 @@
{
"account": "DEX_DEV",
"operator_id": "DEX_DEV",
"json_conf_file_name": "config.json",
"default_priority_fee": 1000
}
@@ -0,0 +1,5 @@
numpy==2.2.6
pybit==5.11.0
requests==2.31.0
websocket-client==1.8.0
websockets==15.0.1
@@ -0,0 +1,76 @@
JetFIX {
value {
connection {
name: "orca_1"
venue: VENUE_ORCA
max_order_msgs_per_window: 300
window_duration_msecs: 1000
}
connection {
name: "okx_1"
venue: VENUE_OKX
max_order_msgs_per_window: 300
window_duration_msecs: 1000
}
}
}
PublisherOptions {
value {
network_interface: "0.0.0.0"
heartbeat_ms: 5000
heartbeat_boot_seconds: 60
skip_options: true
hot_initial: 1024
hot_growth: 1024
cold_initial: 8192
cold_growth: 2048
channel_prefix: ""
disable_heartbeat: true
}
}
MarketDataConfig {
value {
dex {
venue: VENUE_ORCA
num_depth_levels: 10
json_conf_file_name: "config.json"
security_criteria {
clauses {
security_types: SECURITY_TYPE_SPOT
symbols: @SymbolLists.list[Binance].symbol
name_regexps: "USDC|USDT$"
}
}
}
okx {
num_depth_levels: 5
url_rest_base: "https://www.okx.com"
url_ws_base: "wss://ws.okx.com:8443"
security_criteria {
clauses {
security_types: SECURITY_TYPE_SPOT
symbols: @SymbolLists.list[Okx].symbol
}
}
is_application_ping_pong_enabled: true
application_ping_interval_ms: 20000
application_pong_timeout_ms: 10000
is_protocol_ping_pong_enabled: true
protocol_ping_interval_ms: 60000
protocol_pong_timeout_ms: 30000
}
dex {
venue: VENUE_RAYDIUM
num_depth_levels: 10
json_conf_file_name: "config.json"
security_criteria {
clauses {
security_types: SECURITY_TYPE_SPOT
symbols: @SymbolLists.list[Binance].symbol
name_regexps: "USDC|USDT$"
}
}
}
}
}
@@ -0,0 +1,76 @@
SymbolLists {
value {
list {
key: "Binance"
symbol: "SOL"
symbol: "USDC"
#symbol: "BNB"
symbol: "BTC"
#symbol: "DOGE"
#symbol: "EIGEN"
symbol: "ETH"
symbol: "XRP"
}
list {
key: "ORCA"
#symbol: "AAVE"
#symbol: "AVAX"
#symbol: "BNB"
symbol: "BTC"
symbol: "USDC"
symbol: "SOL"
symbol: "ETH"
symbol: "XRP"
}
list {
key: "RAYDIUM"
symbol: "SOL"
symbol: "USDC"
symbol: "ETH"
symbol: "BTC"
symbol: "XRP"
}
list {
key: "BinanceUsdsFutures"
symbol: "SOL"
symbol: "BTC"
symbol: "USDC"
symbol: "ETH"
symbol: "XRP"
}
list {
key: "Okx"
symbol: "SOL"
symbol: "USDC"
symbol: "BTC"
symbol: "ETH"
symbol: "XRP"
}
}
}
Symbols {
value {
# venues {
# name: "VENUE_BINANCE"
# spot: @SymbolLists.list[Binance].symbol
# }
#venues {
# name: "VENUE_BINANCE_USDS_FUTURES"
# future: @SymbolLists.list[BinanceUsdsFutures].symbol
#}
venues {
name: "VENUE_ORCA"
spot: @SymbolLists.list[ORCA].symbol
}
# venues {
# name: "VENUE_RAYDIUM"
# spot: @SymbolLists.list[RAYDIUM].symbol
# }
venues {
name: "VENUE_OKX"
spot: @SymbolLists.list[Okx].symbol
# future: @SymbolLists.list[Okx].symbol
}
}
}
+3
View File
@@ -42,7 +42,10 @@ roll_log() {
# Acquire sudo permission once for the script (needed for a few calls)
echo bts | sudo -S -v
data_root="${HOME:-/home/bts}/.local/share/bts/sparkd"
api_server_data_root="${HOME:-/home/bts}/.local/share/bts/spark_api_server"
sudo -n chown -R bts:bts "$data_root"
sudo -n mkdir -p "$api_server_data_root"
sudo -n chown -R bts:bts "$api_server_data_root"
load_env "$data_root/conf/.env.local"
init_timezone "$TZ"
roll_log "$data_root/data/logging.sparkd/log"
@@ -25,9 +25,12 @@ EnumOptionNames<api::Venue> VenueOption::names() {
NAME_PAIR(api::VENUE_TWSE),
NAME_PAIR(api::VENUE_BYBIT),
NAME_PAIR(api::VENUE_GATEIO),
NAME_PAIR(api::VENUE_OKX)};
NAME_PAIR(api::VENUE_OKX),
NAME_PAIR(api::VENUE_ORCA),
NAME_PAIR(api::VENUE_RAYDIUM)};
}
std::string VenueOption::name(api::Venue venue) {
switch (venue) {
case api::VENUE_BTS:
@@ -58,6 +61,10 @@ std::string VenueOption::name(api::Venue venue) {
return "cboe";
case api::VENUE_OKX:
return "okx";
case api::VENUE_RAYDIUM:
return "raydium";
case api::VENUE_ORCA:
return "orca";
case api::VENUE_TAIFEX:
return "taifex";
case api::VENUE_TWSE:
@@ -4,6 +4,7 @@
#include "binance/spots/spots_gateway.h"
#include "binance/usds_futures/usds_futures_gateway.h"
#include "okx_execution/okx_execution_gateway.h"
#include "sparkd/execution/rakurai/rakurai_execution_gateway.h"
#include "onixs_fix/OnixFIXAdapter.h"
#include "python_interop/plugin_manager.h"
#include "spark/api/domain/Venue.h"
@@ -119,6 +120,18 @@ struct ExecutionGatewayProvider::PImpl {
conn, conn.venue(), std::make_unique<OnixFIXAdapter>(o), o);
case api::VENUE_OKX:
return make_gw<OkxExecutionGateway>(conn, o);
case api::VENUE_ORCA:
case api::VENUE_RAYDIUM: {
auto cfg_j = load_config_json(conn.name());
bts_assert_msgf(
cfg_j,
"Failed to load execution gateway config. name: {}",
conn.name());
auto cfg = cfg_j->get<rakurai::GatewayConfig>();
cfg.expand_env();
return make_gw<rakurai::RakuraiExecutionGateway>(
conn, o, std::move(cfg), conn);
}
case api::VENUE_BINANCE: {
auto cfg =
parse_config_json<binance::GatewayConfig>(conn.name());
@@ -0,0 +1,742 @@
#include "sparkd/execution/rakurai/rakurai_execution_gateway.h"
#include "spark/config/config.h"
#include "spark/config/protos/system_config.pb.h"
#include "spark/domain/DomainSecurityUtils.h"
#include "spark/domain/Price.h"
#include "spark/domain/Quantity.h"
#include "spark/domain_model/Security.h"
#include "spark/logging/logging.h"
#include "spark/md/SecurityOracle.h"
#include "spark/orderman/CancelOrderMsg.h"
#include "spark/orderman/CancelReplaceOrderMsg.h"
#include "spark/orderman/CreateOrderMsg.h"
#include "spark/orderman/RFQMsg.h"
#include "spark/orderman/events/NewAdminDataEvent.h"
#include "spark/orderman/events/NewSecuritiesEvent.h"
#include "spark/orderman/orderman_utils/CancelReject.h"
#include "spark/orderman/orderman_utils/ExecutionReport.h"
#include "spark/orderman/orderman_utils/orderman_common.h"
#include "spark/utils/bts_assert.h"
#include "sparkd/rakurai/RakuraiSharedRuntime.h"
#include <boost/filesystem/path.hpp>
#include <cmath>
#include <cstdint>
#include <deque>
#include <limits>
#include <memory>
#include <mutex>
#include <optional>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
namespace bts::orderman::rakurai {
namespace {
#define LOG_FN(level, fn, ...) \
LOG(OrderMan, level, "RakuraiExecutionGateway::" fn ". " __VA_ARGS__)
#define WARN_FN(fn, ...) LOG_FN(Warn, fn, __VA_ARGS__)
using ConnectorNewOrder = ::rakurai::dex_connector::order_entry::new_order;
using ConnectorOrderState = ::rakurai::dex_connector::order_entry::order_state;
using ConnectorSide = ::rakurai::dex_connector::order_entry::side;
std::optional<std::string> extract_short_pool_id(const api::Security& security) {
std::string desc;
if (!domain::DomainSecurityUtils::get_security_desc(security, desc)) {
return std::nullopt;
}
const auto pos = desc.rfind('-');
if (pos == std::string::npos || pos + 1 >= desc.size()) {
return std::nullopt;
}
return desc.substr(pos + 1);
}
api::orderman::NativeOrderStatus to_order_status(ConnectorOrderState state) {
using Status = ::rakurai::dex_connector::data_types::txn_status_t;
switch (state.tx_status) {
case Status::accepted:
return api::orderman::NATIVE_ORDER_STATUS_NEW;
case Status::confirmed:
return state.is_pass ? api::orderman::NATIVE_ORDER_STATUS_FILLED
: api::orderman::NATIVE_ORDER_STATUS_REJECTED;
case Status::failed:
return api::orderman::NATIVE_ORDER_STATUS_REJECTED;
case Status::pending:
return api::orderman::NATIVE_ORDER_STATUS_PENDING_NEW;
}
return api::orderman::NATIVE_ORDER_STATUS_UNDEFINED;
}
api::orderman::ExecType to_exec_type(ConnectorOrderState state) {
using Status = ::rakurai::dex_connector::data_types::txn_status_t;
switch (state.tx_status) {
case Status::accepted:
return api::orderman::EXEC_TYPE_NEW;
case Status::confirmed:
return state.is_pass ? api::orderman::EXEC_TYPE_FILL
: api::orderman::EXEC_TYPE_REJECTED;
case Status::failed:
return api::orderman::EXEC_TYPE_REJECTED;
case Status::pending:
return api::orderman::EXEC_TYPE_PENDING_NEW;
}
return api::orderman::EXEC_TYPE_UNDEFINED;
}
std::string txn_status_name(
::rakurai::dex_connector::data_types::txn_status_t status) {
using Status = ::rakurai::dex_connector::data_types::txn_status_t;
switch (status) {
case Status::accepted:
return "accepted";
case Status::confirmed:
return "confirmed";
case Status::failed:
return "failed";
case Status::pending:
return "pending";
}
return "unknown";
}
std::string json_for_log(const nlohmann::json& j) {
try {
return j.dump();
} catch (const std::exception& ex) {
return fmt::format("<json dump failed: {}>", ex.what());
}
}
} // namespace
struct RakuraiExecutionGateway::PImpl {
struct ActiveOrder {
CreateOrderMsg create_msg;
trace::CorID cor_id;
std::array<std::uint8_t, 32> pool_id{};
double quantity_scale = 1.0;
std::uint64_t raw_quantity = 0;
std::uint64_t priority_fee = 0;
std::uint64_t client_id = 0;
std::uint64_t txn_id = 0;
};
struct BufferedReport {
std::shared_ptr<const ExecutionReport> report;
trace::CorID cor_id;
};
explicit PImpl(
md::SecurityOracle,
GatewayConfig cfg,
config::JetFIX_Connection jetfix_connection)
: config(std::move(cfg)),
jetfix_connection(std::make_shared<config::JetFIX_Connection>(
std::move(jetfix_connection))) {
config.expand_env();
venue = this->jetfix_connection->venue();
bts_assert(venue == api::VENUE_ORCA || venue == api::VENUE_RAYDIUM);
LOG_FN(
Info,
"ctor",
"session:{} venue:{} account:{} operator_id:{} json_conf_file_name:{} default_priority_fee:{} connector_config:{}",
this->jetfix_connection->name(),
api::Venue_Name(venue),
config.account,
config.operator_id,
config.json_conf_file_name,
config.default_priority_fee,
json_for_log(config.connector_config));
}
GatewayConfig config;
std::shared_ptr<config::JetFIX_Connection> jetfix_connection;
api::Venue venue = api::VENUE_UNSET;
std::mutex mutex;
std::unordered_map<OrderKey, ActiveOrder> active_orders;
std::unordered_map<std::uint64_t, OrderKey> client_id_to_order_key;
std::deque<OrderKey> pending_client_bindings;
std::deque<BufferedReport> buffered_reports;
std::size_t order_update_subscription_id = 0;
std::size_t status_subscription_id = 0;
bool started = false;
bool connected = false;
OnExeReport exe_report;
OnGwStatusChange gw_status_change;
OnCancelReject cancel_reject;
OnExchangePositionState exchange_position_state;
OnExchangeOpenOrdersState exchange_open_orders;
void start() {
{
std::lock_guard<std::mutex> lock(mutex);
if (started) {
LOG_FN(
Info, "start", "already started. venue:{}",
api::Venue_Name(venue));
return;
}
started = true;
}
LOG_FN(
Info,
"start",
"starting. session:{} venue:{} account:{} operator_id:{} json_conf_file_name:{} default_priority_fee:{}",
jetfix_connection->name(),
api::Venue_Name(venue),
config.account,
config.operator_id,
config.json_conf_file_name,
config.default_priority_fee);
auto& runtime = bts::rakurai::RakuraiSharedRuntime::instance();
order_update_subscription_id =
runtime.add_order_update_subscriber([this](ConnectorOrderState state) {
this->on_order_update(state);
});
status_subscription_id =
runtime.add_status_subscriber([this](bool is_connected_now) {
this->on_status_change(is_connected_now);
});
try {
if (!config.json_conf_file_name.empty()) {
const auto config_path =
boost::filesystem::path{
config::get<config::SystemConfig>().config_path()} /
config.json_conf_file_name;
LOG_FN(
Info,
"start",
"ensuring shared runtime from file. path:{}",
config_path.string());
runtime.ensure_started_from_file(config_path.string());
} else {
LOG_FN(
Info,
"start",
"ensuring shared runtime from inline config. config:{}",
json_for_log(config.connector_config));
runtime.ensure_started_from_json(
config.connector_config, jetfix_connection->name());
}
LOG_FN(
Info,
"start",
"shared runtime start request submitted. order_update_sub_id:{} status_sub_id:{} connected_now:{}",
order_update_subscription_id,
status_subscription_id,
runtime.is_connected());
} catch (...) {
runtime.remove_order_update_subscriber(order_update_subscription_id);
runtime.remove_status_subscriber(status_subscription_id);
order_update_subscription_id = 0;
status_subscription_id = 0;
{
std::lock_guard<std::mutex> lock(mutex);
started = false;
}
throw;
}
}
void stop() {
LOG_FN(
Info,
"stop",
"stopping. order_update_sub_id:{} status_sub_id:{}",
order_update_subscription_id,
status_subscription_id);
auto& runtime = bts::rakurai::RakuraiSharedRuntime::instance();
if (order_update_subscription_id != 0) {
runtime.remove_order_update_subscriber(order_update_subscription_id);
order_update_subscription_id = 0;
}
if (status_subscription_id != 0) {
runtime.remove_status_subscriber(status_subscription_id);
status_subscription_id = 0;
}
}
void on_status_change(bool is_connected_now) {
LOG_FN(
Info,
"on_status_change",
"status update. venue:{} old_connected:{} new_connected:{} active_orders:{} pending_bindings:{} buffered_reports:{}",
api::Venue_Name(venue),
connected,
is_connected_now,
active_orders.size(),
pending_client_bindings.size(),
buffered_reports.size());
{
std::lock_guard<std::mutex> lock(mutex);
connected = is_connected_now;
}
if (gw_status_change) {
gw_status_change(is_connected_now);
}
}
void on_order_update(ConnectorOrderState order_state) {
std::lock_guard<std::mutex> lock(mutex);
LOG_FN(
Info,
"on_order_update",
"incoming order update. client_id:{} txn_id:{} txn_status:{} is_pass:{} active_orders:{} pending_bindings:{} mapped_clients:{}",
order_state.client_id,
order_state.txn_id,
txn_status_name(order_state.tx_status),
order_state.is_pass,
active_orders.size(),
pending_client_bindings.size(),
client_id_to_order_key.size());
auto key_it = client_id_to_order_key.find(order_state.client_id);
if (key_it == client_id_to_order_key.end()) {
if (pending_client_bindings.empty()) {
LOG_FN(
Warn,
"on_order_update",
"dropping order update with unknown client_id:{} because there are no pending client bindings",
order_state.client_id);
return;
}
const auto key = pending_client_bindings.front();
pending_client_bindings.pop_front();
client_id_to_order_key.emplace(order_state.client_id, key);
LOG_FN(
Info,
"on_order_update",
"bound client_id:{} to order_key:{}",
order_state.client_id,
key.value());
key_it = client_id_to_order_key.find(order_state.client_id);
}
auto order_it = active_orders.find(key_it->second);
if (order_it == active_orders.end()) {
LOG_FN(
Warn,
"on_order_update",
"dropping order update for client_id:{} mapped_order_key:{} because active order was not found",
order_state.client_id,
key_it->second.value());
return;
}
auto& active = order_it->second;
active.client_id = order_state.client_id;
active.txn_id = order_state.txn_id;
auto report = build_report_locked(active, order_state);
buffered_reports.push_back({std::move(report), active.cor_id});
LOG_FN(
Info,
"on_order_update",
"buffered execution report. order_key:{} client_id:{} txn_id:{} buffered_reports:{} order_status:{} exec_type:{}",
active.create_msg.key.value(),
active.client_id,
active.txn_id,
buffered_reports.size(),
api::orderman::NativeOrderStatus_Name(
buffered_reports.back().report->order_status),
api::orderman::ExecType_Name(buffered_reports.back().report->exec_type));
const auto order_status = buffered_reports.back().report->order_status;
if (order_status == api::orderman::NATIVE_ORDER_STATUS_FILLED ||
order_status == api::orderman::NATIVE_ORDER_STATUS_REJECTED) {
LOG_FN(
Info,
"on_order_update",
"removing terminal active order. order_key:{} client_id:{} status:{}",
active.create_msg.key.value(),
order_state.client_id,
api::orderman::NativeOrderStatus_Name(order_status));
client_id_to_order_key.erase(order_state.client_id);
active_orders.erase(order_it);
}
}
void send_create(const CreateOrderMsg& msg, trace::CorID cor_id) {
std::lock_guard<std::mutex> lock(mutex);
std::string security_desc = "<null>";
if (msg.security != nullptr) {
std::string desc;
if (domain::DomainSecurityUtils::get_security_desc(*msg.security, desc)) {
security_desc = std::move(desc);
} else {
security_desc = "<unresolved>";
}
}
LOG_FN(
Info,
"send_create",
"create request. order_key:{} venue:{} side:{} qty:{} price:{} tif:{} order_type:{} connected:{} security_present:{} desc:{}",
msg.key.value(),
api::Venue_Name(msg.venue()),
api::Side_Name(msg.side),
msg.quantity.full_value(),
msg.price,
api::orderman::TimeInForce_Name(msg.time_in_force),
api::orderman::NativeOrderType_Name(msg.order_type),
connected,
msg.security != nullptr,
security_desc);
if (msg.security == nullptr) {
enqueue_reject_locked(msg, cor_id, "Rakurai order missing security");
return;
}
if (msg.venue() != venue) {
enqueue_reject_locked(
msg,
cor_id,
fmt::format(
"Rakurai gateway venue mismatch. expected:{} actual:{}",
api::Venue_Name(venue),
api::Venue_Name(msg.venue())));
return;
}
if (msg.order_type != api::orderman::NATIVE_ORDER_TYPE_MARKET) {
enqueue_reject_locked(
msg,
cor_id,
"Rakurai DEX gateway currently supports market orders only");
return;
}
if (!connected) {
enqueue_reject_locked(
msg, cor_id, "Rakurai execution session is not connected");
return;
}
const auto short_pool_id = extract_short_pool_id(*msg.security);
if (!short_pool_id.has_value()) {
enqueue_reject_locked(
msg, cor_id, "Unable to derive Rakurai pool alias from security");
return;
}
const auto metadata =
bts::rakurai::RakuraiSharedRuntime::instance().find_pool_metadata(
venue, *short_pool_id);
if (!metadata.has_value()) {
LOG_FN(
Warn,
"send_create",
"pool metadata missing. venue:{} short_pool_id:{}",
api::Venue_Name(venue),
*short_pool_id);
enqueue_reject_locked(
msg,
cor_id,
"Rakurai pool metadata not available yet; wait for pool definitions");
return;
}
const auto raw_qty_f =
msg.quantity.full_value() * metadata->quantity_scale;
if (!std::isfinite(raw_qty_f) || raw_qty_f <= 0.0) {
enqueue_reject_locked(
msg, cor_id, "Rakurai order quantity must be positive");
return;
}
ConnectorNewOrder order;
order.order_def.pool_id = metadata->security.pool_id;
order.order_def.dex = metadata->security.venue;
order.order_def.order_side =
msg.side == api::SIDE_BUY ? ConnectorSide::buy : ConnectorSide::sell;
order.qty = static_cast<std::uint64_t>(std::llround(raw_qty_f));
order.priority_fee = config.default_priority_fee;
LOG_FN(
Info,
"send_create",
"launching new order. order_key:{} short_pool_id:{} raw_qty:{} priority_fee:{} dex:{} quantity_scale:{}",
msg.key.value(),
*short_pool_id,
order.qty,
order.priority_fee,
static_cast<int>(metadata->security.venue),
metadata->quantity_scale);
try {
bts::rakurai::RakuraiSharedRuntime::instance().launch_new_order(order);
} catch (const std::exception& ex) {
WARN_FN(
"send_create",
"launch_new_order threw. order_key:{} reason:{}",
msg.key.value(),
std::string(ex.what()));
enqueue_reject_locked(msg, cor_id, ex.what());
return;
}
active_orders.emplace(
msg.key,
ActiveOrder{
msg,
cor_id,
metadata->security.pool_id,
metadata->quantity_scale,
order.qty,
order.priority_fee});
pending_client_bindings.push_back(msg.key);
LOG_FN(
Info,
"send_create",
"create request accepted locally. order_key:{} active_orders:{} pending_bindings:{}",
msg.key.value(),
active_orders.size(),
pending_client_bindings.size());
}
void send_cancel_reject(OrderKey key, const std::string& reason) {
WARN_FN(
"send_cancel_reject",
"cancel reject. order_key:{} reason:{}",
key.value(),
reason);
if (cancel_reject) {
cancel_reject({key, CancelRejectReason::Other, reason});
}
}
std::shared_ptr<const ExecutionReport> build_report_locked(
const ActiveOrder& active,
ConnectorOrderState state) const {
auto report = std::make_shared<ExecutionReport>();
report->account_name = account_name_for(active.create_msg);
report->operator_id = operator_id_for(active.create_msg);
report->order_key = active.create_msg.key;
report->msg_seq_num = 0;
report->security = active.create_msg.security;
report->leg_security = nullptr;
report->side = active.create_msg.side;
report->time_in_force = active.create_msg.time_in_force;
report->order_type = active.create_msg.order_type;
report->manual_order_indicator = active.create_msg.manual_order_indicator;
report->exchange = venue;
report->cfg = jetfix_connection.get();
report->exchange_order_id = ExchangeOrderID64(
active.client_id != 0 ? active.client_id : state.txn_id);
report->quantity = active.create_msg.quantity;
report->price = active.create_msg.price;
report->stop_price = active.create_msg.stop_px;
report->max_show = active.create_msg.display_quantity;
report->transact_time =
ExchangeTimeStamp64(static_cast<std::uint64_t>(datetime::unix_nano_now()));
report->exec_type = to_exec_type(state);
report->order_status = to_order_status(state);
report->filled_quantity =
report->order_status == api::orderman::NATIVE_ORDER_STATUS_FILLED
? active.create_msg.quantity
: Quantity(0.0f);
report->last_shares = report->filled_quantity;
report->last_price = active.create_msg.price;
report->avg_price = active.create_msg.price;
report->msg = fmt::format(
"Rakurai txn_status={} txn_id={} pass={}",
txn_status_name(state.tx_status),
state.txn_id,
state.is_pass);
LOG_FN(
Info,
"build_report_locked",
"built report. order_key:{} client_id:{} txn_id:{} exec_type:{} order_status:{} message:{}",
active.create_msg.key.value(),
active.client_id,
state.txn_id,
api::orderman::ExecType_Name(report->exec_type),
api::orderman::NativeOrderStatus_Name(report->order_status),
report->msg);
return report;
}
void enqueue_reject_locked(
const CreateOrderMsg& msg,
trace::CorID cor_id,
const std::string& reason) {
auto report = std::make_shared<ExecutionReport>();
report->account_name = account_name_for(msg);
report->operator_id = operator_id_for(msg);
report->order_key = msg.key;
report->msg_seq_num = 0;
report->security = msg.security;
report->leg_security = nullptr;
report->side = msg.side;
report->time_in_force = msg.time_in_force;
report->order_type = msg.order_type;
report->manual_order_indicator = msg.manual_order_indicator;
report->exchange = venue;
report->cfg = jetfix_connection.get();
report->exchange_order_id = ExchangeOrderID64(0);
report->quantity = msg.quantity;
report->filled_quantity = Quantity(0.0f);
report->last_shares = Quantity(0.0f);
report->price = msg.price;
report->stop_price = msg.stop_px;
report->max_show = msg.display_quantity;
report->last_price = Price(std::numeric_limits<double>::quiet_NaN());
report->avg_price = Price(std::numeric_limits<double>::quiet_NaN());
report->transact_time =
ExchangeTimeStamp64(static_cast<std::uint64_t>(datetime::unix_nano_now()));
report->exec_type = api::orderman::EXEC_TYPE_REJECTED;
report->order_status = api::orderman::NATIVE_ORDER_STATUS_REJECTED;
report->msg = reason;
buffered_reports.push_back({std::move(report), cor_id});
WARN_FN(
"enqueue_reject_locked",
"buffered reject. order_key:{} reason:{} buffered_reports:{}",
msg.key.value(),
reason,
buffered_reports.size());
}
AccountName account_name_for(const CreateOrderMsg& msg) const {
const auto account = msg.routing_record != nullptr &&
!msg.routing_record->account().empty()
? msg.routing_record->account()
: config.account;
AccountName result;
result->copy_from(account);
return result;
}
OperatorID operator_id_for(const CreateOrderMsg& msg) const {
const auto operator_id =
msg.routing_record != nullptr &&
!msg.routing_record->operator_id().empty()
? msg.routing_record->operator_id()
: config.operator_id;
OperatorID result;
orderman::to_array(operator_id, &result.value());
return result;
}
};
RakuraiExecutionGateway::RakuraiExecutionGateway(
md::SecurityOracle sec_oracle,
GatewayConfig config,
config::JetFIX_Connection jetfix_connection)
: me_(std::make_unique<PImpl>(
std::move(sec_oracle),
std::move(config),
std::move(jetfix_connection))) {}
RakuraiExecutionGateway::~RakuraiExecutionGateway() { me_->stop(); }
void RakuraiExecutionGateway::start() { me_->start(); }
void RakuraiExecutionGateway::dispatch_incoming_fix_messages() {
std::deque<PImpl::BufferedReport> reports;
{
std::lock_guard<std::mutex> lock(me_->mutex);
reports.swap(me_->buffered_reports);
}
if (!reports.empty()) {
LOG_FN(
Info,
"dispatch_incoming_fix_messages",
"dispatching buffered reports. count:{} has_callback:{}",
reports.size(),
me_->exe_report != nullptr);
}
if (!me_->exe_report) {
return;
}
for (auto& buffered : reports) {
LOG_FN(
Info,
"dispatch_incoming_fix_messages",
"publishing execution report. order_key:{} exec_type:{} order_status:{} msg:{}",
buffered.report->order_key.value(),
api::orderman::ExecType_Name(buffered.report->exec_type),
api::orderman::NativeOrderStatus_Name(buffered.report->order_status),
buffered.report->msg);
me_->exe_report(std::move(buffered.report), buffered.cor_id);
}
}
void RakuraiExecutionGateway::on_securities(
const mercury::Update<NewSecuritiesEvent>& update) {
LOG_FN(
Info,
"on_securities",
"received securities update. type:{} has_events:{} count:{}",
static_cast<int>(update.event_type),
update.events != nullptr,
update.events != nullptr ? update.events->size() : 0);
}
void RakuraiExecutionGateway::on_admin_data(
const mercury::Update<NewAdminDataEvent>& update) {
LOG_FN(
Info,
"on_admin_data",
"received admin data update. type:{} has_events:{} count:{}",
static_cast<int>(update.event_type),
update.events != nullptr,
update.events != nullptr ? update.events->size() : 0);
}
void RakuraiExecutionGateway::send_fix(const jetfix::OutFIXPtr&, trace::CorID) {
WARN_FN("send_fix", "FIX transport is not supported for Rakurai");
}
void RakuraiExecutionGateway::send_msgs(
std::vector<OrderMsg> const& msgs,
trace::CorID cor_id) {
LOG_FN(
Info,
"send_msgs",
"received outbound message batch. count:{}",
msgs.size());
for (const auto& msg : msgs) {
std::visit(
[&](const auto& order_msg) {
using T = std::decay_t<decltype(order_msg)>;
if constexpr (std::is_same_v<T, CreateOrderMsg>) {
me_->send_create(order_msg, cor_id);
} else if constexpr (std::is_same_v<T, CancelOrderMsg>) {
me_->send_cancel_reject(
order_msg.key,
"Rakurai DEX gateway does not support order cancel");
} else if constexpr (std::is_same_v<T, CancelReplaceOrderMsg>) {
me_->send_cancel_reject(
order_msg.create_fields.key,
"Rakurai DEX gateway does not support order replace");
} else if constexpr (std::is_same_v<T, RFQMsg>) {
WARN_FN("send_msgs", "RFQ is not supported for Rakurai");
}
},
msg);
}
}
OnExeReport& RakuraiExecutionGateway::on_exe_report() { return me_->exe_report; }
OnGwStatusChange& RakuraiExecutionGateway::on_gw_status_change() {
return me_->gw_status_change;
}
OnCancelReject& RakuraiExecutionGateway::on_cancel_reject() {
return me_->cancel_reject;
}
OnExchangePositionState& RakuraiExecutionGateway::on_exchange_position_state() {
return me_->exchange_position_state;
}
OnExchangeOpenOrdersState& RakuraiExecutionGateway::on_exchange_open_orders() {
return me_->exchange_open_orders;
}
} // namespace bts::orderman::rakurai
@@ -0,0 +1,76 @@
#pragma once
#include "spark/io/JsonReader.h"
#include "spark/orderman/gateways/IExecutionGateway.h"
#include "spark/os/os.h"
#include <memory>
#include <nlohmann/json.hpp>
namespace bts::config {
class JetFIX_Connection;
}
namespace bts::md {
class SecurityOracle;
}
namespace bts::orderman::rakurai {
struct GatewayConfig {
std::string account;
std::string operator_id;
std::string json_conf_file_name;
std::uint64_t default_priority_fee = 1000;
nlohmann::json connector_config;
void expand_env() {
bts::os::expand_env_in_place(account);
bts::os::expand_env_in_place(operator_id);
bts::os::expand_env_in_place(json_conf_file_name);
}
};
inline void from_json(const nlohmann::json& j, GatewayConfig& cfg) {
cfg.connector_config = j;
const bts::io::JsonReader jr{&j};
jr.read_or("account", cfg.account);
jr.read_or("operator_id", cfg.operator_id);
jr.read_or("json_conf_file_name", cfg.json_conf_file_name);
jr.read_or("default_priority_fee", cfg.default_priority_fee);
}
class RakuraiExecutionGateway : public IExecutionGateway {
public:
explicit RakuraiExecutionGateway(
bts::md::SecurityOracle sec_oracle,
GatewayConfig config,
bts::config::JetFIX_Connection jetfix_connection);
~RakuraiExecutionGateway() override;
void start() override;
void dispatch_incoming_fix_messages() override;
void on_securities(
const bts::mercury::Update<bts::orderman::NewSecuritiesEvent>&)
override;
void on_admin_data(
const bts::mercury::Update<bts::orderman::NewAdminDataEvent>&) override;
void send_fix(const bts::jetfix::OutFIXPtr&, bts::trace::CorID) override;
void send_msgs(
std::vector<bts::orderman::OrderMsg> const&,
bts::trace::CorID) override;
bts::orderman::OnExeReport& on_exe_report() override;
bts::orderman::OnGwStatusChange& on_gw_status_change() override;
bts::orderman::OnCancelReject& on_cancel_reject() override;
bts::orderman::OnExchangePositionState&
on_exchange_position_state() override;
bts::orderman::OnExchangeOpenOrdersState& on_exchange_open_orders() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::orderman::rakurai
@@ -25,6 +25,7 @@
#include "sparkd/md/ccapi/CCApiMarketDataProvider.h"
#include "sparkd/md/databento/DatabentoMarketDataProvider.h"
#include "sparkd/md/emumba/EmumbaMarketDataProvider.h"
#include "sparkd/md/rakurai/RakuraiMarketDataProvider.h"
#include "sparkd/md/taifex/TaifexMarketDataProvider.h"
#include <bits/stdint-uintn.h>
@@ -81,6 +82,8 @@ struct MarketDataGateway::PImpl {
std::make_unique<OnixsMarketDataProvider>(this->config));
providers.push_back(std::make_unique<DatabentoMarketDataProvider>(
this->config, this->sec_id_store));
providers.push_back(std::make_unique<RakuraiMarketDataProvider>(
this->config, this->sec_id_store));
providers.push_back(std::make_unique<TaifexMarketDataProvider>(
this->config, this->sec_id_store));
providers.push_back(std::make_unique<PyMarketDataProvider>(
@@ -29,7 +29,6 @@ using std::vector;
struct OnixsGlbxSubscription::PImpl {
GlbxMarketDataConfig config;
shared_ptr<IDGenerator> sec_id_store;
shared_ptr<IMarketDataListener> listener;
PImpl(shared_ptr<IDGenerator> sec_id_store, GlbxMarketDataConfig config)
: config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
@@ -67,8 +66,7 @@ struct OnixsGlbxSubscription::PImpl {
FMT_JOIN(symbols, ", "));
vector<shared_ptr<IMDPlatform>> platforms;
auto main = std::make_shared<MDP3Main>();
main->start(
{symbols.begin(), symbols.end()}, nullptr, nullptr, listener);
main->start({symbols.begin(), symbols.end()}, nullptr, nullptr);
platforms.emplace_back(std::move(main));
dispatcher = &utils::Dispatcher::current();
heartbeat_timer.emplace(*dispatcher, 250, [&]() {
@@ -109,10 +107,9 @@ OnixsGlbxSubscription::OnixsGlbxSubscription(
OnixsGlbxSubscription::~OnixsGlbxSubscription() = default;
void OnixsGlbxSubscription::start(
shared_ptr<IMarketDataListener> listener) {
shared_ptr<IMarketDataListener> /*listener*/) {
std::atomic_thread_fence(std::memory_order_release);
bts_assert(!me_->dispatcher_thread.joinable());
me_->listener = std::move(listener);
me_->dispatcher_thread = std::thread{[&]() { me_->run(); }};
}
@@ -12,8 +12,6 @@ struct SecurityData {
: id(id), handle(h), mbo_state(h), electronic_volume(0) {}
uint32_t id; // exchange id
MDProducerHandle handle;
// Exception for now: MBO still maintains per-security working state locally
// in MDP3Listener until GMDL grows equivalent MBO support.
MBOWorkingState mbo_state;
uint32_t electronic_volume;
};
@@ -1,428 +1,347 @@
#include "MDP3Listener.h"
#include "spark/config/protos/market_data.pb.h"
#include "spark/domain/Book.h"
#include "spark/domain/Security.h"
#include "spark/domain/TradeRecord.h"
#include "spark/md/protos/co2.pb.h"
#include "spark/utils/datetime.h"
#include "spark/utils/helper_macros.h"
#include <vector>
using namespace std;
using namespace bts::co2;
namespace bts {
namespace md {
namespace {
domain::AggressorSide to_domain_aggressor(onixs::AggressorSide::Enum agside) {
switch (agside) {
case onixs::AggressorSide::Buy:
return domain::AggressorSide::Buy;
case onixs::AggressorSide::Sell:
return domain::AggressorSide::Sell;
default:
return domain::AggressorSide::None;
}
}
domain::Price to_domain_price(const PriceSize &ps) {
if (ps.is_nan()) {
return domain::Price::nan();
}
return domain::Price::from_double(ps.price());
}
domain::BookLevel to_domain_book_level(const PriceSize &ps) {
if (ps.is_nan() || ps.size() == 0) {
return domain::BookLevel::empty();
}
domain::BookLevel level;
level.price = to_domain_price(ps);
level.total_quantity = domain::Quantity(ps.size());
level.direct_quantity = domain::Quantity(ps.direct_size());
return level;
}
void copy_book_side(
const std::array<PriceSize, COCONUT_MAX_DEPTH> &from,
domain::BookSide &to) {
for (size_t i = 0; i < std::min(from.size(), to.size()); ++i) {
to[i] = to_domain_book_level(from[i]);
}
}
} // namespace
MDP3Listener::MDP3Listener(
CMESecurityIndex &index,
MDProducer &producer,
MBOWriter *mbo,
const std::vector<MDProducerChannel> channels,
TradeSummarySignalDispatcher *trade_summary_dispatcher,
std::shared_ptr<IMarketDataListener> market_data_listener)
: _index(index), _producer(producer),
_verbose_ticker(config::get<CMEOptions>().verbose_ticker()),
_packet_arrived(), _latest_sequence(0),
// _settlement_recorder(settles),
_symbol_filter_storage(), _symbol_filter(), _channels(channels),
_concurrency_monitor(0), _book_updates(), _mbo(mbo),
_trade_summary_dispatcher(trade_summary_dispatcher),
_market_data_listener(std::move(market_data_listener)) {
_book_updates.reserve(32);
}
MDP3Listener::~MDP3Listener() {}
//
// Warnings/Errors
void MDP3Listener::onError(onixs::Handler &, const onixs::ErrorArgs &error) {
tracepoint(
bts_trace, trace_order, (char *)"onError", _latest_sequence,
(uint16_t)_latest_sequence, 0);
LOG(md, Warn, "OnixS Error: {} - {}", error.description(), error.source());
}
void MDP3Listener::onWarning(
onixs::Handler &, const onixs::WarningArgs &warning) {
tracepoint(
bts_trace, trace_order, (char *)"onWarning", _latest_sequence,
(uint16_t)_latest_sequence, 0);
LOG(md, Warn, "OnixS Warn: {} - {}", warning.description(),
warning.source());
}
void MDP3Listener::onInstrumentRecovery(onixs::Handler &h) {
tracepoint(
bts_trace, trace_order, (char *)"onInstrumentRecovery", _latest_sequence,
(uint16_t)_latest_sequence, 0);
LOG(md, Info, "Beginning security definition recovery for channel {}",
h.settings().channel());
}
void MDP3Listener::onEndOfInstrumentRecovery(
onixs::Handler &h, const onixs::RecoveryCompletionArgs &) {
tracepoint(
bts_trace, trace_order, (char *)"onEndOfInstrumentRecovery",
_latest_sequence, (uint16_t)_latest_sequence, 0);
LOG(md, Info, "Finished Security definition recovery for channel {}",
h.settings().channel());
}
//
// Gap Recovery
void MDP3Listener::onMarketRecovery(onixs::Handler &h) {
tracepoint(
bts_trace, trace_order, (char *)"onMarketRecovery", _latest_sequence,
(uint16_t)_latest_sequence, 0);
update_all_status(COCONUTSECURITYSTATUS_PACKET_LOSS);
for (auto &c : _channels) {
c.gap();
}
LOG(md, Info, "Beginning market recovery for channel {}",
h.settings().channel());
}
void MDP3Listener::onEndOfMarketRecovery(
onixs::Handler &h, const onixs::RecoveryCompletionArgs &) {
tracepoint(
bts_trace, trace_order, (char *)"onEndOfMarketRecovery",
_latest_sequence, (uint16_t)_latest_sequence, 0);
update_all_status(COCONUTSECURITYSTATUS_OK);
LOG(md, Info, "Finished market recovery for channel {}",
h.settings().channel());
}
void MDP3Listener::onTcpRecovery(onixs::Handler &h) {
tracepoint(
bts_trace, trace_order, (char *)"onTcpRecovery", _latest_sequence,
(uint16_t)_latest_sequence, 0);
update_all_status(COCONUTSECURITYSTATUS_PACKET_LOSS);
LOG(md, Info, "Beginning tcp recovery for channel {}",
h.settings().channel());
}
void MDP3Listener::onEndOfTcpRecovery(
onixs::Handler &h, const onixs::RecoveryCompletionArgs &) {
tracepoint(
bts_trace, trace_order, (char *)"onEndOfTcpRecovery", _latest_sequence,
(uint16_t)_latest_sequence, 0);
update_all_status(COCONUTSECURITYSTATUS_OK);
LOG(md, Info, "Finished tcp recovery for channel {}",
h.settings().channel());
}
//
// Mark the start/end of a packet
void MDP3Listener::onPacket(onixs::Handler &, const onixs::PacketArgs &args) {
// Mark the arrival of the packet
_packet_arrived = TimeStamp::now();
auto seq = args.seqNumber();
if (!_book_updates.empty()) {
if (_latest_sequence != seq) {
LOG(md, Error,
"Expected sequence {} got {} did not flush {} book updates.",
_latest_sequence, seq, _book_updates.size());
}
}
_latest_sequence = seq;
auto onixs_zero = args.receiveTime().sinceEpoch();
auto onixs_now = OnixS::CME::MDH::UtcWatch::now().sinceEpoch();
// needed for clang compilation
UNUSED(onixs_zero);
UNUSED(onixs_now);
// was needed for clang compilation
tracepoint(
bts_trace, trace_order, (char *)"onPacket", _latest_sequence,
(uint16_t)_latest_sequence, onixs_now - onixs_zero);
}
void MDP3Listener::onEndOfPacket(onixs::Handler &, const onixs::PacketArgs &) {
tracepoint(
bts_trace, trace_order, (char *)"onEndOfPacket.begin", _latest_sequence,
(uint16_t)_latest_sequence, 0);
flush_books();
tracepoint(
bts_trace, trace_order, (char *)"onEndOfPacket.end", _latest_sequence,
(uint16_t)_latest_sequence, 0);
}
//
// Book updates
void MDP3Listener::onBookUpdate(
onixs::Handler &,
const onixs::Security &sec,
const onixs::DirectBook &book) {
onBookUpdate<COCONUT_MAX_DEPTH>(sec, book, book_type_direct);
}
void MDP3Listener::onBookUpdate(
onixs::Handler &,
const onixs::Security &sec,
const onixs::ImpliedBook &book) {
onBookUpdate<COCONUT_MAX_IMPLIED_DEPTH>(sec, book, book_type_implied);
}
void MDP3Listener::onBookUpdate(
onixs::Handler &, const onixs::Security &, const onixs::MboBook &) {
#if 0
if (_mbo) {
auto data = _index.get_security(sec.id());
if (data == nullptr) return;
auto key = data->handle.key();
if (key == invalid_security_key) return;
auto* snapshot = _mbo->allocate_snapshot();
// copy from onixs book
for (const auto& bid : book.bids()) {
snapshot->bids.push_back({ ExchangeOrderKey(bid.id()), convert_price(bid.price(), 0), bid.quantity() });
}
for (const auto& ask : book.offers()) {
snapshot->asks.push_back({ ExchangeOrderKey(ask.id()), convert_price(ask.price(),0), ask.quantity() });
}
_mbo->validate_snapshot(sec.id(), SecurityKey(key), snapshot);
}
#endif
}
//
// Trades
// Called from onTradeSummary and onTradeRecovery, ends up calling
// MDProducerHandle::add_trade
void on_trade(
SecurityData *data,
const onixs::Decimal &price,
int size,
onixs::AggressorSide::Enum agside,
const onixs::Timestamp &onixs_date,
IMarketDataListener *market_data_listener) {
auto ts = utils::TimeStamp::now();
PriceSize ps = pricesize_of_price_size(
price, data->handle.security().display_factor, size);
if (market_data_listener != nullptr) {
domain::TradeRecord trade;
trade.sec_id = domain::SecurityID(data->handle.key());
trade.time_stamp = core::TimeStamp::now();
trade.agg_side = to_domain_aggressor(agside);
trade.trade_quantity = domain::Quantity(size);
trade.trade_price = to_domain_price(ps);
std::vector<domain::TradeRecord const *> trades{&trade};
// GMDL!!
market_data_listener->on_trades(trades);
return;
}
auto date = ll_of_timestamp(onixs_date);
data->handle.add_trade(
ts, ps, date, map_aggressor(agside), data->electronic_volume + size);
}
//
// RFQ
void MDP3Listener::onQuoteRequest(
onixs::Handler &,
const onixs::Security &sec,
const onixs::QuoteRequest39RelatedSymArgs &arg) {
tracepoint(
bts_trace, trace_order, (char *)"onQuoteRequest", _latest_sequence,
(uint16_t)_latest_sequence, 0);
auto *data = _index.get_security(sec.id());
if (data != nullptr) {
auto ts = utils::TimeStamp::now();
co2::QuoteRequestMessage message;
message.set_key(data->handle.key());
auto time = ll_of_timestamp(arg.message().transactTime());
message.set_time(time);
onixs::StrRef id = arg.message().quoteReqId();
message.set_request_id(id.items(), id.size());
// https://www.cmegroup.com/confluence/display/EPICSANDBOX/MDP+3.0+-+Request+for+Quote
int8_t side;
if (arg.entry().side(side)) {
switch (side) {
case 1: // buy
case '1':
message.set_side(co2::QuoteRequestMessage::BUY);
break;
case 2: // sell
case '2':
message.set_side(co2::QuoteRequestMessage::SELL);
break;
case 8: // cross
case '8':
message.set_side(co2::QuoteRequestMessage::CROSS);
break;
}
}
message.set_tradable(arg.entry().quoteType() == 1);
int32_t size;
if (arg.entry().orderQty(size)) {
message.set_size(size);
}
data->handle.add_rfq(ts, message);
}
}
//
// Volume
void MDP3Listener::onElectronicVolume(
onixs::Handler &,
const onixs::Security &sec,
const onixs::Volume37Args &args) {
tracepoint(
bts_trace, trace_order, (char *)"onElectronicVolume.volume",
_latest_sequence, (uint16_t)_latest_sequence, 0);
auto *data = _index.get_security(sec.id());
if (data != nullptr) {
bts_assert_d(args.entry().entryType().value() == 'e');
bts_assert_d(
args.entry().updateAction() == onixs::UpdateAction::New ||
args.entry().updateAction() == onixs::UpdateAction::Change);
data->electronic_volume = args.entry().entrySize();
}
}
//
// Non-onixs calls
domain::Book MDP3Listener::to_domain_book(const BookData &entry) const {
domain::Book book = domain::Book::empty(domain::SecurityID(entry.handle.key()));
copy_book_side(entry.bids.first, book.bids());
copy_book_side(entry.asks.first, book.asks());
return book;
}
void MDP3Listener::add_filter_symbol(const std::string &symbol) {
LOG(md, Info, "MDP3Listener::add_filter_symbol symbol: {}", symbol);
_symbol_filter_storage.push_back(symbol);
_symbol_filter.clear();
for (const auto &str : _symbol_filter_storage) {
_symbol_filter.insert(onixs::StrRef(str.c_str(), str.size()));
}
}
void MDP3Listener::heartbeat() {
for (auto &c : _channels) {
c.heartbeat();
}
}
// Called by MDP3Main::prepare_listener as part of pre-load. Also called by
// MDP3Listener::onFuture, onOption and onSpread
void MDP3Listener::add_security(
const FixedSecurity &security, co2::TradingStatus status) {
LOG(md, Info, "MDP3Listener::add_security id: {}", security.numeric_id);
bts_assert(security.numeric_id != 0);
// Set up the shared memory for this security
auto handle = _producer.add_security(security);
// Add it to the index so it can receive messages
_index.add_security(security.numeric_id, handle);
if (_market_data_listener != nullptr) {
domain::Security domain_security(convert_security(security));
std::vector<domain::Security const *> securities{&domain_security};
// GMDL!!
_market_data_listener->on_securities(securities);
}
handle.update_exchange_status(utils::TimeStamp::now(), status);
}
// Private functions
void MDP3Listener::flush_books() {
// This is called at the end of the packet
// Write all book updates to shared memory and reset for the next packet
for (auto &entry : _book_updates) {
uint32_t sequence = _latest_sequence;
if (_market_data_listener != nullptr) {
if (entry.type_flags & book_type_direct) {
// GMDL!!
_market_data_listener->on_book(to_domain_book(entry), _packet_arrived);
}
continue;
}
if (entry.type_flags == book_type_direct) {
entry.handle.update_direct(
_packet_arrived, entry.bids.first, entry.asks.first, sequence);
} else if (entry.type_flags == book_type_implied) {
entry.handle.update_implied(
_packet_arrived, entry.bids.second, entry.asks.second, sequence);
} else if (entry.type_flags == book_type_both) {
entry.handle.update_books(
_packet_arrived, entry.bids.first, entry.asks.first,
entry.bids.second, entry.asks.second, sequence);
}
}
// Reset pending updates
_book_updates.clear();
}
void MDP3Listener::update_all_status(CoconutSecurityStatus status) {
_index.walk_security_list(
[=](SecurityData &data) { data.handle.set_status(status); });
}
void MDP3Listener::update_group_exchange_status(
const onixs::StrRef &group, co2::TradingStatus status) {
_index.walk_security_list([=](SecurityData &data) {
if (data.handle.security().alt_symbol() == group) {
data.handle.update_exchange_status(utils::TimeStamp::now(), status);
}
});
}
void MDP3Listener::update_asset_exchange_status(
const onixs::StrRef &asset, co2::TradingStatus status) {
_index.walk_security_list([=](SecurityData &data) {
if (data.handle.security().symbol() == asset) {
data.handle.update_exchange_status(utils::TimeStamp::now(), status);
}
});
}
md::MDP3Listener::BookData::BookData(MDProducerHandle &handle)
: handle(handle), type_flags(0), bids(), asks() {}
} // namespace md
} // namespace bts
#include "MDP3Listener.h"
#include "spark/config/protos/market_data.pb.h"
#include "spark/md/protos/co2.pb.h"
#include "spark/utils/datetime.h"
#include "spark/utils/helper_macros.h"
#include <vector>
using namespace std;
using namespace bts::co2;
namespace bts {
namespace md {
MDP3Listener::MDP3Listener(
CMESecurityIndex &index,
MDProducer &producer,
MBOWriter *mbo,
const std::vector<MDProducerChannel> channels,
TradeSummarySignalDispatcher *trade_summary_dispatcher)
: _index(index), _producer(producer),
_verbose_ticker(config::get<CMEOptions>().verbose_ticker()),
_packet_arrived(), _latest_sequence(0),
// _settlement_recorder(settles),
_symbol_filter_storage(), _symbol_filter(), _channels(channels),
_concurrency_monitor(0), _book_updates(), _mbo(mbo),
_trade_summary_dispatcher(trade_summary_dispatcher) {
_book_updates.reserve(32);
}
MDP3Listener::~MDP3Listener() {}
//
// Warnings/Errors
void MDP3Listener::onError(onixs::Handler &, const onixs::ErrorArgs &error) {
tracepoint(
bts_trace, trace_order, (char *)"onError", _latest_sequence,
(uint16_t)_latest_sequence, 0);
LOG(md, Warn, "OnixS Error: {} - {}", error.description(), error.source());
}
void MDP3Listener::onWarning(
onixs::Handler &, const onixs::WarningArgs &warning) {
tracepoint(
bts_trace, trace_order, (char *)"onWarning", _latest_sequence,
(uint16_t)_latest_sequence, 0);
LOG(md, Warn, "OnixS Warn: {} - {}", warning.description(),
warning.source());
}
void MDP3Listener::onInstrumentRecovery(onixs::Handler &h) {
tracepoint(
bts_trace, trace_order, (char *)"onInstrumentRecovery", _latest_sequence,
(uint16_t)_latest_sequence, 0);
LOG(md, Info, "Beginning security definition recovery for channel {}",
h.settings().channel());
}
void MDP3Listener::onEndOfInstrumentRecovery(
onixs::Handler &h, const onixs::RecoveryCompletionArgs &) {
tracepoint(
bts_trace, trace_order, (char *)"onEndOfInstrumentRecovery",
_latest_sequence, (uint16_t)_latest_sequence, 0);
LOG(md, Info, "Finished Security definition recovery for channel {}",
h.settings().channel());
}
//
// Gap Recovery
void MDP3Listener::onMarketRecovery(onixs::Handler &h) {
tracepoint(
bts_trace, trace_order, (char *)"onMarketRecovery", _latest_sequence,
(uint16_t)_latest_sequence, 0);
update_all_status(COCONUTSECURITYSTATUS_PACKET_LOSS);
for (auto &c : _channels) {
c.gap();
}
LOG(md, Info, "Beginning market recovery for channel {}",
h.settings().channel());
}
void MDP3Listener::onEndOfMarketRecovery(
onixs::Handler &h, const onixs::RecoveryCompletionArgs &) {
tracepoint(
bts_trace, trace_order, (char *)"onEndOfMarketRecovery",
_latest_sequence, (uint16_t)_latest_sequence, 0);
update_all_status(COCONUTSECURITYSTATUS_OK);
LOG(md, Info, "Finished market recovery for channel {}",
h.settings().channel());
}
void MDP3Listener::onTcpRecovery(onixs::Handler &h) {
tracepoint(
bts_trace, trace_order, (char *)"onTcpRecovery", _latest_sequence,
(uint16_t)_latest_sequence, 0);
update_all_status(COCONUTSECURITYSTATUS_PACKET_LOSS);
LOG(md, Info, "Beginning tcp recovery for channel {}",
h.settings().channel());
}
void MDP3Listener::onEndOfTcpRecovery(
onixs::Handler &h, const onixs::RecoveryCompletionArgs &) {
tracepoint(
bts_trace, trace_order, (char *)"onEndOfTcpRecovery", _latest_sequence,
(uint16_t)_latest_sequence, 0);
update_all_status(COCONUTSECURITYSTATUS_OK);
LOG(md, Info, "Finished tcp recovery for channel {}",
h.settings().channel());
}
//
// Mark the start/end of a packet
void MDP3Listener::onPacket(onixs::Handler &, const onixs::PacketArgs &args) {
// Mark the arrival of the packet
_packet_arrived = TimeStamp::now();
auto seq = args.seqNumber();
if (!_book_updates.empty()) {
if (_latest_sequence != seq) {
LOG(md, Error,
"Expected sequence {} got {} did not flush {} book updates.",
_latest_sequence, seq, _book_updates.size());
}
}
_latest_sequence = seq;
auto onixs_zero = args.receiveTime().sinceEpoch();
auto onixs_now = OnixS::CME::MDH::UtcWatch::now().sinceEpoch();
// needed for clang compilation
UNUSED(onixs_zero);
UNUSED(onixs_now);
// was needed for clang compilation
tracepoint(
bts_trace, trace_order, (char *)"onPacket", _latest_sequence,
(uint16_t)_latest_sequence, onixs_now - onixs_zero);
}
void MDP3Listener::onEndOfPacket(onixs::Handler &, const onixs::PacketArgs &) {
tracepoint(
bts_trace, trace_order, (char *)"onEndOfPacket.begin", _latest_sequence,
(uint16_t)_latest_sequence, 0);
flush_books();
tracepoint(
bts_trace, trace_order, (char *)"onEndOfPacket.end", _latest_sequence,
(uint16_t)_latest_sequence, 0);
}
//
// Book updates
void MDP3Listener::onBookUpdate(
onixs::Handler &,
const onixs::Security &sec,
const onixs::DirectBook &book) {
onBookUpdate<COCONUT_MAX_DEPTH>(sec, book, book_type_direct);
}
void MDP3Listener::onBookUpdate(
onixs::Handler &,
const onixs::Security &sec,
const onixs::ImpliedBook &book) {
onBookUpdate<COCONUT_MAX_IMPLIED_DEPTH>(sec, book, book_type_implied);
}
void MDP3Listener::onBookUpdate(
onixs::Handler &, const onixs::Security &, const onixs::MboBook &) {
#if 0
if (_mbo) {
auto data = _index.get_security(sec.id());
if (data == nullptr) return;
auto key = data->handle.key();
if (key == invalid_security_key) return;
auto* snapshot = _mbo->allocate_snapshot();
// copy from onixs book
for (const auto& bid : book.bids()) {
snapshot->bids.push_back({ ExchangeOrderKey(bid.id()), convert_price(bid.price(), 0), bid.quantity() });
}
for (const auto& ask : book.offers()) {
snapshot->asks.push_back({ ExchangeOrderKey(ask.id()), convert_price(ask.price(),0), ask.quantity() });
}
_mbo->validate_snapshot(sec.id(), SecurityKey(key), snapshot);
}
#endif
}
//
// Trades
// Called from onTradeSummary and onTradeRecovery, ends up calling
// MDProducerHandle::add_trade
void on_trade(
SecurityData *data,
const onixs::Decimal &price,
int size,
onixs::AggressorSide::Enum agside,
const onixs::Timestamp &onixs_date) {
auto ts = utils::TimeStamp::now();
PriceSize ps = pricesize_of_price_size(
price, data->handle.security().display_factor, size);
auto date = ll_of_timestamp(onixs_date);
data->handle.add_trade(
ts, ps, date, map_aggressor(agside), data->electronic_volume + size);
}
//
// RFQ
void MDP3Listener::onQuoteRequest(
onixs::Handler &,
const onixs::Security &sec,
const onixs::QuoteRequest39RelatedSymArgs &arg) {
tracepoint(
bts_trace, trace_order, (char *)"onQuoteRequest", _latest_sequence,
(uint16_t)_latest_sequence, 0);
auto *data = _index.get_security(sec.id());
if (data != nullptr) {
auto ts = utils::TimeStamp::now();
co2::QuoteRequestMessage message;
message.set_key(data->handle.key());
auto time = ll_of_timestamp(arg.message().transactTime());
message.set_time(time);
onixs::StrRef id = arg.message().quoteReqId();
message.set_request_id(id.items(), id.size());
// https://www.cmegroup.com/confluence/display/EPICSANDBOX/MDP+3.0+-+Request+for+Quote
int8_t side;
if (arg.entry().side(side)) {
switch (side) {
case 1: // buy
case '1':
message.set_side(co2::QuoteRequestMessage::BUY);
break;
case 2: // sell
case '2':
message.set_side(co2::QuoteRequestMessage::SELL);
break;
case 8: // cross
case '8':
message.set_side(co2::QuoteRequestMessage::CROSS);
break;
}
}
message.set_tradable(arg.entry().quoteType() == 1);
int32_t size;
if (arg.entry().orderQty(size)) {
message.set_size(size);
}
data->handle.add_rfq(ts, message);
}
}
//
// Volume
void MDP3Listener::onElectronicVolume(
onixs::Handler &,
const onixs::Security &sec,
const onixs::Volume37Args &args) {
tracepoint(
bts_trace, trace_order, (char *)"onElectronicVolume.volume",
_latest_sequence, (uint16_t)_latest_sequence, 0);
auto *data = _index.get_security(sec.id());
if (data != nullptr) {
bts_assert_d(args.entry().entryType().value() == 'e');
bts_assert_d(
args.entry().updateAction() == onixs::UpdateAction::New ||
args.entry().updateAction() == onixs::UpdateAction::Change);
data->electronic_volume = args.entry().entrySize();
}
}
//
// Non-onixs calls
void MDP3Listener::add_filter_symbol(const std::string &symbol) {
LOG(md, Info, "MDP3Listener::add_filter_symbol symbol: {}", symbol);
_symbol_filter_storage.push_back(symbol);
_symbol_filter.clear();
for (const auto &str : _symbol_filter_storage) {
_symbol_filter.insert(onixs::StrRef(str.c_str(), str.size()));
}
}
void MDP3Listener::heartbeat() {
for (auto &c : _channels) {
c.heartbeat();
}
}
// Called by MDP3Main::prepare_listener as part of pre-load. Also called by
// MDP3Listener::onFuture, onOption and onSpread
void MDP3Listener::add_security(
const FixedSecurity &security, co2::TradingStatus status) {
bts_assert(security.numeric_id != 0);
// Set up the shared memory for this security
auto handle = _producer.add_security(security);
// Add it to the index so it can receive messages
_index.add_security(security.numeric_id, handle);
handle.update_exchange_status(utils::TimeStamp::now(), status);
}
// Private functions
void MDP3Listener::flush_books() {
// This is called at the end of the packet
// Write all book updates to shared memory and reset for the next packet
for (auto &entry : _book_updates) {
uint32_t sequence = _latest_sequence;
if (entry.type_flags == book_type_direct) {
entry.handle.update_direct(
_packet_arrived, entry.bids.first, entry.asks.first, sequence);
} else if (entry.type_flags == book_type_implied) {
entry.handle.update_implied(
_packet_arrived, entry.bids.second, entry.asks.second, sequence);
} else if (entry.type_flags == book_type_both) {
entry.handle.update_books(
_packet_arrived, entry.bids.first, entry.asks.first,
entry.bids.second, entry.asks.second, sequence);
}
}
// Reset pending updates
_book_updates.clear();
}
void MDP3Listener::update_all_status(CoconutSecurityStatus status) {
_index.walk_security_list(
[=](SecurityData &data) { data.handle.set_status(status); });
}
void MDP3Listener::update_group_exchange_status(
const onixs::StrRef &group, co2::TradingStatus status) {
_index.walk_security_list([=](SecurityData &data) {
if (data.handle.security().alt_symbol() == group) {
data.handle.update_exchange_status(utils::TimeStamp::now(), status);
}
});
}
void MDP3Listener::update_asset_exchange_status(
const onixs::StrRef &asset, co2::TradingStatus status) {
_index.walk_security_list([=](SecurityData &data) {
if (data.handle.security().symbol() == asset) {
data.handle.update_exchange_status(utils::TimeStamp::now(), status);
}
});
}
md::MDP3Listener::BookData::BookData(MDProducerHandle &handle)
: handle(handle), type_flags(0), bids(), asks() {}
} // namespace md
} // namespace bts
File diff suppressed because it is too large. Load diff
@@ -1,5 +1,4 @@
#include "MDP3Main.h"
#include "ScriptedMdp3Feeder.h"
#include "spark/config/config.h"
#include "spark/config/protos/market_data.pb.h"
#include "spark/config/protos/system_config.pb.h"
@@ -17,8 +16,6 @@
#include "sparkd/md/onixs/OnixsUtils.h"
#include <boost/filesystem.hpp>
#include <thread>
#include <chrono>
#include <memory>
#include <set>
#include <sstream>
@@ -43,8 +40,6 @@ struct MDP3Main::PImpl {
std::map<std::string, std::unique_ptr<onixs::Handler>> handlers;
std::vector<std::unique_ptr<MDP3Listener>> listeners;
std::vector<std::unique_ptr<onixs::TcpRecoveryService>> recovery;
std::atomic<bool> scripted_running{false};
std::thread scripted_thread;
PImpl()
: index(), recorder(
nullptr,
@@ -58,12 +53,7 @@ struct MDP3Main::PImpl {
MDP3Main::MDP3Main()
: me_(std::make_unique<PImpl>()) {}
MDP3Main::~MDP3Main() {
me_->scripted_running = false;
if (me_->scripted_thread.joinable()) {
me_->scripted_thread.join();
}
}
MDP3Main::~MDP3Main() = default;
const CmeChannelConfig channel_settings_for(const string &id) {
const auto &config = config::get<CMEOptions>();
@@ -188,8 +178,7 @@ onixs::SessionSettings session_settings_for_channel(
void MDP3Main::start(
const std::vector<UserSymbol> &symbols,
MBOWriter *mbo,
TradeSummarySignalDispatcher *tssd,
std::shared_ptr<IMarketDataListener> market_data_listener) {
TradeSummarySignalDispatcher *tssd) {
std::ostringstream oss;
for (const auto &us : symbols) {
oss << us << " ";
@@ -202,8 +191,7 @@ void MDP3Main::start(
me_->engines = create_feed_engines(channels);
for (const auto &channel : channels) {
unique_ptr<onixs::Handler> phandler = prepare_handler(channel.id, mbo);
me_->listeners.push_back(prepare_listener(
*phandler, channel, mbo, tssd, market_data_listener));
me_->listeners.push_back(prepare_listener(*phandler, channel, mbo, tssd));
me_->handlers.insert(std::make_pair(channel.id, std::move(phandler)));
std::stringstream symstr;
for (const auto &sym : channel.symbols) {
@@ -222,98 +210,6 @@ void MDP3Main::start(
LOG(md, Info, "** CME MDP3 Publisher started **");
}
void MDP3Main::start_scripted(
const std::vector<UserSymbol> &symbols,
MBOWriter *mbo,
TradeSummarySignalDispatcher *tssd,
std::shared_ptr<IMarketDataListener> market_data_listener) {
std::ostringstream oss;
for (const auto &us : symbols) {
oss << us << " ";
}
LOG(md, Info, "MDP3Main::start_scripted symbols: {}", oss.str());
const auto &channels = parse_channels(symbols);
if (channels.empty()) {
LOG(md, Warn, "MDP3Main::start_scripted found no CME channels");
return;
}
auto channel_it = std::find_if(
channels.begin(), channels.end(), [](const CMEChannel &channel) {
return std::any_of(
channel.symbols.begin(), channel.symbols.end(),
[](const UserSymbol &symbol) {
return symbol.symbol() == "ES" &&
symbol.type() == api::SECURITYTYPE_FUTURE;
});
});
if (channel_it == channels.end()) {
channel_it = channels.begin();
LOG(md, Warn,
"MDP3Main::start_scripted did not find an ES future channel; using channel {}",
channel_it->id);
}
const auto &channel = *channel_it;
std::vector<MDProducerChannel> producer_channels;
for (const auto &symbol : channel.symbols) {
producer_channels.push_back(me_->producer.get_channel(
symbol.symbol().c_str(), symbol.type(), symbol.exchange()));
}
auto listener = std::make_unique<MDP3Listener>(
me_->index, me_->producer, mbo, producer_channels, tssd,
market_data_listener);
for (const auto &symbol : channel.symbols) {
listener->add_filter_symbol(symbol.symbol());
}
ScriptedMdp3Feeder feeder(*listener);
feeder.publish_hard_coded_values();
me_->scripted_running = false;
if (me_->scripted_thread.joinable()) {
me_->scripted_thread.join();
}
me_->scripted_running = true;
auto *listener_ptr = listener.get();
me_->scripted_thread = std::thread([this, listener_ptr]() {
ScriptedMdp3Feeder loop_feeder(*listener_ptr);
std::uint32_t sequence = 2;
int tick_offset = 0;
int tick_direction = -1;
while (me_->scripted_running.load()) {
std::this_thread::sleep_for(std::chrono::seconds(1));
if (!me_->scripted_running.load()) {
break;
}
tick_offset += tick_direction;
if (tick_offset <= -10) {
tick_offset = -10;
tick_direction = 1;
} else if (tick_offset >= 0) {
tick_offset = 0;
tick_direction = -1;
}
loop_feeder.publish_hard_coded_direct_book(
sequence++, 0.25 * tick_offset);
}
});
std::stringstream symstr;
for (const auto &sym : channel.symbols) {
symstr << sym.symbol() << "-" << SecurityType_Name(sym.type()) << " ";
}
LOG(md, Info, "CME: Initialized scripted channel {} ({})", channel.id,
symstr.str());
me_->listeners.push_back(std::move(listener));
LOG(md, Info, "** CME MDP3 Scripted Publisher started **");
}
void MDP3Main::send_heartbeats() {
for (const auto &listener : me_->listeners) {
listener->heartbeat();
@@ -490,8 +386,7 @@ unique_ptr<MDP3Listener> MDP3Main::prepare_listener(
onixs::Handler &handler,
const CMEChannel &channel,
MBOWriter *mbo,
TradeSummarySignalDispatcher *trade_summary_dispatcher,
std::shared_ptr<IMarketDataListener> market_data_listener) {
TradeSummarySignalDispatcher *trade_summary_dispatcher) {
LOG(md, Info, "MDP3Main::prepare_listener channel: {}", channel);
// We need to get a handler to the producer channels so we can produce
// heartbeats
@@ -503,7 +398,7 @@ unique_ptr<MDP3Listener> MDP3Main::prepare_listener(
unique_ptr<MDP3Listener> listener(new MDP3Listener(
me_->index, me_->producer, mbo, producer_channels,
trade_summary_dispatcher, std::move(market_data_listener)));
trade_summary_dispatcher));
for (auto &symbol : channel.symbols) {
listener->add_filter_symbol(symbol.symbol());
}
@@ -2,16 +2,13 @@
#include "CMESecurityIndex.h"
#include "MDP3Listener.h"
#include "spark/md/IMarketDataListener.h"
#include "spark/md/coconut2.h"
#include "spark/utils/bts_no_copy.h"
#include "onixs.h"
#include "sparkd/md/platforms/IMDPlatform.h"
#include <atomic>
#include <map>
#include <memory>
#include <thread>
#include <vector>
namespace bts {
@@ -40,13 +37,7 @@ class MDP3Main : public IMDPlatform {
void start(
const std::vector<UserSymbol> &symbols,
MBOWriter *mbo,
TradeSummarySignalDispatcher *tssd,
std::shared_ptr<IMarketDataListener> market_data_listener = nullptr);
void start_scripted(
const std::vector<UserSymbol> &symbols,
MBOWriter *mbo,
TradeSummarySignalDispatcher *tssd,
std::shared_ptr<IMarketDataListener> market_data_listener = nullptr);
TradeSummarySignalDispatcher *tssd);
void send_heartbeats() override;
void initialize(const std::string &license_store);
@@ -56,8 +47,7 @@ class MDP3Main : public IMDPlatform {
onixs::Handler &handler,
const CMEChannel &channel,
MBOWriter *,
TradeSummarySignalDispatcher *,
std::shared_ptr<IMarketDataListener> market_data_listener = nullptr);
TradeSummarySignalDispatcher *);
private:
struct PImpl;
@@ -1,188 +0,0 @@
#include "ScriptedMdp3Feeder.h"
#include "spark/api/domain/Venue.pb.h"
#include "spark/api/domain/SecurityType.pb.h"
#include "spark/md/IntPrice.h"
#include "spark/md/common.h"
#include <OnixS/CME/MDH/DirectBook.h>
#include <OnixS/CME/MDH/Handler.h>
#include <OnixS/CME/MDH/HandlerSettings.h>
#include <OnixS/CME/MDH/Packet.h>
#include <OnixS/CME/MDH/Security.h>
#include <cmath>
#include <memory>
namespace bts::md {
namespace {
constexpr std::uint32_t scripted_exchange_security_id = 1000001;
class ScriptedSecurity : public onixs::Security {
public:
explicit ScriptedSecurity(uint32_t id) : onixs::Security(id) {}
using onixs::Security::asset;
using onixs::Security::group;
using onixs::Security::symbol;
void sequence(std::uint32_t value) { seqNumber(value); }
};
struct ScriptedPacketHeader {
onixs::SequenceNumber seqNumber = 0;
onixs::Timestamp sendingTime;
};
PriceSize make_level(double price, float size) {
if (size == 0) {
return PriceSize();
}
return PriceSize(IntPrice_from_double<7>(price), size);
}
onixs::Decimal to_onixs_decimal(const PriceSize &level) {
// MDP3Listener applies the security display_factor when converting OnixS
// prices, so the scripted price must be expressed in pre-display-factor
// exchange units here. For ES with display_factor=-2, 7000.00 becomes
// mantissa 700000 with exponent 0.
return onixs::Decimal(
static_cast<onixs::Decimal::Mantissa>(std::llround(level.price() * 100.0)),
0);
}
using DirectBookPtr = std::unique_ptr<onixs::DirectBook, void (*)(onixs::DirectBook *)>;
DirectBookPtr make_direct_book(const ScriptedMdp3Feeder::DirectBookUpdate &update) {
DirectBookPtr book(
onixs::DirectBookFactory::construct(COCONUT_MAX_DEPTH),
&onixs::DirectBookFactory::destruct);
std::size_t bid_depth = 0;
std::size_t ask_depth = 0;
while (bid_depth < update.bids.size() && !update.bids[bid_depth].is_nan() &&
update.bids[bid_depth].size() != 0) {
++bid_depth;
}
while (ask_depth < update.asks.size() && !update.asks[ask_depth].is_nan() &&
update.asks[ask_depth].size() != 0) {
++ask_depth;
}
book->depth(static_cast<onixs::MbpBookDepth>(std::max(bid_depth, ask_depth)));
book->bids().resize(static_cast<onixs::MbpBookDepth>(bid_depth));
book->offers().resize(static_cast<onixs::MbpBookDepth>(ask_depth));
for (std::size_t i = 0; i < bid_depth; ++i) {
book->bids()[i] = onixs::DirectPriceLevel(
to_onixs_decimal(update.bids[i]),
static_cast<int>(update.bids[i].size()),
1);
}
for (std::size_t i = 0; i < ask_depth; ++i) {
book->offers()[i] = onixs::DirectPriceLevel(
to_onixs_decimal(update.asks[i]),
static_cast<int>(update.asks[i].size()),
1);
}
return book;
}
onixs::Handler &scripted_handler() {
static onixs::HandlerSettings settings;
static bool initialized = false;
if (!initialized) {
settings.channel(1);
initialized = true;
}
static onixs::Handler handler(settings);
return handler;
}
ScriptedSecurity &scripted_security() {
static ScriptedSecurity security(scripted_exchange_security_id);
static bool initialized = false;
if (!initialized) {
security.symbol(onixs::StrRef("ESM6", 4));
security.group(onixs::StrRef("ES", 2));
security.asset(onixs::StrRef("ES", 2));
initialized = true;
}
return security;
}
} // namespace
ScriptedMdp3Feeder::ScriptedMdp3Feeder(MDP3Listener &listener)
: _listener(listener) {}
void ScriptedMdp3Feeder::publish_hard_coded_future() const {
FixedFutureSecurity security;
fill_hard_coded_future(security);
_listener.add_security(security, co2::TRADINGSTATUS_NORMAL);
}
void ScriptedMdp3Feeder::publish_hard_coded_direct_book() const {
publish_hard_coded_direct_book(1, 0.0);
}
void ScriptedMdp3Feeder::publish_hard_coded_direct_book(
std::uint32_t sequence, double mid_price_shift) const {
const auto update = hard_coded_direct_book(sequence, mid_price_shift);
auto book = make_direct_book(update);
auto &handler = scripted_handler();
auto &security = scripted_security();
security.sequence(sequence);
ScriptedPacketHeader packet_header;
packet_header.seqNumber = sequence;
onixs::PacketArgs packet(&packet_header, sizeof(packet_header));
packet.receiveTime(onixs::Timestamp(update.packet_time.to_uint64()));
_listener.onPacket(handler, packet);
_listener.onBookUpdate(handler, security, *book);
_listener.onEndOfPacket(handler, packet);
}
void ScriptedMdp3Feeder::publish_hard_coded_values() const {
publish_hard_coded_future();
publish_hard_coded_direct_book();
}
void ScriptedMdp3Feeder::fill_hard_coded_future(FixedFutureSecurity &security) {
security.key = invalid_security_key;
security.numeric_id = scripted_exchange_security_id;
security.type = api::SECURITYTYPE_FUTURE;
security.exchange = api::Venue::VENUE_GLBX;
security.symbol("ES");
security.alt_symbol("ES");
security.security_desc("ESM6");
security.currency("USD");
security.display_factor = -2;
security.low_tick = 25.0;
security.high_tick = 25.0;
security.price_break = 0.0;
security.cash_per_point = 50.0;
security.expiration_year = 2026;
security.expiration_month = 6;
security.expiration_day = 19;
}
ScriptedMdp3Feeder::DirectBookUpdate
ScriptedMdp3Feeder::hard_coded_direct_book(
std::uint32_t sequence, double mid_price_shift) {
DirectBookUpdate update;
const double best_bid = 7000.0 + mid_price_shift;
const double best_ask = 7001.0 + mid_price_shift;
update.exchange_security_id = scripted_exchange_security_id;
update.bids[0] = make_level(best_bid, 10);
update.bids[1] = make_level(best_bid - 0.25, 10);
update.bids[2] = make_level(best_bid - 0.50, 10);
update.asks[0] = make_level(best_ask, 10);
update.asks[1] = make_level(best_ask + 0.25, 10);
update.asks[2] = make_level(best_ask + 0.50, 10);
update.packet_time = utils::TimeStamp::now();
update.sequence = sequence;
return update;
}
} // namespace bts::md
@@ -1,39 +0,0 @@
#pragma once
#include "MDP3Listener.h"
#include "spark/md/PriceSize.h"
#include "spark/utils/TimeStamp.h"
#include <array>
#include <cstdint>
namespace bts::md {
class ScriptedMdp3Feeder {
public:
struct DirectBookUpdate {
uint32_t exchange_security_id = 0;
std::array<PriceSize, COCONUT_MAX_DEPTH> bids = {};
std::array<PriceSize, COCONUT_MAX_DEPTH> asks = {};
utils::TimeStamp packet_time = utils::TimeStamp::now();
uint32_t sequence = 1;
};
explicit ScriptedMdp3Feeder(MDP3Listener &listener);
void publish_hard_coded_future() const;
void publish_hard_coded_direct_book() const;
void publish_hard_coded_direct_book(std::uint32_t sequence,
double mid_price_shift) const;
void publish_hard_coded_values() const;
static void fill_hard_coded_future(FixedFutureSecurity &security);
static DirectBookUpdate hard_coded_direct_book(
std::uint32_t sequence = 1, double mid_price_shift = 0.0);
private:
MDP3Listener &_listener;
};
} // namespace bts::md
@@ -0,0 +1,816 @@
#include "sparkd/md/rakurai/RakuraiDexDataHandler.h"
#include "spark/api/domain/SecurityType.pb.h"
#include "spark/domain/Book.h"
#include "spark/domain/Price.h"
#include "spark/domain/Quantity.h"
#include "spark/domain/Security.h"
#include "spark/logging/logging.h"
#include "spark/persistence/SqliteStore.h"
#include "spark/utils/PBUtils.h"
#include "sparkd/md/MarketDataError.h"
#include <dex_handler.h>
#include <dex_listener.h>
#include <dex_utils.h>
#include <rapidjson/document.h>
#include <rapidjson/istreamwrapper.h>
#include <algorithm>
#include <array>
#include <cstdint>
#include <csignal>
#include <execinfo.h>
#include <fstream>
#include <iomanip>
#include <list>
#include <memory>
#include <mutex>
#include <optional>
#include <sstream>
#include <stdexcept>
#include <string>
#include <string_view>
#include <unordered_map>
#include <unordered_set>
#include <unistd.h>
#include <vector>
#include <arpa/inet.h>
#include <netdb.h>
#include <sys/socket.h>
#include <sys/types.h>
namespace bts::md::rakurai {
namespace {
using ConnectorVenue = ::rakurai::dex_connector::data_types::venue;
using ConnectorSecurity = ::rakurai::dex_connector::data_types::security;
using ConnectorBookLevel = ::rakurai::dex_connector::data_types::book_level;
using ConnectorDexConfig = ::rakurai::dex_connector::configuration::dex_config;
struct ParsedWsUri {
std::string scheme;
std::string host;
std::string port;
std::string path;
};
struct WsProbeResult {
bool tcp_connected = false;
bool sent_request = false;
bool received_response = false;
std::string summary;
};
std::optional<ParsedWsUri> parse_ws_uri(const std::string& uri) {
const auto scheme_pos = uri.find("://");
if (scheme_pos == std::string::npos) {
return std::nullopt;
}
ParsedWsUri parsed;
parsed.scheme = uri.substr(0, scheme_pos);
auto remainder = uri.substr(scheme_pos + 3);
const auto path_pos = remainder.find('/');
if (path_pos == std::string::npos) {
parsed.path = "/";
} else {
parsed.path = remainder.substr(path_pos);
remainder = remainder.substr(0, path_pos);
}
const auto colon_pos = remainder.rfind(':');
if (colon_pos == std::string::npos) {
parsed.host = remainder;
parsed.port = parsed.scheme == "wss" ? "443" : "80";
} else {
parsed.host = remainder.substr(0, colon_pos);
parsed.port = remainder.substr(colon_pos + 1);
}
if (parsed.host.empty() || parsed.port.empty()) {
return std::nullopt;
}
return parsed;
}
WsProbeResult probe_ws_endpoint(const std::string& uri) {
WsProbeResult result;
const auto parsed = parse_ws_uri(uri);
if (!parsed.has_value()) {
result.summary = "invalid websocket URI";
return result;
}
if (parsed->scheme != "ws") {
result.summary = "probe only supports ws:// URIs";
return result;
}
struct addrinfo hints {};
hints.ai_family = AF_UNSPEC;
hints.ai_socktype = SOCK_STREAM;
struct addrinfo* addr_list = nullptr;
const int gai_rc = ::getaddrinfo(
parsed->host.c_str(), parsed->port.c_str(), &hints, &addr_list);
if (gai_rc != 0) {
result.summary =
std::string("DNS resolution failed: ") + ::gai_strerror(gai_rc);
return result;
}
int fd = -1;
for (auto* ai = addr_list; ai != nullptr; ai = ai->ai_next) {
fd = ::socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol);
if (fd < 0) {
continue;
}
timeval timeout {};
timeout.tv_sec = 3;
timeout.tv_usec = 0;
::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout));
::setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(timeout));
if (::connect(fd, ai->ai_addr, ai->ai_addrlen) == 0) {
result.tcp_connected = true;
break;
}
::close(fd);
fd = -1;
}
::freeaddrinfo(addr_list);
if (fd < 0) {
result.summary = "TCP connect failed";
return result;
}
const std::string request =
"GET " + parsed->path + " HTTP/1.1\r\n" +
"Host: " + parsed->host + ":" + parsed->port + "\r\n" +
"Upgrade: websocket\r\n"
"Connection: Upgrade\r\n"
"Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n"
"Sec-WebSocket-Version: 13\r\n\r\n";
const ssize_t sent = ::send(fd, request.data(), request.size(), 0);
if (sent == static_cast<ssize_t>(request.size())) {
result.sent_request = true;
} else {
result.summary = "failed to send websocket probe request";
::close(fd);
return result;
}
std::array<char, 1024> buffer {};
const ssize_t received = ::recv(fd, buffer.data(), buffer.size() - 1, 0);
::close(fd);
if (received <= 0) {
result.summary = "no HTTP response received";
return result;
}
result.received_response = true;
std::string response(buffer.data(), static_cast<std::size_t>(received));
const auto line_end = response.find("\r\n");
if (line_end != std::string::npos) {
response.resize(line_end);
}
result.summary = response;
return result;
}
void validate_ws_endpoint_or_throw(
const std::string& label, const std::string& uri) {
const auto probe = probe_ws_endpoint(uri);
LOG(md, Info,
"Rakurai websocket probe. endpoint:{} uri:{} tcp_connected:{} sent_request:{} received_response:{} summary:{}",
label,
uri,
probe.tcp_connected,
probe.sent_request,
probe.received_response,
probe.summary);
if (!probe.tcp_connected || !probe.sent_request || !probe.received_response) {
throw std::runtime_error(
std::string("Rakurai websocket endpoint probe failed for ") + label +
" (" + uri + "): " + probe.summary);
}
if (probe.summary.rfind("HTTP/1.1 101", 0) != 0 &&
probe.summary.rfind("HTTP/1.0 101", 0) != 0) {
throw std::runtime_error(
std::string("Rakurai websocket endpoint returned unexpected handshake for ") +
label + " (" + uri + "): " + probe.summary);
}
}
void write_crash_text(std::string_view text) {
if(::write(STDERR_FILENO, text.data(), text.size()) ==-1) {
// If write fails, there's not much we can do, but we don't want to get
// stuck in a loop trying to write. So we just ignore the error.
};
}
void rakurai_crash_signal_handler(int signo, siginfo_t* info, void* /*ucontext*/) {
constexpr int max_frames = 64;
void* frames[max_frames];
const int frame_count = ::backtrace(frames, max_frames);
write_crash_text("\n=== Rakurai crash signal caught ===\n");
if (signo == SIGSEGV) {
write_crash_text("signal: SIGSEGV\n");
} else if (signo == SIGBUS) {
write_crash_text("signal: SIGBUS\n");
} else {
write_crash_text("signal: unknown\n");
}
if (info != nullptr) {
if (info->si_addr != nullptr) {
write_crash_text("fault address captured\n");
}
if (info->si_code != 0) {
write_crash_text("signal code captured\n");
}
}
write_crash_text("backtrace:\n");
::backtrace_symbols_fd(frames, frame_count, STDERR_FILENO);
_Exit(128 + signo);
}
void install_rakurai_crash_signal_handlers() {
static std::once_flag once;
std::call_once(once, []() {
struct sigaction sa {};
sa.sa_flags = SA_SIGINFO | SA_RESETHAND;
sa.sa_sigaction = rakurai_crash_signal_handler;
sigemptyset(&sa.sa_mask);
sigaction(SIGSEGV, &sa, nullptr);
sigaction(SIGBUS, &sa, nullptr);
});
}
std::string bytes_to_hex(const std::array<std::uint8_t, 32>& bytes) {
std::ostringstream oss;
oss << std::hex << std::setfill('0');
for (std::uint8_t byte : bytes) {
oss << std::setw(2) << static_cast<int>(byte);
}
return oss.str();
}
std::string short_hex(const std::array<std::uint8_t, 32>& bytes) {
const auto hex = bytes_to_hex(bytes);
return hex.substr(0, std::min<std::size_t>(hex.size(), 12));
}
const char* connector_venue_name(ConnectorVenue venue) {
switch (venue) {
case ConnectorVenue::orca_clmm:
return "orca_clmm";
case ConnectorVenue::orca_amm:
return "orca_amm";
case ConnectorVenue::raydium_amm:
return "raydium_amm";
case ConnectorVenue::raydium_cpmm:
return "raydium_cpmm";
case ConnectorVenue::raydium_clmm:
return "raydium_clmm";
// case ConnectorVenue::meterora_dlmm:
// return "meterora_dlmm";
default:
return "invalid";
}
}
bool venue_matches(api::domain::Venue spark_venue, ConnectorVenue connector_venue) {
switch (spark_venue) {
case api::domain::VENUE_ORCA:
return connector_venue == ConnectorVenue::orca_clmm ||
connector_venue == ConnectorVenue::orca_amm;
case api::domain::VENUE_RAYDIUM:
return connector_venue == ConnectorVenue::raydium_amm ||
connector_venue == ConnectorVenue::raydium_cpmm ||
connector_venue == ConnectorVenue::raydium_clmm;
default:
return false;
}
}
std::string token_display_name(const ::rakurai::dex_connector::data_types::token_info& token) {
return token.name.empty() ? short_hex(token.mint_address) : token.name;
}
std::string build_security_desc(
const ConnectorSecurity& security, std::string_view short_pool_id) {
return fmt::format(
"{}-{}-{}",
token_display_name(security.token_a),
token_display_name(security.token_b),
short_pool_id);
}
double quantity_scale_from_decimals(std::uint8_t decimals) {
double scale = 1.0;
for (std::uint8_t i = 0; i < decimals; ++i) {
scale *= 10.0;
}
return scale;
}
domain::BookLevel to_book_level(
const ConnectorBookLevel& source,
double quantity_scale) {
domain::BookLevel level;
level.price = domain::Price::from_double(source.price);
const auto normalized_quantity =
quantity_scale > 0.0
? static_cast<double>(source.quantity) / quantity_scale
: static_cast<double>(source.quantity);
level.total_quantity =
domain::Quantity(static_cast<float>(normalized_quantity));
level.direct_quantity = level.total_quantity;
level.n_orders = 0;
return level;
}
std::list<ConnectorDexConfig> parse_dex_configs(
const rapidjson::Document& config,
api::domain::Venue venue_filter) {
std::list<ConnectorDexConfig> dex_config_list;
if (!config.HasMember("dex") || !config["dex"].IsArray()) {
return dex_config_list;
}
for (const auto& dex_value : config["dex"].GetArray()) {
if (!dex_value.IsObject() || !dex_value.HasMember("id")) {
continue;
}
ConnectorDexConfig dex_config;
const std::string dex = dex_value["id"].GetString();
if (dex == "orca_clmm") {
if (venue_filter != api::domain::VENUE_ORCA) {
continue;
}
dex_config.dex_name = ConnectorVenue::orca_clmm;
dex_config.pool_typ =
::rakurai::dex_connector::data_types::pool_type::clmm;
} else if (dex == "orca_amm") {
if (venue_filter != api::domain::VENUE_ORCA) {
continue;
}
dex_config.dex_name = ConnectorVenue::orca_amm;
dex_config.pool_typ =
::rakurai::dex_connector::data_types::pool_type::amm;
} else if (dex == "raydium_amm") {
if (venue_filter != api::domain::VENUE_RAYDIUM) {
continue;
}
dex_config.dex_name = ConnectorVenue::raydium_amm;
dex_config.pool_typ =
::rakurai::dex_connector::data_types::pool_type::amm;
} else if (dex == "raydium_cpmm") {
if (venue_filter != api::domain::VENUE_RAYDIUM) {
continue;
}
dex_config.dex_name = ConnectorVenue::raydium_cpmm;
dex_config.pool_typ =
::rakurai::dex_connector::data_types::pool_type::cpmm;
} else if (dex == "raydium_clmm") {
if (venue_filter != api::domain::VENUE_RAYDIUM) {
continue;
}
dex_config.dex_name = ConnectorVenue::raydium_clmm;
dex_config.pool_typ =
::rakurai::dex_connector::data_types::pool_type::clmm;
} else {
LOG(md, Warn, "Skipping unsupported Rakurai dex config '{}'", dex);
continue;
}
if (dex_value.HasMember("pool_ids") && dex_value["pool_ids"].IsArray()) {
for (const auto& pool_value : dex_value["pool_ids"].GetArray()) {
dex_config.interested_pools.emplace_back(
::rakurai::dex_connector::solanaAddressToBytes(
pool_value.GetString()));
}
}
dex_config_list.push_back(dex_config);
}
return dex_config_list;
}
class ListenerAdapter final : public ::rakurai::dex_connector::dex_listener {
public:
ListenerAdapter(
std::shared_ptr<IMarketDataListener> listener,
std::shared_ptr<persistence::IDGenerator> sec_id_store,
api::domain::Venue venue,
std::uint32_t num_depth_levels,
const std::vector<std::string>& excluded_securities)
: listener_(std::move(listener)),
sec_id_store_(std::move(sec_id_store)),
venue_(venue),
depth_limit_(
num_depth_levels == 0
? domain::Book::max_depth()
: std::min<std::size_t>(
num_depth_levels, domain::Book::max_depth())),
excluded_securities_(
excluded_securities.begin(), excluded_securities.end()) {}
std::string get_or_add_short_pool_id(const std::string& pool_id) {
constexpr std::string_view kPoolAliasNamespace = "rakurai_pool_alias";
const auto [exists, rowid, _] =
pool_alias_store_.retrieve_one(std::string(kPoolAliasNamespace), pool_id);
if (exists) {
return std::to_string(rowid);
}
const auto new_id =
pool_alias_store_.insert(std::string(kPoolAliasNamespace), pool_id, "");
return std::to_string(new_id);
}
void on_pool_definition(
ConnectorVenue connector_venue,
const std::vector<ConnectorSecurity*>& securities) override {
std::lock_guard<std::mutex> lock(callback_mutex_);
LOG(md, Info,
"Rakurai listener on_pool_definition start. venue:{} connector_venue:{} securities:{}",
int(venue_),
std::string(connector_venue_name(connector_venue)),
securities.size());
if (!venue_matches(venue_, connector_venue)) {
LOG(md, Info,
"Rakurai listener on_pool_definition skipped. venue:{} connector_venue:{}",
int(venue_),
std::string(connector_venue_name(connector_venue)));
return;
}
std::vector<const domain::Security*> emitted;
emitted.reserve(securities.size());
for (const ConnectorSecurity* security : securities) {
if (security == nullptr || !venue_matches(venue_, security->venue)) {
continue;
}
const auto pool_id = bytes_to_hex(security->pool_id);
const auto short_pool_id = get_or_add_short_pool_id(pool_id);
const auto desc = build_security_desc(*security, short_pool_id);
const auto symbol = token_display_name(security->token_a);
// const auto token_a_cpp = security->token_a.decimals;
const auto currency = token_display_name(security->token_b);
if (excluded_securities_.contains(pool_id) ||
excluded_securities_.contains(short_pool_id) ||
excluded_securities_.contains(desc)) {
continue;
}
domain::Security emitted_security;
emitted_security.security = domain::SpotSecurity{};
emitted_security.security_common.venue = venue_;
emitted_security.security_common.exchange_sec_id = short_pool_id;
emitted_security.security_common.symbol = symbol;
emitted_security.security_common.currency = currency;
emitted_security.security_common.asset = symbol;
emitted_security.security_common.group =
connector_venue_name(security->venue);
emitted_security.security_common.desc = desc;
emitted_security.security_common.min_price_increment =
security->min_tick > 0.0 ? security->min_tick : 0.0;
emitted_security.security_common.min_quantity_increment =
static_cast<double>(security->min_trade_amount);
emitted_security.security_common.min_quantity =
static_cast<double>(security->min_trade_amount);
*emitted_security.security_common.id = sec_id_store_->get_or_add(
venue_, api::domain::SECURITY_TYPE_SPOT, pool_id);
auto [it, is_new] = securities_.try_emplace(
pool_id, std::make_unique<domain::Security>(std::move(emitted_security)));
if (!is_new) {
*it->second = std::move(emitted_security);
}
pool_to_security_id_[pool_id] = it->second->id();
pool_quantity_scales_[pool_id] =
quantity_scale_from_decimals(security->token_a.decimals);
emitted.push_back(it->second.get());
}
if (emitted.empty()) {
LOG(md, Info,
"Rakurai listener on_pool_definition produced no securities. venue:{} connector_venue:{}",
int(venue_),
std::string(connector_venue_name(connector_venue)));
return;
}
LOG(md, Info,
"Rakurai listener on_pool_definition emitting securities. venue:{} connector_venue:{} emitted:{} tracked:{}",
int(venue_),
std::string(connector_venue_name(connector_venue)),
emitted.size(),
securities_.size());
listener_->on_securities(emitted);
LOG(md, Info,
"Rakurai listener on_pool_definition complete. venue:{} connector_venue:{} emitted:{}",
int(venue_),
std::string(connector_venue_name(connector_venue)),
emitted.size());
}
void on_book(
ConnectorVenue connector_venue,
const std::array<std::uint8_t, 32>& pool_id,
const std::vector<ConnectorBookLevel>& bids,
const std::vector<ConnectorBookLevel>& asks) override {
std::lock_guard<std::mutex> lock(callback_mutex_);
LOG(md, Diag,
"Rakurai listener on_book start. venue:{} connector_venue:{} pool:{} bids:{} asks:{}",
int(venue_),
std::string(connector_venue_name(connector_venue)),
short_hex(pool_id),
bids.size(),
asks.size());
if (!venue_matches(venue_, connector_venue)) {
return;
}
const auto pool_key = bytes_to_hex(pool_id);
const auto it = pool_to_security_id_.find(pool_key);
if (it == pool_to_security_id_.end()) {
LOG(md, Warn,
"Received Rakurai book before security definition for pool {}",
pool_key);
return;
}
const auto quantity_scale_it = pool_quantity_scales_.find(pool_key);
const auto quantity_scale =
quantity_scale_it != pool_quantity_scales_.end()
? quantity_scale_it->second
: 1.0;
auto book = domain::Book::empty(it->second);
const auto bid_depth = std::min<std::size_t>(depth_limit_, bids.size());
const auto ask_depth = std::min<std::size_t>(depth_limit_, asks.size());
for (std::size_t i = 0; i < bid_depth; ++i) {
book.bids()[i] = to_book_level(bids[i], quantity_scale);
}
for (std::size_t i = 0; i < ask_depth; ++i) {
book.asks()[i] = to_book_level(asks[i], quantity_scale);
}
listener_->on_book(book);
LOG(md, Diag,
"Rakurai listener on_book complete. venue:{} connector_venue:{} pool:{}",
int(venue_),
std::string(connector_venue_name(connector_venue)),
short_hex(pool_id));
}
void on_order_update(
::rakurai::dex_connector::order_entry::order_state order_state) override {
std::lock_guard<std::mutex> lock(callback_mutex_);
LOG(md, Info,
"Rakurai order update. client_id:{} txn_id:{} status:{}",
order_state.client_id,
order_state.txn_id,
static_cast<int>(order_state.tx_status));
}
void on_ata_verification_status(
const std::array<std::uint8_t, 32>& token_id,
const std::string& ata_address,
::rakurai::dex_connector::ata_status status) override {
std::lock_guard<std::mutex> lock(callback_mutex_);
LOG(md, Info,
"Rakurai ATA verification. token:{} ata:{} status:{}",
short_hex(token_id),
ata_address,
static_cast<int>(status));
}
private:
std::shared_ptr<IMarketDataListener> listener_;
std::shared_ptr<persistence::IDGenerator> sec_id_store_;
api::domain::Venue venue_;
std::size_t depth_limit_;
std::mutex callback_mutex_;
std::unordered_set<std::string> excluded_securities_;
persistence::SqliteStore pool_alias_store_{"rakurai_pool_alias"};
std::unordered_map<std::string, std::unique_ptr<domain::Security>> securities_;
std::unordered_map<std::string, domain::SecurityID> pool_to_security_id_;
std::unordered_map<std::string, double> pool_quantity_scales_;
};
} // namespace
void RakuraiDexDataHandler::run(
std::shared_ptr<IMarketDataListener> listener,
std::shared_ptr<persistence::IDGenerator> sec_id_store,
api::domain::Venue venue,
const std::string& config_path,
std::uint32_t num_depth_levels,
const std::vector<std::string>& excluded_securities) {
try {
install_rakurai_crash_signal_handlers();
LOG(md, Info,
"Rakurai DEX handler run start. venue:{} config:{} depth:{} excluded:{}",
int(venue),
config_path,
num_depth_levels,
excluded_securities.size());
std::ifstream config_file(config_path);
if (!config_file.is_open()) {
throw std::runtime_error("Unable to open Rakurai config: " + config_path);
}
rapidjson::IStreamWrapper isw{config_file};
rapidjson::Document config;
if (config.ParseStream(isw).HasParseError()) {
throw std::runtime_error(
"Unable to parse Rakurai config: " + config_path);
}
LOG(md, Info,
"Rakurai DEX handler config parsed. venue:{} config:{}",
int(venue),
config_path);
::rakurai::dex_connector::configuration::logging_config logging_config;
::rakurai::dex_connector::configuration::session_config session_config;
::rakurai::dex_connector::configuration::wallet wallet_config;
bool cluster_mode = false;
if (config.HasMember("app_logging") && config["app_logging"].IsObject()) {
const auto& app_logging = config["app_logging"];
if (app_logging.HasMember("severity")) {
logging_config.severity = app_logging["severity"].GetString();
}
if (app_logging.HasMember("console")) {
logging_config.is_console_enabled = app_logging["console"].GetBool();
}
if (app_logging.HasMember("file")) {
logging_config.is_file_enabled = app_logging["file"].GetBool();
}
if (app_logging.HasMember("queuesize")) {
logging_config.queue_size = app_logging["queuesize"].GetUint();
}
}
if (config.HasMember("cluster_mode")) {
cluster_mode = config["cluster_mode"].GetBool();
}
auto add_uri = [&](const char* md_key, const char* oms_key) {
if (!config.HasMember(md_key) || !config.HasMember(oms_key)) {
throw std::runtime_error(
std::string("Incomplete Rakurai uri config in ") + config_path);
}
::rakurai::dex_connector::configuration::uri_config uri_config;
uri_config.md_server_uri = config[md_key].GetString();
uri_config.node_oms_uri = config[oms_key].GetString();
session_config.uri.push_back(uri_config);
};
add_uri("md_server_uri_1", "node_oms_uri_1");
if (cluster_mode) {
add_uri("md_server_uri_2", "node_oms_uri_2");
}
LOG(md, Info,
"Rakurai DEX handler uri config prepared. venue:{} cluster_mode:{} uri_count:{}",
int(venue),
cluster_mode,
session_config.uri.size());
std::size_t uri_index = 1;
for (const auto& uri_config : session_config.uri) {
const auto index = std::to_string(uri_index);
validate_ws_endpoint_or_throw(
"md_server_uri_" + index, uri_config.md_server_uri);
validate_ws_endpoint_or_throw(
"node_oms_uri_" + index, uri_config.node_oms_uri);
++uri_index;
}
if (config.HasMember("wallet_pub")) {
wallet_config.wallet_pub_key =
::rakurai::dex_connector::solanaAddressToBytes(
config["wallet_pub"].GetString());
}
if (config.HasMember("wallet_pvt") && config["wallet_pvt"].IsArray()) {
const auto& wallet_pvt = config["wallet_pvt"];
if (wallet_pvt.Size() != wallet_config.wallet_pvt_key.size()) {
throw std::runtime_error(
"Rakurai wallet_pvt must contain 64 integer entries");
}
for (rapidjson::SizeType i = 0; i < wallet_pvt.Size(); ++i) {
wallet_config.wallet_pvt_key[i] =
static_cast<std::uint8_t>(wallet_pvt[i].GetUint());
}
} else {
throw std::runtime_error("Missing Rakurai wallet_pvt in " + config_path);
}
if (config.HasMember("core")) {
session_config.core = config["core"].GetUint();
}
LOG(md, Info,
"Rakurai DEX handler session config prepared. venue:{} core:{} console:{} file:{} queue:{} severity:{}",
int(venue),
session_config.core,
logging_config.is_console_enabled,
logging_config.is_file_enabled,
logging_config.queue_size,
logging_config.severity);
auto dex_configs = parse_dex_configs(config, venue);
if (dex_configs.empty()) {
LOG(md, Warn,
"No Rakurai DEX subscriptions matched venue {} in {}",
int(venue),
config_path);
return;
}
LOG(md, Info,
"Rakurai DEX subscriptions prepared. venue:{} dex_configs:{}",
int(venue),
dex_configs.size());
ListenerAdapter adapter(
listener,
sec_id_store,
venue,
num_depth_levels,
excluded_securities);
LOG(md, Info,
"Rakurai listener adapter constructed. venue:{} listener_present:{} sec_id_store_present:{}",
int(venue),
listener != nullptr,
sec_id_store != nullptr);
::rakurai::dex_connector::dex_handler handler(
dex_configs,
wallet_config,
logging_config,
session_config.uri,
session_config.core,
cluster_mode,
&adapter);
LOG(md, Info,
"Rakurai dex_handler constructed. venue:{} uri_count:{} dex_configs:{} cluster_mode:{}",
int(venue),
session_config.uri.size(),
dex_configs.size(),
cluster_mode);
auto trading_session =
std::make_unique<
::rakurai::dex_connector::trading_session_interface>(
handler.create_new_trading_session());
LOG(md, Info,
"Rakurai trading session created. venue:{} session_present:{}",
int(venue),
trading_session != nullptr);
LOG(md, Info,
"Rakurai dex_handler about to run. venue:{}",
int(venue));
handler.run();
LOG(md, Info,
"Rakurai dex_handler run returned. venue:{}",
int(venue));
} catch (const std::exception& ex) {
LOG(md, Error, "Rakurai DEX handler failed: {}", std::string(ex.what()));
if (listener) {
MarketDataError error;
error.type = MarketDataError::DISCONNECTION;
error.message = ex.what();
listener->on_error(error);
}
}
}
} // namespace bts::md::rakurai
@@ -0,0 +1,25 @@
#pragma once
#include "spark/api/domain/Venue.pb.h"
#include "spark/md/IMarketDataListener.h"
#include "spark/persistence/IDGenerator.h"
#include <cstdint>
#include <memory>
#include <string>
#include <vector>
namespace bts::md::rakurai {
class RakuraiDexDataHandler {
public:
static void run(
std::shared_ptr<IMarketDataListener> listener,
std::shared_ptr<persistence::IDGenerator> sec_id_store,
api::domain::Venue venue,
const std::string& config_path,
std::uint32_t num_depth_levels,
const std::vector<std::string>& excluded_securities);
};
} // namespace bts::md::rakurai
@@ -0,0 +1,74 @@
#include "sparkd/md/rakurai/RakuraiDexSubscription.h"
#include "spark/config/config.h"
#include "spark/config/protos/system_config.pb.h"
#include "spark/logging/logging.h"
#include "sparkd/rakurai/RakuraiSharedRuntime.h"
#include <boost/filesystem/path.hpp>
#include <memory>
#include <string>
#include <utility>
#include <vector>
namespace bts::md {
using Path = boost::filesystem::path;
using bts::config::SystemConfig;
struct RakuraiDexSubscription::PImpl {
api::domain::Venue venue;
std::string json_conf_file_name;
std::uint32_t num_depth_levels = 0;
std::vector<std::string> excluded_securities;
std::shared_ptr<persistence::IDGenerator> sec_id_store;
std::size_t subscription_id = 0;
};
RakuraiDexSubscription::RakuraiDexSubscription(
api::domain::Venue venue,
api::md::DexMarketDataConfig config,
std::shared_ptr<persistence::IDGenerator> sec_id_store)
: me_(std::make_unique<PImpl>()) {
me_->venue = venue;
me_->json_conf_file_name = config.json_conf_file_name();
me_->num_depth_levels = config.num_depth_levels();
me_->excluded_securities.assign(
config.excluded_securities().begin(),
config.excluded_securities().end());
me_->sec_id_store = std::move(sec_id_store);
}
RakuraiDexSubscription::~RakuraiDexSubscription() = default;
void RakuraiDexSubscription::start(
std::shared_ptr<IMarketDataListener> listener) {
const auto config_path =
Path{config::get<SystemConfig>().config_path()} / me_->json_conf_file_name;
LOG(md, Info,
"Starting Rakurai DEX market data. venue:{} config:{}",
int(me_->venue),
config_path.string());
me_->subscription_id =
rakurai::RakuraiSharedRuntime::instance().add_market_data_subscriber(
std::move(listener),
me_->sec_id_store,
me_->venue,
me_->num_depth_levels,
me_->excluded_securities);
rakurai::RakuraiSharedRuntime::instance().ensure_started_from_file(
config_path.string());
}
void RakuraiDexSubscription::stop() {
if (me_->subscription_id != 0) {
rakurai::RakuraiSharedRuntime::instance().remove_market_data_subscriber(
me_->subscription_id);
me_->subscription_id = 0;
}
}
} // namespace bts::md
@@ -0,0 +1,31 @@
#pragma once
#include "spark/api/domain/Venue.pb.h"
#include "spark/api/md/DexMarketDataConfig.pb.h"
#include "spark/md/IMarketDataSource.h"
#include <memory>
namespace bts::persistence {
class IDGenerator;
}
namespace bts::md {
class RakuraiDexSubscription : public IMarketDataSource {
public:
RakuraiDexSubscription(
api::domain::Venue venue,
api::md::DexMarketDataConfig config,
std::shared_ptr<persistence::IDGenerator> sec_id_store);
~RakuraiDexSubscription() override;
void start(std::shared_ptr<IMarketDataListener> listener) override;
void stop() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -0,0 +1,58 @@
#include "RakuraiMarketDataProvider.h"
#include "spark/api/domain/Venue.pb.h"
#include "spark/api/md/MarketDataConfig.pb.h"
#include "spark/md/IMarketDataSource.h"
#include "spark/persistence/IDGenerator.h"
#include "spark/logging/logging.h"
#include "sparkd/md/rakurai/RakuraiDexSubscription.h"
#include <memory>
#include <vector>
namespace bts::md {
using bts::api::md::MarketDataConfig;
using bts::persistence::IDGenerator;
using std::pair;
using std::shared_ptr;
using std::string;
using std::vector;
struct RakuraiMarketDataProvider::PImpl {
MarketDataConfig config;
shared_ptr<IDGenerator> sec_id_store;
PImpl(MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: config(std::move(config)), sec_id_store(std::move(sec_id_store)) {}
};
RakuraiMarketDataProvider::RakuraiMarketDataProvider(
MarketDataConfig config, shared_ptr<IDGenerator> sec_id_store)
: me_(std::make_unique<PImpl>(std::move(config), std::move(sec_id_store))) {
}
RakuraiMarketDataProvider::~RakuraiMarketDataProvider() = default;
vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>>
RakuraiMarketDataProvider::get_sources() {
vector<pair<api::domain::Venue, shared_ptr<IMarketDataSource>>> rv;
for (const auto &dex : me_->config.dex()) {
switch (dex.venue()) {
case api::domain::VENUE_ORCA:
case api::domain::VENUE_RAYDIUM:
rv.push_back(
{dex.venue(),
std::make_shared<RakuraiDexSubscription>(
dex.venue(), dex, me_->sec_id_store)});
break;
default:
LOG(md, Warn,
"Skipping unsupported Rakurai dex venue:{}",
int(dex.venue()));
break;
}
}
return rv;
}
} // namespace bts::md
@@ -0,0 +1,34 @@
#pragma once
#include "spark/md/IMarketDataProvider.h"
#include <memory>
namespace bts::persistence {
class IDGenerator;
}
namespace bts::api::md {
class MarketDataConfig;
}
namespace bts::md {
class RakuraiMarketDataProvider : public IMarketDataProvider {
public:
explicit RakuraiMarketDataProvider(
api::md::MarketDataConfig config,
std::shared_ptr<persistence::IDGenerator> sec_id_store);
~RakuraiMarketDataProvider();
std::vector<
std::pair<api::domain::Venue, std::shared_ptr<IMarketDataSource>>>
get_sources() override;
private:
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::md
@@ -0,0 +1,867 @@
#include "sparkd/rakurai/RakuraiSharedRuntime.h"
#include "spark/api/domain/SecurityType.pb.h"
#include "spark/domain/Book.h"
#include "spark/domain/Price.h"
#include "spark/domain/Quantity.h"
#include "spark/domain/Security.h"
#include "spark/logging/logging.h"
#include "spark/md/IMarketDataListener.h"
#include "spark/persistence/IDGenerator.h"
#include "spark/persistence/SqliteStore.h"
#include "sparkd/md/MarketDataError.h"
#include <boost/filesystem/path.hpp>
#include <dex_handler.h>
#include <dex_listener.h>
#include <nlohmann/json.hpp>
#include <algorithm>
#include <array>
#include <cstddef>
#include <cstdint>
#include <fstream>
#include <functional>
#include <memory>
#include <mutex>
#include <optional>
#include <stdexcept>
#include <string>
#include <string_view>
#include <thread>
#include <unordered_map>
#include <unordered_set>
#include <utility>
#include <vector>
namespace bts::rakurai {
namespace {
using ConnectorVenue = ::rakurai::dex_connector::data_types::venue;
using ConnectorSecurity = ::rakurai::dex_connector::data_types::security;
using ConnectorBookLevel = ::rakurai::dex_connector::data_types::book_level;
using ConnectorDexConfig = ::rakurai::dex_connector::configuration::dex_config;
const char* connector_venue_name(ConnectorVenue venue) {
switch (venue) {
case ConnectorVenue::orca_clmm:
return "orca_clmm";
case ConnectorVenue::orca_amm:
return "orca_amm";
case ConnectorVenue::raydium_amm:
return "raydium_amm";
case ConnectorVenue::raydium_cpmm:
return "raydium_cpmm";
case ConnectorVenue::raydium_clmm:
return "raydium_clmm";
default:
return "invalid";
}
}
bool venue_matches(api::domain::Venue spark_venue, ConnectorVenue connector_venue) {
switch (spark_venue) {
case api::domain::VENUE_ORCA:
return connector_venue == ConnectorVenue::orca_clmm ||
connector_venue == ConnectorVenue::orca_amm;
case api::domain::VENUE_RAYDIUM:
return connector_venue == ConnectorVenue::raydium_amm ||
connector_venue == ConnectorVenue::raydium_cpmm ||
connector_venue == ConnectorVenue::raydium_clmm;
default:
return false;
}
}
bool venue_matches(api::Venue spark_venue, ConnectorVenue connector_venue) {
switch (spark_venue) {
case api::VENUE_ORCA:
return connector_venue == ConnectorVenue::orca_clmm ||
connector_venue == ConnectorVenue::orca_amm;
case api::VENUE_RAYDIUM:
return connector_venue == ConnectorVenue::raydium_amm ||
connector_venue == ConnectorVenue::raydium_cpmm ||
connector_venue == ConnectorVenue::raydium_clmm;
default:
return false;
}
}
std::string bytes_to_hex(const std::array<std::uint8_t, 32>& bytes) {
static constexpr char kHex[] = "0123456789abcdef";
std::string out;
out.reserve(bytes.size() * 2);
for (std::uint8_t byte : bytes) {
out.push_back(kHex[(byte >> 4) & 0x0f]);
out.push_back(kHex[byte & 0x0f]);
}
return out;
}
std::string short_hex(const std::array<std::uint8_t, 32>& bytes) {
const auto hex = bytes_to_hex(bytes);
return hex.substr(0, std::min<std::size_t>(hex.size(), 12));
}
std::string token_display_name(
const ::rakurai::dex_connector::data_types::token_info& token) {
return token.name.empty() ? short_hex(token.mint_address) : token.name;
}
std::string build_security_desc(
const ConnectorSecurity& security,
std::string_view short_pool_id) {
return fmt::format(
"{}-{}-{}",
token_display_name(security.token_a),
token_display_name(security.token_b),
short_pool_id);
}
double quantity_scale_from_decimals(std::uint8_t decimals) {
double scale = 1.0;
for (std::uint8_t i = 0; i < decimals; ++i) {
scale *= 10.0;
}
return scale;
}
domain::BookLevel to_book_level(
const ConnectorBookLevel& source,
double quantity_scale) {
domain::BookLevel level;
level.price = domain::Price::from_double(source.price);
const auto normalized_quantity =
quantity_scale > 0.0
? static_cast<double>(source.quantity) / quantity_scale
: static_cast<double>(source.quantity);
level.total_quantity =
domain::Quantity(static_cast<float>(normalized_quantity));
level.direct_quantity = level.total_quantity;
level.n_orders = 0;
return level;
}
std::list<ConnectorDexConfig> parse_dex_configs(const nlohmann::json& config) {
std::list<ConnectorDexConfig> dex_config_list;
const auto dex_it = config.find("dex");
if (dex_it == config.end() || !dex_it->is_array()) {
return dex_config_list;
}
for (const auto& dex_value : *dex_it) {
if (!dex_value.is_object()) {
continue;
}
ConnectorDexConfig dex_config;
const std::string dex = dex_value.value("id", "");
if (dex == "orca_clmm") {
dex_config.dex_name = ConnectorVenue::orca_clmm;
dex_config.pool_typ =
::rakurai::dex_connector::data_types::pool_type::clmm;
} else if (dex == "orca_amm") {
dex_config.dex_name = ConnectorVenue::orca_amm;
dex_config.pool_typ =
::rakurai::dex_connector::data_types::pool_type::amm;
} else if (dex == "raydium_amm") {
dex_config.dex_name = ConnectorVenue::raydium_amm;
dex_config.pool_typ =
::rakurai::dex_connector::data_types::pool_type::amm;
} else if (dex == "raydium_cpmm") {
dex_config.dex_name = ConnectorVenue::raydium_cpmm;
dex_config.pool_typ =
::rakurai::dex_connector::data_types::pool_type::cpmm;
} else if (dex == "raydium_clmm") {
dex_config.dex_name = ConnectorVenue::raydium_clmm;
dex_config.pool_typ =
::rakurai::dex_connector::data_types::pool_type::clmm;
} else {
LOG(md, Warn, "Skipping unsupported Rakurai dex config '{}'", dex);
continue;
}
const auto pools_it = dex_value.find("pool_ids");
if (pools_it != dex_value.end() && pools_it->is_array()) {
for (const auto& pool_value : *pools_it) {
dex_config.interested_pools.emplace_back(
::rakurai::dex_connector::solanaAddressToBytes(
pool_value.get<std::string>()));
}
}
dex_config_list.push_back(std::move(dex_config));
}
return dex_config_list;
}
nlohmann::json load_json_file(const std::string& path) {
LOG(md, Info, "Rakurai shared runtime loading config file: {}", path);
std::ifstream config_file(path);
if (!config_file.is_open()) {
throw std::runtime_error("Unable to open Rakurai config: " + path);
}
nlohmann::json config;
config_file >> config;
const auto nested_config_it = config.find("json_conf_file_name");
if (nested_config_it != config.end() && nested_config_it->is_string() &&
!config.contains("dex")) {
const auto nested_name = nested_config_it->get<std::string>();
auto nested_path = boost::filesystem::path(nested_name);
if (nested_path.is_relative()) {
nested_path = boost::filesystem::path(path).parent_path() / nested_path;
}
LOG(md, Info,
"Rakurai shared runtime resolved wrapper config '{}' -> '{}'. wrapper:{}",
path,
nested_path.string(),
config.dump());
return load_json_file(nested_path.string());
}
LOG(md, Info,
"Rakurai shared runtime loaded connector config from '{}': {}",
path,
config.dump());
return config;
}
} // namespace
struct RakuraiSharedRuntime::PImpl {
struct MarketDataSubscriber {
std::size_t id = 0;
std::shared_ptr<md::IMarketDataListener> listener;
std::shared_ptr<persistence::IDGenerator> sec_id_store;
api::domain::Venue venue = api::domain::VENUE_UNSPECIFIED;
std::size_t depth_limit = domain::Book::max_depth();
std::unordered_set<std::string> excluded_securities;
std::unordered_map<std::string, std::unique_ptr<domain::Security>>
securities;
std::unordered_map<std::string, domain::SecurityID> pool_to_security_id;
std::unordered_map<std::string, double> pool_quantity_scales;
std::mutex mutex;
};
struct ListenerAdapter final : public ::rakurai::dex_connector::dex_listener {
explicit ListenerAdapter(PImpl& owner) : owner(owner) {}
void on_pool_definition(
ConnectorVenue venue,
const std::vector<ConnectorSecurity*>& securities) override {
owner.on_pool_definition(venue, securities);
}
void on_book(
ConnectorVenue venue,
const std::array<std::uint8_t, 32>& pool_id,
const std::vector<ConnectorBookLevel>& bids,
const std::vector<ConnectorBookLevel>& asks) override {
owner.on_book(venue, pool_id, bids, asks);
}
void on_order_update(
::rakurai::dex_connector::order_entry::order_state order_state)
override {
owner.on_order_update(order_state);
}
void on_ata_verification_status(
const std::array<std::uint8_t, 32>& token_id,
const std::string& ata_address,
::rakurai::dex_connector::ata_status status) override {
LOG(md, Info,
"Rakurai ATA verification. token:{} ata:{} status:{}",
short_hex(token_id),
ata_address,
static_cast<int>(status));
}
PImpl& owner;
};
mutable std::mutex mutex;
std::optional<nlohmann::json> config;
std::string config_label;
std::optional<std::thread> run_thread;
std::unique_ptr<::rakurai::dex_connector::dex_handler> handler;
std::unique_ptr<::rakurai::dex_connector::trading_session_interface>
trading_session;
persistence::SqliteStore pool_alias_store{"rakurai_pool_alias"};
ListenerAdapter listener{*this};
std::unordered_map<std::size_t, std::shared_ptr<MarketDataSubscriber>>
market_data_subscribers;
std::unordered_map<std::size_t, OrderUpdateCallback> order_update_subscribers;
std::unordered_map<std::size_t, StatusCallback> status_subscribers;
std::unordered_map<std::string, PoolMetadata> pool_metadata;
std::unordered_map<std::string, std::string> short_pool_to_full_pool;
std::unordered_set<std::string> expected_pool_ids;
std::unordered_set<std::string> resolved_pool_ids;
std::size_t next_subscriber_id = 1;
bool started = false;
bool connected = false;
~PImpl() {
if (run_thread.has_value() && run_thread->joinable()) {
run_thread->detach();
}
}
std::size_t next_id_locked() { return next_subscriber_id++; }
void ensure_started(const nlohmann::json& cfg, const std::string& label) {
std::lock_guard<std::mutex> lock(mutex);
LOG(md, Info,
"Rakurai shared runtime ensure_started requested. label:{} config:{}",
label,
cfg.dump());
if (!config.has_value()) {
config = cfg;
config_label = label;
} else if (*config != cfg) {
throw std::runtime_error(
"Rakurai shared runtime already configured with a different config source: " +
config_label + " vs " + label);
}
if (started) {
LOG(md, Info,
"Rakurai shared runtime already started. label:{} connected:{}",
config_label,
connected);
return;
}
started = true;
LOG(md, Info,
"Rakurai shared runtime launching thread. label:{}",
config_label);
run_thread.emplace([this]() { run(); });
}
std::size_t add_market_data_subscriber(
std::shared_ptr<md::IMarketDataListener> listener_in,
std::shared_ptr<persistence::IDGenerator> sec_id_store,
api::domain::Venue venue,
std::uint32_t num_depth_levels,
const std::vector<std::string>& excluded_securities) {
auto sub = std::make_shared<MarketDataSubscriber>();
sub->listener = std::move(listener_in);
sub->sec_id_store = std::move(sec_id_store);
sub->venue = venue;
sub->depth_limit =
num_depth_levels == 0
? domain::Book::max_depth()
: std::min<std::size_t>(
num_depth_levels, domain::Book::max_depth());
sub->excluded_securities.insert(
excluded_securities.begin(), excluded_securities.end());
std::vector<PoolMetadata> snapshot;
{
std::lock_guard<std::mutex> lock(mutex);
sub->id = next_id_locked();
market_data_subscribers.emplace(sub->id, sub);
snapshot.reserve(pool_metadata.size());
for (const auto& [_, metadata] : pool_metadata) {
if (venue_matches(venue, metadata.security.venue)) {
snapshot.push_back(metadata);
}
}
}
replay_pool_definitions(*sub, snapshot);
return sub->id;
}
void remove_market_data_subscriber(std::size_t id) {
std::lock_guard<std::mutex> lock(mutex);
market_data_subscribers.erase(id);
}
std::size_t add_order_update_subscriber(OrderUpdateCallback callback) {
std::lock_guard<std::mutex> lock(mutex);
const auto id = next_id_locked();
order_update_subscribers.emplace(id, std::move(callback));
return id;
}
void remove_order_update_subscriber(std::size_t id) {
std::lock_guard<std::mutex> lock(mutex);
order_update_subscribers.erase(id);
}
std::size_t add_status_subscriber(StatusCallback callback) {
bool current_connected = false;
std::size_t id = 0;
{
std::lock_guard<std::mutex> lock(mutex);
id = next_id_locked();
status_subscribers.emplace(id, callback);
current_connected = connected;
}
callback(current_connected);
return id;
}
void remove_status_subscriber(std::size_t id) {
std::lock_guard<std::mutex> lock(mutex);
status_subscribers.erase(id);
}
bool is_connected() const {
std::lock_guard<std::mutex> lock(mutex);
return connected;
}
void launch_new_order(const RakuraiSharedRuntime::NewOrder& order) {
std::lock_guard<std::mutex> lock(mutex);
if (!connected || trading_session == nullptr) {
throw std::runtime_error("Rakurai execution session is not connected");
}
trading_session->launch_new_order(order);
}
std::optional<PoolMetadata> find_pool_metadata(
api::Venue venue,
const std::string& short_pool_id) const {
std::lock_guard<std::mutex> lock(mutex);
const auto pool_it = short_pool_to_full_pool.find(short_pool_id);
if (pool_it == short_pool_to_full_pool.end()) {
return std::nullopt;
}
const auto metadata_it = pool_metadata.find(pool_it->second);
if (metadata_it == pool_metadata.end() ||
!venue_matches(venue, metadata_it->second.security.venue)) {
return std::nullopt;
}
return metadata_it->second;
}
void run() {
try {
nlohmann::json connector_config;
{
std::lock_guard<std::mutex> lock(mutex);
connector_config = *config;
}
auto dex_configs = parse_dex_configs(connector_config);
if (dex_configs.empty()) {
throw std::runtime_error("No Rakurai dex pools configured");
}
LOG(md, Info,
"Rakurai shared runtime parsed config. dex_configs:{} config:{}",
dex_configs.size(),
connector_config.dump());
{
std::lock_guard<std::mutex> lock(mutex);
expected_pool_ids.clear();
resolved_pool_ids.clear();
for (const auto& dex_config : dex_configs) {
for (const auto& pool_id : dex_config.interested_pools) {
expected_pool_ids.insert(bytes_to_hex(pool_id));
}
}
}
::rakurai::dex_connector::configuration::logging_config logging_config;
if (connector_config.contains("app_logging")) {
const auto& app_logging = connector_config.at("app_logging");
logging_config.severity =
app_logging.value("severity", logging_config.severity);
logging_config.is_console_enabled =
app_logging.value("console", logging_config.is_console_enabled);
logging_config.is_file_enabled =
app_logging.value("file", logging_config.is_file_enabled);
logging_config.queue_size =
app_logging.value("queuesize", logging_config.queue_size);
}
::rakurai::dex_connector::configuration::wallet wallet_config;
wallet_config.wallet_pub_key =
::rakurai::dex_connector::solanaAddressToBytes(
connector_config.at("wallet_pub").get<std::string>());
const auto& wallet_pvt = connector_config.at("wallet_pvt");
if (!wallet_pvt.is_array() ||
wallet_pvt.size() != wallet_config.wallet_pvt_key.size()) {
throw std::runtime_error(
"Rakurai wallet_pvt must contain 64 integer entries");
}
for (std::size_t i = 0; i < wallet_config.wallet_pvt_key.size(); ++i) {
wallet_config.wallet_pvt_key[i] =
wallet_pvt.at(i).get<std::uint8_t>();
}
std::list<::rakurai::dex_connector::configuration::uri_config> uris;
auto add_uri = [&](const char* md_key, const char* oms_key) {
::rakurai::dex_connector::configuration::uri_config uri;
uri.md_server_uri =
connector_config.at(md_key).get<std::string>();
uri.node_oms_uri =
connector_config.at(oms_key).get<std::string>();
uris.push_back(std::move(uri));
};
add_uri("md_server_uri_1", "node_oms_uri_1");
const bool cluster_mode =
connector_config.value("cluster_mode", false);
if (cluster_mode) {
add_uri("md_server_uri_2", "node_oms_uri_2");
}
const auto core = connector_config.value("core", 0U);
{
std::lock_guard<std::mutex> lock(mutex);
handler = std::make_unique<::rakurai::dex_connector::dex_handler>(
dex_configs,
wallet_config,
logging_config,
uris,
core,
cluster_mode,
&listener);
trading_session = std::make_unique<
::rakurai::dex_connector::trading_session_interface>(
handler->create_new_trading_session());
connected = false;
}
handler->run();
} catch (const std::exception& ex) {
LOG(md, Error, "Rakurai shared runtime failed: {}", std::string(ex.what()));
notify_md_error(ex.what());
}
{
std::lock_guard<std::mutex> lock(mutex);
connected = false;
}
notify_status(false);
}
void notify_status(bool is_connected_now) {
std::vector<StatusCallback> callbacks;
{
std::lock_guard<std::mutex> lock(mutex);
for (const auto& [_, callback] : status_subscribers) {
callbacks.push_back(callback);
}
}
for (auto& callback : callbacks) {
callback(is_connected_now);
}
}
void notify_md_error(const std::string& message) {
std::vector<std::shared_ptr<md::IMarketDataListener>> listeners;
{
std::lock_guard<std::mutex> lock(mutex);
for (const auto& [_, sub] : market_data_subscribers) {
listeners.push_back(sub->listener);
}
}
for (const auto& listener_ptr : listeners) {
if (!listener_ptr) {
continue;
}
md::MarketDataError error;
error.type = md::MarketDataError::DISCONNECTION;
error.message = message;
listener_ptr->on_error(error);
}
}
void load_aliases_locked() {
if (!short_pool_to_full_pool.empty()) {
return;
}
constexpr std::string_view kPoolAliasNamespace = "rakurai_pool_alias";
for (const auto& [ns, rows] : pool_alias_store.retrieve_all()) {
if (ns != kPoolAliasNamespace) {
continue;
}
for (const auto& [full_pool_id, alias] : rows) {
if (!alias.empty()) {
short_pool_to_full_pool.emplace(alias, full_pool_id);
}
}
}
}
std::string get_or_add_short_pool_id_locked(const std::string& pool_id) {
constexpr std::string_view kPoolAliasNamespace = "rakurai_pool_alias";
const auto [exists, rowid, _] =
pool_alias_store.retrieve_one(std::string(kPoolAliasNamespace), pool_id);
if (exists) {
const auto alias = std::to_string(rowid);
short_pool_to_full_pool.emplace(alias, pool_id);
return alias;
}
const auto new_id =
pool_alias_store.insert(std::string(kPoolAliasNamespace), pool_id, "");
const auto alias = std::to_string(new_id);
short_pool_to_full_pool.emplace(alias, pool_id);
return alias;
}
void on_pool_definition(
ConnectorVenue connector_venue,
const std::vector<ConnectorSecurity*>& securities) {
std::vector<std::shared_ptr<MarketDataSubscriber>> subscribers;
std::vector<PoolMetadata> touched;
bool should_notify_ready = false;
std::size_t resolved_pool_count = 0;
std::size_t expected_pool_count = 0;
{
std::lock_guard<std::mutex> lock(mutex);
load_aliases_locked();
for (const auto* security : securities) {
if (security == nullptr) {
continue;
}
const auto pool_id = bytes_to_hex(security->pool_id);
pool_metadata[pool_id] = PoolMetadata{
*security,
pool_id,
get_or_add_short_pool_id_locked(pool_id),
quantity_scale_from_decimals(security->token_a.decimals)};
resolved_pool_ids.insert(pool_id);
touched.push_back(pool_metadata.at(pool_id));
}
const bool has_trading_session = trading_session != nullptr;
const bool has_pool_definitions = !resolved_pool_ids.empty();
const bool all_expected_pools_resolved =
expected_pool_ids.empty()
? has_pool_definitions
: std::all_of(
expected_pool_ids.begin(),
expected_pool_ids.end(),
[this](const std::string& pool_id) {
return resolved_pool_ids.contains(pool_id);
});
if (!connected && has_trading_session && all_expected_pools_resolved) {
connected = true;
should_notify_ready = true;
resolved_pool_count = resolved_pool_ids.size();
expected_pool_count = expected_pool_ids.size();
}
for (const auto& [_, sub] : market_data_subscribers) {
if (venue_matches(sub->venue, connector_venue)) {
subscribers.push_back(sub);
}
}
}
if (should_notify_ready) {
LOG(md, Info,
"Rakurai shared runtime ready. resolved_pools:{} expected_pools:{}",
resolved_pool_count,
expected_pool_count);
notify_status(true);
}
for (const auto& sub : subscribers) {
replay_pool_definitions(*sub, touched);
}
}
void replay_pool_definitions(
MarketDataSubscriber& sub,
const std::vector<PoolMetadata>& metadata_list) {
std::lock_guard<std::mutex> lock(sub.mutex);
std::vector<const domain::Security*> emitted;
for (const auto& metadata : metadata_list) {
if (!venue_matches(sub.venue, metadata.security.venue)) {
continue;
}
const auto& security = metadata.security;
const auto& pool_id = metadata.full_pool_id;
const auto desc = build_security_desc(security, metadata.short_pool_id);
const auto symbol = token_display_name(security.token_a);
const auto currency = token_display_name(security.token_b);
if (sub.excluded_securities.contains(pool_id) ||
sub.excluded_securities.contains(metadata.short_pool_id) ||
sub.excluded_securities.contains(desc)) {
continue;
}
domain::Security emitted_security;
emitted_security.security = domain::SpotSecurity{};
emitted_security.security_common.venue = sub.venue;
emitted_security.security_common.exchange_sec_id = metadata.short_pool_id;
emitted_security.security_common.symbol = symbol;
emitted_security.security_common.currency = currency;
emitted_security.security_common.asset = symbol;
emitted_security.security_common.group =
connector_venue_name(security.venue);
emitted_security.security_common.desc = desc;
emitted_security.security_common.min_price_increment =
security.min_tick > 0.0 ? security.min_tick : 0.0;
emitted_security.security_common.min_quantity_increment =
static_cast<double>(security.min_trade_amount);
emitted_security.security_common.min_quantity =
static_cast<double>(security.min_trade_amount);
*emitted_security.security_common.id = sub.sec_id_store->get_or_add(
sub.venue, api::domain::SECURITY_TYPE_SPOT, pool_id);
auto [it, is_new] = sub.securities.try_emplace(
pool_id, std::make_unique<domain::Security>(std::move(emitted_security)));
if (!is_new) {
*it->second = std::move(emitted_security);
}
sub.pool_to_security_id[pool_id] = it->second->id();
sub.pool_quantity_scales[pool_id] = metadata.quantity_scale;
emitted.push_back(it->second.get());
}
if (!emitted.empty() && sub.listener) {
sub.listener->on_securities(emitted);
}
}
void on_book(
ConnectorVenue connector_venue,
const std::array<std::uint8_t, 32>& pool_id,
const std::vector<ConnectorBookLevel>& bids,
const std::vector<ConnectorBookLevel>& asks) {
std::vector<std::shared_ptr<MarketDataSubscriber>> subscribers;
{
std::lock_guard<std::mutex> lock(mutex);
for (const auto& [_, sub] : market_data_subscribers) {
if (venue_matches(sub->venue, connector_venue)) {
subscribers.push_back(sub);
}
}
}
const auto pool_key = bytes_to_hex(pool_id);
for (const auto& sub : subscribers) {
std::lock_guard<std::mutex> lock(sub->mutex);
const auto it = sub->pool_to_security_id.find(pool_key);
if (it == sub->pool_to_security_id.end()) {
continue;
}
const auto quantity_scale_it = sub->pool_quantity_scales.find(pool_key);
const auto quantity_scale =
quantity_scale_it != sub->pool_quantity_scales.end()
? quantity_scale_it->second
: 1.0;
auto book = domain::Book::empty(it->second);
const auto bid_depth =
std::min<std::size_t>(sub->depth_limit, bids.size());
const auto ask_depth =
std::min<std::size_t>(sub->depth_limit, asks.size());
for (std::size_t i = 0; i < bid_depth; ++i) {
book.bids()[i] = to_book_level(bids[i], quantity_scale);
}
for (std::size_t i = 0; i < ask_depth; ++i) {
book.asks()[i] = to_book_level(asks[i], quantity_scale);
}
if (sub->listener) {
sub->listener->on_book(book);
}
}
}
void on_order_update(
::rakurai::dex_connector::order_entry::order_state order_state) {
std::vector<OrderUpdateCallback> callbacks;
{
std::lock_guard<std::mutex> lock(mutex);
for (const auto& [_, callback] : order_update_subscribers) {
callbacks.push_back(callback);
}
}
for (auto& callback : callbacks) {
callback(order_state);
}
}
};
RakuraiSharedRuntime& RakuraiSharedRuntime::instance() {
static RakuraiSharedRuntime runtime;
return runtime;
}
RakuraiSharedRuntime::RakuraiSharedRuntime()
: me_(std::make_unique<PImpl>()) {}
RakuraiSharedRuntime::~RakuraiSharedRuntime() = default;
std::size_t RakuraiSharedRuntime::add_market_data_subscriber(
std::shared_ptr<md::IMarketDataListener> listener,
std::shared_ptr<persistence::IDGenerator> sec_id_store,
api::domain::Venue venue,
std::uint32_t num_depth_levels,
const std::vector<std::string>& excluded_securities) {
return me_->add_market_data_subscriber(
std::move(listener),
std::move(sec_id_store),
venue,
num_depth_levels,
excluded_securities);
}
void RakuraiSharedRuntime::remove_market_data_subscriber(std::size_t id) {
me_->remove_market_data_subscriber(id);
}
std::size_t RakuraiSharedRuntime::add_order_update_subscriber(
OrderUpdateCallback callback) {
return me_->add_order_update_subscriber(std::move(callback));
}
void RakuraiSharedRuntime::remove_order_update_subscriber(std::size_t id) {
me_->remove_order_update_subscriber(id);
}
std::size_t RakuraiSharedRuntime::add_status_subscriber(StatusCallback callback) {
return me_->add_status_subscriber(std::move(callback));
}
void RakuraiSharedRuntime::remove_status_subscriber(std::size_t id) {
me_->remove_status_subscriber(id);
}
void RakuraiSharedRuntime::ensure_started_from_file(
const std::string& config_path) {
me_->ensure_started(load_json_file(config_path), config_path);
}
void RakuraiSharedRuntime::ensure_started_from_json(
const nlohmann::json& config,
const std::string& config_label) {
me_->ensure_started(config, config_label);
}
bool RakuraiSharedRuntime::is_connected() const { return me_->is_connected(); }
void RakuraiSharedRuntime::launch_new_order(const NewOrder& order) {
me_->launch_new_order(order);
}
std::optional<RakuraiSharedRuntime::PoolMetadata>
RakuraiSharedRuntime::find_pool_metadata(
api::Venue venue,
const std::string& short_pool_id) const {
return me_->find_pool_metadata(venue, short_pool_id);
}
} // namespace bts::rakurai
@@ -0,0 +1,78 @@
#pragma once
#include "spark/api/domain/Venue.pb.h"
#include "spark/domain_model/Security.pb.h"
#include <dex_utils.h>
#include <order_entry_common.h>
#include <cstddef>
#include <functional>
#include <memory>
#include <nlohmann/json_fwd.hpp>
#include <optional>
#include <string>
#include <vector>
namespace bts::md {
class IMarketDataListener;
}
namespace bts::persistence {
class IDGenerator;
}
namespace bts::rakurai {
class RakuraiSharedRuntime {
public:
using OrderUpdate = ::rakurai::dex_connector::order_entry::order_state;
using NewOrder = ::rakurai::dex_connector::order_entry::new_order;
using OrderUpdateCallback = std::function<void(OrderUpdate)>;
using StatusCallback = std::function<void(bool)>;
struct PoolMetadata {
::rakurai::dex_connector::data_types::security security;
std::string full_pool_id;
std::string short_pool_id;
double quantity_scale = 1.0;
};
static RakuraiSharedRuntime& instance();
std::size_t add_market_data_subscriber(
std::shared_ptr<md::IMarketDataListener> listener,
std::shared_ptr<persistence::IDGenerator> sec_id_store,
api::domain::Venue venue,
std::uint32_t num_depth_levels,
const std::vector<std::string>& excluded_securities);
void remove_market_data_subscriber(std::size_t id);
std::size_t add_order_update_subscriber(OrderUpdateCallback callback);
void remove_order_update_subscriber(std::size_t id);
std::size_t add_status_subscriber(StatusCallback callback);
void remove_status_subscriber(std::size_t id);
void ensure_started_from_file(const std::string& config_path);
void ensure_started_from_json(
const nlohmann::json& config,
const std::string& config_label);
bool is_connected() const;
void launch_new_order(const NewOrder& order);
std::optional<PoolMetadata> find_pool_metadata(
api::Venue venue,
const std::string& short_pool_id) const;
private:
RakuraiSharedRuntime();
~RakuraiSharedRuntime();
RakuraiSharedRuntime(const RakuraiSharedRuntime&) = delete;
RakuraiSharedRuntime& operator=(const RakuraiSharedRuntime&) = delete;
struct PImpl;
std::unique_ptr<PImpl> me_;
};
} // namespace bts::rakurai
@@ -8,15 +8,13 @@ using bts.api;
using mbts.analytics;
using mbts;
namespace BCG
{
namespace BCG {
/// For each NMSheetSymbol in an NMSheet gets all the
/// data together (theos, position, vol change etc)
/// and fires a callback.
/// This is where all the data is gathered, the rest
/// is all access to this
public class NMSheetSingleOptionMainDataSource : IDisposable
{
public class NMSheetSingleOptionMainDataSource : IDisposable {
private ThreadedBlackboard _threaded_bb;
private readonly ITag<NMSheetDataRequest> _in_bb_main_sheet_request_tag;
private readonly ITag<ExchangeSecurityMap> _in_bb_secmap_tag =
@@ -26,7 +24,6 @@ namespace BCG
private readonly ITag<OptionTheoInterpolationData> _in_bb_precomputed_sparse_theos_tag;
private readonly ITag<IDictionary<NMSheetExpiration, double>> _in_bb_vol_shift_tag;
private readonly ITag<NMSheetSettings> _in_bb_sheet_settings_tag;
private readonly ITag<IEnumerable<ITradeLogRow>> _in_bb_trade_log_tag = SparkTags.TradeLogTag;
private readonly ITag<VolSurfaceNonVolInput> _in_exp_param_tag = SparkTags.VolSurfaceNonVolInputTag;
private readonly ITag<bool> _in_bb_main_sheet_visibility_tag;
private ITag<SparkTags.Nothing> _in_bb_onefive_value_tag = SparkTags.TC2Tags.VoidTag,
@@ -34,6 +31,7 @@ namespace BCG
_in_bb_symbol_theo_shift_tag = SparkTags.TC2Tags.VoidTag,
_in_bb_rollover_data_tag = SparkTags.TC2Tags.VoidTag,
_in_bb_presaved_aw2_theos_tag = SparkTags.TC2Tags.VoidTag,
_in_bb_position_tag = SparkTags.TC2Tags.VoidTag,
_in_bb_option_driver_detailed_price_tag = SparkTags.TC2Tags.VoidTag;
private readonly NMSheetSymbol _nmsheet_symbol;
private Action<Pair<NMSheetSymbol, NMSheetSingleOptionData>> _cb;
@@ -42,8 +40,7 @@ namespace BCG
private string _electronic_equivelant_symbol;
private NMSheetArchivedVolProvider _archived_vol_provider;
public NMSheetSingleOptionMainDataSource(object req_id, ThreadedBlackboard threaded_bb, NMSheetSymbol option, Action<Pair<NMSheetSymbol, NMSheetSingleOptionData>> callback)
{
public NMSheetSingleOptionMainDataSource(object req_id, ThreadedBlackboard threaded_bb, NMSheetSymbol option, Action<Pair<NMSheetSymbol, NMSheetSingleOptionData>> callback) {
_threaded_bb = threaded_bb;
_nmsheet_symbol = option;
_electronic_equivelant_symbol = option.option_name;
@@ -59,8 +56,7 @@ namespace BCG
_in_bb_sheet_settings_tag = SparkTags.TC2Tags.NMSheetSettingsTag(req_id.ToString());
_in_bb_main_sheet_visibility_tag = SparkTags.TC2Tags.NMSheetVisibilityTag(req_id.ToString());
_archived_vol_provider = new NMSheetArchivedVolProvider(req_id.ToString(), _threaded_bb, option,
() =>
{
() => {
_rollover_data = null;
_compute_and_fire();
});
@@ -80,7 +76,7 @@ namespace BCG
_threaded_bb.subscribe(_in_bb_rollover_data_tag);
_threaded_bb.subscribe(_in_bb_presaved_aw2_theos_tag);
_threaded_bb.subscribe(_in_exp_param_tag);
_threaded_bb.subscribe(_in_bb_trade_log_tag);
_threaded_bb.subscribe(_in_bb_position_tag);
_threaded_bb.subscribe(_in_bb_sheet_settings_tag);
_threaded_bb.subscribe(_in_bb_onefive_value_tag);
_threaded_bb.subscribe(_in_bb_qs_tag);
@@ -88,17 +84,15 @@ namespace BCG
_threaded_bb.subscribe(_in_bb_option_driver_detailed_price_tag);
}
private void _react_to_bb(object tag, object value)
{
private void _react_to_bb(object tag, object value) {
if (_in_bb_main_sheet_request_tag.Equals(tag) || _in_bb_secmap_tag.Equals(tag) || _in_bb_live_theos_tag.Equals(tag) || /*_in_bb_live_driver_bbo_tag.Equals(tag) ||*/ _in_bb_precomputed_sparse_theos_tag.Equals(tag) ||
_in_bb_vol_shift_tag.Equals(tag) || _in_bb_symbol_theo_shift_tag.Equals(tag) || _in_exp_param_tag.Equals(tag) || _in_bb_presaved_aw2_theos_tag.Equals(tag) ||
_in_bb_trade_log_tag.Equals(tag) || _in_bb_rollover_data_tag.Equals(tag) || _in_bb_sheet_settings_tag.Equals(tag) || _in_bb_qs_tag.Equals(tag) ||
_in_bb_position_tag.Equals(tag) || _in_bb_rollover_data_tag.Equals(tag) || _in_bb_sheet_settings_tag.Equals(tag) || _in_bb_qs_tag.Equals(tag) ||
_in_bb_main_sheet_visibility_tag.Equals(tag))
_compute_and_fire();
}
private NMSheetDriverData _get_driver(DriverMD d)
{
private NMSheetDriverData _get_driver(DriverMD d) {
double dr_pr = d.get_driver();
if (d.driver_security.HasValue)
return new NMSheetDriverData(d.driver_security.symbol_or_default(),
@@ -108,10 +102,8 @@ namespace BCG
return new NMSheetDriverData(String.Empty, Venue.Unset, DateTime.MinValue, dr_pr, dr_pr, false);
}
private void _compute_and_fire()
{
if (_is_main_sheet_active())
{
private void _compute_and_fire() {
if (_is_main_sheet_active()) {
var timer = Stopwatch.StartNew();
NMSheetDataRequest data_request = _threaded_bb.read_state(_in_bb_main_sheet_request_tag) ?? new NMSheetDataRequest(new List<Triplet<NMSheetExpiration, bool, double>>());
data_request = NMSheetDataRequest.restrict_for_symbol(data_request, _nmsheet_symbol);
@@ -120,8 +112,7 @@ namespace BCG
NMSheetSettings sheet_settings = _threaded_bb.read_state(_in_bb_sheet_settings_tag) ?? new NMSheetSettings();
ExpStrikeKeyedData<mbts.spark.TheoPoint> shared_live_theo_data_map = _threaded_bb.read_state(_in_bb_live_theos_tag);
if (data_request != null && live_driver_map != null && shared_live_theo_data_map != null)
{
if (data_request != null && live_driver_map != null && shared_live_theo_data_map != null) {
ExpStrikeKeyedData<mbts.spark.TheoPoint> output_theo_data_map = _limit_to_requested_securities(shared_live_theo_data_map, data_request, sheet_settings);
output_theo_data_map = _adjust_for_freezes(output_theo_data_map, data_request);
_add_vol_shift(output_theo_data_map);
@@ -147,17 +138,14 @@ namespace BCG
}
}
private bool _is_main_sheet_active()
{
private bool _is_main_sheet_active() {
return _threaded_bb.read_state(_in_bb_main_sheet_visibility_tag, true);
}
private ExpStrikeKeyedData<mbts.spark.TheoPoint> _limit_to_requested_securities(ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_data, NMSheetDataRequest data_request, NMSheetSettings sheet_settings)
{
private ExpStrikeKeyedData<mbts.spark.TheoPoint> _limit_to_requested_securities(ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_data, NMSheetDataRequest data_request, NMSheetSettings sheet_settings) {
HashSet<DateTime> selected_exp_map = new HashSet<DateTime>();
HashSet<double> selected_strike_map = new HashSet<double>(sheet_settings.setting_unselected_strikes);
foreach (Triplet<NMSheetExpiration, bool, double> t in data_request.expiration_request_list)
{
foreach (Triplet<NMSheetExpiration, bool, double> t in data_request.expiration_request_list) {
NMSheetExpiration nm_exp = t.first;
if (nm_exp != null && nm_exp.option_name == _nmsheet_symbol.option_name)
selected_exp_map.Add(nm_exp.option_expiration);
@@ -165,14 +153,12 @@ namespace BCG
return theo_data.clone_filtered_set(selected_exp_map, true, selected_strike_map, false);
}
private ExpStrikeKeyedData<Triplet<mbts.spark.TheoPoint, uint, uint>> _attach_sec_keys(ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_map)
{
private ExpStrikeKeyedData<Triplet<mbts.spark.TheoPoint, uint, uint>> _attach_sec_keys(ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_map) {
theo_map = theo_map ?? new ExpStrikeKeyedData<mbts.spark.TheoPoint>();
ExchangeSecurityMap sec_map = _threaded_bb.read_state(_in_bb_secmap_tag);
if (sec_map != null)
return sec_map.attach_sec_keys(theo_map, _electronic_equivelant_symbol, _nmsheet_symbol.option_exchange);
else
{
else {
ExpStrikeKeyedData<Triplet<mbts.spark.TheoPoint, uint, uint>> rv = new ExpStrikeKeyedData<Triplet<mbts.spark.TheoPoint, uint, uint>>();
foreach (Triplet<DateTime, double, mbts.spark.TheoPoint> t in theo_map.get_data_linearly())
rv.set_data(t.first, t.second, new Triplet<mbts.spark.TheoPoint, uint, uint>(t.third, 0, 0));
@@ -180,8 +166,7 @@ namespace BCG
}
}
private ExpStrikeKeyedData<mbts.spark.TheoPoint> _adjust_for_freezes(ExpStrikeKeyedData<mbts.spark.TheoPoint> live_theo_map, NMSheetDataRequest data_request)
{
private ExpStrikeKeyedData<mbts.spark.TheoPoint> _adjust_for_freezes(ExpStrikeKeyedData<mbts.spark.TheoPoint> live_theo_map, NMSheetDataRequest data_request) {
return live_theo_map;
// TODO
// ExpStrikeKeyedData<mbts.spark.TheoPoint> rv = new ExpStrikeKeyedData<mbts.spark.TheoPoint>();
@@ -207,11 +192,9 @@ namespace BCG
private Dictionary<DateTime, NMSheetDriverData> _get_option_exp_driver_data_map(NMSheetDataRequest data_request,
ExpStrikeKeyedData<mbts.spark.TheoPoint> output_theo_data_map, ExpStrikeKeyedData<mbts.spark.TheoPoint> shared_live_theo_data_map,
Dictionary<DateTime, Triplet<(string, Venue), DateTime, double>> live_driver_map)
{
Dictionary<DateTime, Triplet<(string, Venue), DateTime, double>> live_driver_map) {
Dictionary<DateTime, NMSheetDriverData> rv = new Dictionary<DateTime, NMSheetDriverData>();
foreach (Triplet<NMSheetExpiration, bool, double> theo_request in data_request.expiration_request_list)
{
foreach (Triplet<NMSheetExpiration, bool, double> theo_request in data_request.expiration_request_list) {
DateTime option_expiration = theo_request.first.option_expiration;
/// live driver
var live_driver_info = live_driver_map.GetValueOrDefault(option_expiration)
@@ -225,11 +208,9 @@ namespace BCG
private Dictionary<DateTime, NMSheetDriverData>
_get_option_exp_driver_data_map(NMSheetDataRequest data_request,
OptionDriverPriceMap drivers)
{
OptionDriverPriceMap drivers) {
var rv = new Dictionary<DateTime, NMSheetDriverData>();
foreach (var tr in data_request.expiration_request_list)
{
foreach (var tr in data_request.expiration_request_list) {
NMSheetExpiration nm_exp = tr.first;
SecurityExpiration exp = tr.first.to_sec_exp();
drivers.exp_price_map.TryGetValue(exp, out DriverMD dr_md);
@@ -241,8 +222,7 @@ namespace BCG
return rv;
}
private static double _get_driverprice_for_expiration(ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_data_map, DateTime option_expiration)
{
private static double _get_driverprice_for_expiration(ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_data_map, DateTime option_expiration) {
IDictionary<double, mbts.spark.TheoPoint> exp_data = theo_data_map.get_data_for_single_exp(option_expiration);
if (exp_data != null)
foreach (mbts.spark.TheoPoint tp in exp_data.Values)
@@ -250,8 +230,7 @@ namespace BCG
return Double.NaN;
}
private IDictionary<DateTime, ATMData> _get_atm_data(ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_data)
{
private IDictionary<DateTime, ATMData> _get_atm_data(ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_data) {
NMSheetSettings sheet_settings = _threaded_bb.read_state(_in_bb_sheet_settings_tag) ?? new NMSheetSettings();
VolSurfaceNonVolInput vs_nvinp = _threaded_bb.read_state(_in_exp_param_tag);
var option_exp_roll_data = vs_nvinp
@@ -269,43 +248,17 @@ namespace BCG
}
OptionContractKeyedData<PNLRollOverDataUnit> _rollover_data = null;
private OptionContractKeyedData<PNLRollOverDataUnit> _get_rollover_data()
{
private OptionContractKeyedData<PNLRollOverDataUnit> _get_rollover_data() {
if (_rollover_data == null)
_rollover_data = _archived_vol_provider.get_all_data();
return _rollover_data;
}
private ExpStrikeKeyedData<Tuple<double, double>> _get_position_map()
{
var sheet_settings = _threaded_bb.read_state(_in_bb_sheet_settings_tag);
var risk_account_text = sheet_settings?.setting_layout?.risk_account ?? string.Empty;
var risk_accounts = risk_account_text
.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)
.Select(a => a.Trim())
.Where(a => !string.IsNullOrEmpty(a))
.ToHashSet(StringComparer.OrdinalIgnoreCase);
var trades = _threaded_bb.read_state(_in_bb_trade_log_tag) ?? Enumerable.Empty<ITradeLogRow>();
var option_trades = trades.Where(t =>
t.is_active &&
t.security_type == SecurityType.Option &&
t.symbol == _nmsheet_symbol.option_name &&
t.venue == _nmsheet_symbol.option_exchange &&
(risk_accounts.Count == 0 || risk_accounts.Contains(t.account)));
var pos_data = SecurityPositionData.from_trades(option_trades);
var rv = new ExpStrikeKeyedData<Tuple<double, double>>();
foreach (var (sec_exp, strike_map) in pos_data.option_pos_map)
{
foreach (var (strike, strike_pos) in strike_map)
{
rv.set_data(sec_exp.expiration, strike, Tuple.Create(strike_pos.call, strike_pos.put));
}
}
return rv;
private ExpStrikeKeyedData<Tuple<double, double>> _get_position_map() {
return new ExpStrikeKeyedData<Tuple<double, double>>();
}
private double _get_onefive()
{
private double _get_onefive() {
// object dp_obj = _threaded_bb.sec_thread_retrieve_from_cache(_in_bb_onefive_value_tag);
// if (dp_obj != null) {
// DataPacket dp = dp_obj as DataPacket;
@@ -317,29 +270,22 @@ namespace BCG
#region Vol Shift
private void _add_vol_shift(ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_data)
{
private void _add_vol_shift(ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_data) {
var vshift_map = _threaded_bb.read_state(_in_bb_vol_shift_tag);
if (vshift_map != null)
add_vol_shift(_nmsheet_symbol, theo_data, vshift_map);
}
internal static void add_vol_shift(NMSheetSymbol apply_to_symbol, ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_data, IDictionary<NMSheetExpiration, double> exp_shift_map)
{
foreach (KeyValuePair<NMSheetExpiration, double> p in exp_shift_map)
{
if (p.Key.nmsheet_symbol.Equals(apply_to_symbol))
{
internal static void add_vol_shift(NMSheetSymbol apply_to_symbol, ExpStrikeKeyedData<mbts.spark.TheoPoint> theo_data, IDictionary<NMSheetExpiration, double> exp_shift_map) {
foreach (KeyValuePair<NMSheetExpiration, double> p in exp_shift_map) {
if (p.Key.nmsheet_symbol.Equals(apply_to_symbol)) {
DateTime option_expiration = p.Key.option_expiration;
double shift = p.Value;
if (shift != 0)
{
if (shift != 0) {
IDictionary<double, mbts.spark.TheoPoint> exp_theos = theo_data.get_data_for_single_exp(option_expiration);
if (exp_theos != null)
{
if (exp_theos != null) {
Dictionary<double, mbts.spark.TheoPoint> shifted_exp_theos = new Dictionary<double, mbts.spark.TheoPoint>();
foreach (KeyValuePair<double, mbts.spark.TheoPoint> strike_tp in exp_theos)
{
foreach (KeyValuePair<double, mbts.spark.TheoPoint> strike_tp in exp_theos) {
mbts.spark.TheoPoint new_tp = (mbts.spark.TheoPoint)strike_tp.Value.Clone();
add_vol_shift(new_tp, shift);
shifted_exp_theos[strike_tp.Key] = new_tp;
@@ -351,8 +297,7 @@ namespace BCG
}
}
internal static void add_vol_shift(mbts.spark.TheoPoint v, double shift)
{
internal static void add_vol_shift(mbts.spark.TheoPoint v, double shift) {
v.call += v.cvega * shift;
v.put += v.pvega * shift;
}
@@ -360,8 +305,7 @@ namespace BCG
#endregion
#region IDisposable
public void Dispose()
{
public void Dispose() {
_threaded_bb.data_callback -= _react_to_bb;
}
#endregion
@@ -288,7 +288,12 @@ namespace bts.websocket_core {
}
// If the message is the end of the message, handle the message and reset the request string
if (result.EndOfMessage) {
gtwy.HandleMessage(new WebsocketIncomingMessage(id, request));
System.Console.WriteLine($"WebSocketHandler received message from {id}: {request}");
try {
gtwy.HandleMessage(new WebsocketIncomingMessage(id, request));
} catch (Exception ex) {
System.Console.WriteLine($"WebSocketHandler failed to dispatch message from {id}: {ex}");
}
request = "";
}
}
@@ -298,6 +303,9 @@ namespace bts.websocket_core {
var log_txt = $"WebsocketGateway.ProcessWebSocketRequestAsync connection closed Error: {ex_str}";
gtwy.HandleDisconnection(id, log_txt);
} catch (OperationCanceledException) {//not sure about this, read that this should just be ignored
} catch (Exception ex) {
System.Console.WriteLine($"WebSocketHandler.ProcessWebSocketRequestAsync failed for {id}: {ex}");
gtwy.HandleDisconnection(id, ex.Message);
}
}
/// <summary>
@@ -15,6 +15,8 @@ namespace bts.websocket_core {
using Spark.Core.Disposables;
using Spark.Core.Reactive;
using mbts.config;
using System.Collections.Concurrent;
using System.Reactive.Linq;
using System.Text.Json;
@@ -30,6 +32,7 @@ namespace bts.websocket_core {
readonly ISecurityResolver securityResolver;
IReadOnlyDictionary<uint, ExecutionStrategyImmutable> synthStrategies = new Dictionary<uint, ExecutionStrategyImmutable>(0);
IDisposable? synthStrategiesSub;
static readonly TimeSpan SyntheticSpreadCreateTimeout = TimeSpan.FromSeconds(15);
public WebsocketDataSync(Spark.Api.IServiceProvider service_ctx, IMarketDataClient mdClient, ISecurityResolver securityResolver) {
_service_ctx = service_ctx;
@@ -225,17 +228,478 @@ namespace bts.websocket_core {
// broadcast.Start();
}
private uint get_security(Venue v, string sec_name) {
var securites = _securities;
if (securites.Values.TryGetFirst(sec =>
(
sec.Exchange == v &&
sec.ToString() == sec_name),
out var sec
)) {
return sec.Id;
private readonly struct SyntheticSpreadLegRequest {
public readonly Venue Exchange;
public readonly string ExchangeSecurityName;
public readonly double Quantity;
public readonly double PriceMultiplier;
public readonly double LegDelta;
public readonly Side Side;
public readonly bool IsActiveQuoting;
public readonly bool EnableHedging;
public readonly LegSpreaderParams.Types.MarketDataType MarketDataType;
public SyntheticSpreadLegRequest(
Venue exchange,
string exchangeSecurityName,
double quantity,
double priceMultiplier,
double legDelta,
Side side,
bool isActiveQuoting,
bool enableHedging,
LegSpreaderParams.Types.MarketDataType marketDataType) {
Exchange = exchange;
ExchangeSecurityName = exchangeSecurityName;
Quantity = quantity;
PriceMultiplier = priceMultiplier;
LegDelta = legDelta;
Side = side;
IsActiveQuoting = isActiveQuoting;
EnableHedging = enableHedging;
MarketDataType = marketDataType;
}
throw new ArgumentException($"{sec_name} at {v} not found in securites");
}
private static string SanitizeIdentifier(string value) =>
string.Concat(value.Select(ch => char.IsLetterOrDigit(ch) ? ch : '_'));
private static bool TryGetOptionalString(
JsonObject parameters,
string key,
out string? value,
out string message) {
value = null;
if (!WebsocketJsonParsers.TryParse<WebsocketJsonParsers.StringWrapper>(
parameters,
key,
out WebsocketJsonParsers.StringWrapper? wrapper,
out message)) {
return false;
}
value = wrapper?.Val;
return true;
}
private static bool TryGetRequiredString(
JsonObject parameters,
string key,
out string value,
out string message) {
value = string.Empty;
if (!TryGetOptionalString(parameters, key, out string? parsed, out message) ||
string.IsNullOrWhiteSpace(parsed)) {
message = string.IsNullOrWhiteSpace(message) ? $"{key} is required" : message;
return false;
}
value = parsed.Trim();
return true;
}
private static bool TryGetOptionalDouble(
JsonObject parameters,
string key,
double defaultValue,
out double value,
out string message) {
value = defaultValue;
if (!WebsocketJsonParsers.TryParse<double>(
parameters,
key,
out double? parsed,
out message)) {
return false;
}
value = parsed ?? defaultValue;
return true;
}
private static bool TryGetOptionalBool(
JsonObject parameters,
string key,
bool defaultValue,
out bool value,
out string message) {
value = defaultValue;
if (!WebsocketJsonParsers.TryParse<bool>(
parameters,
key,
out bool? parsed,
out message)) {
return false;
}
value = parsed ?? defaultValue;
return true;
}
private static bool TryGetOptionalEnum<TEnum>(
JsonObject parameters,
string key,
TEnum defaultValue,
out TEnum value,
out string message) where TEnum : struct {
value = defaultValue;
if (!WebsocketJsonParsers.TryParse<TEnum>(
parameters,
key,
out TEnum? parsed,
out message)) {
return false;
}
value = parsed ?? defaultValue;
return true;
}
private static bool TryGetRequiredVenue(
JsonObject parameters,
string key,
out Venue value,
out string message) {
value = default;
if (!WebsocketJsonParsers.TryParse<Venue>(
parameters,
key,
out Venue? parsed,
out message) ||
parsed is null) {
message = string.IsNullOrWhiteSpace(message) ? $"{key} is required" : message;
return false;
}
value = parsed.Value;
return true;
}
private static bool IsWildcardSecurityName(string? value) =>
string.IsNullOrWhiteSpace(value) || value.Trim() == "*";
private static bts.config.SyntheticSpreadPreset? ResolvePreset(
bts.config.SyntheticSpreadConfig config,
Venue exchange,
string exchangeSecurityName) {
var normalized = exchangeSecurityName.Trim();
return config.Preset.FirstOrDefault(p =>
p.FirstLegExchange == exchange &&
string.Equals(p.FirstLegExchangeSecurityName, normalized, StringComparison.OrdinalIgnoreCase))
?? config.Preset.FirstOrDefault(p =>
p.FirstLegExchange == exchange &&
IsWildcardSecurityName(p.FirstLegExchangeSecurityName))
?? config.Preset.FirstOrDefault(p =>
p.FirstLegExchange == Venue.Unset &&
IsWildcardSecurityName(p.FirstLegExchangeSecurityName));
}
private static double ResolveDouble(double requested, double configured, double fallback) =>
requested != 0 ? requested : (configured != 0 ? configured : fallback);
private static uint ResolveUInt32(uint requested, uint configured, uint fallback) =>
requested != 0 ? requested : (configured != 0 ? configured : fallback);
private static string BuildSyntheticName(
string? requestedName,
string _configuredName,
ContractLeg leg1,
ContractLeg leg2) {
if (!string.IsNullOrWhiteSpace(requestedName)) {
return requestedName.Trim();
}
var leg1Security = new SecurityImmutable(leg1.Security);
var leg2Security = new SecurityImmutable(leg2.Security);
var leg1Desc = leg1Security.security_desc().ToLowerInvariant();
var leg2Desc = leg2Security.security_desc().ToLowerInvariant();
var leg1Exchange = leg1Security.Exchange.ToString().ToLowerInvariant();
var leg2Exchange = leg2Security.Exchange.ToString().ToLowerInvariant();
return $"{leg1Exchange}:{leg1Desc}_{leg2Exchange}:{leg2Desc}";
}
private static string BuildSyntheticSymbol(
string? requestedSymbol,
string configuredSymbol,
string firstLegSymbol,
string syntheticName) {
if (!string.IsNullOrWhiteSpace(requestedSymbol)) {
return requestedSymbol.Trim();
}
if (!string.IsNullOrWhiteSpace(configuredSymbol)) {
return configuredSymbol.Trim();
}
if (!string.IsNullOrWhiteSpace(firstLegSymbol)) {
return firstLegSymbol.Trim();
}
return SanitizeIdentifier(syntheticName);
}
private static ContractSpecification BuildContractSpec(
double tickSize,
double cashPerPoint,
double priceMultiplier,
uint mainFraction,
uint subFraction,
uint priceDisplayFormat,
bool supportsFractionalQuantity) {
var specification = new ContractSpecification {
LowTick = tickSize,
HighTick = tickSize,
PriceBreak = double.MaxValue,
CashPerPoint = cashPerPoint,
PriceMultiplier = priceMultiplier,
MainFraction = mainFraction,
SubFraction = subFraction,
PriceDisplayFormat = priceDisplayFormat
};
if (supportsFractionalQuantity) {
specification.QuantityInfos.Add(new ContractQuantityInfo {
MinValue = 0.00001,
MaxValue = double.PositiveInfinity,
TickUnit = 1,
TickBase = 100000,
MaxPrice = float.MaxValue
});
}
return specification;
}
private static bool SupportsFractionalQuantity(SecurityImmutable security) =>
security.HasContractSpec && security.ContractSpec.QuantityInfos.Any(info =>
info.TickBase > 0 && info.TickUnit > 0 && info.TickUnit < info.TickBase);
private static ContractLeg BuildLeg(
SecurityImmutable security,
SyntheticSpreadLegRequest leg) =>
new ContractLeg {
Security = security.CloneToMutable(),
Quantity = leg.Quantity,
PriceMultiplier = leg.PriceMultiplier,
LegDelta = leg.LegDelta,
Side = leg.Side
};
private static LegSpreaderParams BuildLegParams(
double leanQuantityRatio,
SyntheticSpreadLegRequest leg) =>
new LegSpreaderParams {
IsActiveQuoting = leg.IsActiveQuoting,
EnableHedging = leg.EnableHedging,
MarketDataType = leg.MarketDataType,
QuoteParams = new QuoteParams {
LeanQuantityRatio = leanQuantityRatio
},
PreHedgeParams = new PreHedgeParams(),
PostHedgeParams = new PostHedgeParams()
};
private static async Task<T> WaitForSyntheticSpreadOperation<T>(Task<T> operation, string operationName) {
try {
return await operation.WaitAsync(SyntheticSpreadCreateTimeout);
} catch (TimeoutException) {
throw new TimeoutException(
$"{operationName} timed out after {SyntheticSpreadCreateTimeout.TotalSeconds:0} seconds");
}
}
private static bool TryGetSyntheticSpreadLeg(
JsonObject parameters,
string legPrefix,
bts.config.SyntheticSpreadLegDefaults defaults,
Side fallbackSide,
out SyntheticSpreadLegRequest leg,
out string message) {
leg = default;
if (!TryGetRequiredVenue(parameters, $"{legPrefix}_exchange", out var exchange, out message) ||
!TryGetRequiredString(parameters, $"{legPrefix}_exchange_security_name", out var secName, out message)) {
return false;
}
double quantity = defaults.Quantity != 0 ? defaults.Quantity : 1.0;
double priceMultiplier = defaults.PriceMultiplier != 0 ? defaults.PriceMultiplier : 1.0;
double legDelta = defaults.LegDelta;
bool isActiveQuoting = defaults.IsActiveQuoting;
bool enableHedging = defaults.EnableHedging;
uint marketDataTypeValue = defaults.MarketDataType;
var side = defaults.Side != Side.Undefined ? defaults.Side : fallbackSide;
if (!TryGetOptionalDouble(parameters, $"{legPrefix}_quantity", quantity, out quantity, out message) ||
!TryGetOptionalDouble(parameters, $"{legPrefix}_price_multiplier", priceMultiplier, out priceMultiplier, out message) ||
!TryGetOptionalDouble(parameters, $"{legPrefix}_leg_delta", legDelta, out legDelta, out message) ||
!TryGetOptionalBool(parameters, $"{legPrefix}_is_active_quoting", isActiveQuoting, out isActiveQuoting, out message) ||
!TryGetOptionalBool(parameters, $"{legPrefix}_enable_hedging", enableHedging, out enableHedging, out message) ||
!TryGetOptionalEnum(parameters, $"{legPrefix}_market_data_type", marketDataTypeValue, out marketDataTypeValue, out message) ||
!TryGetOptionalEnum(parameters, $"{legPrefix}_side", side, out side, out message)) {
return false;
}
leg = new SyntheticSpreadLegRequest(
exchange,
secName,
quantity,
priceMultiplier,
legDelta,
side,
isActiveQuoting,
enableHedging,
(LegSpreaderParams.Types.MarketDataType)marketDataTypeValue);
return true;
}
private bool TryResolveSecurity(
Venue exchange,
string exchangeSecurityName,
out SecurityImmutable security,
out string message) {
if (_securities.Values.TryGetFirst(sec =>
sec.Exchange == exchange &&
sec.ToString() == exchangeSecurityName,
out security)) {
message = string.Empty;
return true;
}
message = $"{exchangeSecurityName} at {exchange} not found in securities";
return false;
}
private async Task<(bool success, uint? id, string status)> CreateSyntheticSpreadAsync(WebsocketRequest req) {
if (req.parameters == null) {
return (false, null, "parameters not present");
}
if (_synth_sec_client is null) {
return (false, null, "synthetic security client is not initialized");
}
var config = Configuration.get<bts.config.SyntheticSpreadConfig>();
if (!TryGetRequiredVenue(req.parameters, "leg1_exchange", out var leg1Exchange, out var message) ||
!TryGetRequiredString(req.parameters, "leg1_exchange_security_name", out var leg1SecurityName, out message) ||
!TryGetRequiredVenue(req.parameters, "leg2_exchange", out var leg2Exchange, out message) ||
!TryGetRequiredString(req.parameters, "leg2_exchange_security_name", out var leg2SecurityName, out message)) {
return (false, null, message);
}
var preset = ResolvePreset(config, leg1Exchange, leg1SecurityName);
if (preset is null) {
return (false, null, $"No synthetic spread preset found for {leg1Exchange}:{leg1SecurityName}");
}
string? requestedName = null;
string? requestedSymbol = null;
string? requestedCurrency = null;
double requestedTickSize = 0;
double requestedCashPerPoint = 0;
double requestedPriceMultiplier = 0;
uint requestedMainFraction = 0;
uint requestedSubFraction = 0;
uint requestedPriceDisplayFormat = 0;
uint requestedPriceCalcValue = preset.PriceCalc;
bool requestedCancelHedgesOnCancel = preset.CancelHedgesOnCancel;
uint requestedHedgePricingValue = preset.HedgePricing;
uint requestedSpreaderTypeValue = preset.SpreaderType;
double requestedLeanQuantityRatio = 0;
if (!TryGetOptionalString(req.parameters, "name", out requestedName, out message) ||
!TryGetOptionalString(req.parameters, "symbol", out requestedSymbol, out message) ||
!TryGetOptionalString(req.parameters, "currency", out requestedCurrency, out message) ||
!TryGetOptionalDouble(req.parameters, "tick_size", preset.TickSize != 0 ? preset.TickSize : config.DefaultTickSize, out requestedTickSize, out message) ||
!TryGetOptionalDouble(req.parameters, "cash_per_point", preset.CashPerPoint != 0 ? preset.CashPerPoint : config.DefaultCashPerPoint, out requestedCashPerPoint, out message) ||
!TryGetOptionalDouble(req.parameters, "price_multiplier", preset.PriceMultiplier != 0 ? preset.PriceMultiplier : config.DefaultPriceMultiplier, out requestedPriceMultiplier, out message) ||
!TryGetOptionalEnum(req.parameters, "main_fraction", 1U, out requestedMainFraction, out message) ||
!TryGetOptionalEnum(req.parameters, "sub_fraction", 100000U, out requestedSubFraction, out message) ||
!TryGetOptionalEnum(req.parameters, "price_display_format", (preset.PriceDisplayFormat != 0 ? preset.PriceDisplayFormat : config.DefaultPriceDisplayFormat), out requestedPriceDisplayFormat, out message) ||
!TryGetOptionalEnum(req.parameters, "price_calc", requestedPriceCalcValue, out requestedPriceCalcValue, out message) ||
!TryGetOptionalBool(req.parameters, "cancel_hedges_on_cancel", requestedCancelHedgesOnCancel, out requestedCancelHedgesOnCancel, out message) ||
!TryGetOptionalEnum(req.parameters, "hedge_pricing", requestedHedgePricingValue, out requestedHedgePricingValue, out message) ||
!TryGetOptionalEnum(req.parameters, "spreader_type", requestedSpreaderTypeValue, out requestedSpreaderTypeValue, out message) ||
!TryGetOptionalDouble(req.parameters, "lean_quantity_ratio", preset.LeanQuantityRatio != 0 ? preset.LeanQuantityRatio : config.DefaultLeanQuantityRatio, out requestedLeanQuantityRatio, out message)) {
return (false, null, message);
}
var leg2Defaults = preset.Leg2 ?? new bts.config.SyntheticSpreadLegDefaults();
leg2Defaults.PriceMultiplier = -1;
if (!TryGetSyntheticSpreadLeg(req.parameters, "leg1", preset.Leg1 ?? new bts.config.SyntheticSpreadLegDefaults(), Side.Buy, out var leg1, out message) ||
!TryGetSyntheticSpreadLeg(req.parameters, "leg2", leg2Defaults, Side.Sell, out var leg2, out message)) {
return (false, null, message);
}
if (!TryResolveSecurity(leg1.Exchange, leg1.ExchangeSecurityName, out var sec1, out message)) {
return (false, null, message);
}
if (!TryResolveSecurity(leg2.Exchange, leg2.ExchangeSecurityName, out var sec2, out message)) {
return (false, null, message);
}
var builtLeg1 = BuildLeg(sec1, leg1);
var builtLeg2 = BuildLeg(sec2, leg2);
string resolvedName = BuildSyntheticName(requestedName, preset.Name, builtLeg1, builtLeg2);
string resolvedSymbol = BuildSyntheticSymbol(requestedSymbol, preset.Symbol, sec1.Symbol, resolvedName);
string resolvedCurrency = !string.IsNullOrWhiteSpace(requestedCurrency)
? requestedCurrency.Trim()
: !string.IsNullOrWhiteSpace(preset.Currency)
? preset.Currency.Trim()
: !string.IsNullOrWhiteSpace(config.DefaultCurrency)
? config.DefaultCurrency.Trim()
: sec1.Currency;
double resolvedTickSize = ResolveDouble(requestedTickSize, preset.TickSize != 0 ? preset.TickSize : config.DefaultTickSize, sec1.ContractSpec.LowTick);
double resolvedCashPerPoint = ResolveDouble(requestedCashPerPoint, preset.CashPerPoint != 0 ? preset.CashPerPoint : config.DefaultCashPerPoint, sec1.ContractSpec.CashPerPoint);
double resolvedPriceMultiplier = ResolveDouble(requestedPriceMultiplier, preset.PriceMultiplier != 0 ? preset.PriceMultiplier : config.DefaultPriceMultiplier, sec1.ContractSpec.PriceMultiplier);
uint resolvedMainFraction = ResolveUInt32(requestedMainFraction, 1, sec1.ContractSpec.MainFraction);
uint resolvedSubFraction = ResolveUInt32(requestedSubFraction, 100000, sec1.ContractSpec.SubFraction);
uint resolvedPriceDisplayFormat = ResolveUInt32(requestedPriceDisplayFormat, preset.PriceDisplayFormat != 0 ? preset.PriceDisplayFormat : config.DefaultPriceDisplayFormat, sec1.ContractSpec.PriceDisplayFormat);
var resolvedPriceCalc = (SyntheticContract.Types.PriceCalc)requestedPriceCalcValue;
var resolvedCancelHedgesOnCancel = requestedCancelHedgesOnCancel;
var resolvedHedgePricing = (ExecutionStrategy.Types.HedgePricing)requestedHedgePricingValue;
var resolvedSpreaderType = (ExecutionStrategy.Types.SpreaderType)requestedSpreaderTypeValue;
var resolvedLeanQuantityRatio = requestedLeanQuantityRatio;
bool supportsFractionalQuantity =
SupportsFractionalQuantity(sec1) || SupportsFractionalQuantity(sec2);
var security = new Security {
Exchange = Venue.Bts,
Symbol = resolvedSymbol,
Currency = resolvedCurrency,
ContractSpec = BuildContractSpec(
resolvedTickSize,
resolvedCashPerPoint,
resolvedPriceMultiplier,
resolvedMainFraction,
resolvedSubFraction,
resolvedPriceDisplayFormat,
supportsFractionalQuantity),
Synthetic = new SyntheticContract {
Name = resolvedName,
PriceCalc = resolvedPriceCalc
}
};
security.Synthetic.Legs.Add(builtLeg1);
security.Synthetic.Legs.Add(builtLeg2);
uint securityKey;
try {
bts.Log.Gen.info($"CREATE_SYNTHETIC_SPREAD creating synthetic '{resolvedName}'");
securityKey = await WaitForSyntheticSpreadOperation(
Synth_Sec_Client.create_synthetic_security(new SecurityImmutable(security)),
"create_synthetic_security");
} catch (Exception ex) {
return (false, null, ex.Message);
}
var strategy = new ExecutionStrategy {
CancelHedgesOnCancel = resolvedCancelHedgesOnCancel,
HedgePricing = resolvedHedgePricing,
SpreaderType = resolvedSpreaderType
};
strategy.LegParamsMap[sec1.Id] = BuildLegParams(resolvedLeanQuantityRatio, leg1);
strategy.LegParamsMap[sec2.Id] = BuildLegParams(resolvedLeanQuantityRatio, leg2);
try {
bts.Log.Gen.info($"CREATE_SYNTHETIC_SPREAD updating strategy for security_key {securityKey}");
var status = await WaitForSyntheticSpreadOperation(
Synth_Sec_Client.update_synthetic_strategy(securityKey, new ExecutionStrategyImmutable(strategy)),
"update_synthetic_strategy");
if (status != null) {
return (false, null, status);
}
} catch (Exception ex) {
return (false, null, ex.Message);
}
return (true, securityKey, "OK");
}
public void handle_connection(WebsocketClientID client_id,
WebsocketIncomingMessage? optional_message) {
@@ -264,10 +728,10 @@ namespace bts.websocket_core {
handle_incorrect_request(id, message.sender_client_id, error);
return;
}
handle_request(req);
var log_txt = $"WebsocketDataSync.handle_client_message message: {req.type}";
var log_txt = $"WebsocketDataSync.handle_client_message message type: {req.type}";
bts.Log.Gen.info(log_txt);
System.Console.WriteLine(log_txt);
handle_request(req);
}
public void handle_request(WebsocketRequest req) {
@@ -296,6 +760,10 @@ namespace bts.websocket_core {
handle_set_theos(req);
break;
}
case WebsocketRequest.REQUEST_TYPES.CREATE_SYNTHETIC_SPREAD: {
handle_create_synthetic_spread(req);
break;
}
case WebsocketRequest.REQUEST_TYPES.GET: {
handle_get(req);
break;
@@ -325,6 +793,19 @@ namespace bts.websocket_core {
public async void handle_delete_order(WebsocketRequest req) { await HandleRestRequest(req,WebsocketRestHandler.DeleteOrderAsync);}
public async void handle_set_theos(WebsocketRequest req) { await HandleRestRequest(req,(req, Orders_Client) => WebsocketRestHandler.SetTheosAsync(req, Orders_Client, _securities));}
public async void handle_get(WebsocketRequest req) { await HandleRestRequest(req,WebsocketRestHandler.Get);}
public async void handle_create_synthetic_spread(WebsocketRequest req) {
try {
System.Console.WriteLine($"CREATE_SYNTHETIC_SPREAD request received id: {req.id}");
var (success, id, status) = await CreateSyntheticSpreadAsync(req);
if (!success) {
handle_incorrect_request(req, status);
return;
}
handle_correct_request(req.id, status, new { security_key = id }, req.client_id);
} catch (Exception ex) {
handle_incorrect_request(req, ex.Message);
}
}
public async void handle_enter_order(WebsocketRequest req) {
await HandleRestRequest(req, (req, Orders_Client) => WebsocketRestHandler.EnterOrderAsync(
req, Orders_Client, _securities, adminService, Dynamic_Descriptors, synthStrategies
@@ -375,7 +856,16 @@ namespace bts.websocket_core {
send_message(ms, cid);
}
public void send_message(object value, WebsocketClientID id) {
var requestJson = System.Text.Json.JsonSerializer.Serialize(value);
string requestJson;
try {
requestJson = System.Text.Json.JsonSerializer.Serialize(value);
}
catch (ArgumentException ex) {
// System.Text.Json rejects NaN and +/-Infinity by default. Do not let
// one malformed market-data update terminate the API server.
bts.Log.Gen.error($"Websocket JSON serialization failed for client {id}: {ex.Message}");
return;
}
Gateway.send_to_clients(
new List<WebsocketOutgoingMessage>() {
new WebsocketOutgoingMessage(id, requestJson)
@@ -14,6 +14,7 @@ namespace bts.websocket_core {
DELETE_ORDER,
UPDATE_ORDER,
SET_PRECOMPUTED_THEOS,
CREATE_SYNTHETIC_SPREAD,
DELETE,
GET,
PATCH,
@@ -30,10 +31,12 @@ namespace bts.websocket_core {
type = REQUEST_TYPES.UNKOWN;
JsonObject? request;
_message_id = "missing";
json = json.Replace('\'', '"');
try {
request = JsonSerializer.Deserialize<JsonObject>(json);
} catch (JsonException ex) {
message = $"Incorrect Json: {ex.Path}";
bts.Log.Gen.error($"Message: |{json}| is Inccorect json: {ex.Path}");
message = $"Incorrect Json: Path: {ex.Path}";
return false;
}
if (request is null) {
@@ -192,6 +192,9 @@
<Proto Include="../../lib_spark/spark/config/protos/quickbus_cfg.proto" CSName="QuickbusCfg">
<OutDir>$(ProtoOutputPath)/lib_spark/spark/config/protos</OutDir>
</Proto>
<Proto Include="../../lib_spark/spark/config/protos/synthetic_spread.proto" CSName="SyntheticSpread">
<OutDir>$(ProtoOutputPath)/lib_spark/spark/config/protos</OutDir>
</Proto>
<Proto Include="../../lib_spark/spark/config/protos/symbols.proto" CSName="Symbols">
<OutDir>$(ProtoOutputPath)/lib_spark/spark/config/protos</OutDir>
</Proto>
@@ -190,10 +190,6 @@ namespace mbts.spark {
result = SingleSecurityCriteria.Empty;
return false;
}
if (id is null && (name is null || venue is null)) {
result = SingleSecurityCriteria.Empty;
return false;
}
result = new(venue, type, asset, name, id);
return true;
}
+3 -3
View File
@@ -126,12 +126,12 @@ namespace bts.api {
tick_sz = s.ContractSpec.LowTick,
leg_1_symbol = leg_1.Security.Symbol,
leg_1_exchange = leg_1.Security.Exchange.ToString(),
leg_1_min_quantity = leg_1.Security.MinOrderQuantity(),
leg_1_min_quantity = Math.Round(leg_1.Security.MinOrderQuantity(),9),
leg_1_tick_sz = leg_1.Security.ContractSpec.LowTick,
leg_1_cpp = leg_1.Security.ContractSpec.CashPerPoint,
leg_2_symbol = leg_2.Security.Symbol,
leg_2_exchange = leg_2.Security.Exchange.ToString(),
leg_2_min_quantity = leg_2.Security.MinOrderQuantity(),
leg_2_min_quantity = Math.Round(leg_2.Security.MinOrderQuantity(),9),
leg_2_tick_sz = leg_2.Security.ContractSpec.LowTick,
leg_2_cpp = leg_2.Security.ContractSpec.CashPerPoint
};
@@ -141,7 +141,7 @@ namespace bts.api {
security = s.security_desc(),
security_type = sec_type(s),
exchange = s.Exchange.ToString(),
min_quantity = s.MinOrderQuantity(),
min_quantity = Math.Round(s.MinOrderQuantity(),9),
tick_sz = s.ContractSpec.LowTick,
cpp = s.ContractSpec.CashPerPoint
};
@@ -182,6 +182,8 @@ namespace bts.api {
Security.ExchangeDataOneofCase.None => security.Synthetic.Name,
Security.ExchangeDataOneofCase.Bybit => security.Bybit.SecurityDesc,
Security.ExchangeDataOneofCase.Gateio => security.Gateio.SecurityDesc,
Security.ExchangeDataOneofCase.Orca => security.Orca.SecurityDesc,
Security.ExchangeDataOneofCase.Raydium => security.Raydium.SecurityDesc,
_ => throw new NotImplementedException("security_desc not implemented for exchange")
};
@@ -235,6 +237,8 @@ namespace bts.api {
domain.Venue.Twse => Venue.Twse,
domain.Venue.Bybit => Venue.Bybit,
domain.Venue.Gateio => Venue.Gateio,
domain.Venue.Orca => Venue.Orca,
domain.Venue.Raydium => Venue.Raydium,
domain.Venue.Unspecified => Venue.Unset,
_ => throw new NotImplementedException(),
};
@@ -257,6 +261,8 @@ namespace bts.api {
Venue.Twse => domain.Venue.Twse,
Venue.Bybit => domain.Venue.Bybit,
Venue.Gateio => domain.Venue.Gateio,
Venue.Orca => domain.Venue.Orca,
Venue.Raydium => domain.Venue.Raydium,
Venue.Unset => domain.Venue.Unspecified,
_ => throw new NotImplementedException(),
};
@@ -30,4 +30,6 @@ enum Venue {
VENUE_TWSE = 18;
VENUE_BYBIT = 19;
VENUE_GATEIO = 20;
VENUE_ORCA = 21;
VENUE_RAYDIUM = 22;
}
@@ -0,0 +1,17 @@
syntax = "proto3";
import "spark/api/domain/SecurityCriteria.proto";
import "spark/api/domain/Venue.proto";
package bts.api.md;
option csharp_namespace = "bts.api.md";
option cc_enable_arenas = true;
message DexMarketDataConfig {
uint32 num_depth_levels = 1;
bts.api.domain.SecurityCriteria security_criteria = 2;
repeated string excluded_securities = 3;
string json_conf_file_name = 4;
bts.api.domain.Venue venue = 5;
}
@@ -18,6 +18,7 @@ import "spark/api/md/BybitMarketDataConfig.proto";
import "spark/api/md/GateioMarketDataConfig.proto";
import "spark/api/md/CboeMarketDataConfig.proto";
import "spark/api/md/PyMarketDataProviderConfig.proto";
import "spark/api/md/DexMarketDataConfig.proto";
option csharp_namespace = "bts.api.md";
@@ -38,6 +39,7 @@ message MarketDataConfig {
repeated TwseMarketDataConfig twse = 12;
repeated BybitMarketDataConfig bybit = 13;
repeated GateioMarketDataConfig gateio = 14;
repeated DexMarketDataConfig dex = 15;
// Add an entry here for all the exchanges that need their market data to be recorded
repeated EmumbaDeribitMarketDataConfig emumba_deribit = 100;
@@ -27,12 +27,27 @@ std::set<bts::api::Venue> MarketDataConfigUtils::get_live_exchanges(
rv.insert(bts::api::VENUE_TWSE);
if (cfg.cboe_size()>0)
rv.insert(bts::api::VENUE_CBOE);
if (cfg.okx_size() > 0) {
rv.insert(bts::api::VENUE_OKX);
}
if (cfg.bybit_size() > 0) {
rv.insert(bts::api::VENUE_BYBIT);
}
if (cfg.gateio_size() > 0) {
rv.insert(bts::api::VENUE_GATEIO);
}
for (const auto &dex_cfg : cfg.dex()) {
switch (dex_cfg.venue()) {
case bts::api::domain::VENUE_ORCA:
rv.insert(bts::api::VENUE_ORCA);
break;
case bts::api::domain::VENUE_RAYDIUM:
rv.insert(bts::api::VENUE_RAYDIUM);
break;
default:
break;
}
}
return rv;
}
@@ -71,6 +71,8 @@ bool AdminServiceUtils::has_exchange(
case api::Venue::VENUE_COINBASE:
case api::Venue::VENUE_BINANCE_USDS_FUTURES:
case api::Venue::VENUE_OKX:
case api::Venue::VENUE_RAYDIUM:
case api::Venue::VENUE_ORCA:
case api::Venue::VENUE_BTS:
case api::Venue::VENUE_CBOE:
case api::Venue::VENUE_TWSE:
@@ -188,6 +190,8 @@ AdminServiceUtils::resolve_connection(
case api::Venue::VENUE_DERIBIT:
case api::Venue::VENUE_FTX:
case api::Venue::VENUE_OKX:
case api::Venue::VENUE_ORCA:
case api::Venue::VENUE_RAYDIUM:
case api::Venue::VENUE_BINANCE:
case api::Venue::VENUE_BINANCE_COIN_FUTURES:
case api::Venue::VENUE_BINANCE_US:
@@ -467,6 +471,8 @@ std::string AdminServiceUtils::find_account_exchange_account(
case api::Venue::VENUE_COINBASE:
case api::Venue::VENUE_BINANCE_USDS_FUTURES:
case api::Venue::VENUE_OKX:
case api::Venue::VENUE_RAYDIUM:
case api::Venue::VENUE_ORCA:
case api::Venue::VENUE_BTS:
case api::Venue::VENUE_CBOE:
case api::Venue::VENUE_TWSE:
@@ -790,6 +796,20 @@ bts::api::orderman::RoutingRecord AdminServiceUtils::get_routing_record(
out_result.set_is_success(true);
return rv;
}
case api::VENUE_RAYDIUM: {
rv.set_session("raydium_1");
rv.set_account(account_ptr->name());
rv.set_operator_id(user_ptr->name());
out_result.set_is_success(true);
return rv;
}
case api::VENUE_ORCA: {
rv.set_session("orca_1");
rv.set_account(account_ptr->name());
rv.set_operator_id(user_ptr->name());
out_result.set_is_success(true);
return rv;
}
case api::VENUE_BYBIT: {
rv.set_session("bybit_1");
rv.set_account(account_ptr->name());
@@ -1204,6 +1224,8 @@ std::string AdminServiceUtils::find_user_exchange_id(
case api::Venue::VENUE_COINBASE:
case api::Venue::VENUE_BINANCE_USDS_FUTURES:
case api::Venue::VENUE_OKX:
case api::Venue::VENUE_RAYDIUM:
case api::Venue::VENUE_ORCA:
case api::Venue::VENUE_UNSET:
case api::Venue::VENUE_BTS:
case api::Venue::VENUE_CBOE:
@@ -1243,6 +1265,8 @@ std::string AdminServiceUtils::find_user_company_id(
case api::Venue::VENUE_COINBASE:
case api::Venue::VENUE_BINANCE_USDS_FUTURES:
case api::Venue::VENUE_OKX:
case api::Venue::VENUE_ORCA:
case api::Venue::VENUE_RAYDIUM:
case api::Venue::VENUE_SIM:
case api::Venue::VENUE_SIMICE:
case api::Venue::VENUE_GLBX:
@@ -4034,6 +4034,16 @@ JetFIX {
max_order_msgs_per_window: 300
venue: VENUE_OKX
}
connection {
name: "orca_1"
max_order_msgs_per_window: 300
venue: VENUE_OKX
}
connection {
name: "raydium_1"
max_order_msgs_per_window: 300
venue: VENUE_OKX
}
connection {
name: "binance_usds_futures_1"
max_order_msgs_per_window: 300
@@ -60,7 +60,15 @@ SymbolLists {
list {
key: "OKX"
symbol: "BTC"
symbol: "ETH"
symbol: "SOL"
}
list {
key: "ORCA"
symbol: "SOL"
}
list {
key: "RAYDIUM"
symbol: "SOL"
}
list {
key: "BYBIT"
@@ -199,6 +207,14 @@ Symbols {
future: @SymbolLists.list[OKX].symbol
option: @SymbolLists.list[OKX].symbol
}
venues {
name: "VENUE_ORCA"
spot: @SymbolLists.list[ORCA].symbol
}
venues {
name: "VENUE_RAYDIUM"
spot: @SymbolLists.list[RAYDIUM].symbol
}
venues {
name: "VENUE_BYBIT"
spot: @SymbolLists.list[BYBIT].symbol
@@ -0,0 +1,53 @@
SyntheticSpreadConfig {
value {
default_currency: "USD"
default_tick_size: 0.00001
default_cash_per_point: 1.0
default_price_multiplier: 1.0
default_main_fraction: 1
default_sub_fraction: 100000
default_price_display_format: 0
default_price_calc: 0
default_cancel_hedges_on_cancel: true
default_hedge_pricing: 1
default_spreader_type: 0
default_lean_quantity_ratio: 1.0
preset {
first_leg_exchange: VENUE_UNSET
first_leg_exchange_security_name: ""
name: "Synthetic"
symbol: ""
currency: "USD"
tick_size: 0.00001
cash_per_point: 1.0
price_multiplier: 1.0
main_fraction: 1
sub_fraction: 100000
price_display_format: 0
price_calc: 0
cancel_hedges_on_cancel: true
hedge_pricing: 1
spreader_type: 0
lean_quantity_ratio: 1.0
leg1 {
quantity: 1.0
price_multiplier: 1.0
leg_delta: 0.0
side: SIDE_BUY
is_active_quoting: true
enable_hedging: true
market_data_type: 0
}
leg2 {
quantity: 1.0
price_multiplier: 1.0
leg_delta: 0.0
side: SIDE_SELL
is_active_quoting: true
enable_hedging: true
market_data_type: 0
}
}
}
}
@@ -0,0 +1,58 @@
syntax = "proto3";
import "spark/domain_model/Security.proto";
import "spark/sparkproto/SparkProtos.proto";
package bts.config;
option csharp_namespace = "bts.config";
message SyntheticSpreadConfig {
string default_currency = 1;
double default_tick_size = 2;
double default_cash_per_point = 3;
double default_price_multiplier = 4;
uint32 default_main_fraction = 5;
uint32 default_sub_fraction = 6;
uint32 default_price_display_format = 7;
uint32 default_price_calc = 8;
bool default_cancel_hedges_on_cancel = 9;
uint32 default_hedge_pricing = 10;
uint32 default_spreader_type = 11;
double default_lean_quantity_ratio = 12;
repeated SyntheticSpreadPreset preset = 20;
}
message SyntheticSpreadPreset {
bts.api.Venue first_leg_exchange = 1;
string first_leg_exchange_security_name = 2;
string name = 10;
string symbol = 11;
string currency = 12;
double tick_size = 13;
double cash_per_point = 14;
double price_multiplier = 15;
uint32 main_fraction = 16;
uint32 sub_fraction = 17;
uint32 price_display_format = 18;
uint32 price_calc = 19;
bool cancel_hedges_on_cancel = 20;
uint32 hedge_pricing = 21;
uint32 spreader_type = 22;
double lean_quantity_ratio = 23;
SyntheticSpreadLegDefaults leg1 = 30;
SyntheticSpreadLegDefaults leg2 = 31;
}
message SyntheticSpreadLegDefaults {
double quantity = 1;
double price_multiplier = 2;
double leg_delta = 3;
bts.api.Side side = 4;
bool is_active_quoting = 5;
bool enable_hedging = 6;
uint32 market_data_type = 7;
}
@@ -31,6 +31,10 @@ api::domain::Venue DomainSecurityUtils::convert(bts::api::Venue venue) {
return DOMAIN_VENUE(BTS);
case bts::api::Venue::VENUE_SIM:
return DOMAIN_VENUE(SIM);
case bts::api::Venue::VENUE_RAYDIUM:
return DOMAIN_VENUE(RAYDIUM);
case bts::api::Venue::VENUE_ORCA:
return DOMAIN_VENUE(ORCA);
case bts::api::Venue::VENUE_BINANCE:
return DOMAIN_VENUE(BINANCE);
case bts::api::Venue::VENUE_BINANCE_COIN_FUTURES:
@@ -85,6 +89,10 @@ bts::api::Venue DomainSecurityUtils::convert(api::domain::Venue venue) {
return api::Venue::VENUE_BYBIT;
case DOMAIN_VENUE(GATEIO):
return api::Venue::VENUE_GATEIO;
case DOMAIN_VENUE(ORCA):
return api::Venue::VENUE_ORCA;
case DOMAIN_VENUE(RAYDIUM):
return api::Venue::VENUE_RAYDIUM;
case DOMAIN_VENUE(SIM):
return api::Venue::VENUE_SIM;
case DOMAIN_VENUE(COINBASE):
@@ -245,6 +253,12 @@ bool DomainSecurityUtils::get_security_desc(
case bts::api::Security::ExchangeDataCase::kOkx:
desc = security.okx().security_desc();
break;
case bts::api::Security::ExchangeDataCase::kRaydium:
desc = security.raydium().security_desc();
break;
case bts::api::Security::ExchangeDataCase::kOrca:
desc = security.orca().security_desc();
break;
case bts::api::Security::ExchangeDataCase::kFtx:
desc = security.ftx().security_desc();
break;
@@ -502,6 +502,29 @@ void from_json(const json &j, OkxExchangeData &data) {
PB_JSON_READ_STRING(j, data, security_desc, it);
}
void to_json(json &j, const OrcaExchangeData &data) {
PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
}
void from_json(const json &j, OrcaExchangeData &data) {
json::const_iterator it;
PB_JSON_READ_VALUE(j, data, exchange_id, it);
PB_JSON_READ_STRING(j, data, security_desc, it);
}
void to_json(json &j, const RaydiumExchangeData &data) {
PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
}
void from_json(const json &j, RaydiumExchangeData &data) {
json::const_iterator it;
PB_JSON_READ_VALUE(j, data, exchange_id, it);
PB_JSON_READ_STRING(j, data, security_desc, it);
}
void to_json(json &j, const CoinbaseExchangeData &data) {
PB_JSON_WRITE_IF_NOT_ZERO(j, data, exchange_id);
PB_JSON_WRITE_IF_NOT_EMPTY(j, data, security_desc);
@@ -535,6 +558,8 @@ void to_json(json &j, const Security &s) {
oneof(j, s, BinanceUs, binance_us) \
oneof(j, s, Coinbase, coinbase) \
oneof(j, s, Okx, okx) \
oneof(j, s, Raydium, raydium) \
oneof(j, s, Orca, orca) \
oneof(j, s, Taifex, taifex)
// clang-format on
@@ -569,6 +594,8 @@ void from_json(const json &j, Security &s) {
oneof(j, s, binance_us, it); \
oneof(j, s, coinbase, it); \
oneof(j, s, okx, it); \
oneof(j, s, orca, it); \
oneof(j, s, raydium, it); \
oneof(j, s, taifex, it);
PB_JSON_READ_VALUE(j, s, id, it);
@@ -88,6 +88,12 @@ void from_json(const nlohmann::json &j, CoinbaseExchangeData &data);
void to_json(nlohmann::json &j, const Security &s);
void from_json(const nlohmann::json &j, Security &s);
void to_json(nlohmann::json &j, const OrcaExchangeData &data);
void from_json(const nlohmann::json &j, OrcaExchangeData &data);
void to_json(nlohmann::json &j, const RaydiumExchangeData &data);
void from_json(const nlohmann::json &j, RaydiumExchangeData &data);
PB_ENUM_FROM_JSON(SyntheticContract_PriceCalc);
PB_ENUM_TO_JSON(SyntheticContract_PriceCalc);
@@ -371,6 +377,10 @@ inline const std::string &descriptor(
return sec.gateio().security_desc();
case api::Security::kOkx:
return sec.okx().security_desc();
case api::Security::kRaydium:
return sec.raydium().security_desc();
case api::Security::kOrca:
return sec.orca().security_desc();
case api::Security::kTwse:
return sec.twse().security_desc();
case api::Security::kTaifex:
@@ -24,6 +24,8 @@ enum Venue {
VENUE_TWSE = 600;
VENUE_BYBIT = 610;
VENUE_GATEIO = 620;
VENUE_ORCA = 630;
VENUE_RAYDIUM = 640;
}
enum SecurityType {
@@ -141,6 +143,16 @@ message OkxExchangeData {
string security_desc = 2;
}
message RaydiumExchangeData {
int64 exchange_id = 1;
string security_desc = 2;
}
message OrcaExchangeData {
int64 exchange_id = 1;
string security_desc = 2;
}
message BybitExchangeData {
int64 exchange_id = 10;
string security_desc = 20;
@@ -282,6 +294,8 @@ message Security {
TWSEExchangeData twse = 500;
BybitExchangeData bybit = 510;
GateioExchangeData gateio = 520;
RaydiumExchangeData raydium = 530;
OrcaExchangeData orca = 540;
}
}
message SecurityList {
@@ -54,7 +54,6 @@
#include <google/protobuf/timestamp.pb.h>
#include <google/protobuf/util/time_util.h>
#include <memory>
#include <mutex>
#include <optional>
#include <sstream>
#include <string>
@@ -296,7 +295,6 @@ struct GeneralMarketDataListener::PImpl {
// (a flag is used to tell if bts::domain::SecurityID can be used as bts id
// as well)
std::unordered_map<bts::domain::SecurityID, uint64_t> cc_sec_id_map = {};
std::mutex callback_mutex;
MDArray bids = {};
MDArray asks = {};
IncrementalBookData inc_book_data = {};
@@ -389,13 +387,10 @@ GeneralMarketDataListener::~GeneralMarketDataListener() {}
void GeneralMarketDataListener::on_securities(
const std::vector<domain::Security const *> &complete_security_list) {
std::lock_guard<std::mutex> lock(me->callback_mutex);
LOG_SECURITIES_EVENT(Info, complete_security_list.size());
for (auto s_ptr : complete_security_list) {
const domain::Security &sec = *s_ptr;
CCSecSearchResult search_result = _find_sec(sec, true);
LOG(md, Info, "GeneralMarketDataListener::on_securities bef: {} after: {} key: {}", search_result.existed_before,
search_result.exists_now, search_result.key);
if (search_result.existed_before) {
LOG_SECURITY(
Warn, sec.venue(), sec.type(), sec.desc(), *sec.id(), 0,
@@ -429,7 +424,6 @@ void GeneralMarketDataListener::on_security_state(
void GeneralMarketDataListener::on_security_statistics(
const domain::SecurityMarketDataStatistics &sec_stats) {
std::lock_guard<std::mutex> lock(me->callback_mutex);
auto handle = _find_handle(sec_stats.security_id);
bts_assert_msgf(
handle != nullptr,
@@ -448,7 +442,6 @@ void GeneralMarketDataListener::on_security_statistics(
void GeneralMarketDataListener::on_book(
const bts::domain::Book &book, std::optional<utils::TimeStamp> time) {
std::lock_guard<std::mutex> lock(me->callback_mutex);
LOG_BOOK(Diag, book, BookLog_Status::PROCESSING);
// in case this is called externally in conjunction iwth inc book udpates
me->inc_book_data.overwrite_book(book);
@@ -457,7 +450,6 @@ void GeneralMarketDataListener::on_book(
void GeneralMarketDataListener::on_book_increment(
const IMarketDataListener::BookOp &book_op) {
std::lock_guard<std::mutex> lock(me->callback_mutex);
LOG_BOOK_OP(Diag, book_op, BookOpLog_Status::PROCESSING);
if (!me->inc_book_data.update_incremental(book_op)) {
return;
@@ -496,7 +488,6 @@ co2::AggressorSide try_fill_agg_side(
}
void GeneralMarketDataListener::on_trades(
const std::vector<domain::TradeRecord const *> &trades) {
std::lock_guard<std::mutex> lock(me->callback_mutex);
utils::Stopwatch sw;
for (auto tp : trades) {
sw.lap();
@@ -639,8 +630,6 @@ SecurityIndexData *GeneralMarketDataListener::_find_handle(
void GeneralMarketDataListener::_add_future(const bts::domain::Security &sec) {
bts_assert(sec.type() == SEC_TYPE(FUTURE));
auto search_result = _find_sec(sec, false);
LOG(md, Info, "GeneralMarketDataListener::_add_future bef: {} after: {} key: {}", search_result.existed_before,
search_result.exists_now, search_result.key);
bts_assert(search_result.existed_before);
FixedFutureSecurity ff_sec;
_copy_security_common(sec, search_result.key, ff_sec);
@@ -717,7 +706,6 @@ void GeneralMarketDataListener::_add_spot(const bts::domain::Security &sec) {
void GeneralMarketDataListener::_commit_security(const FixedSecurity &sec) {
// Set up the shared memory for this security
LOG(md, Info, "GeneralMarketDataListener::_commit_security numeric_id: {}", sec.numeric_id);
bts_assert(sec.numeric_id != 0);
auto handle = me->md_producer.add_security(sec);
// Add it to the index so it can receive messages
+1 -1
View File
@@ -174,7 +174,7 @@ MDProducerImpl &MDProducer::get_producer(
MDProducerHandle MDProducer::add_security(const FixedSecurity &security) {
bts_assert_d(strlen(security.symbol()) > 0);
bts_assert(security.size >= sizeof(FixedSecurity));
LOG(md, Info, "MDProducer::add_security desc:{} key: {} num_id: {}",
LOG(md, Diag, "MDProducer::add_security desc:{} key: {} num_id: {}",
std::string(security.security_desc()), security.key,
security.numeric_id);
std::lock_guard<std::mutex> lock(*_mutex);
@@ -225,6 +225,14 @@ void convert_security(const FixedSecurity &source, Security *presult) {
dest.mutable_okx()->set_security_desc(source.security_desc());
dest.mutable_okx()->set_exchange_id(source.numeric_id);
break;
case Venue::VENUE_RAYDIUM:
dest.mutable_raydium()->set_security_desc(source.security_desc());
dest.mutable_raydium()->set_exchange_id(source.numeric_id);
break;
case Venue::VENUE_ORCA:
dest.mutable_orca()->set_security_desc(source.security_desc());
dest.mutable_orca()->set_exchange_id(source.numeric_id);
break;
case Venue::VENUE_TWSE:
dest.mutable_twse()->set_security_desc(source.security_desc());
dest.mutable_twse()->set_exchange_id(source.numeric_id);
@@ -349,6 +357,10 @@ uint64_t exchange_id(const Security &security) {
return security.binance_usds_futures().exchange_id();
case Security::ExchangeDataCase::kOkx:
return security.okx().exchange_id();
case Security::ExchangeDataCase::kRaydium:
return security.raydium().exchange_id();
case Security::ExchangeDataCase::kOrca:
return security.orca().exchange_id();
case Security::ExchangeDataCase::kTwse:
return security.twse().exchange_id();
case Security::ExchangeDataCase::kTaifex:
@@ -449,6 +461,14 @@ void convert_security(
dest.numeric_id = src.okx().exchange_id();
dest.security_desc(src.okx().security_desc());
break;
case Security::ExchangeDataCase::kRaydium:
dest.numeric_id = src.raydium().exchange_id();
dest.security_desc(src.raydium().security_desc());
break;
case Security::ExchangeDataCase::kOrca:
dest.numeric_id = src.orca().exchange_id();
dest.security_desc(src.orca().security_desc());
break;
case Security::ExchangeDataCase::kBybit:
dest.numeric_id = src.bybit().exchange_id();
dest.security_desc(src.bybit().security_desc());
@@ -194,6 +194,12 @@ struct MDConsumerGroup {
case api::VENUE_OKX:
this->mic = "OKX";
break;
case api::VENUE_RAYDIUM:
this->mic = "RAYDIUM";
break;
case api::VENUE_ORCA:
this->mic = "ORCA";
break;
case api::VENUE_CBOE:
this->mic = "CBOE";
break;
+7
View File
@@ -24,6 +24,9 @@ class VenueIndex {
std::optional<T> coinbase;
std::optional<T> bybit;
std::optional<T> gateio;
std::optional<T> orca;
std::optional<T> raydium;
public:
std::optional<T> &get(api::Venue venue) { return get_(*this, venue); }
@@ -73,6 +76,10 @@ class VenueIndex {
return self.bybit;
case api::VENUE_GATEIO:
return self.gateio;
case api::VENUE_RAYDIUM:
return self.raydium;
case api::VENUE_ORCA:
return self.orca;
case api::VENUE_BTS:
return self.bts;
case api::VENUE_FTX:
@@ -395,14 +395,9 @@ void SimExecutionGateway::send_msgs(
bts::os::get_thread_id(), bts::os::get_thread_name(), msg_vec.size());
for (auto &m : msg_vec) {
auto mtype = msg_type(m);
std::visit(
[&](auto const &typed_msg) {
LOG(OrderMan, Info,
"SimExecutionGateway::send_msgs tid: {} tname: {} m_type: {} msg: {}",
bts::os::get_thread_id(), bts::os::get_thread_name(), mtype,
typed_msg);
},
m);
LOG(OrderMan, Info,
"SimExecutionGateway::send_msgs tid: {} tname: {} m_type: {}",
bts::os::get_thread_id(), bts::os::get_thread_name(), mtype);
me->buffered_order_ops.push(m);
}
}
@@ -39,6 +39,8 @@ bool validate(
VENUE_CASE(BINANCE_US, BinanceUs)
VENUE_CASE(COINBASE, Coinbase)
VENUE_CASE(OKX, Okx)
VENUE_CASE(ORCA, Orca)
VENUE_CASE(RAYDIUM, Raydium)
VENUE_CASE(TAIFEX, Taifex)
VENUE_CASE(TWSE, Twse)
case api::VENUE_BTS:
@@ -520,6 +520,12 @@ OrderOpResult EMNativeOrder::_on_activate(
case api::VENUE_OKX:
session_name="okx_1";
break;
case api::VENUE_RAYDIUM:
session_name="raydium_1";
break;
case api::VENUE_ORCA:
session_name="orca_1";
break;
case api::VENUE_BINANCE_USDS_FUTURES:
session_name = "binance_usds_futures_1";
break;
@@ -670,11 +676,11 @@ void EMNativeOrder::_create_create_msg(CreateOrderMsg &c) {
c.side = me->definition.side();
c.order_type = get_native_ord_type(me->definition);
c.time_in_force = me->definition.time_in_force();
c.order_type = api::orderman::NATIVE_ORDER_TYPE_LIMIT;
c.manual_order_indicator = me->definition.manual_order_indicator();
c.routing_record = &me->routing_record;
LOG(OrderMan, Info, "EMNativeOrder::_create_create_msg c.qty: {}",
c.quantity);
LOG(OrderMan, Info,
"EMNativeOrder::_create_create_msg order_type:{} c.qty:{}",
api::orderman::NativeOrderType_Name(c.order_type), c.quantity);
}
void EMNativeOrder::_on_exe_report(const NewExecutionReportEvent &er) {
@@ -27,7 +27,8 @@ bool is_crypto_leg(const LegPropertyData &l) {
return v == api::VENUE_BINANCE || v == api::VENUE_BINANCE_USDS_FUTURES ||
v == api::VENUE_BINANCE_COIN_FUTURES ||
v == api::Venue::VENUE_BINANCE_US || v == api::VENUE_DERIBIT ||
v == api::Venue::VENUE_DERIBIT || v == api::Venue::VENUE_OKX;
v == api::Venue::VENUE_DERIBIT || v == api::Venue::VENUE_OKX ||
v == api::Venue::VENUE_ORCA || v == api::Venue::VENUE_RAYDIUM;
}
bool has_crypto_leg(const std::vector<LegPropertyData> &legs) {
@@ -16,6 +16,23 @@
namespace bts {
namespace orderman {
namespace {
bool should_send_hedge_as_market_order(
const spark::LegSpreaderParams &leg_params) {
if (leg_params.has_pre_hedge_params() &&
leg_params.pre_hedge_params().has_send_hedge_as_market_order()) {
return true || leg_params.pre_hedge_params()
.send_hedge_as_market_order()
.send_hedge_as_market_order();
}
// Temporary default until the spreader config/UI is wired to this flag.
return true;
}
} // namespace
// computes the book (last arg) resulting from from a sweep
bool SpreaderUtils::compute_book_from_sweep(
const Book &book,
@@ -162,9 +179,16 @@ bool SpreaderUtils::resolve_synthetic_spread(
// hedger order def
api::orderman::OrderDefinition hedger_order_def =
quoter_order_def;
hedger_order_def.mutable_custom_def()
->mutable_limit()
->MergeFrom(api::orderman::LimitOrderDefinition());
if (should_send_hedge_as_market_order(leg_params)) {
hedger_order_def.mutable_custom_def()
->mutable_market()
->MergeFrom(api::orderman::MarketOrderDefinition());
} else {
hedger_order_def.mutable_custom_def()
->mutable_limit()
->MergeFrom(api::orderman::LimitOrderDefinition());
}
//quoter_order_def.set_time_in_force(order_def.time_in_force());
hedger_order_def.set_sub_type(
bts::api::orderman::OrderSubType::