Compare commits
16
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 John Colvin
|
0c0d2ac911 | ||
|
John Colvin
|
c55d6447f4 | ||
|
John Colvin
|
48c2641069 | ||
|
John Colvin
|
303e4be09e | ||
|
John Colvin
|
45ff7fea95 | ||
|
John Colvin
|
6462a6a56c | ||
|
John Colvin
|
f842a0528f | ||
|
John Colvin
|
74c9023eac | ||
|
John Colvin
|
c7edd8403f | ||
|
John Colvin
|
0a8e066c86 | ||
|
John Colvin
|
2db4d23ca9 | ||
|
John Colvin
|
f876061236 | ||
|
John Colvin
|
40482d51e4 | ||
|
John Colvin
|
28e899fd10 | ||
|
John Colvin
|
2f7e75fec0 | ||
|
John Colvin
|
749fed15ad |
No files matched your search
@@ -29,9 +29,6 @@
|
||||
# tools/deploy.sh --help
|
||||
#
|
||||
|
||||
### Host/Runtime ###
|
||||
TZ=Etc/UTC
|
||||
|
||||
### Sparkd ###
|
||||
# The internal image for sparkd to use. For more info, view:
|
||||
# src/server/sparkd/Dockerfile
|
||||
@@ -41,9 +38,6 @@ SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
|
||||
# Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
|
||||
# Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
|
||||
# SPARKD__DATA_DIR=/home/bts/sparkdata/data
|
||||
# Mount volume or dir to "/md_recording" inside the docker. Defaults to volume
|
||||
# "sparkd_md_recording_<config_port>" (e.g., sparkd_md_recording_8000).
|
||||
# SPARKD__MD_RECORDING_DIR=/home/bts/md_recording
|
||||
SPARKD__SOURCE_DIR=install/sparkd-release-gcc
|
||||
SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
|
||||
SPARKD__CONFIG_PORT=8000
|
||||
@@ -51,8 +45,6 @@ SPARKD__QUICKBUS_PORT=8001
|
||||
SPARKD__SIM_FIX_PORT=8002
|
||||
SPARKD__SHM_SIZE=4gb
|
||||
SPARKD__PYTHON_REQUIREMENTS=requirements.txt
|
||||
SPARKD__SPARKD_ENTRYPOINT=src/app_sparkd/sparkd-entrypoint.sh
|
||||
SPARKD__MAIN_ENTRYPOINT=./sparkd-entrypoint.sh
|
||||
|
||||
### PubSubServer ###
|
||||
PUBSUB__CONFIG_DIR=src/DataStreamSuite/PubSubServer/Configurations/Development
|
||||
|
||||
@@ -17,6 +17,14 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk", "src\cs\Bts
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk.Demo", "src\cs\Bts.Spark.Sdk.Demo\Bts.Spark.Sdk.Demo.csproj", "{FCFDBDBE-74B0-41C4-9011-C651B21056D4}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Collections", "src\cs\Bts.Collections\Bts.Collections.csproj", "{3BF4DF96-5AFF-4674-866A-71A8090627FF}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Core", "src\cs\Bts.Core\Bts.Core.csproj", "{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Disposables", "src\cs\Bts.Disposables\Bts.Disposables.csproj", "{62067659-AA2E-4CB5-B7CE-0B38EAA92080}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Numerics", "src\cs\Bts.Numerics\Bts.Numerics.csproj", "{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
@@ -87,6 +95,54 @@ Global
|
||||
{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x64.Build.0 = Release|Any CPU
|
||||
{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{FCFDBDBE-74B0-41C4-9011-C651B21056D4}.Release|x86.Build.0 = Release|Any CPU
|
||||
{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|x64.Build.0 = Release|Any CPU
|
||||
{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{3BF4DF96-5AFF-4674-866A-71A8090627FF}.Release|x86.Build.0 = Release|Any CPU
|
||||
{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|x64.Build.0 = Release|Any CPU
|
||||
{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B}.Release|x86.Build.0 = Release|Any CPU
|
||||
{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|x64.Build.0 = Release|Any CPU
|
||||
{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{62067659-AA2E-4CB5-B7CE-0B38EAA92080}.Release|x86.Build.0 = Release|Any CPU
|
||||
{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|x64.Build.0 = Release|Any CPU
|
||||
{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE}.Release|x86.Build.0 = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
@@ -98,5 +154,9 @@ Global
|
||||
{37E4E890-D025-4729-90F9-6F95862F47BC} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
|
||||
{A5FF3FAF-43CB-496B-AF4B-F61EFD645579} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
|
||||
{FCFDBDBE-74B0-41C4-9011-C651B21056D4} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
|
||||
{3BF4DF96-5AFF-4674-866A-71A8090627FF} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
|
||||
{CD0A3EEF-303A-44EE-85F2-76F4CE632A3B} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
|
||||
{62067659-AA2E-4CB5-B7CE-0B38EAA92080} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
|
||||
{2FE1E5D0-168A-4D83-9B18-0D48250B4BFE} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
|
||||
EndGlobalSection
|
||||
EndGlobal
|
||||
+106
-28
@@ -51,9 +51,13 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "src\cs\Bts.Net\B
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "src\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{BC277499-8215-4F9F-B9AB-07B6391110BF}"
|
||||
EndProject
|
||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "client", "client", "{70E3DCF5-A3C3-45A0-8F18-B01B2D446532}"
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Collections", "src\cs\Bts.Collections\Bts.Collections.csproj", "{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_command_bridge", "src\client\spark_command_bridge\spark_command_bridge.csproj", "{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}"
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Core", "src\cs\Bts.Core\Bts.Core.csproj", "{802ECF0B-7264-4168-95D1-158E47FB65CC}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Disposables", "src\cs\Bts.Disposables\Bts.Disposables.csproj", "{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Numerics", "src\cs\Bts.Numerics\Bts.Numerics.csproj", "{0354655D-2394-42BB-B1D4-2D75B9BCABF1}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
@@ -575,30 +579,102 @@ Global
|
||||
{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
|
||||
{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
|
||||
{BC277499-8215-4F9F-B9AB-07B6391110BF}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
|
||||
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Debug|x64.ActiveCfg = Debug|x64
|
||||
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Debug|x64.Build.0 = Debug|x64
|
||||
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Debug|Any CPU.ActiveCfg = Debug|x64
|
||||
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Debug|Any CPU.Build.0 = Debug|x64
|
||||
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Debug|x86.ActiveCfg = Debug|x64
|
||||
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Debug|x86.Build.0 = Debug|x64
|
||||
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.MinSizeRel|x64.ActiveCfg = Debug|x64
|
||||
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.MinSizeRel|x64.Build.0 = Debug|x64
|
||||
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.MinSizeRel|Any CPU.ActiveCfg = Debug|x64
|
||||
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.MinSizeRel|Any CPU.Build.0 = Debug|x64
|
||||
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.MinSizeRel|x86.ActiveCfg = Debug|x64
|
||||
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.MinSizeRel|x86.Build.0 = Debug|x64
|
||||
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Release|x64.ActiveCfg = Release|x64
|
||||
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Release|x64.Build.0 = Release|x64
|
||||
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Release|Any CPU.ActiveCfg = Release|x64
|
||||
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Release|Any CPU.Build.0 = Release|x64
|
||||
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Release|x86.ActiveCfg = Release|x64
|
||||
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.Release|x86.Build.0 = Release|x64
|
||||
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
|
||||
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.RelWithDebInfo|x64.Build.0 = Debug|x64
|
||||
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|x64
|
||||
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.RelWithDebInfo|Any CPU.Build.0 = Debug|x64
|
||||
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.RelWithDebInfo|x86.ActiveCfg = Debug|x64
|
||||
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED}.RelWithDebInfo|x86.Build.0 = Debug|x64
|
||||
{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||
{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||
{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
|
||||
{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
|
||||
{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.MinSizeRel|x86.Build.0 = Debug|Any CPU
|
||||
{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|x64.Build.0 = Release|Any CPU
|
||||
{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.Release|x86.Build.0 = Release|Any CPU
|
||||
{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||
{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||
{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
|
||||
{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
|
||||
{C42F4CCB-9E65-4370-939A-527ACDBC4DA6}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
|
||||
{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{802ECF0B-7264-4168-95D1-158E47FB65CC}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||
{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||
{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
|
||||
{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
|
||||
{802ECF0B-7264-4168-95D1-158E47FB65CC}.MinSizeRel|x86.Build.0 = Debug|Any CPU
|
||||
{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|x64.Build.0 = Release|Any CPU
|
||||
{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{802ECF0B-7264-4168-95D1-158E47FB65CC}.Release|x86.Build.0 = Release|Any CPU
|
||||
{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||
{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||
{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
|
||||
{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
|
||||
{802ECF0B-7264-4168-95D1-158E47FB65CC}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
|
||||
{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||
{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||
{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
|
||||
{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
|
||||
{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.MinSizeRel|x86.Build.0 = Debug|Any CPU
|
||||
{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|x64.Build.0 = Release|Any CPU
|
||||
{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.Release|x86.Build.0 = Release|Any CPU
|
||||
{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||
{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||
{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
|
||||
{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
|
||||
{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
|
||||
{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||
{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||
{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
|
||||
{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
|
||||
{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.MinSizeRel|x86.Build.0 = Debug|Any CPU
|
||||
{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|x64.Build.0 = Release|Any CPU
|
||||
{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.Release|x86.Build.0 = Release|Any CPU
|
||||
{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||
{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||
{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
|
||||
{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
|
||||
{0354655D-2394-42BB-B1D4-2D75B9BCABF1}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
@@ -620,8 +696,10 @@ Global
|
||||
{385C4948-E080-56A6-E29E-8B0F4913175C} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
|
||||
{43FB297B-7A4D-4835-A878-E6DF345B580C} = {385C4948-E080-56A6-E29E-8B0F4913175C}
|
||||
{BC277499-8215-4F9F-B9AB-07B6391110BF} = {385C4948-E080-56A6-E29E-8B0F4913175C}
|
||||
{70E3DCF5-A3C3-45A0-8F18-B01B2D446532} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
|
||||
{9CD13227-B4D7-45BC-B1C5-1D2ECD2E3AED} = {70E3DCF5-A3C3-45A0-8F18-B01B2D446532}
|
||||
{C42F4CCB-9E65-4370-939A-527ACDBC4DA6} = {385C4948-E080-56A6-E29E-8B0F4913175C}
|
||||
{802ECF0B-7264-4168-95D1-158E47FB65CC} = {385C4948-E080-56A6-E29E-8B0F4913175C}
|
||||
{95C5F6C0-88E5-4338-9A4F-614CF6C03CC0} = {385C4948-E080-56A6-E29E-8B0F4913175C}
|
||||
{0354655D-2394-42BB-B1D4-2D75B9BCABF1} = {385C4948-E080-56A6-E29E-8B0F4913175C}
|
||||
EndGlobalSection
|
||||
GlobalSection(ExtensibilityGlobals) = postSolution
|
||||
SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
|
||||
|
||||
@@ -40,13 +40,7 @@ function(init_python_module)
|
||||
add_custom_command(
|
||||
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
|
||||
DEPENDS ${_name} ${module_proj}
|
||||
COMMAND
|
||||
${CMAKE_COMMAND} -E env
|
||||
"PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}"
|
||||
${Python3_EXECUTABLE} -m pybind11_stubgen
|
||||
--numpy-array-use-type-var
|
||||
--ignore-invalid-expressions '.*'
|
||||
${_name}_module
|
||||
COMMAND PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR} pybind11-stubgen --numpy-array-use-type-var --ignore-invalid-expressions '.*' ${_name}_module
|
||||
COMMENT "Generating py stubs"
|
||||
)
|
||||
add_custom_target(
|
||||
|
||||
+37
-67
@@ -2,7 +2,6 @@ services:
|
||||
sparkd:
|
||||
volumes:
|
||||
- ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
|
||||
- ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
|
||||
build:
|
||||
dockerfile: src/app_sparkd/Dockerfile
|
||||
args:
|
||||
@@ -10,38 +9,24 @@ services:
|
||||
- SOURCE_DIR=${SPARKD__SOURCE_DIR}
|
||||
- CONFIG_DIR=${SPARKD__CONFIG_DIR}
|
||||
- PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
|
||||
- SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
|
||||
shm_size: "${SPARKD__SHM_SIZE:-512mb}"
|
||||
env_file:
|
||||
- .env
|
||||
- ${SPARKD__CONFIG_DIR}/.env
|
||||
ports:
|
||||
- "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
|
||||
- "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
|
||||
- "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
|
||||
- 8008:8008
|
||||
- 8080:8080
|
||||
- 9443:9443
|
||||
entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: "${SPARKD__MEMORY:-4gb}"
|
||||
|
||||
sparkd-shell:
|
||||
volumes:
|
||||
- ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
|
||||
- ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
|
||||
build:
|
||||
dockerfile: docker/images/bluetradingsystems/linux_shell/Dockerfile
|
||||
args:
|
||||
- IMAGE=${SPARKD__IMAGE}
|
||||
- SOURCE_DIR=${SPARKD__SOURCE_DIR}
|
||||
- CONFIG_DIR=${SPARKD__CONFIG_DIR}
|
||||
- PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
|
||||
- SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
|
||||
shm_size: "${SPARKD__SHM_SIZE:-512mb}"
|
||||
command: "do sleep 1; done"
|
||||
command: >
|
||||
sh -c "
|
||||
echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd/data
|
||||
if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
|
||||
mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
|
||||
fi
|
||||
if [ -e "/home/bts/.local/share/bts/conf/.env.local" ]; then
|
||||
echo 'Sourcing /home/bts/.local/share/bts/conf/.env.local'
|
||||
set -a
|
||||
. /home/bts/.local/share/bts/sparkd/conf/.env.local
|
||||
set +a
|
||||
fi
|
||||
if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then
|
||||
echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)'
|
||||
cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/
|
||||
fi
|
||||
./sparkd run"
|
||||
env_file:
|
||||
- .env
|
||||
- ${SPARKD__CONFIG_DIR}/.env
|
||||
@@ -57,7 +42,6 @@ services:
|
||||
sparkd-symbols:
|
||||
volumes:
|
||||
- ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
|
||||
- ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
|
||||
- ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
|
||||
build:
|
||||
dockerfile: src/app_sparkd/Dockerfile
|
||||
@@ -66,8 +50,26 @@ services:
|
||||
- SOURCE_DIR=${SPARKD__SOURCE_DIR}
|
||||
- CONFIG_DIR=${SPARKD__CONFIG_DIR}
|
||||
- PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
|
||||
- SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
|
||||
shm_size: "${SPARKD__SHM_SIZE:-512mb}"
|
||||
command: >
|
||||
sh -c "
|
||||
echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd
|
||||
if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
|
||||
mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
|
||||
fi
|
||||
ls -alh /home/bts/.local/share/bts/sparkd/conf/
|
||||
cat /home/bts/.local/share/bts/sparkd/conf/.env.local
|
||||
if [ -e "/home/bts/.local/share/bts/sparkd/conf/.env.local" ]; then
|
||||
echo 'Sourcing /home/bts/.local/share/bts/sparkd/conf/.env.local'
|
||||
set -a
|
||||
. /home/bts/.local/share/bts/sparkd/conf/.env.local
|
||||
set +a
|
||||
fi
|
||||
if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then
|
||||
echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)'
|
||||
cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/
|
||||
fi
|
||||
./sparkd run"
|
||||
env_file:
|
||||
- .env
|
||||
- ${SPARKD__CONFIG_DIR}/.env
|
||||
@@ -75,36 +77,6 @@ services:
|
||||
- "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
|
||||
- "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
|
||||
- "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
|
||||
- 8008:8008
|
||||
- 8080:8080
|
||||
- 9443:9443
|
||||
entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
|
||||
deploy:
|
||||
restart_policy:
|
||||
condition: any
|
||||
delay: 3s
|
||||
resources:
|
||||
limits:
|
||||
memory: "${SPARKD__MEMORY:-4gb}"
|
||||
sparkd-multicast:
|
||||
volumes:
|
||||
- ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
|
||||
- ${SPARKD__MD_RECORDING_DIR:-md_recording}:/md_recording
|
||||
- ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
|
||||
build:
|
||||
dockerfile: src/app_sparkd/Dockerfile
|
||||
args:
|
||||
- IMAGE=${SPARKD__IMAGE}
|
||||
- SOURCE_DIR=${SPARKD__SOURCE_DIR}
|
||||
- CONFIG_DIR=${SPARKD__CONFIG_DIR}
|
||||
- PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
|
||||
- SPARKD_ENTRYPOINT=${SPARKD__SPARKD_ENTRYPOINT}
|
||||
shm_size: "${SPARKD__SHM_SIZE:-2048mb}"
|
||||
env_file:
|
||||
- .env
|
||||
- ${SPARKD__CONFIG_DIR}/.env
|
||||
network_mode: "host"
|
||||
entrypoint: ${SPARKD__MAIN_ENTRYPOINT}
|
||||
deploy:
|
||||
restart_policy:
|
||||
condition: any
|
||||
@@ -112,6 +84,7 @@ services:
|
||||
resources:
|
||||
limits:
|
||||
memory: "${SPARKD__MEMORY:-4gb}"
|
||||
|
||||
sparkd-tail:
|
||||
image: bluetradingsystems/linux_dotnet8
|
||||
volumes:
|
||||
@@ -222,6 +195,3 @@ volumes:
|
||||
name: "md_publisher_22019"
|
||||
historic_md:
|
||||
name: "historic_md"
|
||||
md_recording:
|
||||
name: "sparkd_md_recording_${SPARKD__CONFIG_PORT:-8000}"
|
||||
external: true
|
||||
@@ -8,7 +8,6 @@ RUN apt-get update && apt-get install -y ca-certificates
|
||||
RUN apt-get update && apt-get install -y libssh2-1-dev
|
||||
RUN apt-get update && apt-get install -y software-properties-common
|
||||
RUN apt-get update && apt-get install -y sudo
|
||||
RUN apt-get update && apt-get install -y tzdata
|
||||
|
||||
# GCC
|
||||
ARG gcc_major=13
|
||||
@@ -19,13 +18,6 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y dotnet8
|
||||
|
||||
RUN apt-get update && apt-get install -y libpq-dev
|
||||
RUN apt-get update && apt-get install -y libsodium-dev
|
||||
RUN apt-get update && apt-get install -y fish
|
||||
RUN apt-get update && apt-get install -y less
|
||||
RUN apt-get update && apt-get install -y vim
|
||||
|
||||
|
||||
RUN locale-gen en_US.UTF-8
|
||||
|
||||
ARG docker_user="bts"
|
||||
|
||||
@@ -98,7 +98,7 @@ RUN apt-get update && apt-get install -y libpcap0.8-dev
|
||||
RUN apt-get update && apt-get -y install libzstd-dev
|
||||
|
||||
# Rakurai
|
||||
ARG RAKURAI_DEB_URL="https://api.bitbucket.org/2.0/repositories/bluetradesys/spark/downloads/rakurai-dex-connector_0.0.12_amd64.deb"
|
||||
ARG RAKURAI_DEB_URL="https://api.bitbucket.org/2.0/repositories/bluetradesys/spark/downloads/rakurai-dex-connector_0.0.5_amd64.deb"
|
||||
RUN --mount=type=secret,id=bitbucket_access_token \
|
||||
set -eux \
|
||||
&& apt-get update \
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
FROM ubuntu:24.04
|
||||
|
||||
ARG DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt-get update && apt-get install -y \
|
||||
fish \
|
||||
less \
|
||||
vim
|
||||
|
||||
USER bts
|
||||
WORKDIR /home/bts
|
||||
+1
-1
Submodule externals updated: 2755fd6b8f...c7478ef5cd.
@@ -49,6 +49,14 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{2B8243FF-60C6-4607-8AAE-7762D29B0A48}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Collections", "src\cs\Bts.Collections\Bts.Collections.csproj", "{00D21F82-61F4-467F-B125-268018A185E5}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Core", "src\cs\Bts.Core\Bts.Core.csproj", "{513A3F0D-46B5-4AC2-B700-8671933F6347}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Disposables", "src\cs\Bts.Disposables\Bts.Disposables.csproj", "{D419D996-9870-4FF5-82AD-08370D1C945F}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Numerics", "src\cs\Bts.Numerics\Bts.Numerics.csproj", "{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
@@ -275,6 +283,102 @@ Global
|
||||
{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x64.Build.0 = Release|x64
|
||||
{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x86.ActiveCfg = Release|x64
|
||||
{2B8243FF-60C6-4607-8AAE-7762D29B0A48}.Release|x86.Build.0 = Release|x64
|
||||
{00D21F82-61F4-467F-B125-268018A185E5}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{00D21F82-61F4-467F-B125-268018A185E5}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{00D21F82-61F4-467F-B125-268018A185E5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{00D21F82-61F4-467F-B125-268018A185E5}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{00D21F82-61F4-467F-B125-268018A185E5}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{00D21F82-61F4-467F-B125-268018A185E5}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||
{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||
{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
|
||||
{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
|
||||
{00D21F82-61F4-467F-B125-268018A185E5}.MinSizeRel|x86.Build.0 = Debug|Any CPU
|
||||
{00D21F82-61F4-467F-B125-268018A185E5}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{00D21F82-61F4-467F-B125-268018A185E5}.Release|x64.Build.0 = Release|Any CPU
|
||||
{00D21F82-61F4-467F-B125-268018A185E5}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{00D21F82-61F4-467F-B125-268018A185E5}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{00D21F82-61F4-467F-B125-268018A185E5}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{00D21F82-61F4-467F-B125-268018A185E5}.Release|x86.Build.0 = Release|Any CPU
|
||||
{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||
{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||
{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
|
||||
{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
|
||||
{00D21F82-61F4-467F-B125-268018A185E5}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
|
||||
{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{513A3F0D-46B5-4AC2-B700-8671933F6347}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||
{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||
{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
|
||||
{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
|
||||
{513A3F0D-46B5-4AC2-B700-8671933F6347}.MinSizeRel|x86.Build.0 = Debug|Any CPU
|
||||
{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|x64.Build.0 = Release|Any CPU
|
||||
{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{513A3F0D-46B5-4AC2-B700-8671933F6347}.Release|x86.Build.0 = Release|Any CPU
|
||||
{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||
{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||
{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
|
||||
{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
|
||||
{513A3F0D-46B5-4AC2-B700-8671933F6347}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
|
||||
{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{D419D996-9870-4FF5-82AD-08370D1C945F}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||
{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||
{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
|
||||
{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
|
||||
{D419D996-9870-4FF5-82AD-08370D1C945F}.MinSizeRel|x86.Build.0 = Debug|Any CPU
|
||||
{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|x64.Build.0 = Release|Any CPU
|
||||
{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{D419D996-9870-4FF5-82AD-08370D1C945F}.Release|x86.Build.0 = Release|Any CPU
|
||||
{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||
{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||
{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
|
||||
{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
|
||||
{D419D996-9870-4FF5-82AD-08370D1C945F}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
|
||||
{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
||||
{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
||||
{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU
|
||||
{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU
|
||||
{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.MinSizeRel|x86.Build.0 = Debug|Any CPU
|
||||
{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|x64.Build.0 = Release|Any CPU
|
||||
{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.Release|x86.Build.0 = Release|Any CPU
|
||||
{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||
{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||
{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|Any CPU.Build.0 = Debug|Any CPU
|
||||
{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU
|
||||
{697BC3A3-1A98-43D3-A6E8-BB39A05EA049}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
@@ -302,5 +406,9 @@ Global
|
||||
{D432F3E9-3D2F-4623-B7B9-CEBA6701FFC3} = {42C014A0-00C5-0620-8552-38912E891A7F}
|
||||
{DD00F5D5-6C2F-4FDA-AE3D-25CF01A9A224} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
|
||||
{2B8243FF-60C6-4607-8AAE-7762D29B0A48} = {C5A3B5EA-BE9F-76E5-E9B8-8A22B289B295}
|
||||
{00D21F82-61F4-467F-B125-268018A185E5} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
|
||||
{513A3F0D-46B5-4AC2-B700-8671933F6347} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
|
||||
{D419D996-9870-4FF5-82AD-08370D1C945F} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
|
||||
{697BC3A3-1A98-43D3-A6E8-BB39A05EA049} = {D7C6B9C7-1058-B1EA-FEE6-34C56B05AEDB}
|
||||
EndGlobalSection
|
||||
EndGlobal
|
||||
@@ -15,21 +15,18 @@ static class ArgumentComparer {
|
||||
}
|
||||
var obj1 = value1.GetValue(env);
|
||||
var obj2 = value2.GetValue(env);
|
||||
if (isNullLike(obj1)) {
|
||||
if (isNullLike(obj2)) {
|
||||
if (obj1 is null) {
|
||||
if (obj2 is null) {
|
||||
return 0;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
if (isNullLike(obj2)) {
|
||||
if (obj2 is null) {
|
||||
return 1;
|
||||
}
|
||||
return compareObjs(obj1!, obj2!);
|
||||
return compareObjs(obj1, obj2);
|
||||
}
|
||||
|
||||
static bool isNullLike(object? value) =>
|
||||
value is null || DBNull.Value.Equals(value);
|
||||
|
||||
static int compareObjs(object obj1, object obj2) {
|
||||
try {
|
||||
var typ1 = obj1.GetType();
|
||||
|
||||
@@ -15,7 +15,7 @@ namespace bts.BlueOrb {
|
||||
private int _size = 0;
|
||||
|
||||
public TimeSeriesStream(ViewStream stream)
|
||||
: this(stream, 1000) { }
|
||||
: this(stream, Int32.MaxValue) { }
|
||||
|
||||
public TimeSeriesStream(ViewStream stream, int size) {
|
||||
_stream = stream;
|
||||
@@ -44,7 +44,7 @@ namespace bts.BlueOrb {
|
||||
ar.setValue("id", _id++);
|
||||
_table.add_row(ar);
|
||||
}
|
||||
var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
|
||||
var dp_out = BlueDataUtility.build_dp(_table, myLastValue as DataPacket);
|
||||
myFireEvent(dp_out);
|
||||
}
|
||||
}
|
||||
|
||||
+3
-3
@@ -102,10 +102,10 @@ namespace BCG
|
||||
List<ColumnInfo> col_info = new List<ColumnInfo>();
|
||||
col_info.Add(new ColumnInfo("id", col_index++, typeof(int)));
|
||||
foreach (Pair<string, Type> p in group_by_col_info)
|
||||
col_info.Add(new ColumnInfo(p.first, col_index++, p.second));
|
||||
col_info.Add(new ColumnInfo(p.first ?? string.Empty, col_index++, ExceptionAssert.NotNull(p.second)));
|
||||
foreach (string tc_prefix in tc_prefixes)
|
||||
foreach (Pair<string, Type> p in _cols_per_transposed)
|
||||
col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first ?? string.Empty), col_index++, p.second));
|
||||
col_info.Add(new ColumnInfo(_get_full_column_name(tc_prefix, p.first ?? string.Empty), col_index++, ExceptionAssert.NotNull(p.second)));
|
||||
IBlueTableSchema schema = new BlueTableSchema(col_info.ToArray(), "xy_transposed_" + table_name);
|
||||
if (_schema == null)
|
||||
_schema = schema;
|
||||
@@ -182,7 +182,7 @@ namespace BCG
|
||||
if (_transpsed_col_value_comparer_wrapper != null)
|
||||
tc_values.Sort(_transpsed_col_value_comparer_wrapper);
|
||||
List<string> prefixes = new List<string>();
|
||||
foreach (Pair<int, object[]> p in tc_values)
|
||||
foreach (Pair<int, object?[]> p in tc_values)
|
||||
prefixes.Add(tc_prefixes[p.first]);
|
||||
_set_schema(prefixes, group_by_col_info, table_name);
|
||||
BlueTable result = new BlueTable(_schema);
|
||||
|
||||
+25
-19
@@ -1,17 +1,17 @@
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Text;
|
||||
|
||||
using BCG;
|
||||
using BCG.Utilities;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
public class ConcatenateColumnsMapper : IStreamMapper
|
||||
{
|
||||
protected IDictionary myTempColMap = new SortedList ();
|
||||
protected ArrayList myColMap;
|
||||
protected BlueTableSchema mySchema;
|
||||
protected ArrayList? myColMap;
|
||||
protected BlueTableSchema? mySchema;
|
||||
|
||||
public ConcatenateColumnsMapper(params string [] columnDefs)
|
||||
{
|
||||
@@ -55,11 +55,7 @@ namespace BCG
|
||||
myColMap = new ArrayList (myTempColMap.Count);
|
||||
foreach (DictionaryEntry de in myTempColMap)
|
||||
{
|
||||
string[] cols = de.Value as string[];
|
||||
ArrayList temp = new ArrayList ();
|
||||
foreach (string col in cols)
|
||||
temp.Add (inputSchema[col].colNo);
|
||||
myColMap.Add (temp.ToArray(typeof(int)) as int[]);
|
||||
myColMap.Add(Array.ConvertAll((string[])de.Value!, c => inputSchema[c].colNo));
|
||||
}
|
||||
myTempColMap.Clear ();
|
||||
}
|
||||
@@ -72,6 +68,8 @@ namespace BCG
|
||||
|
||||
public virtual IBlueRow map (IBlueRow row)
|
||||
{
|
||||
requireSchema();
|
||||
requireColMap();
|
||||
object[] retVal = new object[mySchema.columnInfo.Length];
|
||||
row.CopyTo (retVal, myColMap.Count);
|
||||
for (int colNo = 0; colNo < myColMap.Count; colNo ++)
|
||||
@@ -90,11 +88,21 @@ namespace BCG
|
||||
/// <returns></returns>
|
||||
protected virtual object myGetConcatValue (IBlueRow row, int colNo)
|
||||
{
|
||||
requireColMap();
|
||||
StringBuilder b = new StringBuilder();
|
||||
foreach (int cn in myColMap[colNo] as int[])
|
||||
b.Append(row[cn].ToString());
|
||||
foreach (int cn in (int[])myColMap[colNo]!) {
|
||||
b.Append(row[cn]?.ToString());
|
||||
}
|
||||
return b.ToString ();
|
||||
}
|
||||
|
||||
[MemberNotNull(nameof(myColMap))]
|
||||
protected void requireColMap() =>
|
||||
ExceptionAssert.NotNull(myColMap);
|
||||
|
||||
[MemberNotNull(nameof(mySchema))]
|
||||
void requireSchema() =>
|
||||
ExceptionAssert.NotNull(mySchema);
|
||||
}
|
||||
|
||||
public class ConcatenateColumnsMapperForSameDataSet : ConcatenateColumnsMapper
|
||||
@@ -107,23 +115,21 @@ namespace BCG
|
||||
myDicts = new HashTableND [columnDefs.Length];
|
||||
int i = 0;
|
||||
foreach (DictionaryEntry de in myTempColMap)
|
||||
myDicts[i++] = new HashTableND ((de.Value as string[]).Length);
|
||||
myDicts[i++] = new HashTableND(((string[])de.Value!).Length);
|
||||
}
|
||||
|
||||
protected override object myGetConcatValue (IBlueRow row, int colNo)
|
||||
{
|
||||
int[] cols = myColMap[colNo] as int[];
|
||||
object[] vals = new object[cols.Length];
|
||||
for (int i = 0; i < vals.Length; i++)
|
||||
vals[i] = row[cols[i]];
|
||||
requireColMap();
|
||||
HashTableND ht = myDicts[colNo];
|
||||
object rv = ht.getValue (vals);
|
||||
var vals = Array.ConvertAll((int[])myColMap[colNo]!, c => row[c]);
|
||||
var rv = ht.getValue(vals);
|
||||
if (rv != null)
|
||||
return rv;
|
||||
else
|
||||
{
|
||||
rv = base.myGetConcatValue (row, colNo);
|
||||
object[] args = new object[vals.Length + 1];
|
||||
var args = new object?[vals.Length + 1];
|
||||
vals.CopyTo (args, 0);
|
||||
args[args.Length-1] = rv;
|
||||
ht.add (args);
|
||||
|
||||
@@ -14,8 +14,10 @@ namespace BCG
|
||||
public DistinctRowEvaluator(string colMapExpression, string[] columnNames)
|
||||
: base (colMapExpression, null, columnNames) {}
|
||||
|
||||
public override IBlueRow? evaluate(IDataPacket packet) {
|
||||
myOutputTable ??= new GroupByOutputTable(packet.Schema);
|
||||
|
||||
public override IBlueRow evaluate (DataPacket packet)
|
||||
{
|
||||
myOutputTable = myOutputTable == null ? new GroupByOutputTable (packet.schema) : myOutputTable;
|
||||
string hashString = GroupByStream.getRepString (packet, myGroupByCols);
|
||||
if (packet.newRows.Count == 0)
|
||||
//if there are no rows for this group we remove the group from the output table
|
||||
|
||||
@@ -67,7 +67,7 @@ namespace BCG {
|
||||
}
|
||||
#endregion
|
||||
|
||||
public event FilterChangeDelegate onFilterChange;
|
||||
public event FilterChangeDelegate? onFilterChange;
|
||||
|
||||
#region IStreamFilter
|
||||
public bool filter(IBlueRow row) {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
using System;
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
@@ -10,8 +10,8 @@ namespace BCG
|
||||
{
|
||||
protected string[] myGroupByCols;
|
||||
protected string myOutputColumn;
|
||||
protected GroupByOutputTable myOutputTable;
|
||||
protected string myColMapExpression;
|
||||
protected GroupByOutputTable? myOutputTable;
|
||||
protected string? myColMapExpression;
|
||||
|
||||
#region constructors
|
||||
public GroupByEvaluator (string outputCol, string[] groupByCols)
|
||||
@@ -42,7 +42,7 @@ namespace BCG
|
||||
return retVal;
|
||||
}
|
||||
|
||||
public abstract IBlueRow? evaluate(IDataPacket dp);
|
||||
public abstract IBlueRow evaluate (DataPacket dp) ;
|
||||
#endregion
|
||||
|
||||
#region Helper methods
|
||||
@@ -52,21 +52,20 @@ namespace BCG
|
||||
/// <param name="r"></param>
|
||||
/// <param name="id"></param>
|
||||
/// <returns></returns>
|
||||
protected IBlueRow myProject (IBlueRow r, int id)
|
||||
{
|
||||
return r == null ? null :
|
||||
myColMapExpression == null || myColMapExpression.Length < 1 ?
|
||||
BlueDataUtility.getNewReplacedRow (r, id, "id") :
|
||||
BlueDataUtility.getNewReplacedRow(myProjectRow (r, myColMapExpression),
|
||||
id, "id");
|
||||
}
|
||||
|
||||
protected IBlueRow? myProject(IBlueRow r, int id) =>
|
||||
r is null
|
||||
? null
|
||||
: myColMapExpression is null || myColMapExpression.Length < 1
|
||||
? BlueDataUtility.getNewReplacedRow(r, id, "id")
|
||||
: BlueDataUtility.getNewReplacedRow(myProjectRow(r, myColMapExpression), id, "id");
|
||||
|
||||
/// <summary>
|
||||
/// Projects the id, conserving the id
|
||||
/// </summary>
|
||||
/// <param name="r"></param>
|
||||
/// <returns></returns>
|
||||
protected IBlueRow myProject (IBlueRow r) {return myProject (r, r == null ? Int32.MinValue : (int)r["id"]);}
|
||||
protected IBlueRow? myProject(IBlueRow r) =>
|
||||
myProject(r, r is null ? int.MinValue : r.Id());
|
||||
#endregion
|
||||
|
||||
IBlueRow myProjectRow (IBlueRow r, string colMapExpression) =>
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using System;
|
||||
using BCG.Utilities;
|
||||
using System.Collections;
|
||||
|
||||
namespace BCG
|
||||
@@ -22,10 +23,12 @@ namespace BCG
|
||||
/// <param name="groupRepString"></param>
|
||||
public void addRow (IBlueRow row, string groupRepString)
|
||||
{
|
||||
if (myDict[groupRepString] != null)
|
||||
myTable.deleteRow ((IBlueRow)myDict[groupRepString]);
|
||||
var grp = myDict[groupRepString];
|
||||
if (grp is not null) {
|
||||
myTable.deleteRow((IBlueRow)grp);
|
||||
}
|
||||
myTable.addRow (row);
|
||||
myDict[groupRepString] = myTable[(int)row["id"]];
|
||||
myDict[groupRepString] = myTable[row.Id()];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -35,7 +38,7 @@ namespace BCG
|
||||
public void removeRow (string groupRepString)
|
||||
{
|
||||
// BlueEventLog.logAssert (myDict[groupRepString] != null);
|
||||
myTable.deleteRow ((IBlueRow)myDict[groupRepString]);
|
||||
myTable.deleteRow ((IBlueRow)myDict[groupRepString]!);
|
||||
myDict.Remove (groupRepString);
|
||||
}
|
||||
|
||||
@@ -48,11 +51,8 @@ namespace BCG
|
||||
myDict.Clear ();
|
||||
}
|
||||
|
||||
public IBlueRow getRow (string groupRepString)
|
||||
{
|
||||
return myDict[groupRepString] == null ? null :
|
||||
myDict[groupRepString] as IBlueRow;
|
||||
}
|
||||
public IBlueRow? getRow(string groupRepString) =>
|
||||
myDict[groupRepString] as IBlueRow;
|
||||
|
||||
#region IBlueRowCollection members
|
||||
public IBlueRow this[int id] {get{return myTable[id];}}
|
||||
|
||||
@@ -1,20 +1,21 @@
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using BCG.Utilities;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Linq;
|
||||
|
||||
namespace BCG {
|
||||
internal class GroupByStream : MethodDrivenStream {
|
||||
private class MyFuncGroupByEvaluator : IGroupByEvaluator {
|
||||
private string[] _groupby_cols;
|
||||
readonly Func<IDataPacket, IBlueRow> _groupby_func;
|
||||
public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<IDataPacket, IBlueRow> groupby_func) {
|
||||
private Func<DataPacket, IBlueRow> _groupby_func;
|
||||
public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func) {
|
||||
_groupby_cols = groupby_cols.ToArray();
|
||||
_groupby_func = groupby_func;
|
||||
}
|
||||
public IBlueRow evaluate(IDataPacket packet) =>
|
||||
_groupby_func(packet);
|
||||
public IBlueRow evaluate(DataPacket packet) { return _groupby_func(packet); }
|
||||
|
||||
public string[] groupByColumns { get { return _groupby_cols; } }
|
||||
|
||||
@@ -26,14 +27,15 @@ namespace BCG {
|
||||
/// Serves as a quick way to access all rows belonging to the same group
|
||||
/// </summary>
|
||||
private bool myIdGuard = true;
|
||||
private RowDictionaryTable myBookKeepingTable;
|
||||
RowDictionaryTable? myBookKeepingTable;
|
||||
private IGroupByEvaluator myGroupByEvaluator;
|
||||
private IBlueTableSchema mySchema; //input schema, as output schema can differ (IGroupByEvaluator's
|
||||
// input schema, as output schema can differ (IGroupByEvaluator's
|
||||
IBlueTableSchema? mySchema;
|
||||
//can map a schema to another
|
||||
private IBlueTableSchema myOutputSchema;
|
||||
IBlueTableSchema? myOutputSchema;
|
||||
private int myIdGenerator = 0;
|
||||
|
||||
public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<IDataPacket, IBlueRow> groupby_func)
|
||||
public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
|
||||
: this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func)) { }
|
||||
|
||||
public GroupByStream(ViewStream stream, IGroupByEvaluator groupByEvaluator)
|
||||
@@ -46,14 +48,14 @@ namespace BCG {
|
||||
}
|
||||
|
||||
protected override void myHandleFirstCall() {
|
||||
requireLastArrivedPacket();
|
||||
mySchema = myLastArrivedPacket.schema;
|
||||
myOutputSchema = myGroupByEvaluator.getOutputSchema(mySchema);
|
||||
myBookKeepingTable = new RowDictionaryTable(myLastArrivedPacket.newRows,
|
||||
myGroupByEvaluator.groupByColumns);
|
||||
GroupByOutputTable temp = new GroupByOutputTable(myOutputSchema);
|
||||
foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups()) {
|
||||
IBlueRow newRow = myEvaluate(new DataPacket(rowGroup,
|
||||
rowGroup, new BlueTable(mySchema)));
|
||||
var newRow = myEvaluate(new DataPacket(rowGroup, rowGroup, new BlueTable(mySchema)));
|
||||
if (newRow != null)
|
||||
temp.addRow(newRow, myGetGroupString(rowGroup));
|
||||
}
|
||||
@@ -79,15 +81,14 @@ namespace BCG {
|
||||
ArrayList handledAddGroups = new ArrayList();
|
||||
string groupRepString;
|
||||
|
||||
requireBookKeeping();
|
||||
foreach (IBlueRowCollection delGroup in delGroups.getGroups()) {
|
||||
// BlueEventLog.logAssert (delGroup.Count > 0);
|
||||
groupRepString = myGetGroupString(delGroup);
|
||||
IBlueRowCollection addGroup = addGroups.getGroup(groupRepString) == null ?
|
||||
new BlueTable(mySchema) : addGroups.getGroup(groupRepString);
|
||||
IBlueRowCollection addGroup = addGroups.getGroup(groupRepString) ?? new BlueTable(mySchema);
|
||||
handledAddGroups.Add(addGroup);
|
||||
IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString) == null ?
|
||||
new BlueTable(mySchema) : myBookKeepingTable.getGroup(groupRepString);
|
||||
IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup, delGroup));
|
||||
IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString) ?? new BlueTable(mySchema);
|
||||
var newRow = myEvaluate(new DataPacket(aggGroup, addGroup, delGroup));
|
||||
myFireSwitch |= myMergeRow(newRow, groupRepString);
|
||||
}
|
||||
|
||||
@@ -99,16 +100,16 @@ namespace BCG {
|
||||
//the loop above
|
||||
|
||||
groupRepString = myGetGroupString(addGroup);
|
||||
IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString);
|
||||
var aggGroup = ExceptionAssert.NotNull(myBookKeepingTable.getGroup(groupRepString));
|
||||
// BlueEventLog.logAssert (aggGroup != null && aggGroup.Count > 0);
|
||||
IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup,
|
||||
new BlueTable(aggGroup.schema)));
|
||||
var newRow = myEvaluate(new DataPacket(aggGroup, addGroup, new BlueTable(aggGroup.schema)));
|
||||
myFireSwitch |= myMergeRow(newRow, groupRepString);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private bool myMergeRow(IBlueRow newRow, string groupRepString) {
|
||||
private bool myMergeRow(IBlueRow? newRow, string groupRepString) {
|
||||
requireLastFiredPacket();
|
||||
IBlueRow existingRow = ((GroupByOutputTable)myLastFiredPacket.newRows).getRow(groupRepString);
|
||||
//BlueEventLog.logAssert (newRow != null || existingRow != null);
|
||||
if (existingRow == null && newRow == null)
|
||||
@@ -116,6 +117,7 @@ namespace BCG {
|
||||
else if (existingRow != null && newRow != null && BlueDataUtility.compute_equality(existingRow, newRow))
|
||||
return false;
|
||||
else if (existingRow == null) {
|
||||
ArgumentNullException.ThrowIfNull(newRow);
|
||||
//row added for the first time
|
||||
((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString);
|
||||
((BlueTable)myLastFiredPacket.addedRows).addRow(newRow);
|
||||
@@ -138,6 +140,9 @@ namespace BCG {
|
||||
/// <param name="delGroups"></param>
|
||||
/// <param name="addGroups"></param>
|
||||
private void myInitIncrementalChangeData(out RowDictionaryTable delGroups, out RowDictionaryTable addGroups) {
|
||||
ExceptionAssert.NotNull(myBookKeepingTable);
|
||||
requireLastFiredPacket();
|
||||
requireLastArrivedPacket();
|
||||
delGroups =
|
||||
new RowDictionaryTable(myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
|
||||
addGroups =
|
||||
@@ -151,16 +156,19 @@ namespace BCG {
|
||||
}
|
||||
|
||||
protected override void myOnTableClear() {
|
||||
if (myBookKeepingTable.Count == 0)
|
||||
return; //ie there arent any rows, so no need to optimize the 'cleansing' process
|
||||
if (ExceptionAssert.NotNull(myBookKeepingTable).Count == 0) {
|
||||
// ie there arent any rows, so no need to optimize the 'cleansing' process
|
||||
return;
|
||||
}
|
||||
requireLastFiredPacket();
|
||||
myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows);
|
||||
myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
|
||||
(myLastFiredPacket.newRows as GroupByOutputTable).clear();
|
||||
((GroupByOutputTable)myLastFiredPacket.newRows).clear();
|
||||
myBookKeepingTable.clear();
|
||||
myGroupByEvaluator.clear();
|
||||
}
|
||||
|
||||
private IBlueRow myEvaluate(DataPacket dp) {
|
||||
private IBlueRow? myEvaluate(DataPacket dp) {
|
||||
IBlueRow evalRow = myGroupByEvaluator.evaluate(dp);
|
||||
return evalRow == null ? null :
|
||||
myIdGuard ?
|
||||
@@ -188,11 +196,11 @@ namespace BCG {
|
||||
}
|
||||
}
|
||||
|
||||
public static string getRepString(IDataPacket dp, string[] colNames) {
|
||||
public static string getRepString(DataPacket dp, string[] colNames) {
|
||||
return BlueDataUtility.rowToString(getRepRow(dp), colNames);
|
||||
}
|
||||
|
||||
public static IBlueRow getRepRow(IDataPacket dp) {
|
||||
public static IBlueRow? getRepRow(DataPacket dp) {
|
||||
if (dp.newRows.Count > 0)
|
||||
return BlueDataUtility.extractRow(dp.newRows, 0);
|
||||
else if (dp.addedRows.Count > 0)
|
||||
@@ -211,13 +219,13 @@ namespace BCG {
|
||||
/// <param name="newValue"></param>
|
||||
/// <param name="isAdd"></param>
|
||||
/// <returns></returns>
|
||||
public static object incrementalAdd(object lastValue, object newValue, bool isAdd) {
|
||||
public static object? incrementalAdd(object lastValue, object newValue, bool isAdd) {
|
||||
if (lastValue == null || lastValue.Equals(DBNull.Value))
|
||||
return newValue;
|
||||
// BlueEventLog.logAssert (lastValue.GetType().Equals (typeof(Int32)) && newValue.GetType().Equals (typeof(Int32)) ||
|
||||
// lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
|
||||
// lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
|
||||
Object retVal = null;
|
||||
object? retVal = null;
|
||||
if (newValue.GetType().Equals(typeof(int)))
|
||||
retVal = isAdd ? (int)lastValue + (int)newValue : (int)lastValue - (int)newValue;
|
||||
else if (newValue.GetType().Equals(typeof(Double)))
|
||||
@@ -229,6 +237,9 @@ namespace BCG {
|
||||
return retVal;
|
||||
}
|
||||
|
||||
[MemberNotNull(nameof(myBookKeepingTable))]
|
||||
void requireBookKeeping() =>
|
||||
ExceptionAssert.NotNull(myBookKeepingTable);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
using System;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
/// <summary>
|
||||
@@ -12,7 +14,7 @@ namespace BCG
|
||||
/// deletetion) and the added and deletedRows being the increments. If a group has
|
||||
/// to be totally removed then a null should be returned
|
||||
/// </summary>
|
||||
IBlueRow? evaluate(IDataPacket packet);
|
||||
IBlueRow evaluate (DataPacket packet);
|
||||
|
||||
/// <summary>
|
||||
/// Returns the columns (belonging to the original schema) on which this IGroupByEvaluator
|
||||
|
||||
@@ -14,8 +14,9 @@ namespace BCG {
|
||||
public MaxGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
|
||||
: base(colMapEpression, outputCol, groupByCols) { }
|
||||
|
||||
public override IBlueRow? evaluate(IDataPacket dp) {
|
||||
myOutputTable ??= new GroupByOutputTable(dp.Schema);
|
||||
public override IBlueRow evaluate(DataPacket dp) {
|
||||
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) :
|
||||
myOutputTable;
|
||||
string hashString = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
|
||||
IBlueRow currMaxRow = myOutputTable.getRow(hashString);
|
||||
IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using BCG.Utilities;
|
||||
using System.IO;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
|
||||
/// <summary>
|
||||
/// Some basic event handling functionality factorred out of any Stream whose behaviour
|
||||
/// is driven by some interface passed on by the user
|
||||
@@ -12,8 +11,8 @@ namespace BCG
|
||||
public abstract class MethodDrivenStream : ViewStream
|
||||
{
|
||||
protected ViewStream myStream;
|
||||
protected DataPacket myLastArrivedPacket;
|
||||
protected DataPacket myLastFiredPacket;
|
||||
protected DataPacket? myLastArrivedPacket;
|
||||
protected DataPacket? myLastFiredPacket;
|
||||
protected bool myFireSwitch = true;
|
||||
|
||||
public MethodDrivenStream(ViewStream vs) {myStream = vs;}
|
||||
@@ -59,6 +58,7 @@ namespace BCG
|
||||
/// To handle the first call, the LastFiredPacket is supposed to be initialized and filled
|
||||
/// with data.
|
||||
/// </summary>
|
||||
[MemberNotNull(nameof(myLastFiredPacket))]
|
||||
protected abstract void myHandleFirstCall ();
|
||||
|
||||
/// <summary>
|
||||
@@ -89,5 +89,13 @@ namespace BCG
|
||||
{
|
||||
myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
|
||||
}
|
||||
|
||||
[MemberNotNull(nameof(myLastArrivedPacket))]
|
||||
protected void requireLastArrivedPacket() =>
|
||||
ExceptionAssert.NotNull(myLastArrivedPacket);
|
||||
|
||||
[MemberNotNull(nameof(myLastFiredPacket))]
|
||||
protected void requireLastFiredPacket() =>
|
||||
ExceptionAssert.NotNull(myLastFiredPacket);
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,7 @@
|
||||
using System;
|
||||
|
||||
using BCG.Utilities;
|
||||
|
||||
namespace BCG {
|
||||
public class MinGroupByEvaluator : MaxGroupByEvaluator {
|
||||
public MinGroupByEvaluator(string outputCol, string[] groupByCols)
|
||||
@@ -6,8 +10,8 @@ namespace BCG {
|
||||
public MinGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) :
|
||||
base(colMapEpression, outputCol, groupByCols) { }
|
||||
|
||||
public override IBlueRow? evaluate(IDataPacket dp) {
|
||||
myOutputTable ??= new GroupByOutputTable(dp.Schema);
|
||||
public override IBlueRow evaluate(DataPacket dp) {
|
||||
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
|
||||
string stringRep = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
|
||||
IBlueRow currMinRow = myOutputTable.getRow(stringRep);
|
||||
IBlueRow minRow = BlueDataUtility.computeMinRow(dp.deletedRows, myOutputColumn);
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
using System;
|
||||
|
||||
namespace BCG {
|
||||
/// <summary>
|
||||
/// Computes the sum for a given column
|
||||
@@ -8,8 +10,8 @@ namespace BCG {
|
||||
public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
|
||||
: base(colMapEpression, outputCol, groupByCols) { }
|
||||
|
||||
public override IBlueRow evaluate(IDataPacket dp) {
|
||||
myOutputTable ??= new GroupByOutputTable(dp.Schema);
|
||||
public override IBlueRow evaluate(DataPacket dp) {
|
||||
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
|
||||
|
||||
IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
|
||||
return sumRow;
|
||||
@@ -31,7 +33,7 @@ namespace BCG {
|
||||
|
||||
#region IGroupByEvaluator Members
|
||||
|
||||
public IBlueRow evaluate(IDataPacket packet) => BlueDataUtility.computeSum(packet.newRows, myOutputCols);
|
||||
public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum(packet.newRows, myOutputCols); }
|
||||
|
||||
public string[] groupByColumns { get { return myGroupByCols; } }
|
||||
|
||||
@@ -49,8 +51,9 @@ namespace BCG {
|
||||
public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
|
||||
: base(colMapEpression, outputCol, groupByCols) { }
|
||||
|
||||
public override IBlueRow evaluate(IDataPacket dp) {
|
||||
myOutputTable ??= new GroupByOutputTable(dp.Schema);
|
||||
public override IBlueRow evaluate(DataPacket dp) {
|
||||
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
|
||||
|
||||
IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
|
||||
return sumRow;
|
||||
}
|
||||
@@ -65,7 +68,7 @@ namespace BCG {
|
||||
|
||||
#region IGroupByEvaluator Members
|
||||
|
||||
public IBlueRow evaluate(IDataPacket packet) => BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols);
|
||||
public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols); }
|
||||
|
||||
public string[] groupByColumns { get { return myGroupByCols; } }
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
using mbts;
|
||||
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
@@ -35,7 +37,7 @@ namespace BCG {
|
||||
List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
|
||||
foreach (Triplet<ViewStream, string, string> r in join_specs) {
|
||||
string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
|
||||
string[] visible_cols = r.third.Split(' ');
|
||||
string[] visible_cols = ExceptionAssert.NotNull(r.third).Split(' ');
|
||||
List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
|
||||
foreach (string s in visible_cols) {
|
||||
string[] toks = s.Split(':');
|
||||
@@ -47,7 +49,7 @@ namespace BCG {
|
||||
mapped_col_names.Add(toks[1]);
|
||||
}
|
||||
}
|
||||
rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
|
||||
rv.Add(new(ExceptionAssert.NotNull(r.first), keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
|
||||
}
|
||||
return rv.ToArray();
|
||||
}
|
||||
|
||||
@@ -290,6 +290,7 @@ namespace BCG
|
||||
}
|
||||
}
|
||||
}
|
||||
ExceptionAssert.NotNull(myState);
|
||||
}
|
||||
|
||||
private void myAddRows(IBlueRowCollection addedRows, bool isLeft, BlueTable state, BlueTable addedRowsOut)
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
|
||||
namespace BCG
|
||||
@@ -18,16 +20,21 @@ namespace BCG
|
||||
|
||||
public override void onTableClear (bool isLeft)
|
||||
{
|
||||
requireSchema();
|
||||
requireLastFiredPacket();
|
||||
myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
|
||||
myLastFiredPacket.newRows = new JoinForeignTable (mySchema);
|
||||
myLastFiredPacket.addedRows = new BlueTable (mySchema);
|
||||
if (!isLeft)
|
||||
if (!isLeft) {
|
||||
ExceptionAssert.NotNull(myLeftDataPacket);
|
||||
foreach (IBlueRow row in myLeftDataPacket.newRows)
|
||||
((JoinTable)myLastFiredPacket.newRows).addRow (row, null, myLastFiredPacket);
|
||||
}
|
||||
}
|
||||
|
||||
protected override void myInitFiredPacket ()
|
||||
{
|
||||
requireSchema();
|
||||
myLastFiredPacket = new DataPacket ();
|
||||
myLastFiredPacket.newRows = new JoinForeignTable (mySchema);
|
||||
myLastFiredPacket.addedRows = new BlueTable (mySchema);
|
||||
|
||||
@@ -15,7 +15,7 @@ namespace BCG
|
||||
: this (r, null, null)
|
||||
{}
|
||||
|
||||
public SimpleNode(IBlueRow r, SimpleNode? nx, SimpleNode? pv) {
|
||||
public SimpleNode(IBlueRow? r, SimpleNode? nx, SimpleNode? pv) {
|
||||
row = r;
|
||||
next = nx;
|
||||
prev = pv;
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
using bts;
|
||||
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
@@ -60,7 +62,7 @@ namespace BCG {
|
||||
if (columnAndType.Length == 1)
|
||||
ci[n] = new ColumnInfo(c, n, typeof(double));
|
||||
else
|
||||
ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
|
||||
ci[n] = new(columnAndType[0], n, ExceptionAssert.NotNull(Type.GetType("System." + columnAndType[1], true)));
|
||||
n++;
|
||||
}
|
||||
|
||||
|
||||
@@ -2,8 +2,6 @@ using BCG;
|
||||
|
||||
using MongoDB.Driver.Core.WireProtocol.Messages;
|
||||
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
@@ -39,7 +37,7 @@ namespace bts {
|
||||
}
|
||||
|
||||
void _handle_stream_update(object dp_obj) {
|
||||
var dp = ExceptionAssert.Argument.Is<IDataPacket>(dp_obj);
|
||||
DataPacket dp = dp_obj as DataPacket;
|
||||
IDictionary<int, List<IBlueRow>> groups = !String.IsNullOrEmpty(_group_by_col) ?
|
||||
BlueDataUtility.group_together(dp.newRows, new string[] { _group_by_col })
|
||||
:
|
||||
|
||||
@@ -23,6 +23,6 @@ namespace BCG
|
||||
/// fires an event when the mapper
|
||||
/// changes itself
|
||||
/// </summary>
|
||||
event MapperChangeDelegate onMapperChange;
|
||||
event MapperChangeDelegate? onMapperChange;
|
||||
}
|
||||
}
|
||||
@@ -75,6 +75,7 @@ namespace BCG {
|
||||
|
||||
protected override void myHandleFirstCall() {
|
||||
ArgumentNullException.ThrowIfNull(myMapper);
|
||||
requireLastArrivedPacket();
|
||||
IBlueTableSchema outputSchema = myMapper.getOutputSchema(myLastArrivedPacket.newRows.schema);
|
||||
myLastFiredPacket = new DataPacket();
|
||||
myLastFiredPacket.newRows = new BlueTable(outputSchema);
|
||||
@@ -85,10 +86,12 @@ namespace BCG {
|
||||
}
|
||||
|
||||
protected override void myHandleIncrementalAdd() {
|
||||
requireLastArrivedPacket();
|
||||
if (myLastArrivedPacket.addedRows.Count == 0) {
|
||||
Log.Gen.warn("MapStream.myHandleIncrementalAdd called with 0 addedRows");
|
||||
return;
|
||||
}
|
||||
requireLastFiredPacket();
|
||||
ArgumentNullException.ThrowIfNull(myMapper);
|
||||
var addedRows = new BlueTable(myLastFiredPacket.newRows.schema);
|
||||
var newRows = ExceptionAssert.Cast(myLastFiredPacket.newRows).To<BlueTable>();
|
||||
@@ -101,10 +104,12 @@ namespace BCG {
|
||||
}
|
||||
|
||||
protected override void myHandleIncrementalDelete() {
|
||||
requireLastArrivedPacket();
|
||||
if (myLastArrivedPacket.deletedRows.Count == 0) {
|
||||
Log.Gen.warn("MapStream.myHandleIncrementalDelete called with 0 deletedRows");
|
||||
return;
|
||||
}
|
||||
requireLastFiredPacket();
|
||||
var deletedRows = new BlueTable(myLastFiredPacket.newRows.schema);
|
||||
var newRows = ExceptionAssert.Cast(myLastFiredPacket.newRows).To<BlueTable>();
|
||||
foreach (IBlueRow row in myLastArrivedPacket.deletedRows) {
|
||||
@@ -127,6 +132,7 @@ namespace BCG {
|
||||
}
|
||||
|
||||
protected override void myOnTableClear() {
|
||||
requireLastFiredPacket();
|
||||
myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
|
||||
myLastFiredPacket.newRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
|
||||
myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema);
|
||||
|
||||
@@ -47,14 +47,14 @@ namespace BCG {
|
||||
_map_rows = map_rows;
|
||||
}
|
||||
|
||||
event MapperChangeDelegate? ISuperStreamMapper.onMapperChange { add { } remove { } }
|
||||
|
||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return _get_schema(schema); }
|
||||
|
||||
public IBlueRow[] map(IBlueRow r) {
|
||||
IEnumerable<IBlueRow> rows = _map_rows(r);
|
||||
return rows == null ? new IBlueRow[] { } : new List<IBlueRow>(rows).ToArray();
|
||||
}
|
||||
|
||||
public event MapperChangeDelegate? onMapperChange;
|
||||
}
|
||||
|
||||
//maps the id of the underlying stream to the
|
||||
|
||||
@@ -608,8 +608,8 @@ namespace BCG {
|
||||
// 5. deserialize the state into a stream op
|
||||
str_io_msg_p = deser.deserialize(content_p);
|
||||
// 6. check to make sure the state's rows are good
|
||||
passes = passes && str_io_msg_p.op == bts.StreamIOMessage.StreamOp.CREATE &&
|
||||
BlueDataUtility.compute_equality(cr_rows, str_io_msg_p.create_message.rows);
|
||||
passes &= str_io_msg_p.op == StreamIOMessage.StreamOp.CREATE
|
||||
&& BlueDataUtility.compute_equality(cr_rows, ExceptionAssert.NotNull(str_io_msg_p.create_message).rows);
|
||||
return new TestResult {
|
||||
test_result = passes,
|
||||
test_name = "run_pubsub_server_stream_tag_handler_test"
|
||||
@@ -692,7 +692,7 @@ namespace BCG {
|
||||
var content = st_data.get_current_content();
|
||||
var msg_p = deser.deserialize(content);
|
||||
Log.assert(msg_p.op == bts.StreamIOMessage.StreamOp.CREATE);
|
||||
return msg_p.create_message.rows;
|
||||
return ExceptionAssert.NotNull(msg_p.create_message).rows;
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
using Bts.ColumnSortExpressions;
|
||||
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
@@ -39,7 +41,7 @@ public class SortRow : IComparable, IBlueRow {
|
||||
var otherVal = otherRow[col.Name];
|
||||
int rv = myIsNull(this[col.Name]) ?
|
||||
(myIsNull(otherVal) ? 0 : -1) :
|
||||
(myIsNull(otherVal) ? 1 : ((IComparable)this[col.Name]).CompareTo(otherVal));
|
||||
(myIsNull(otherVal) ? 1 : ExceptionAssert.Argument.Is<IComparable>(this[col.Name]).CompareTo(otherVal));
|
||||
rv *= col.IsAscending ? 1 : -1;
|
||||
return rv;
|
||||
}
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
using System;
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
|
||||
@@ -60,13 +62,13 @@ namespace BCG
|
||||
List<string> columnInfoStrParts = new List<string>(columnInfoStr.Split(new char[] { ':' }, StringSplitOptions.None));
|
||||
if (columnInfoStrParts.Count != 2)
|
||||
{
|
||||
throw (new Exception(String.Format("Unable to decode schema string '{0}'", schemaStr)));
|
||||
throw new Exception($"Unable to decode schema string '{schemaStr}'");
|
||||
}
|
||||
|
||||
columnInfos.Add(new ColumnInfo(columnInfoStrParts[0], i, Type.GetType(columnInfoStrParts[1])));
|
||||
columnInfos.Add(new ColumnInfo(columnInfoStrParts[0], i, ExceptionAssert.NotNull(Type.GetType(columnInfoStrParts[1]))));
|
||||
}
|
||||
|
||||
return (new BlueTableSchema(columnInfos.ToArray(), ""));
|
||||
return new BlueTableSchema(columnInfos.ToArray(), string.Empty);
|
||||
}
|
||||
|
||||
private IBlueRow myGetRow(IBlueTableSchema schema, string rowStr)
|
||||
@@ -75,7 +77,7 @@ namespace BCG
|
||||
|
||||
if (schema.columnInfo.Length != valueStrs.Count)
|
||||
{
|
||||
throw (new Exception(String.Format("Unable to decode row string '{0}'", rowStr)));
|
||||
throw new Exception($"Unable to decode row string '{rowStr}'");
|
||||
}
|
||||
|
||||
List<object> values = new List<object>();
|
||||
@@ -84,7 +86,7 @@ namespace BCG
|
||||
values.Add(Convert.ChangeType(valueStrs[i], schema.columnInfo[i].colType));
|
||||
}
|
||||
|
||||
return (new BlueRow(schema, values.ToArray()));
|
||||
return new BlueRow(schema, values.ToArray());
|
||||
}
|
||||
#endregion
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using BCG.Utilities.General;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
|
||||
namespace BCG
|
||||
@@ -97,7 +98,7 @@ namespace BCG
|
||||
// BlueEventLog2.instance().assert(Thread.CurrentThread.ManagedThreadId == _target_thread,
|
||||
// "onTargetDoWork should only be called from target thread");
|
||||
var updates = _queued_updates.dequeueAll();
|
||||
foreach (var u in updates) {
|
||||
foreach (var u in updates.ExceptNull()) {
|
||||
if (u.subscription == _current_target_subscription)
|
||||
myFireEvent(u.dp);
|
||||
}
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
using System;
|
||||
|
||||
using BCG.Utilities.General;
|
||||
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
using System;
|
||||
|
||||
namespace BCG {
|
||||
public abstract class TimeUnitStream : ViewStream, IWorker {
|
||||
protected TimeSpan myOffset;
|
||||
@@ -46,8 +48,8 @@ namespace BCG {
|
||||
myRoundTime(myZeroTime + new TimeSpan(myOffset.Ticks * periods));
|
||||
|
||||
|
||||
if (myGetLatestRow() == null ||
|
||||
!myGetLatestRow()["time"].Equals(dt)) {
|
||||
var latestRow = myGetLatestRow();
|
||||
if (latestRow is null || !ExceptionAssert.Argument.Is<DateTime>(latestRow["time"]).Equals(dt)) {
|
||||
IBlueRow ar = new BlueRow(myCollection.schema,
|
||||
myId++, dt);
|
||||
IBlueRow dr = myCollection.addRowWithReturn(ar);
|
||||
@@ -71,11 +73,10 @@ namespace BCG {
|
||||
|
||||
#region Private Helper methods
|
||||
|
||||
private IBlueRow myGetLatestRow() {
|
||||
return myCollection.Count == 0 ?
|
||||
null :
|
||||
BlueDataUtility.extractRow(myCollection, myCollection.Count - 1);
|
||||
}
|
||||
IBlueRow? myGetLatestRow() =>
|
||||
myCollection.Count == 0
|
||||
? null
|
||||
: BlueDataUtility.extractRow(myCollection, myCollection.Count - 1);
|
||||
|
||||
private IBlueRowCollection myConvertToCollection(IBlueRow r) {
|
||||
BlueTable retVal = new BlueTable(myCollection.schema);
|
||||
|
||||
@@ -1,43 +1,42 @@
|
||||
using System;
|
||||
namespace BCG;
|
||||
|
||||
namespace BCG {
|
||||
public class TopStream : MethodDrivenStream {
|
||||
private int myTopCount;
|
||||
public class TopStream : MethodDrivenStream {
|
||||
int myTopCount;
|
||||
|
||||
public TopStream(ViewStream stream, int topCount) : base(stream) { myTopCount = topCount; }
|
||||
public TopStream(ViewStream stream, int topCount)
|
||||
: base(stream) =>
|
||||
myTopCount = topCount;
|
||||
|
||||
public TopStream(ViewStream stream,
|
||||
int topCount,
|
||||
string sortExpression)
|
||||
:
|
||||
this(sortExpression == null ?
|
||||
stream :
|
||||
new SortStream(stream, sortExpression), topCount) { }
|
||||
public TopStream(ViewStream stream, int topCount, string sortExpression)
|
||||
: this(sortExpression is null ? stream : new SortStream(stream, sortExpression), topCount) { }
|
||||
|
||||
protected override void myOnTableClear() {
|
||||
myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
|
||||
myLastFiredPacket.newRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
|
||||
myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
|
||||
}
|
||||
protected override void myOnTableClear() {
|
||||
requireLastFiredPacket();
|
||||
myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
|
||||
myLastFiredPacket.newRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
|
||||
myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.deletedRows.schema);
|
||||
}
|
||||
|
||||
protected override void myHandleFirstCall() {
|
||||
myLastFiredPacket = new DataPacket();
|
||||
myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
|
||||
myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
|
||||
myLastFiredPacket.deletedRows = new OrderedBlueTable(myLastFiredPacket.newRows.schema);
|
||||
}
|
||||
protected override void myHandleFirstCall() {
|
||||
requireLastArrivedPacket();
|
||||
myLastFiredPacket = new DataPacket();
|
||||
myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
|
||||
myLastFiredPacket.addedRows = new OrderedBlueTable(myLastFiredPacket.newRows);
|
||||
myLastFiredPacket.deletedRows = new OrderedBlueTable(myLastFiredPacket.newRows.schema);
|
||||
}
|
||||
|
||||
protected override void myHandleIncrementalAdd() { }
|
||||
protected override void myHandleIncrementalAdd() { }
|
||||
|
||||
protected override void myHandleIncrementalDelete() {
|
||||
IBlueRowCollection oldRows = myLastFiredPacket.newRows;
|
||||
myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
|
||||
BlueDataUtility.computeDifferences(myLastFiredPacket.newRows,
|
||||
oldRows,
|
||||
myLastFiredPacket.schema.columnNames,
|
||||
out myLastFiredPacket.deletedRows,
|
||||
out myLastFiredPacket.addedRows);
|
||||
myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
|
||||
}
|
||||
}
|
||||
protected override void myHandleIncrementalDelete() {
|
||||
requireLastArrivedPacket();
|
||||
requireLastFiredPacket();
|
||||
IBlueRowCollection oldRows = myLastFiredPacket.newRows;
|
||||
myLastFiredPacket.newRows = new OrderedBlueTable(myLastArrivedPacket.newRows, myTopCount);
|
||||
BlueDataUtility.computeDifferences(myLastFiredPacket.newRows,
|
||||
oldRows,
|
||||
myLastFiredPacket.schema.columnNames,
|
||||
out myLastFiredPacket.deletedRows,
|
||||
out myLastFiredPacket.addedRows);
|
||||
myFireSwitch = myLastFiredPacket.addedRows.Count + myLastFiredPacket.deletedRows.Count == 0 ? false : true;
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,10 @@
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
|
||||
namespace BCG {
|
||||
|
||||
public class AlphabetizeColumnsStream : ViewStream {
|
||||
private ViewStream mySrc;
|
||||
private IBlueTableSchema mySchema = null;
|
||||
IBlueTableSchema? mySchema = null;
|
||||
|
||||
public AlphabetizeColumnsStream(ViewStream src) {
|
||||
mySrc = src;
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
|
||||
namespace BCG
|
||||
{
|
||||
public class AppendStreamModified : ViewStream
|
||||
{
|
||||
private AppendCollection myColl = new AppendCollection ();
|
||||
private ViewStream myLeftStream, myRightstream;
|
||||
|
||||
ViewStream? myLeftStream, myRightstream;
|
||||
|
||||
public AppendStreamModified (ViewStream leftStream, ViewStream rightStream)
|
||||
{
|
||||
myLeftStream = leftStream;
|
||||
@@ -15,11 +15,24 @@ namespace BCG
|
||||
|
||||
public AppendStreamModified (ViewStream[] vss)
|
||||
{
|
||||
switch (vss.Length) {
|
||||
case 0:
|
||||
break;
|
||||
case 1:
|
||||
mySetRealSource (vss[0]);
|
||||
break;
|
||||
default:
|
||||
mySetRealSource(
|
||||
vss[1..].Aggregate(
|
||||
new AppendStreamModified(new[] { vss[0] }),
|
||||
(asm, vs) => new AppendStreamModified(asm, vs)));
|
||||
break;
|
||||
}
|
||||
if (vss.Length == 1)
|
||||
mySetRealSource (vss[0]);
|
||||
else
|
||||
{
|
||||
AppendStreamModified rv = null;
|
||||
AppendStreamModified? rv = null;
|
||||
foreach (ViewStream s in vss)
|
||||
rv = rv == null ? new AppendStreamModified (new ViewStream[] {s}) : new AppendStreamModified (rv, s);
|
||||
mySetRealSource (rv);
|
||||
|
||||
@@ -117,9 +117,6 @@ namespace BCG {
|
||||
addWorker(op.w);
|
||||
else
|
||||
removeWorker(op.w);
|
||||
}
|
||||
catch (InvalidOperationException ex) {
|
||||
bts.Log.Gen.error($"ViewStreamDatabase worker collection was modified during enumeration: {ex}");
|
||||
} finally {
|
||||
_pending_worker_ops.Clear();
|
||||
_is_worker_call_in_progress = false;
|
||||
|
||||
@@ -8,11 +8,11 @@ using Bts.DataStreamCore;
|
||||
using Bts.DataStreamExpressions;
|
||||
using Bts.PubSub;
|
||||
|
||||
using global::Spark.Core.Disposables;
|
||||
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
using Spark.Core.Disposables;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
@@ -8,7 +8,6 @@ using Bts.PubSub.Services;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
|
||||
using Spark.Api.MarketData;
|
||||
using Spark.Api.Services;
|
||||
using Spark.Core.Services;
|
||||
|
||||
|
||||
@@ -43,6 +43,14 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Net", "..\cs\Bts.Net\Bt
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.IO.Compression", "..\cs\Bts.IO.Compression\Bts.IO.Compression.csproj", "{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Collections", "..\cs\Bts.Collections\Bts.Collections.csproj", "{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Core", "..\cs\Bts.Core\Bts.Core.csproj", "{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Disposables", "..\cs\Bts.Disposables\Bts.Disposables.csproj", "{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}"
|
||||
EndProject
|
||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Numerics", "..\cs\Bts.Numerics\Bts.Numerics.csproj", "{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
@@ -293,6 +301,54 @@ Global
|
||||
{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x64.Build.0 = Release|Any CPU
|
||||
{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{3BF059B0-51C0-44A8-8297-9B7464DC5BA3}.Release|x86.Build.0 = Release|Any CPU
|
||||
{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Release|x64.Build.0 = Release|Any CPU
|
||||
{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{ADBEBCB3-5367-431B-AFC8-F57CAF9F029D}.Release|x86.Build.0 = Release|Any CPU
|
||||
{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Release|x64.Build.0 = Release|Any CPU
|
||||
{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{4E3D817D-9A0A-40F4-A1A1-4865375F40DD}.Release|x86.Build.0 = Release|Any CPU
|
||||
{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Release|x64.Build.0 = Release|Any CPU
|
||||
{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{ED034E82-07F4-473D-9C3A-F5EF9C961CFD}.Release|x86.Build.0 = Release|Any CPU
|
||||
{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Debug|x64.ActiveCfg = Debug|Any CPU
|
||||
{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Debug|x64.Build.0 = Debug|Any CPU
|
||||
{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Debug|x86.ActiveCfg = Debug|Any CPU
|
||||
{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Debug|x86.Build.0 = Debug|Any CPU
|
||||
{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Release|x64.ActiveCfg = Release|Any CPU
|
||||
{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Release|x64.Build.0 = Release|Any CPU
|
||||
{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Release|x86.ActiveCfg = Release|Any CPU
|
||||
{9A3E289A-3A88-4A76-AD0B-32C12EBA1B8B}.Release|x86.Build.0 = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
|
||||
@@ -5,7 +5,7 @@ using BCG.Utilities.General;
|
||||
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using global::Spark.Core.Collections;
|
||||
using Spark.Core.Collections;
|
||||
|
||||
public class TestDriverStream : ViewStream, IWorker {
|
||||
readonly IBlueTableSchema schema;
|
||||
|
||||
@@ -22,7 +22,6 @@ namespace BCG {
|
||||
/// with PubSub, as some clients are still using ManagedSubcribers)
|
||||
/// </summary>
|
||||
public class ManagedSubscriber : IManagedSubscriber, IWorker, IDisposable {
|
||||
private static ManagedSubscriber? _self;
|
||||
private ISubscriberCore _subcore;
|
||||
// private IDictionary myTypedDispatchTable = new Hashtable(),
|
||||
// myRawDispatchTable = new Hashtable();
|
||||
@@ -34,7 +33,6 @@ namespace BCG {
|
||||
ManagedSubscriber(ISubscriberCore subsCore) { _subcore = subsCore; }
|
||||
|
||||
public void doWork() {
|
||||
var msg_map = new Dictionary<string, List<bts.PubSubServerMessage>>();
|
||||
var updates = _subcore.get_updates(TimeSpan.Zero);
|
||||
if (updates.Count() == 0)
|
||||
return;
|
||||
|
||||
@@ -45,11 +45,7 @@ namespace BCG
|
||||
get { return myConnInfo; }
|
||||
}
|
||||
|
||||
override public string ToString()
|
||||
{
|
||||
Func<byte[], int> len = (byte[] m) => { return m == null ? 0 : m.Length; };
|
||||
return $"type: {myMsgType} conn: {myConnInfo} msg: {len(myMsg)}";
|
||||
}
|
||||
|
||||
public override string ToString() =>
|
||||
$"type: {myMsgType} conn: {myConnInfo} msg: {myMsg?.Length ?? 0}";
|
||||
}
|
||||
}
|
||||
@@ -46,7 +46,7 @@ namespace bts {
|
||||
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(5));
|
||||
_accept_socket.BeginAccept(new AsyncCallback(_on_accept), new object[] { });
|
||||
if (ex is OutOfMemoryException)
|
||||
throw ex;
|
||||
throw;
|
||||
return;
|
||||
}
|
||||
new_sock.LingerState = new LingerOption(false, 0);
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
using bts;
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Collections;
|
||||
using bts;
|
||||
using System.Threading;
|
||||
using System.Reflection;
|
||||
using System.Text;
|
||||
|
||||
namespace BCG {
|
||||
/// <summary>
|
||||
@@ -69,7 +68,8 @@ namespace BCG {
|
||||
|
||||
public static byte[] getBytesFromStream(params Stream[] streams) {
|
||||
long totalSize = 0;
|
||||
BinaryWriter bw = new BinaryWriter(new MemoryStream());
|
||||
using var ms = new MemoryStream();
|
||||
using var bw = new BinaryWriter(ms, Encoding.UTF8, true);
|
||||
foreach (Stream s in streams)
|
||||
totalSize += s.Length - s.Position;
|
||||
bw.Write(Convert.ToInt32(totalSize));
|
||||
@@ -78,7 +78,8 @@ namespace BCG {
|
||||
int lengthToReadForThisStream = Convert.ToInt32(s.Length - s.Position);
|
||||
bw.Write(br.ReadBytes(lengthToReadForThisStream));
|
||||
}
|
||||
return (bw.BaseStream as MemoryStream).ToArray();
|
||||
bw.Flush();
|
||||
return ms.ToArray();
|
||||
}
|
||||
|
||||
public static byte[] read_bytes(Stream str) {
|
||||
|
||||
@@ -92,7 +92,7 @@ namespace BCG {
|
||||
|
||||
public void publish(PubSubClientMessage msg) {
|
||||
Log.assert(msg.msg_type == PubSubMessageType.PUBLISH);
|
||||
Log.assert(!String.IsNullOrEmpty(msg.pub_msg.tag_type));
|
||||
Log.assert(!string.IsNullOrEmpty(msg.pub_msg?.tag_type));
|
||||
bts.Log.Gen.info($"PublisherCore.publish msg: {msg}");
|
||||
_main_queue.enqueue(msg);
|
||||
}
|
||||
|
||||
@@ -143,7 +143,7 @@ namespace BCG {
|
||||
public void doWork() {
|
||||
if (_connection_cb != null) {
|
||||
var conns = _conn_callback_queue.dequeueAll();
|
||||
foreach (var ci in conns) {
|
||||
foreach (var ci in conns.ExceptNull()) {
|
||||
_connection_cb(ci.client_url, ci.status);
|
||||
}
|
||||
}
|
||||
@@ -175,7 +175,7 @@ namespace BCG {
|
||||
case RemoteMessageType.Data:
|
||||
// if callback is registered fire it right away, else enqueue it
|
||||
if (_data_cb != null)
|
||||
_data_cb(sender_rci.ToString(), message.message);
|
||||
_data_cb(sender_rci.ToString(), message.message ?? Array.Empty<byte>());
|
||||
else
|
||||
_received_data_queue.enqueue(new Pair<string, byte[]>(sender_rci.ToString(), message.message));
|
||||
break;
|
||||
@@ -258,7 +258,7 @@ namespace BCG {
|
||||
// (which gets called in another thread)
|
||||
private void _handle_new_connections() {
|
||||
var new_conns = _new_connections.dequeueAll();
|
||||
foreach (ITCPConnection conn in new_conns) {
|
||||
foreach (var conn in new_conns.ExceptNull()) {
|
||||
var peer_rci = conn.get_peer_address();
|
||||
bts.Log.Gen.info($"ManagedTCPMessageHub._handle_new_connections conn: {conn}");
|
||||
///to stop it from being garbabge collected immediatly
|
||||
|
||||
@@ -16,8 +16,8 @@ public interface IPubSubState {
|
||||
|
||||
ICollection<string> all_tags { get; }
|
||||
|
||||
byte[] get_content(string tag);
|
||||
string getTagType(string tag);
|
||||
byte[]? get_content(string tag);
|
||||
string? getTagType(string tag);
|
||||
void get_all_connections(
|
||||
string tag,
|
||||
out Dictionary<string, string> subscribers,
|
||||
|
||||
@@ -188,11 +188,11 @@ namespace BCG {
|
||||
_state.handle_sub(sub_msg); // state keeps the list of all subscribers
|
||||
///to all these new subscriber s, send the existing data
|
||||
foreach (string tag in sub_msg.tags.empty_if_null()) {
|
||||
byte[] data = _state.get_content(tag);
|
||||
byte[]? data = _state.get_content(tag);
|
||||
//ie data has been published once
|
||||
if (data != null) {
|
||||
bts.Log.Gen.info($"ManagedTCPMessgeHub._handle_sub get_content tag: {tag} data len: {data.Length}");
|
||||
string tag_type_ = _state.getTagType(tag);
|
||||
string? tag_type_ = _state.getTagType(tag);
|
||||
var msg_to_send = new PubSubServerMessage() {
|
||||
tag = tag,
|
||||
tag_type = tag_type_,
|
||||
|
||||
@@ -148,7 +148,7 @@ namespace BCG {
|
||||
return new string[] { };
|
||||
}
|
||||
|
||||
public byte[] get_content(string tag) {
|
||||
public byte[]? get_content(string tag) {
|
||||
if (!_tag_to_data_map.ContainsKey(tag))
|
||||
return null;
|
||||
var rv = _tag_to_data_map[tag].get_current_content();
|
||||
@@ -157,7 +157,7 @@ namespace BCG {
|
||||
return rv;
|
||||
}
|
||||
|
||||
public string getTagType(string tag) {
|
||||
public string? getTagType(string tag) {
|
||||
if (!_tag_to_data_map.ContainsKey(tag))
|
||||
return null;
|
||||
else
|
||||
|
||||
@@ -20,7 +20,7 @@ namespace bts {
|
||||
public class SingleTagData {
|
||||
private readonly IPubSubServerTagHandler? _tag_handler;
|
||||
|
||||
public SingleTagData(string tag, string tag_type, IPubSubServerTagHandler tag_handler) {
|
||||
public SingleTagData(string tag, string tag_type, IPubSubServerTagHandler? tag_handler) {
|
||||
bts.Log.Gen.info($"SingleTagData constructed for tag: {tag} tag_type: {tag_type}");
|
||||
this.tag = tag;
|
||||
this.tag_type = tag_type;
|
||||
|
||||
@@ -46,7 +46,6 @@ namespace BCG {
|
||||
}
|
||||
|
||||
public void onConnection(ITCPConnection connection) {
|
||||
string ip; int port;
|
||||
var peer_rci = connection.get_peer_address();
|
||||
bts.Log.Gen.info($"SimpleTCPClient.onConnection connected to server: {peer_rci}, starting task to send data in a loop");
|
||||
if (_listener != null) {
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
namespace bts.BlueOrb {
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
|
||||
using BCG;
|
||||
|
||||
@@ -12,14 +10,7 @@ namespace bts.BlueOrb {
|
||||
|
||||
[LiveMarketDataStream]
|
||||
public class CrossExchangeSymbolMatcherStream : ViewStream {
|
||||
const string CanonicalSymbolColumn = "canonical_symbol";
|
||||
readonly CrossExchangeSymbolAliasResolver symbolAliasResolver;
|
||||
|
||||
public CrossExchangeSymbolMatcherStream([MarketDataClientParam] IMarketDataClient mdClient, IWorkScheduler scheduler)
|
||||
: this(mdClient, scheduler, null) { }
|
||||
|
||||
public CrossExchangeSymbolMatcherStream([MarketDataClientParam] IMarketDataClient mdClient, IWorkScheduler scheduler, string? symbolAliasConfigFile) {
|
||||
symbolAliasResolver = CrossExchangeSymbolAliasResolver.Load(ResolveSymbolAliasConfigFile(symbolAliasConfigFile));
|
||||
public CrossExchangeSymbolMatcherStream([MarketDataClientParam] IMarketDataClient mdClient, IWorkScheduler scheduler) {
|
||||
ViewStream r;
|
||||
ViewStream okx_usdc;
|
||||
ViewStream okx_usdt;
|
||||
@@ -33,9 +24,6 @@ namespace bts.BlueOrb {
|
||||
r = new InstrumentsStream(mdClient, scheduler);
|
||||
r = new DateTimeExpanstionStream(r,
|
||||
"expiration");
|
||||
r = new MapStream(r,
|
||||
_add_canonical_symbol_schema,
|
||||
_add_canonical_symbol_row);
|
||||
okx_usdc = new FilterStream(r,
|
||||
"venue='Okx' and sec~='USDC'");
|
||||
okx_usdc = new AddColumnsStream(okx_usdc,
|
||||
@@ -75,7 +63,7 @@ namespace bts.BlueOrb {
|
||||
r,
|
||||
"");
|
||||
r = new FilterStream(r,
|
||||
"left_venue!=right_venue and left_canonical_symbol=right_canonical_symbol and left_cur=right_cur and left_sec_type=right_sec_type");
|
||||
"left_venue!=right_venue and left_symbol=right_symbol and left_cur=right_cur and left_sec_type=right_sec_type");
|
||||
perps = new FilterStream(r,
|
||||
"left_expiration_year=2100 and right_expiration_year=2100");
|
||||
perps = new FilterStream(perps,
|
||||
@@ -90,127 +78,5 @@ namespace bts.BlueOrb {
|
||||
|
||||
mySetRealSource(r);
|
||||
}
|
||||
|
||||
static string ResolveSymbolAliasConfigFile(string? symbolAliasConfigFile) {
|
||||
if (!string.IsNullOrWhiteSpace(symbolAliasConfigFile)) {
|
||||
return symbolAliasConfigFile;
|
||||
}
|
||||
|
||||
var home = Environment.GetEnvironmentVariable("HOME");
|
||||
if (!string.IsNullOrWhiteSpace(home)) {
|
||||
var sparkdConf = Path.Combine(home, ".local", "share", "bts", "sparkd", "conf", "cross_exchange_symbol_aliases.conf");
|
||||
if (File.Exists(sparkdConf)) {
|
||||
return sparkdConf;
|
||||
}
|
||||
}
|
||||
|
||||
var appData = Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData);
|
||||
if (!string.IsNullOrWhiteSpace(appData)) {
|
||||
var appDataConf = Path.Combine(appData, "bts", "conf", "cross_exchange_symbol_aliases.conf");
|
||||
if (File.Exists(appDataConf)) {
|
||||
return appDataConf;
|
||||
}
|
||||
}
|
||||
|
||||
return !string.IsNullOrWhiteSpace(home)
|
||||
? Path.Combine(home, ".local", "share", "bts", "sparkd", "conf", "cross_exchange_symbol_aliases.conf")
|
||||
: "cross_exchange_symbol_aliases.conf";
|
||||
}
|
||||
|
||||
IBlueTableSchema _add_canonical_symbol_schema(IBlueTableSchema schema) =>
|
||||
BlueDataUtility.add_column(schema, CanonicalSymbolColumn, typeof(string));
|
||||
|
||||
IBlueRow _add_canonical_symbol_row(IBlueRow row) {
|
||||
var rv = new MutableBlueRow(_add_canonical_symbol_schema(row.schema), row);
|
||||
rv.setValue(CanonicalSymbolColumn, symbolAliasResolver.Canonicalize(row.get<string>("symbol")));
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class CrossExchangeSymbolAliasResolver {
|
||||
static readonly System.Text.RegularExpressions.Regex TokenRegex =
|
||||
new(@"[A-Za-z0-9]+|[^A-Za-z0-9]+", System.Text.RegularExpressions.RegexOptions.Compiled);
|
||||
|
||||
readonly IReadOnlyDictionary<string, string> aliases;
|
||||
|
||||
CrossExchangeSymbolAliasResolver(IReadOnlyDictionary<string, string> aliases) =>
|
||||
this.aliases = aliases;
|
||||
|
||||
public static CrossExchangeSymbolAliasResolver Load(string configFilePath) {
|
||||
if (!File.Exists(configFilePath)) {
|
||||
Log.Gen.warn($"CrossExchangeSymbolMatcherStream alias config not found: {configFilePath}");
|
||||
return new CrossExchangeSymbolAliasResolver(new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
var aliases = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var rawLine in File.ReadLines(configFilePath)) {
|
||||
var line = StripComments(rawLine).Trim();
|
||||
if (line.Length == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!TryParseAlias(line, out var alias, out var canonical)) {
|
||||
Log.Gen.warn($"CrossExchangeSymbolMatcherStream skipping invalid alias line: '{rawLine}'");
|
||||
continue;
|
||||
}
|
||||
|
||||
aliases[NormalizeToken(alias)] = NormalizeToken(canonical);
|
||||
}
|
||||
|
||||
return new CrossExchangeSymbolAliasResolver(aliases);
|
||||
}
|
||||
|
||||
public string Canonicalize(string? symbol) {
|
||||
if (string.IsNullOrWhiteSpace(symbol)) {
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var rv = new System.Text.StringBuilder(symbol.Length);
|
||||
foreach (System.Text.RegularExpressions.Match match in TokenRegex.Matches(symbol)) {
|
||||
var token = match.Value;
|
||||
if (IsToken(token)) {
|
||||
var normalized = NormalizeToken(token);
|
||||
rv.Append(aliases.TryGetValue(normalized, out var canonical) ? canonical : normalized);
|
||||
} else {
|
||||
rv.Append(token);
|
||||
}
|
||||
}
|
||||
return rv.ToString();
|
||||
}
|
||||
|
||||
static bool TryParseAlias(string line, out string alias, out string canonical) {
|
||||
var match = System.Text.RegularExpressions.Regex.Match(line, @"^(?<alias>.+?)\s*(?:=|:|->)\s*(?<canonical>.+)$");
|
||||
if (match.Success) {
|
||||
alias = match.Groups["alias"].Value.Trim();
|
||||
canonical = match.Groups["canonical"].Value.Trim();
|
||||
return alias.Length > 0 && canonical.Length > 0;
|
||||
}
|
||||
|
||||
var parts = line.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries);
|
||||
if (parts.Length == 2) {
|
||||
alias = parts[0];
|
||||
canonical = parts[1];
|
||||
return true;
|
||||
}
|
||||
|
||||
alias = string.Empty;
|
||||
canonical = string.Empty;
|
||||
return false;
|
||||
}
|
||||
|
||||
static string StripComments(string line) {
|
||||
var commentIdx = line.IndexOf('#');
|
||||
var slashIdx = line.IndexOf("//", StringComparison.Ordinal);
|
||||
if (slashIdx >= 0 && (commentIdx < 0 || slashIdx < commentIdx)) {
|
||||
commentIdx = slashIdx;
|
||||
}
|
||||
return commentIdx >= 0 ? line[..commentIdx] : line;
|
||||
}
|
||||
|
||||
static bool IsToken(string token) =>
|
||||
token.Length > 0 && char.IsLetterOrDigit(token[0]);
|
||||
|
||||
static string NormalizeToken(string token) =>
|
||||
token.Trim().ToUpperInvariant();
|
||||
}
|
||||
}
|
||||
-31
@@ -1,31 +0,0 @@
|
||||
namespace bts.BlueOrb {
|
||||
using BCG;
|
||||
using Bts.DataStreamCore;
|
||||
using Bts.PubSub;
|
||||
using Spark.Api.MarketData;
|
||||
|
||||
[LiveMarketDataStream]
|
||||
public class FeeAdjustedCrossExchangeArbDiscoveryStream : ViewStream {
|
||||
public FeeAdjustedCrossExchangeArbDiscoveryStream(
|
||||
[MarketDataClientParam] IMarketDataClient mdClient,
|
||||
IWorkScheduler scheduler) {
|
||||
ViewStream r = new CrossExchangeArbDiscoveryStream(mdClient, scheduler);
|
||||
r = new AddColumnsStream(r, "fee:0.0002");
|
||||
r = new AddColumnsStream(r, "capital_available:100000");
|
||||
r = new MathStream(r, """
|
||||
left_leg_cost = left_bid_price*left_bid_qty*left_cpp;
|
||||
right_leg_cost = right_bid_price*right_bid_qty*right_cpp;
|
||||
total_cost = left_leg_cost + right_leg_cost;
|
||||
total_fees = total_cost * fee;
|
||||
fee_adjusted_arb = arb-total_fees;
|
||||
possible_arb = fee_adjusted_arb*(Min(capital_available,total_cost)/total_cost);
|
||||
""");
|
||||
r = new FilterStream(r, "arb>0");
|
||||
r = new TimeSeriesStream(r);
|
||||
r = new ProjectStream(r,
|
||||
"spread_name, left_sec_name,left_venue,right_sec_name,right_venue,arb, fee_adjusted_arb, timestamp");
|
||||
|
||||
mySetRealSource(r);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,9 +5,10 @@ using BCG;
|
||||
using bts.api;
|
||||
using bts.md;
|
||||
|
||||
using Bts.DataStream;
|
||||
using Bts.DataStreamCore;
|
||||
|
||||
using global::Spark.Api.MarketData;
|
||||
using Spark.Api.MarketData;
|
||||
|
||||
public class BookSchemaBuilder {
|
||||
public int NumLevels { get; set; } = 5;
|
||||
|
||||
@@ -2,7 +2,7 @@ namespace Bts.SharedStreams;
|
||||
|
||||
using bts.api;
|
||||
|
||||
using global::Spark.Api.Orders;
|
||||
using Spark.Api.Orders;
|
||||
|
||||
public static class WellKnownColumns {
|
||||
public static (string Name, Type Type) AskNotional => (WellKnownColumnNames.AskNotional, typeof(double));
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
[DataStreamUserRespresentation("nl")]
|
||||
public class BlueOrbNLStream : ViewStream {
|
||||
private readonly string _nl_prompt;
|
||||
private ViewStream _nl_to_dsl_stream, _dsl_stream;
|
||||
private ViewStream? _nl_to_dsl_stream, _dsl_stream;
|
||||
private IDictionary<string, object> _dp_metadata = new Dictionary<string, object>();
|
||||
public BlueOrbNLStream(string nl_prompt) {
|
||||
_nl_prompt = nl_prompt;
|
||||
|
||||
@@ -85,7 +85,8 @@ namespace bts.BlueOrb {
|
||||
.GetProperty("choices")[0]
|
||||
.GetProperty("message")
|
||||
.GetProperty("content")
|
||||
.GetString();
|
||||
.GetString()
|
||||
?? string.Empty;
|
||||
|
||||
//return gpt_response;
|
||||
DateTime t1 = DateTime.Now;
|
||||
|
||||
@@ -4,6 +4,8 @@ using BCG.Utilities.General;
|
||||
using Bts.PubSub;
|
||||
using Bts.SharedStreams.Tables;
|
||||
|
||||
using Spark.Core.Diagnostics;
|
||||
|
||||
namespace bts {
|
||||
// TODO: Use dependency injection to instantiate this stream, and others like it that depend on
|
||||
// ISubscriberCore(Factory). Until then, hide it from DSQA.
|
||||
@@ -40,7 +42,7 @@ namespace bts {
|
||||
foreach (var u in updates) {
|
||||
if (u.tag != "System_AllStatistics")
|
||||
continue;
|
||||
var tag_infos = TagStatsInfo.deserialize(u.data);
|
||||
var tag_infos = TagStatsInfo.deserialize(ExceptionAssert.NotNull(u.data));
|
||||
foreach (var ti in tag_infos) {
|
||||
_tag_map[ti.tag] = ti;
|
||||
}
|
||||
|
||||
@@ -187,32 +187,4 @@
|
||||
Name="Test"
|
||||
DependsOnTargets="Build"></Target>
|
||||
|
||||
<ItemGroup Condition="Exists('@(NativeWindowsAsset)')">
|
||||
<Content Include="@(NativeWindowsAsset)">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
<TargetPath>%(FileName)%(Extension)</TargetPath>
|
||||
<Pack>false</Pack>
|
||||
</Content>
|
||||
<Content Include="@(NativeWindowsAsset)">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
<TargetPath>runtimes/win-x64/native/%(FileName)%(Extension)</TargetPath>
|
||||
<Pack>true</Pack>
|
||||
<PackagePath>runtimes/win-x64/native</PackagePath>
|
||||
</Content>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup Condition="Exists('@(NativeLinuxAsset)')">
|
||||
<Content Include="@(NativeLinuxAsset)">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
<TargetPath>%(FileName)%(Extension)</TargetPath>
|
||||
<Pack>false</Pack>
|
||||
</Content>
|
||||
<Content Include="@(NativeLinuxAsset)">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
<TargetPath>runtimes/linux-x64/native/%(FileName)%(Extension)</TargetPath>
|
||||
<Pack>true</Pack>
|
||||
<PackagePath>runtimes/linux-x64/native</PackagePath>
|
||||
</Content>
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -17,9 +17,7 @@ type Tab =
|
||||
| PositionManagementTab
|
||||
| MarginTab
|
||||
|
||||
// BEGIN GENERATED ADMIN VENUES
|
||||
let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.BYBIT; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.GATEIO; Venue.GLBX; Venue.ICE; Venue.OKX; Venue.ORCA; Venue.PHOENIX; Venue.RAYDIUM; Venue.SIM; Venue.TAIFEX; Venue.TWSE; |]
|
||||
// END GENERATED ADMIN VENUES
|
||||
let venues = [| Venue.BINANCE; Venue.BinanceUs; Venue.BinanceUsdsFutures; Venue.BYBIT; Venue.CBOE; Venue.COINBASE; Venue.DERIBIT; Venue.GATEIO; Venue.OKX; Venue.GLBX; Venue.ICE; Venue.TAIFEX; Venue.TWSE; Venue.SIM; |]
|
||||
|
||||
let securityTypes = [| SecurityType.SPOT; SecurityType.FUTURE; SecurityType.OPTION; SecurityType.SPREAD; |]
|
||||
|
||||
@@ -96,16 +94,6 @@ type OkxExchangeField =
|
||||
| Origin of CustomerOrFirm
|
||||
| ClearingAccount of string
|
||||
|
||||
type OrcaExchangeField =
|
||||
| CTI of CTICode
|
||||
| Origin of CustomerOrFirm
|
||||
| ClearingAccount of string
|
||||
|
||||
type RaydiumExchangeField =
|
||||
| CTI of CTICode
|
||||
| Origin of CustomerOrFirm
|
||||
| ClearingAccount of string
|
||||
|
||||
type SIMExchangeField =
|
||||
| CTI of CTICode
|
||||
| Origin of CustomerOrFirm
|
||||
|
||||
@@ -801,11 +801,8 @@ let venueToString (venue:bts.api.Venue) =
|
||||
| bts.api.Venue.GATEIO -> "GATEIO"
|
||||
| bts.api.Venue.FTX -> "FTX"
|
||||
| bts.api.Venue.OKX -> "OKX"
|
||||
| bts.api.Venue.ORCA -> "ORCA"
|
||||
| bts.api.Venue.RAYDIUM -> "RAYDIUM"
|
||||
| bts.api.Venue.TAIFEX -> "TAIFEX"
|
||||
| bts.api.Venue.TWSE -> "TWSE"
|
||||
| value -> sprintf "%O" value
|
||||
|
||||
let exchange_display expirationNameList =
|
||||
let s = sprintf "%O" expirationNameList.product.exchange
|
||||
|
||||
@@ -37,45 +37,6 @@ protobuf_generate_cpp(
|
||||
)
|
||||
add_executable(sparkd ${${PROJECT_NAME}_SOURCES} ${${PROJECT_NAME}_PROTO_CXX_SOURCES})
|
||||
|
||||
find_package(PkgConfig REQUIRED)
|
||||
pkg_check_modules(
|
||||
emumba-deribit-connector-library
|
||||
REQUIRED
|
||||
IMPORTED_TARGET
|
||||
emumba-deribit-connector
|
||||
)
|
||||
pkg_check_modules(
|
||||
rakurai-dex-connector-library
|
||||
REQUIRED
|
||||
IMPORTED_TARGET
|
||||
rakurai-dex-connector
|
||||
)
|
||||
# Rakurai's static dex SDK uses hidapi's libudev backend. Keep this explicit so
|
||||
# existing CMake build trees also pick up the dependency when the connector's
|
||||
# pkg-config metadata changes between package versions.
|
||||
find_library(RAKURAI_UDEV_LIBRARY NAMES udev libudev.so.1 REQUIRED)
|
||||
|
||||
# The Rakurai package currently publishes a bare `udev` dependency. Minimal
|
||||
# runtime images only contain libudev.so.1, so replace that entry with the
|
||||
# resolved library while preserving its position after Rakurai's static SDK.
|
||||
get_target_property(
|
||||
RAKURAI_LINK_LIBRARIES
|
||||
PkgConfig::rakurai-dex-connector-library
|
||||
INTERFACE_LINK_LIBRARIES
|
||||
)
|
||||
set(RAKURAI_RESOLVED_LINK_LIBRARIES)
|
||||
foreach(RAKURAI_LINK_LIBRARY IN LISTS RAKURAI_LINK_LIBRARIES)
|
||||
if(RAKURAI_LINK_LIBRARY STREQUAL "udev" OR RAKURAI_LINK_LIBRARY STREQUAL "-ludev")
|
||||
list(APPEND RAKURAI_RESOLVED_LINK_LIBRARIES ${RAKURAI_UDEV_LIBRARY})
|
||||
else()
|
||||
list(APPEND RAKURAI_RESOLVED_LINK_LIBRARIES ${RAKURAI_LINK_LIBRARY})
|
||||
endif()
|
||||
endforeach()
|
||||
set_property(
|
||||
TARGET PkgConfig::rakurai-dex-connector-library
|
||||
PROPERTY INTERFACE_LINK_LIBRARIES ${RAKURAI_RESOLVED_LINK_LIBRARIES}
|
||||
)
|
||||
|
||||
# include paths
|
||||
target_include_directories(
|
||||
sparkd
|
||||
@@ -83,9 +44,11 @@ target_include_directories(
|
||||
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
|
||||
$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>/include
|
||||
$<BUILD_INTERFACE:${emumba-deribit-connector-library_INCLUDE_DIRS}>
|
||||
$<BUILD_INTERFACE:${rakurai-dex-connector-library_INCLUDE_DIRS}>
|
||||
)
|
||||
|
||||
find_package(PkgConfig REQUIRED)
|
||||
pkg_check_modules(emumba-deribit-connector-library REQUIRED emumba-deribit-connector)
|
||||
|
||||
# This dir contains rapidjsonConfig.cmake
|
||||
list(APPEND CMAKE_PREFIX_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules)
|
||||
find_package(rapidjson REQUIRED)
|
||||
@@ -164,8 +127,7 @@ target_link_libraries(
|
||||
python_interop
|
||||
python_interop_module
|
||||
databento::databento
|
||||
PkgConfig::emumba-deribit-connector-library
|
||||
PkgConfig::rakurai-dex-connector-library
|
||||
${emumba-deribit-connector-library_LIBRARIES}
|
||||
tbb
|
||||
quickfix
|
||||
benchmark
|
||||
@@ -178,11 +140,6 @@ target_link_libraries(
|
||||
-static-libstdc++
|
||||
-pthread)
|
||||
|
||||
# Both vendor static connector archives currently export the same global
|
||||
# `print_books` symbol. Allow that duplicate so sparkd can link until the
|
||||
# upstream packages make the symbol internal or rename it.
|
||||
target_link_options(${PROJECT_NAME} PRIVATE -Wl,--allow-multiple-definition)
|
||||
|
||||
# Onixs
|
||||
target_link_libraries(
|
||||
${PROJECT_NAME}
|
||||
|
||||
@@ -2,14 +2,12 @@ ARG IMAGE="sparkd"
|
||||
ARG CONFIG_DIR="src/app_sparkd/configurations/dev"
|
||||
ARG SOURCE_DIR="install/sparkd-release-gcc"
|
||||
ARG PYTHON_REQUIREMENTS="requirements.txt"
|
||||
ARG SPARKD_ENTRYPOINT="src/app_sparkd/sparkd-entrypoint.sh"
|
||||
|
||||
# sparkd_base
|
||||
FROM bluetradingsystems/linux_dotnet8 AS sparkd_base
|
||||
ARG CONFIG_DIR
|
||||
ARG SOURCE_DIR
|
||||
ARG PYTHON_REQUIREMENTS
|
||||
ARG SPARKD_ENTRYPOINT
|
||||
|
||||
USER "root"
|
||||
RUN apt-get update && apt-get install -y liblttng-ust-dev
|
||||
@@ -23,7 +21,6 @@ RUN pip3 install -r "/home/bts/bin/requirements.txt"
|
||||
USER "bts"
|
||||
COPY --chown=bts:bts "${SOURCE_DIR}" "."
|
||||
COPY --chown=bts:bts "${CONFIG_DIR}" "/home/bts/.local/share/bts/sparkd/conf"
|
||||
COPY --chown=bts:bts "${SPARKD_ENTRYPOINT}" "/home/bts/bin/sparkd-entrypoint.sh"
|
||||
WORKDIR "/home/bts/bin"
|
||||
|
||||
# sparkd
|
||||
@@ -38,10 +35,6 @@ COPY --chown=bts:bts "${MD_RECORDING_DIR}" "/home/bts/.local/share/bts/sparkd/da
|
||||
FROM bluetradingsystems/linux_dotnet8 AS sparkd_symbols
|
||||
ARG SOURCE_DIR
|
||||
ARG PYTHON_REQUIREMENTS="requirements.txt"
|
||||
ARG SPARKD_ENTRYPOINT
|
||||
|
||||
ENV DOTNET_ENVIRONMENT=Development
|
||||
RUN dotnet dev-certs https --trust
|
||||
|
||||
USER "root"
|
||||
RUN apt-get update && apt-get install -y liblttng-ust-dev
|
||||
@@ -54,7 +47,6 @@ RUN pip3 install -r "/home/bts/bin/requirements.txt"
|
||||
|
||||
USER "bts"
|
||||
COPY --chown=bts:bts "${SOURCE_DIR}" "."
|
||||
COPY --chown=bts:bts "${SPARKD_ENTRYPOINT}" "/home/bts/bin/sparkd-entrypoint.sh"
|
||||
WORKDIR "/home/bts/bin"
|
||||
|
||||
# selected image
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dex_dev
|
||||
SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/dex_dev/requirements.txt
|
||||
SPARKD__SHM_SIZE=2gb
|
||||
@@ -1,2 +0,0 @@
|
||||
* Connects to raydium and orca for dex
|
||||
* Execution is SIM
|
||||
@@ -1,22 +0,0 @@
|
||||
{
|
||||
"app_logging": {
|
||||
"severity": "trace",
|
||||
"file": false,
|
||||
"console": true
|
||||
},
|
||||
"dex": [
|
||||
{
|
||||
"id": "orca_clmm",
|
||||
"pool_ids": ["Czfq3xZZDmsdGdUyrNLtRhGc47cXcZtLG4crryfu44zE", "4HppGTweoGQ8ZZ6UcCgwJKfi5mJD9Dqwy6htCpnbfBLW"
|
||||
]
|
||||
}
|
||||
],
|
||||
"cluster_mode": false,
|
||||
"wallet_pub": "HFru3Rc3Rf6UjZtrYPsBSKvmq5uyha2DWefV77rFXU64",
|
||||
"wallet_pvt": [168,123,234,72,218,209,180,225,42,193,240,65,11,15,105,91,95,255,231,214,188,119,90,45,119,144,177,207,203,245,250,27,241,137,138,138,103,247,64,134,70,212,107,134,75,97,87,167,53,32,169,108,75,126,254,17,213,216,74,14,245,28,231,161],
|
||||
"md_server_uri_1": "ws://185.191.117.159:8130",
|
||||
"node_oms_uri_1": "ws://185.191.117.159:8133",
|
||||
"md_server_uri_2": "ws://189.1.171.199:8130",
|
||||
"node_oms_uri_2": "ws://189.1.171.199:8133",
|
||||
"core": 1
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
{
|
||||
"account": "DEX_DEV",
|
||||
"operator_id": "DEX_DEV",
|
||||
"json_conf_file_name": "config.json",
|
||||
"default_priority_fee": 1000
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
{
|
||||
"account": "DEX_DEV",
|
||||
"operator_id": "DEX_DEV",
|
||||
"json_conf_file_name": "config.json",
|
||||
"default_priority_fee": 1000
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
numpy==2.2.6
|
||||
pybit==5.11.0
|
||||
requests==2.31.0
|
||||
websocket-client==1.8.0
|
||||
websockets==15.0.1
|
||||
@@ -1,76 +0,0 @@
|
||||
|
||||
JetFIX {
|
||||
value {
|
||||
connection {
|
||||
name: "orca_1"
|
||||
venue: VENUE_ORCA
|
||||
max_order_msgs_per_window: 300
|
||||
window_duration_msecs: 1000
|
||||
}
|
||||
connection {
|
||||
name: "okx_1"
|
||||
venue: VENUE_OKX
|
||||
max_order_msgs_per_window: 300
|
||||
window_duration_msecs: 1000
|
||||
}
|
||||
}
|
||||
}
|
||||
PublisherOptions {
|
||||
value {
|
||||
network_interface: "0.0.0.0"
|
||||
heartbeat_ms: 5000
|
||||
heartbeat_boot_seconds: 60
|
||||
skip_options: true
|
||||
hot_initial: 1024
|
||||
hot_growth: 1024
|
||||
cold_initial: 8192
|
||||
cold_growth: 2048
|
||||
channel_prefix: ""
|
||||
disable_heartbeat: true
|
||||
}
|
||||
}
|
||||
MarketDataConfig {
|
||||
value {
|
||||
dex {
|
||||
venue: VENUE_ORCA
|
||||
num_depth_levels: 10
|
||||
json_conf_file_name: "config.json"
|
||||
security_criteria {
|
||||
clauses {
|
||||
security_types: SECURITY_TYPE_SPOT
|
||||
symbols: @SymbolLists.list[Binance].symbol
|
||||
name_regexps: "USDC|USDT$"
|
||||
}
|
||||
}
|
||||
}
|
||||
okx {
|
||||
num_depth_levels: 5
|
||||
url_rest_base: "https://www.okx.com"
|
||||
url_ws_base: "wss://ws.okx.com:8443"
|
||||
security_criteria {
|
||||
clauses {
|
||||
security_types: SECURITY_TYPE_SPOT
|
||||
symbols: @SymbolLists.list[Okx].symbol
|
||||
}
|
||||
}
|
||||
is_application_ping_pong_enabled: true
|
||||
application_ping_interval_ms: 20000
|
||||
application_pong_timeout_ms: 10000
|
||||
is_protocol_ping_pong_enabled: true
|
||||
protocol_ping_interval_ms: 60000
|
||||
protocol_pong_timeout_ms: 30000
|
||||
}
|
||||
dex {
|
||||
venue: VENUE_RAYDIUM
|
||||
num_depth_levels: 10
|
||||
json_conf_file_name: "config.json"
|
||||
security_criteria {
|
||||
clauses {
|
||||
security_types: SECURITY_TYPE_SPOT
|
||||
symbols: @SymbolLists.list[Binance].symbol
|
||||
name_regexps: "USDC|USDT$"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,84 +0,0 @@
|
||||
SymbolLists {
|
||||
value {
|
||||
list {
|
||||
key: "Binance"
|
||||
symbol: "SOL"
|
||||
symbol: "USDC"
|
||||
#symbol: "BNB"
|
||||
symbol: "BTC"
|
||||
#symbol: "DOGE"
|
||||
#symbol: "EIGEN"
|
||||
symbol: "ETH"
|
||||
symbol: "XRP"
|
||||
}
|
||||
list {
|
||||
key: "ORCA"
|
||||
#symbol: "AAVE"
|
||||
#symbol: "AVAX"
|
||||
#symbol: "BNB"
|
||||
symbol: "BTC"
|
||||
symbol: "USDC"
|
||||
symbol: "SOL"
|
||||
symbol: "ETH"
|
||||
symbol: "XRP"
|
||||
}
|
||||
list {
|
||||
key: "RAYDIUM"
|
||||
symbol: "SOL"
|
||||
symbol: "USDC"
|
||||
symbol: "ETH"
|
||||
symbol: "BTC"
|
||||
symbol: "XRP"
|
||||
}
|
||||
list {
|
||||
key: "PHOENIX"
|
||||
symbol: "SOL"
|
||||
}
|
||||
list {
|
||||
key: "BinanceUsdsFutures"
|
||||
symbol: "SOL"
|
||||
symbol: "BTC"
|
||||
symbol: "USDC"
|
||||
symbol: "ETH"
|
||||
symbol: "XRP"
|
||||
}
|
||||
list {
|
||||
key: "Okx"
|
||||
symbol: "SOL"
|
||||
symbol: "USDC"
|
||||
symbol: "BTC"
|
||||
symbol: "ETH"
|
||||
symbol: "XRP"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Symbols {
|
||||
value {
|
||||
# venues {
|
||||
# name: "VENUE_BINANCE"
|
||||
# spot: @SymbolLists.list[Binance].symbol
|
||||
# }
|
||||
#venues {
|
||||
# name: "VENUE_BINANCE_USDS_FUTURES"
|
||||
# future: @SymbolLists.list[BinanceUsdsFutures].symbol
|
||||
#}
|
||||
venues {
|
||||
name: "VENUE_ORCA"
|
||||
spot: @SymbolLists.list[ORCA].symbol
|
||||
}
|
||||
# venues {
|
||||
# name: "VENUE_RAYDIUM"
|
||||
# spot: @SymbolLists.list[RAYDIUM].symbol
|
||||
# }
|
||||
venues {
|
||||
name: "VENUE_PHOENIX"
|
||||
future: @SymbolLists.list[PHOENIX].symbol
|
||||
}
|
||||
venues {
|
||||
name: "VENUE_OKX"
|
||||
spot: @SymbolLists.list[Okx].symbol
|
||||
# future: @SymbolLists.list[Okx].symbol
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_sim_live_wb
|
||||
SPARKD__DATA_DIR=/home/wedbushprod/8000/sparkdata/data
|
||||
SPARKD__IMAGE=sparkd_symbols
|
||||
SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_sim_live_wb/requirements.txt
|
||||
SPARKD__RUNTIME_CONF_DIR=/home/wedbushprod/8000/sparkdata/conf
|
||||
@@ -1,5 +0,0 @@
|
||||
numpy==2.2.6
|
||||
pybit==5.11.0
|
||||
requests==2.31.0
|
||||
websocket-client==1.8.0
|
||||
websockets==15.0.1
|
||||
@@ -1,116 +0,0 @@
|
||||
JetFIX {
|
||||
value {
|
||||
connection {
|
||||
name: "glbx_1"
|
||||
venue: VENUE_GLBX
|
||||
max_order_msgs_per_window: 300
|
||||
window_duration_msecs: 1000
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
PublisherOptions {
|
||||
value {
|
||||
network_interface_a: 'enp193s0f0np0'
|
||||
network_interface_b: 'enp193s0f0np0'
|
||||
#these get overriden if not entered here
|
||||
heartbeat_ms: 5000
|
||||
heartbeat_boot_seconds: 60
|
||||
skip_options: false
|
||||
hot_initial: 1024
|
||||
hot_growth: 1024
|
||||
cold_initial: 8192
|
||||
cold_growth: 2048
|
||||
channel_prefix: ""
|
||||
disable_heartbeat: false
|
||||
}
|
||||
}
|
||||
|
||||
CMEOptions {
|
||||
value {
|
||||
feed_engine {
|
||||
name: "mdp3feed"
|
||||
spin_time: 0
|
||||
channels_per_thread: 3
|
||||
data_wait: 10
|
||||
udp_socket_buffer_size: 268435456
|
||||
}
|
||||
channel {
|
||||
id: "*"
|
||||
feed_engine: "mdp3feed"
|
||||
use_tcp_replay: false
|
||||
recovery_queue_size: 640000
|
||||
lost_packet_wait_timeout_ms: 10
|
||||
onixs_log_mode: CME_ONIXS_LOG_MODE_UNSPECIFIED:q
|
||||
#max_gap_recovery_cache_size: 4
|
||||
use_natural_refresh: false
|
||||
join_snapshot_on_gap: false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Logging {
|
||||
value {
|
||||
initial_queue_size: 120001
|
||||
max_queue_growth: 4
|
||||
console {
|
||||
default_level: "Info"
|
||||
subsystem {
|
||||
name: "md"
|
||||
level: "Info"
|
||||
}
|
||||
subsystem {
|
||||
name: "OrderMan"
|
||||
level: "Info"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
MarketDataConfig {
|
||||
value {
|
||||
glbx {
|
||||
unique_check_value: 15
|
||||
use_databento: false
|
||||
}
|
||||
recording{
|
||||
venue: VENUE_GLBX
|
||||
num_of_seconds_to_run_file_writer: 300
|
||||
num_of_seconds_to_run_db_writer : 0
|
||||
n_book_levels: 10
|
||||
batch_size: 100 #for the db writer
|
||||
enable_book_recording: true
|
||||
enable_trade_recording: true
|
||||
enable_bbo_recording: false
|
||||
enable_tbbo_recording: false
|
||||
enable_misc_recording: true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# JetFIX {
|
||||
# value {
|
||||
# #connection: @JetFIXSimConnection.sim
|
||||
# connection {
|
||||
# name: "mlx_004_54"
|
||||
# venue: VENUE_GLBX
|
||||
# socket_connect_host_ip: "69.50.112.191"
|
||||
# socket_connect_port: 42876
|
||||
# sender_compid: "MLX004"
|
||||
# target_compid: "CME"
|
||||
# target_subid: "54"
|
||||
# cme_market_segment_id: 54
|
||||
# start_time: "00:10:00"
|
||||
# end_time: "23:55:00"
|
||||
# start_day: "sunday"
|
||||
# end_day: "friday"
|
||||
# seconds_between_hbs: 30
|
||||
# sender_subid: "BTS"
|
||||
# use_ilink3: true
|
||||
# # raw_data: "S4PMA"
|
||||
# # auto_fix_seq_num: true
|
||||
# }
|
||||
# }
|
||||
# }
|
||||
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
SymbolLists {
|
||||
value {
|
||||
list {
|
||||
key: "GlbxFutures"
|
||||
symbol: "ES"
|
||||
}
|
||||
list {
|
||||
key: "GlbxOptions"
|
||||
symbol: "E1A"
|
||||
symbol: "E1B"
|
||||
symbol: "E1C"
|
||||
symbol: "E1D"
|
||||
symbol: "E2A"
|
||||
symbol: "E2B"
|
||||
symbol: "E2C"
|
||||
symbol: "E2D"
|
||||
symbol: "E3A"
|
||||
symbol: "E3B"
|
||||
symbol: "E3C"
|
||||
symbol: "E3D"
|
||||
symbol: "E4A"
|
||||
symbol: "E4B"
|
||||
symbol: "E4C"
|
||||
symbol: "E4D"
|
||||
symbol: "E5A"
|
||||
symbol: "E5B"
|
||||
symbol: "E5C"
|
||||
symbol: "E5D"
|
||||
symbol: "EW1"
|
||||
symbol: "EW2"
|
||||
symbol: "EW3"
|
||||
symbol: "EW4"
|
||||
symbol: "ESH9"
|
||||
symbol: "ESZ6"
|
||||
symbol: "ESH0"
|
||||
symbol: "ESM6"
|
||||
symbol: "ESZ7"
|
||||
symbol: "ESH6"
|
||||
symbol: "ESM7"
|
||||
symbol: "ESZ8"
|
||||
symbol: "ESH7"
|
||||
symbol: "ESH8"
|
||||
symbol: "ESU6"
|
||||
symbol: "ESZ5"
|
||||
symbol: "ESZ9"
|
||||
symbol: "E1D"
|
||||
symbol: "ESZ5"
|
||||
symbol: "ES"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Symbols {
|
||||
value {
|
||||
venues {
|
||||
name: "VENUE_GLBX"
|
||||
future: @SymbolLists.list[GlbxFutures].symbol
|
||||
option: @SymbolLists.list[GlbxOptions].symbol
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_sim_replay
|
||||
SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_sim_replay/requirements.txt
|
||||
TZ=America/Chicago
|
||||
SPARKD__MD_RECORDING_DIR=/home/wedbushprod/8000/md_recording
|
||||
@@ -1 +0,0 @@
|
||||
requests==2.31.0
|
||||
@@ -1,33 +0,0 @@
|
||||
JetFIX {
|
||||
value {
|
||||
connection {
|
||||
name: "glbx_1"
|
||||
venue: VENUE_GLBX
|
||||
max_order_msgs_per_window: 300
|
||||
window_duration_msecs: 1000
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
PublisherOptions {
|
||||
value {
|
||||
network_interface: "0.0.0.0"
|
||||
heartbeat_ms: 5000
|
||||
heartbeat_boot_seconds: 60
|
||||
skip_options: true
|
||||
hot_initial: 1024
|
||||
hot_growth: 1024
|
||||
cold_initial: 8192
|
||||
cold_growth: 2048
|
||||
channel_prefix: ""
|
||||
disable_heartbeat: true
|
||||
}
|
||||
}
|
||||
|
||||
MarketDataConfig {
|
||||
value {
|
||||
replay {
|
||||
venue: VENUE_GLBX
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
SymbolLists {
|
||||
value {
|
||||
list {
|
||||
key: "GlbxFutures"
|
||||
symbol: "ES"
|
||||
}
|
||||
list {
|
||||
key: "GlbxOptions"
|
||||
symbol: "E1A"
|
||||
symbol: "E1B"
|
||||
symbol: "E1C"
|
||||
symbol: "E1D"
|
||||
symbol: "E2A"
|
||||
symbol: "E2B"
|
||||
symbol: "E2C"
|
||||
symbol: "E2D"
|
||||
symbol: "E3A"
|
||||
symbol: "E3B"
|
||||
symbol: "E3C"
|
||||
symbol: "E3D"
|
||||
symbol: "E4A"
|
||||
symbol: "E4B"
|
||||
symbol: "E4C"
|
||||
symbol: "E4D"
|
||||
symbol: "E5A"
|
||||
symbol: "E5B"
|
||||
symbol: "E5C"
|
||||
symbol: "E5D"
|
||||
symbol: "ES"
|
||||
symbol: "EW1"
|
||||
symbol: "EW2"
|
||||
symbol: "EW3"
|
||||
symbol: "EW4"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Symbols {
|
||||
value {
|
||||
venues {
|
||||
name: "VENUE_GLBX"
|
||||
future: @SymbolLists.list[GlbxFutures].symbol
|
||||
option: @SymbolLists.list[GlbxOptions].symbol
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
SPARKD__CONFIG_DIR=src/app_sparkd/configurations/ophir_sim_replay_wb
|
||||
SPARKD__DATA_DIR=/home/wedbushprod/9000/sparkdata/data
|
||||
SPARKD__IMAGE=sparkd_symbols
|
||||
SPARKD__PYTHON_REQUIREMENTS=src/app_sparkd/configurations/ophir_sim_replay_wb/requirements.txt
|
||||
SPARKD__RUNTIME_CONF_DIR=/home/wedbushprod/9000/sparkdata/conf
|
||||
SPARKD__MD_RECORDING_DIR=/home/wedbushprod/md_recording
|
||||
SPARKD__CONFIG_PORT=9000
|
||||
SPARKD__QUICKBUS_PORT=9001
|
||||
SPARKD__SIM_FIX_PORT=9002
|
||||
@@ -1,5 +0,0 @@
|
||||
numpy==2.2.6
|
||||
pybit==5.11.0
|
||||
requests==2.31.0
|
||||
websocket-client==1.8.0
|
||||
websockets==15.0.1
|
||||
@@ -1,46 +0,0 @@
|
||||
JetFIX {
|
||||
value {
|
||||
connection {
|
||||
name: "glbx_1"
|
||||
venue: VENUE_GLBX
|
||||
max_order_msgs_per_window: 300
|
||||
window_duration_msecs: 1000
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ports {
|
||||
value {
|
||||
config_port: 9000
|
||||
qbus_port: 9001
|
||||
glbx_sim_fix_port: 9002
|
||||
ice_sim_fix_port: 9003
|
||||
}
|
||||
}
|
||||
|
||||
Logging {
|
||||
value {
|
||||
initial_queue_size: 120001
|
||||
max_queue_growth: 4
|
||||
console {
|
||||
default_level: "Info"
|
||||
subsystem {
|
||||
name: "md"
|
||||
level: "Info"
|
||||
}
|
||||
subsystem {
|
||||
name: "OrderMan"
|
||||
level: "Info"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
MarketDataConfig {
|
||||
value {
|
||||
replay {
|
||||
venue: VENUE_GLBX
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
SymbolLists {
|
||||
value {
|
||||
list {
|
||||
key: "GlbxFutures"
|
||||
symbol: "ES"
|
||||
}
|
||||
list {
|
||||
key: "GlbxOptions"
|
||||
symbol: "E1A"
|
||||
symbol: "E1B"
|
||||
symbol: "E1C"
|
||||
symbol: "E1D"
|
||||
symbol: "E2A"
|
||||
symbol: "E2B"
|
||||
symbol: "E2C"
|
||||
symbol: "E2D"
|
||||
symbol: "E3A"
|
||||
symbol: "E3B"
|
||||
symbol: "E3C"
|
||||
symbol: "E3D"
|
||||
symbol: "E4A"
|
||||
symbol: "E4B"
|
||||
symbol: "E4C"
|
||||
symbol: "E4D"
|
||||
symbol: "E5A"
|
||||
symbol: "E5B"
|
||||
symbol: "E5C"
|
||||
symbol: "E5D"
|
||||
symbol: "EW1"
|
||||
symbol: "EW2"
|
||||
symbol: "EW3"
|
||||
symbol: "EW4"
|
||||
symbol: "ESH9"
|
||||
symbol: "ESZ6"
|
||||
symbol: "ESH0"
|
||||
symbol: "ESM6"
|
||||
symbol: "ESZ7"
|
||||
symbol: "ESH6"
|
||||
symbol: "ESM7"
|
||||
symbol: "ESZ8"
|
||||
symbol: "ESH7"
|
||||
symbol: "ESH8"
|
||||
symbol: "ESU6"
|
||||
symbol: "ESZ5"
|
||||
symbol: "ESZ9"
|
||||
symbol: "E1D"
|
||||
symbol: "ESZ5"
|
||||
symbol: "ES"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Symbols {
|
||||
value {
|
||||
venues {
|
||||
name: "VENUE_GLBX"
|
||||
future: @SymbolLists.list[GlbxFutures].symbol
|
||||
option: @SymbolLists.list[GlbxOptions].symbol
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,58 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
init_timezone() {
|
||||
tz="${1:-$TZ}"
|
||||
zoneinfo_dir=/usr/share/zoneinfo
|
||||
if [ -f "$zoneinfo_dir/$tz" -a "$(readlink -f /etc/localtime)" != "$zoneinfo_dir/$tz" ]; then
|
||||
echo "Setting locales and timezone to $tz"
|
||||
if [ -e /etc/localtime ]; then
|
||||
sudo -n rm -f /etc/localtime
|
||||
fi
|
||||
sudo -n ln -snf "$zoneinfo_dir/$tz" /etc/localtime
|
||||
echo $tz | sudo -n tee /etc/timezone > /dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
load_env() {
|
||||
f="$1"
|
||||
if [ -e "$f" ]; then
|
||||
echo "Sourcing $f"
|
||||
set -a
|
||||
. "$f"
|
||||
set +a
|
||||
fi
|
||||
}
|
||||
|
||||
symlink() {
|
||||
src="$1"
|
||||
dst="$2"
|
||||
if [ -L "$dst" ]; then
|
||||
rm "$dst"
|
||||
fi
|
||||
ln -s "$src" "$dst"
|
||||
}
|
||||
|
||||
roll_log() {
|
||||
f="$1"
|
||||
if [ -e "$f" ]; then
|
||||
mv "$f" "${f}_$(date '+%Y%m%d-%H%M')"
|
||||
fi
|
||||
}
|
||||
|
||||
# Acquire sudo permission once for the script (needed for a few calls)
|
||||
echo bts | sudo -S -v
|
||||
data_root="${HOME:-/home/bts}/.local/share/bts/sparkd"
|
||||
api_server_data_root="${HOME:-/home/bts}/.local/share/bts/spark_api_server"
|
||||
sudo -n chown -R bts:bts "$data_root"
|
||||
sudo -n mkdir -p "$api_server_data_root"
|
||||
sudo -n chown -R bts:bts "$api_server_data_root"
|
||||
load_env "$data_root/conf/.env.local"
|
||||
init_timezone "$TZ"
|
||||
roll_log "$data_root/data/logging.sparkd/log"
|
||||
symlink "$data_root/conf/plugins" /home/bts/bin/plugins
|
||||
|
||||
if [ -e "/md_recording" -a ! -e "/home/bts/.local/share/bts/sparkd/data/md_recording" ]; then
|
||||
symlink "/md_recording" /home/bts/.local/share/bts/sparkd/data/md_recording
|
||||
fi
|
||||
|
||||
bash -c "${@:-./sparkd run}"
|
||||
@@ -17,6 +17,7 @@
|
||||
#include "spark/orderman/OrderServiceServer.h"
|
||||
#include "spark/orderman/events/TradeSummarySignalDispatcher.h"
|
||||
#include "spark/orderman/order_factory/DynamicOrderRegistry.h"
|
||||
#include "spark/os/os.h"
|
||||
#include "spark/os/spark_data_directory.h"
|
||||
#include "spark/persistence/PersistenceServiceServer.h"
|
||||
#include "spark/plugin/SparkServices.h"
|
||||
@@ -31,7 +32,6 @@
|
||||
#include "sparkd/execution/execution_gateway_provider.h"
|
||||
#include "sparkd/http/WebServer.h"
|
||||
#include "sparkd/md/MarketDataAuxiliaryServiceServer.h"
|
||||
#include "sparkd/md/market_data_session.h"
|
||||
#include "sparkd/md/recording/MDRecorder.h"
|
||||
#include "sparkd/position/MarginServiceServer.h"
|
||||
#include "sparkd/position/PositionServiceServer.h"
|
||||
@@ -46,7 +46,6 @@
|
||||
namespace bts::cmds {
|
||||
|
||||
using bts::api::ServiceContext;
|
||||
using bts::md::MarketDataSession;
|
||||
using bts::md::MDClient;
|
||||
using bts::md::TradeSummarySignalDispatcher;
|
||||
using bts::metrics::MetricsContainer;
|
||||
@@ -108,7 +107,7 @@ static void execute() {
|
||||
auto py_plugin_man = std::make_shared<python_interop::PluginManager>();
|
||||
// start work loop in background
|
||||
py_plugin_man->start_work_thread();
|
||||
MarketDataSession md_session{py_plugin_man};
|
||||
md::cmds::Run::run(py_plugin_man);
|
||||
|
||||
api::synthetic::SyntheticSecurityService synthetic_service(sc);
|
||||
bts::md::MDClient mdclient(
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
#include "DumpSecurity.h"
|
||||
#include "spark/api/domain/SecurityType.h"
|
||||
#include "spark/config/protos/market_data.pb.h"
|
||||
#include "spark/core/TableFormatter.h"
|
||||
#include "spark/domain_model/Security.h"
|
||||
#include "spark/domain_model/Security.pb.h"
|
||||
#include "spark/domain_model/SecurityType.h"
|
||||
#include "spark/domain_model/Venue.h"
|
||||
#include "spark/format/table.h"
|
||||
#include "spark/md/Diag/ShmDumper.h"
|
||||
#include "spark/md/SecurityDB.h"
|
||||
#include "spark/md/SymbolLoader.h"
|
||||
@@ -23,8 +23,7 @@ using bts::api::SecurityType;
|
||||
using bts::api::SecurityType_Name;
|
||||
using bts::api::Venue;
|
||||
using bts::api::venue_parse_smart;
|
||||
using bts::format::format_table;
|
||||
using bts::format::detail::TableSchema;
|
||||
using bts::core::TableFormatter;
|
||||
using bts::md::securitytype_of;
|
||||
using bts::md::SymbolLoader;
|
||||
using bts::md::db::load_metadata;
|
||||
@@ -70,7 +69,7 @@ void print_securities(function<bool(const Security &)> predicate) {
|
||||
|
||||
void print_db_securities(function<bool(const Security &)> predicate) {
|
||||
vector<Security> secs;
|
||||
const TableSchema<Security> schema{
|
||||
const TableFormatter::Schema<Security> schema{
|
||||
make_tuple(
|
||||
"id", 12, [](const Security &s) { return std::to_string(s.id()); }),
|
||||
make_tuple(
|
||||
@@ -94,7 +93,7 @@ void print_db_securities(function<bool(const Security &)> predicate) {
|
||||
secs.push_back(sec);
|
||||
};
|
||||
load_securities([](auto) { return true; }, on_sec);
|
||||
format_table(std::cout, secs, schema);
|
||||
TableFormatter::format(std::cout, secs, schema);
|
||||
}
|
||||
|
||||
template <class T, class ParseT = function<bool(const string &, T *)>>
|
||||
|
||||
@@ -1,34 +1,74 @@
|
||||
#include "VenueOption.h"
|
||||
#include "spark/domain_model/Security.pb.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
|
||||
namespace bts::cmds::md {
|
||||
|
||||
#define NAME_PAIR(val) \
|
||||
{ name(val), val }
|
||||
|
||||
EnumOptionNames<api::Venue> VenueOption::names() {
|
||||
EnumOptionNames<api::Venue> result;
|
||||
const auto *descriptor = api::Venue_descriptor();
|
||||
for (int i = 0; i < descriptor->value_count(); ++i) {
|
||||
const auto venue = static_cast<api::Venue>(descriptor->value(i)->number());
|
||||
if (venue != api::VENUE_UNSET) {
|
||||
result.emplace(name(venue), venue);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
return {
|
||||
NAME_PAIR(api::VENUE_BTS),
|
||||
NAME_PAIR(api::VENUE_GLBX),
|
||||
NAME_PAIR(api::VENUE_ICE),
|
||||
NAME_PAIR(api::VENUE_SIM),
|
||||
NAME_PAIR(api::VENUE_SIMICE),
|
||||
NAME_PAIR(api::VENUE_DERIBIT),
|
||||
NAME_PAIR(api::VENUE_FTX),
|
||||
NAME_PAIR(api::VENUE_BINANCE),
|
||||
NAME_PAIR(api::VENUE_BINANCE_COIN_FUTURES),
|
||||
NAME_PAIR(api::VENUE_BINANCE_US),
|
||||
NAME_PAIR(api::VENUE_COINBASE),
|
||||
NAME_PAIR(api::VENUE_BINANCE_USDS_FUTURES),
|
||||
NAME_PAIR(api::VENUE_CBOE),
|
||||
NAME_PAIR(api::VENUE_TAIFEX),
|
||||
NAME_PAIR(api::VENUE_TWSE),
|
||||
NAME_PAIR(api::VENUE_BYBIT),
|
||||
NAME_PAIR(api::VENUE_GATEIO),
|
||||
NAME_PAIR(api::VENUE_OKX)};
|
||||
}
|
||||
|
||||
std::string VenueOption::name(api::Venue venue) {
|
||||
if (!api::Venue_IsValid(venue)) {
|
||||
switch (venue) {
|
||||
case api::VENUE_BTS:
|
||||
return "bts";
|
||||
case api::VENUE_GLBX:
|
||||
return "glbx";
|
||||
case api::VENUE_ICE:
|
||||
return "ice";
|
||||
case api::VENUE_SIM:
|
||||
return "sim";
|
||||
case api::VENUE_SIMICE:
|
||||
return "simice";
|
||||
case api::VENUE_DERIBIT:
|
||||
return "deribit";
|
||||
case api::VENUE_FTX:
|
||||
return "ftx";
|
||||
case api::VENUE_BINANCE:
|
||||
return "binance";
|
||||
case api::VENUE_BINANCE_COIN_FUTURES:
|
||||
return "binance_coin_futures";
|
||||
case api::VENUE_BINANCE_US:
|
||||
return "binance_us";
|
||||
case api::VENUE_COINBASE:
|
||||
return "coinbase";
|
||||
case api::VENUE_BINANCE_USDS_FUTURES:
|
||||
return "binance_usds_futures";
|
||||
case api::VENUE_CBOE:
|
||||
return "cboe";
|
||||
case api::VENUE_OKX:
|
||||
return "okx";
|
||||
case api::VENUE_TAIFEX:
|
||||
return "taifex";
|
||||
case api::VENUE_TWSE:
|
||||
return "twse";
|
||||
case api::VENUE_BYBIT:
|
||||
return "bybit";
|
||||
case api::VENUE_GATEIO:
|
||||
return "gateio";
|
||||
default:
|
||||
return std::to_string(static_cast<int32_t>(venue));
|
||||
}
|
||||
std::string result = api::Venue_Name(venue);
|
||||
constexpr auto prefix_size = sizeof("VENUE_") - 1;
|
||||
result.erase(0, prefix_size);
|
||||
std::transform(result.begin(), result.end(), result.begin(), [](char value) {
|
||||
return static_cast<char>(std::tolower(static_cast<unsigned char>(value)));
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
} // namespace bts::cmds::md
|
||||
@@ -4,7 +4,6 @@
|
||||
#include "binance/spots/spots_gateway.h"
|
||||
#include "binance/usds_futures/usds_futures_gateway.h"
|
||||
#include "okx_execution/okx_execution_gateway.h"
|
||||
#include "sparkd/execution/rakurai/rakurai_execution_gateway.h"
|
||||
#include "onixs_fix/OnixFIXAdapter.h"
|
||||
#include "python_interop/plugin_manager.h"
|
||||
#include "spark/api/domain/Venue.h"
|
||||
@@ -120,18 +119,6 @@ struct ExecutionGatewayProvider::PImpl {
|
||||
conn, conn.venue(), std::make_unique<OnixFIXAdapter>(o), o);
|
||||
case api::VENUE_OKX:
|
||||
return make_gw<OkxExecutionGateway>(conn, o);
|
||||
case api::VENUE_ORCA:
|
||||
case api::VENUE_RAYDIUM: {
|
||||
auto cfg_j = load_config_json(conn.name());
|
||||
bts_assert_msgf(
|
||||
cfg_j,
|
||||
"Failed to load execution gateway config. name: {}",
|
||||
conn.name());
|
||||
auto cfg = cfg_j->get<rakurai::GatewayConfig>();
|
||||
cfg.expand_env();
|
||||
return make_gw<rakurai::RakuraiExecutionGateway>(
|
||||
conn, o, std::move(cfg), conn);
|
||||
}
|
||||
case api::VENUE_BINANCE: {
|
||||
auto cfg =
|
||||
parse_config_json<binance::GatewayConfig>(conn.name());
|
||||
|
||||
@@ -23,7 +23,7 @@
|
||||
#include "spark/api/services/native_order_type.h"
|
||||
#include "spark/api/services/time_in_force.h"
|
||||
#include "spark/config/protos/jetfix.pb.h"
|
||||
#include "spark/core/time_stamp.h"
|
||||
#include "spark/core/TimeStamp.h"
|
||||
#include "spark/core/start_stop_state_machine.h"
|
||||
#include "spark/domain/tick_utils.h"
|
||||
#include "spark/domain_model/Security.h"
|
||||
|
||||
@@ -1,742 +0,0 @@
|
||||
#include "sparkd/execution/rakurai/rakurai_execution_gateway.h"
|
||||
|
||||
#include "spark/config/config.h"
|
||||
#include "spark/config/protos/system_config.pb.h"
|
||||
#include "spark/domain/DomainSecurityUtils.h"
|
||||
#include "spark/domain/Price.h"
|
||||
#include "spark/domain/Quantity.h"
|
||||
#include "spark/domain_model/Security.h"
|
||||
#include "spark/logging/logging.h"
|
||||
#include "spark/md/SecurityOracle.h"
|
||||
#include "spark/orderman/CancelOrderMsg.h"
|
||||
#include "spark/orderman/CancelReplaceOrderMsg.h"
|
||||
#include "spark/orderman/CreateOrderMsg.h"
|
||||
#include "spark/orderman/RFQMsg.h"
|
||||
#include "spark/orderman/events/NewAdminDataEvent.h"
|
||||
#include "spark/orderman/events/NewSecuritiesEvent.h"
|
||||
#include "spark/orderman/orderman_utils/CancelReject.h"
|
||||
#include "spark/orderman/orderman_utils/ExecutionReport.h"
|
||||
#include "spark/orderman/orderman_utils/orderman_common.h"
|
||||
#include "spark/utils/bts_assert.h"
|
||||
#include "sparkd/rakurai/RakuraiSharedRuntime.h"
|
||||
|
||||
#include <boost/filesystem/path.hpp>
|
||||
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
#include <deque>
|
||||
#include <limits>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace bts::orderman::rakurai {
|
||||
|
||||
namespace {
|
||||
|
||||
#define LOG_FN(level, fn, ...) \
|
||||
LOG(OrderMan, level, "RakuraiExecutionGateway::" fn ". " __VA_ARGS__)
|
||||
#define WARN_FN(fn, ...) LOG_FN(Warn, fn, __VA_ARGS__)
|
||||
|
||||
using ConnectorNewOrder = ::rakurai::dex_connector::order_entry::new_order;
|
||||
using ConnectorOrderState = ::rakurai::dex_connector::order_entry::order_state;
|
||||
using ConnectorSide = ::rakurai::dex_connector::order_entry::side;
|
||||
|
||||
std::optional<std::string> extract_short_pool_id(const api::Security& security) {
|
||||
std::string desc;
|
||||
if (!domain::DomainSecurityUtils::get_security_desc(security, desc)) {
|
||||
return std::nullopt;
|
||||
}
|
||||
const auto pos = desc.rfind('-');
|
||||
if (pos == std::string::npos || pos + 1 >= desc.size()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
return desc.substr(pos + 1);
|
||||
}
|
||||
|
||||
api::orderman::NativeOrderStatus to_order_status(ConnectorOrderState state) {
|
||||
using Status = ::rakurai::dex_connector::data_types::txn_status_t;
|
||||
switch (state.tx_status) {
|
||||
case Status::accepted:
|
||||
return api::orderman::NATIVE_ORDER_STATUS_NEW;
|
||||
case Status::confirmed:
|
||||
return state.is_pass ? api::orderman::NATIVE_ORDER_STATUS_FILLED
|
||||
: api::orderman::NATIVE_ORDER_STATUS_REJECTED;
|
||||
case Status::failed:
|
||||
return api::orderman::NATIVE_ORDER_STATUS_REJECTED;
|
||||
case Status::pending:
|
||||
return api::orderman::NATIVE_ORDER_STATUS_PENDING_NEW;
|
||||
}
|
||||
return api::orderman::NATIVE_ORDER_STATUS_UNDEFINED;
|
||||
}
|
||||
|
||||
api::orderman::ExecType to_exec_type(ConnectorOrderState state) {
|
||||
using Status = ::rakurai::dex_connector::data_types::txn_status_t;
|
||||
switch (state.tx_status) {
|
||||
case Status::accepted:
|
||||
return api::orderman::EXEC_TYPE_NEW;
|
||||
case Status::confirmed:
|
||||
return state.is_pass ? api::orderman::EXEC_TYPE_FILL
|
||||
: api::orderman::EXEC_TYPE_REJECTED;
|
||||
case Status::failed:
|
||||
return api::orderman::EXEC_TYPE_REJECTED;
|
||||
case Status::pending:
|
||||
return api::orderman::EXEC_TYPE_PENDING_NEW;
|
||||
}
|
||||
return api::orderman::EXEC_TYPE_UNDEFINED;
|
||||
}
|
||||
|
||||
std::string txn_status_name(
|
||||
::rakurai::dex_connector::data_types::txn_status_t status) {
|
||||
using Status = ::rakurai::dex_connector::data_types::txn_status_t;
|
||||
switch (status) {
|
||||
case Status::accepted:
|
||||
return "accepted";
|
||||
case Status::confirmed:
|
||||
return "confirmed";
|
||||
case Status::failed:
|
||||
return "failed";
|
||||
case Status::pending:
|
||||
return "pending";
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
std::string json_for_log(const nlohmann::json& j) {
|
||||
try {
|
||||
return j.dump();
|
||||
} catch (const std::exception& ex) {
|
||||
return fmt::format("<json dump failed: {}>", ex.what());
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
struct RakuraiExecutionGateway::PImpl {
|
||||
struct ActiveOrder {
|
||||
CreateOrderMsg create_msg;
|
||||
trace::CorID cor_id;
|
||||
std::array<std::uint8_t, 32> pool_id{};
|
||||
double quantity_scale = 1.0;
|
||||
std::uint64_t raw_quantity = 0;
|
||||
std::uint64_t priority_fee = 0;
|
||||
std::uint64_t client_id = 0;
|
||||
std::uint64_t txn_id = 0;
|
||||
};
|
||||
|
||||
struct BufferedReport {
|
||||
std::shared_ptr<const ExecutionReport> report;
|
||||
trace::CorID cor_id;
|
||||
};
|
||||
|
||||
explicit PImpl(
|
||||
md::SecurityOracle,
|
||||
GatewayConfig cfg,
|
||||
config::JetFIX_Connection jetfix_connection)
|
||||
: config(std::move(cfg)),
|
||||
jetfix_connection(std::make_shared<config::JetFIX_Connection>(
|
||||
std::move(jetfix_connection))) {
|
||||
config.expand_env();
|
||||
venue = this->jetfix_connection->venue();
|
||||
bts_assert(venue == api::VENUE_ORCA || venue == api::VENUE_RAYDIUM);
|
||||
LOG_FN(
|
||||
Info,
|
||||
"ctor",
|
||||
"session:{} venue:{} account:{} operator_id:{} json_conf_file_name:{} default_priority_fee:{} connector_config:{}",
|
||||
this->jetfix_connection->name(),
|
||||
api::Venue_Name(venue),
|
||||
config.account,
|
||||
config.operator_id,
|
||||
config.json_conf_file_name,
|
||||
config.default_priority_fee,
|
||||
json_for_log(config.connector_config));
|
||||
}
|
||||
|
||||
GatewayConfig config;
|
||||
std::shared_ptr<config::JetFIX_Connection> jetfix_connection;
|
||||
api::Venue venue = api::VENUE_UNSET;
|
||||
std::mutex mutex;
|
||||
std::unordered_map<OrderKey, ActiveOrder> active_orders;
|
||||
std::unordered_map<std::uint64_t, OrderKey> client_id_to_order_key;
|
||||
std::deque<OrderKey> pending_client_bindings;
|
||||
std::deque<BufferedReport> buffered_reports;
|
||||
std::size_t order_update_subscription_id = 0;
|
||||
std::size_t status_subscription_id = 0;
|
||||
bool started = false;
|
||||
bool connected = false;
|
||||
OnExeReport exe_report;
|
||||
OnGwStatusChange gw_status_change;
|
||||
OnCancelReject cancel_reject;
|
||||
OnExchangePositionState exchange_position_state;
|
||||
OnExchangeOpenOrdersState exchange_open_orders;
|
||||
|
||||
void start() {
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mutex);
|
||||
if (started) {
|
||||
LOG_FN(
|
||||
Info, "start", "already started. venue:{}",
|
||||
api::Venue_Name(venue));
|
||||
return;
|
||||
}
|
||||
started = true;
|
||||
}
|
||||
LOG_FN(
|
||||
Info,
|
||||
"start",
|
||||
"starting. session:{} venue:{} account:{} operator_id:{} json_conf_file_name:{} default_priority_fee:{}",
|
||||
jetfix_connection->name(),
|
||||
api::Venue_Name(venue),
|
||||
config.account,
|
||||
config.operator_id,
|
||||
config.json_conf_file_name,
|
||||
config.default_priority_fee);
|
||||
|
||||
auto& runtime = bts::rakurai::RakuraiSharedRuntime::instance();
|
||||
order_update_subscription_id =
|
||||
runtime.add_order_update_subscriber([this](ConnectorOrderState state) {
|
||||
this->on_order_update(state);
|
||||
});
|
||||
status_subscription_id =
|
||||
runtime.add_status_subscriber([this](bool is_connected_now) {
|
||||
this->on_status_change(is_connected_now);
|
||||
});
|
||||
|
||||
try {
|
||||
if (!config.json_conf_file_name.empty()) {
|
||||
const auto config_path =
|
||||
boost::filesystem::path{
|
||||
config::get<config::SystemConfig>().config_path()} /
|
||||
config.json_conf_file_name;
|
||||
LOG_FN(
|
||||
Info,
|
||||
"start",
|
||||
"ensuring shared runtime from file. path:{}",
|
||||
config_path.string());
|
||||
runtime.ensure_started_from_file(config_path.string());
|
||||
} else {
|
||||
LOG_FN(
|
||||
Info,
|
||||
"start",
|
||||
"ensuring shared runtime from inline config. config:{}",
|
||||
json_for_log(config.connector_config));
|
||||
runtime.ensure_started_from_json(
|
||||
config.connector_config, jetfix_connection->name());
|
||||
}
|
||||
LOG_FN(
|
||||
Info,
|
||||
"start",
|
||||
"shared runtime start request submitted. order_update_sub_id:{} status_sub_id:{} connected_now:{}",
|
||||
order_update_subscription_id,
|
||||
status_subscription_id,
|
||||
runtime.is_connected());
|
||||
} catch (...) {
|
||||
runtime.remove_order_update_subscriber(order_update_subscription_id);
|
||||
runtime.remove_status_subscriber(status_subscription_id);
|
||||
order_update_subscription_id = 0;
|
||||
status_subscription_id = 0;
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mutex);
|
||||
started = false;
|
||||
}
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
void stop() {
|
||||
LOG_FN(
|
||||
Info,
|
||||
"stop",
|
||||
"stopping. order_update_sub_id:{} status_sub_id:{}",
|
||||
order_update_subscription_id,
|
||||
status_subscription_id);
|
||||
auto& runtime = bts::rakurai::RakuraiSharedRuntime::instance();
|
||||
if (order_update_subscription_id != 0) {
|
||||
runtime.remove_order_update_subscriber(order_update_subscription_id);
|
||||
order_update_subscription_id = 0;
|
||||
}
|
||||
if (status_subscription_id != 0) {
|
||||
runtime.remove_status_subscriber(status_subscription_id);
|
||||
status_subscription_id = 0;
|
||||
}
|
||||
}
|
||||
|
||||
void on_status_change(bool is_connected_now) {
|
||||
LOG_FN(
|
||||
Info,
|
||||
"on_status_change",
|
||||
"status update. venue:{} old_connected:{} new_connected:{} active_orders:{} pending_bindings:{} buffered_reports:{}",
|
||||
api::Venue_Name(venue),
|
||||
connected,
|
||||
is_connected_now,
|
||||
active_orders.size(),
|
||||
pending_client_bindings.size(),
|
||||
buffered_reports.size());
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mutex);
|
||||
connected = is_connected_now;
|
||||
}
|
||||
if (gw_status_change) {
|
||||
gw_status_change(is_connected_now);
|
||||
}
|
||||
}
|
||||
|
||||
void on_order_update(ConnectorOrderState order_state) {
|
||||
std::lock_guard<std::mutex> lock(mutex);
|
||||
LOG_FN(
|
||||
Info,
|
||||
"on_order_update",
|
||||
"incoming order update. client_id:{} txn_id:{} txn_status:{} is_pass:{} active_orders:{} pending_bindings:{} mapped_clients:{}",
|
||||
order_state.client_id,
|
||||
order_state.txn_id,
|
||||
txn_status_name(order_state.tx_status),
|
||||
order_state.is_pass,
|
||||
active_orders.size(),
|
||||
pending_client_bindings.size(),
|
||||
client_id_to_order_key.size());
|
||||
auto key_it = client_id_to_order_key.find(order_state.client_id);
|
||||
if (key_it == client_id_to_order_key.end()) {
|
||||
if (pending_client_bindings.empty()) {
|
||||
LOG_FN(
|
||||
Warn,
|
||||
"on_order_update",
|
||||
"dropping order update with unknown client_id:{} because there are no pending client bindings",
|
||||
order_state.client_id);
|
||||
return;
|
||||
}
|
||||
const auto key = pending_client_bindings.front();
|
||||
pending_client_bindings.pop_front();
|
||||
client_id_to_order_key.emplace(order_state.client_id, key);
|
||||
LOG_FN(
|
||||
Info,
|
||||
"on_order_update",
|
||||
"bound client_id:{} to order_key:{}",
|
||||
order_state.client_id,
|
||||
key.value());
|
||||
key_it = client_id_to_order_key.find(order_state.client_id);
|
||||
}
|
||||
|
||||
auto order_it = active_orders.find(key_it->second);
|
||||
if (order_it == active_orders.end()) {
|
||||
LOG_FN(
|
||||
Warn,
|
||||
"on_order_update",
|
||||
"dropping order update for client_id:{} mapped_order_key:{} because active order was not found",
|
||||
order_state.client_id,
|
||||
key_it->second.value());
|
||||
return;
|
||||
}
|
||||
|
||||
auto& active = order_it->second;
|
||||
active.client_id = order_state.client_id;
|
||||
active.txn_id = order_state.txn_id;
|
||||
auto report = build_report_locked(active, order_state);
|
||||
buffered_reports.push_back({std::move(report), active.cor_id});
|
||||
LOG_FN(
|
||||
Info,
|
||||
"on_order_update",
|
||||
"buffered execution report. order_key:{} client_id:{} txn_id:{} buffered_reports:{} order_status:{} exec_type:{}",
|
||||
active.create_msg.key.value(),
|
||||
active.client_id,
|
||||
active.txn_id,
|
||||
buffered_reports.size(),
|
||||
api::orderman::NativeOrderStatus_Name(
|
||||
buffered_reports.back().report->order_status),
|
||||
api::orderman::ExecType_Name(buffered_reports.back().report->exec_type));
|
||||
|
||||
const auto order_status = buffered_reports.back().report->order_status;
|
||||
if (order_status == api::orderman::NATIVE_ORDER_STATUS_FILLED ||
|
||||
order_status == api::orderman::NATIVE_ORDER_STATUS_REJECTED) {
|
||||
LOG_FN(
|
||||
Info,
|
||||
"on_order_update",
|
||||
"removing terminal active order. order_key:{} client_id:{} status:{}",
|
||||
active.create_msg.key.value(),
|
||||
order_state.client_id,
|
||||
api::orderman::NativeOrderStatus_Name(order_status));
|
||||
client_id_to_order_key.erase(order_state.client_id);
|
||||
active_orders.erase(order_it);
|
||||
}
|
||||
}
|
||||
|
||||
void send_create(const CreateOrderMsg& msg, trace::CorID cor_id) {
|
||||
std::lock_guard<std::mutex> lock(mutex);
|
||||
std::string security_desc = "<null>";
|
||||
if (msg.security != nullptr) {
|
||||
std::string desc;
|
||||
if (domain::DomainSecurityUtils::get_security_desc(*msg.security, desc)) {
|
||||
security_desc = std::move(desc);
|
||||
} else {
|
||||
security_desc = "<unresolved>";
|
||||
}
|
||||
}
|
||||
LOG_FN(
|
||||
Info,
|
||||
"send_create",
|
||||
"create request. order_key:{} venue:{} side:{} qty:{} price:{} tif:{} order_type:{} connected:{} security_present:{} desc:{}",
|
||||
msg.key.value(),
|
||||
api::Venue_Name(msg.venue()),
|
||||
api::Side_Name(msg.side),
|
||||
msg.quantity.full_value(),
|
||||
msg.price,
|
||||
api::orderman::TimeInForce_Name(msg.time_in_force),
|
||||
api::orderman::NativeOrderType_Name(msg.order_type),
|
||||
connected,
|
||||
msg.security != nullptr,
|
||||
security_desc);
|
||||
if (msg.security == nullptr) {
|
||||
enqueue_reject_locked(msg, cor_id, "Rakurai order missing security");
|
||||
return;
|
||||
}
|
||||
if (msg.venue() != venue) {
|
||||
enqueue_reject_locked(
|
||||
msg,
|
||||
cor_id,
|
||||
fmt::format(
|
||||
"Rakurai gateway venue mismatch. expected:{} actual:{}",
|
||||
api::Venue_Name(venue),
|
||||
api::Venue_Name(msg.venue())));
|
||||
return;
|
||||
}
|
||||
if (msg.order_type != api::orderman::NATIVE_ORDER_TYPE_MARKET) {
|
||||
enqueue_reject_locked(
|
||||
msg,
|
||||
cor_id,
|
||||
"Rakurai DEX gateway currently supports market orders only");
|
||||
return;
|
||||
}
|
||||
if (!connected) {
|
||||
enqueue_reject_locked(
|
||||
msg, cor_id, "Rakurai execution session is not connected");
|
||||
return;
|
||||
}
|
||||
|
||||
const auto short_pool_id = extract_short_pool_id(*msg.security);
|
||||
if (!short_pool_id.has_value()) {
|
||||
enqueue_reject_locked(
|
||||
msg, cor_id, "Unable to derive Rakurai pool alias from security");
|
||||
return;
|
||||
}
|
||||
|
||||
const auto metadata =
|
||||
bts::rakurai::RakuraiSharedRuntime::instance().find_pool_metadata(
|
||||
venue, *short_pool_id);
|
||||
if (!metadata.has_value()) {
|
||||
LOG_FN(
|
||||
Warn,
|
||||
"send_create",
|
||||
"pool metadata missing. venue:{} short_pool_id:{}",
|
||||
api::Venue_Name(venue),
|
||||
*short_pool_id);
|
||||
enqueue_reject_locked(
|
||||
msg,
|
||||
cor_id,
|
||||
"Rakurai pool metadata not available yet; wait for pool definitions");
|
||||
return;
|
||||
}
|
||||
|
||||
const auto raw_qty_f =
|
||||
msg.quantity.full_value() * metadata->quantity_scale;
|
||||
if (!std::isfinite(raw_qty_f) || raw_qty_f <= 0.0) {
|
||||
enqueue_reject_locked(
|
||||
msg, cor_id, "Rakurai order quantity must be positive");
|
||||
return;
|
||||
}
|
||||
|
||||
ConnectorNewOrder order;
|
||||
order.order_def.pool_id = metadata->security.pool_id;
|
||||
order.order_def.dex = metadata->security.venue;
|
||||
order.order_def.order_side =
|
||||
msg.side == api::SIDE_BUY ? ConnectorSide::buy : ConnectorSide::sell;
|
||||
order.qty = static_cast<std::uint64_t>(std::llround(raw_qty_f));
|
||||
order.priority_fee = config.default_priority_fee;
|
||||
LOG_FN(
|
||||
Info,
|
||||
"send_create",
|
||||
"launching new order. order_key:{} short_pool_id:{} raw_qty:{} priority_fee:{} dex:{} quantity_scale:{}",
|
||||
msg.key.value(),
|
||||
*short_pool_id,
|
||||
order.qty,
|
||||
order.priority_fee,
|
||||
static_cast<int>(metadata->security.venue),
|
||||
metadata->quantity_scale);
|
||||
|
||||
try {
|
||||
bts::rakurai::RakuraiSharedRuntime::instance().launch_new_order(order);
|
||||
} catch (const std::exception& ex) {
|
||||
WARN_FN(
|
||||
"send_create",
|
||||
"launch_new_order threw. order_key:{} reason:{}",
|
||||
msg.key.value(),
|
||||
std::string(ex.what()));
|
||||
enqueue_reject_locked(msg, cor_id, ex.what());
|
||||
return;
|
||||
}
|
||||
|
||||
active_orders.emplace(
|
||||
msg.key,
|
||||
ActiveOrder{
|
||||
msg,
|
||||
cor_id,
|
||||
metadata->security.pool_id,
|
||||
metadata->quantity_scale,
|
||||
order.qty,
|
||||
order.priority_fee});
|
||||
pending_client_bindings.push_back(msg.key);
|
||||
LOG_FN(
|
||||
Info,
|
||||
"send_create",
|
||||
"create request accepted locally. order_key:{} active_orders:{} pending_bindings:{}",
|
||||
msg.key.value(),
|
||||
active_orders.size(),
|
||||
pending_client_bindings.size());
|
||||
}
|
||||
|
||||
void send_cancel_reject(OrderKey key, const std::string& reason) {
|
||||
WARN_FN(
|
||||
"send_cancel_reject",
|
||||
"cancel reject. order_key:{} reason:{}",
|
||||
key.value(),
|
||||
reason);
|
||||
if (cancel_reject) {
|
||||
cancel_reject({key, CancelRejectReason::Other, reason});
|
||||
}
|
||||
}
|
||||
|
||||
std::shared_ptr<const ExecutionReport> build_report_locked(
|
||||
const ActiveOrder& active,
|
||||
ConnectorOrderState state) const {
|
||||
auto report = std::make_shared<ExecutionReport>();
|
||||
report->account_name = account_name_for(active.create_msg);
|
||||
report->operator_id = operator_id_for(active.create_msg);
|
||||
report->order_key = active.create_msg.key;
|
||||
report->msg_seq_num = 0;
|
||||
report->security = active.create_msg.security;
|
||||
report->leg_security = nullptr;
|
||||
report->side = active.create_msg.side;
|
||||
report->time_in_force = active.create_msg.time_in_force;
|
||||
report->order_type = active.create_msg.order_type;
|
||||
report->manual_order_indicator = active.create_msg.manual_order_indicator;
|
||||
report->exchange = venue;
|
||||
report->cfg = jetfix_connection.get();
|
||||
report->exchange_order_id = ExchangeOrderID64(
|
||||
active.client_id != 0 ? active.client_id : state.txn_id);
|
||||
report->quantity = active.create_msg.quantity;
|
||||
report->price = active.create_msg.price;
|
||||
report->stop_price = active.create_msg.stop_px;
|
||||
report->max_show = active.create_msg.display_quantity;
|
||||
report->transact_time =
|
||||
ExchangeTimeStamp64(static_cast<std::uint64_t>(datetime::unix_nano_now()));
|
||||
report->exec_type = to_exec_type(state);
|
||||
report->order_status = to_order_status(state);
|
||||
report->filled_quantity =
|
||||
report->order_status == api::orderman::NATIVE_ORDER_STATUS_FILLED
|
||||
? active.create_msg.quantity
|
||||
: Quantity(0.0f);
|
||||
report->last_shares = report->filled_quantity;
|
||||
report->last_price = active.create_msg.price;
|
||||
report->avg_price = active.create_msg.price;
|
||||
report->msg = fmt::format(
|
||||
"Rakurai txn_status={} txn_id={} pass={}",
|
||||
txn_status_name(state.tx_status),
|
||||
state.txn_id,
|
||||
state.is_pass);
|
||||
LOG_FN(
|
||||
Info,
|
||||
"build_report_locked",
|
||||
"built report. order_key:{} client_id:{} txn_id:{} exec_type:{} order_status:{} message:{}",
|
||||
active.create_msg.key.value(),
|
||||
active.client_id,
|
||||
state.txn_id,
|
||||
api::orderman::ExecType_Name(report->exec_type),
|
||||
api::orderman::NativeOrderStatus_Name(report->order_status),
|
||||
report->msg);
|
||||
return report;
|
||||
}
|
||||
|
||||
void enqueue_reject_locked(
|
||||
const CreateOrderMsg& msg,
|
||||
trace::CorID cor_id,
|
||||
const std::string& reason) {
|
||||
auto report = std::make_shared<ExecutionReport>();
|
||||
report->account_name = account_name_for(msg);
|
||||
report->operator_id = operator_id_for(msg);
|
||||
report->order_key = msg.key;
|
||||
report->msg_seq_num = 0;
|
||||
report->security = msg.security;
|
||||
report->leg_security = nullptr;
|
||||
report->side = msg.side;
|
||||
report->time_in_force = msg.time_in_force;
|
||||
report->order_type = msg.order_type;
|
||||
report->manual_order_indicator = msg.manual_order_indicator;
|
||||
report->exchange = venue;
|
||||
report->cfg = jetfix_connection.get();
|
||||
report->exchange_order_id = ExchangeOrderID64(0);
|
||||
report->quantity = msg.quantity;
|
||||
report->filled_quantity = Quantity(0.0f);
|
||||
report->last_shares = Quantity(0.0f);
|
||||
report->price = msg.price;
|
||||
report->stop_price = msg.stop_px;
|
||||
report->max_show = msg.display_quantity;
|
||||
report->last_price = Price(std::numeric_limits<double>::quiet_NaN());
|
||||
report->avg_price = Price(std::numeric_limits<double>::quiet_NaN());
|
||||
report->transact_time =
|
||||
ExchangeTimeStamp64(static_cast<std::uint64_t>(datetime::unix_nano_now()));
|
||||
report->exec_type = api::orderman::EXEC_TYPE_REJECTED;
|
||||
report->order_status = api::orderman::NATIVE_ORDER_STATUS_REJECTED;
|
||||
report->msg = reason;
|
||||
buffered_reports.push_back({std::move(report), cor_id});
|
||||
WARN_FN(
|
||||
"enqueue_reject_locked",
|
||||
"buffered reject. order_key:{} reason:{} buffered_reports:{}",
|
||||
msg.key.value(),
|
||||
reason,
|
||||
buffered_reports.size());
|
||||
}
|
||||
|
||||
AccountName account_name_for(const CreateOrderMsg& msg) const {
|
||||
const auto account = msg.routing_record != nullptr &&
|
||||
!msg.routing_record->account().empty()
|
||||
? msg.routing_record->account()
|
||||
: config.account;
|
||||
AccountName result;
|
||||
result->copy_from(account);
|
||||
return result;
|
||||
}
|
||||
|
||||
OperatorID operator_id_for(const CreateOrderMsg& msg) const {
|
||||
const auto operator_id =
|
||||
msg.routing_record != nullptr &&
|
||||
!msg.routing_record->operator_id().empty()
|
||||
? msg.routing_record->operator_id()
|
||||
: config.operator_id;
|
||||
OperatorID result;
|
||||
orderman::to_array(operator_id, &result.value());
|
||||
return result;
|
||||
}
|
||||
};
|
||||
|
||||
RakuraiExecutionGateway::RakuraiExecutionGateway(
|
||||
md::SecurityOracle sec_oracle,
|
||||
GatewayConfig config,
|
||||
config::JetFIX_Connection jetfix_connection)
|
||||
: me_(std::make_unique<PImpl>(
|
||||
std::move(sec_oracle),
|
||||
std::move(config),
|
||||
std::move(jetfix_connection))) {}
|
||||
|
||||
RakuraiExecutionGateway::~RakuraiExecutionGateway() { me_->stop(); }
|
||||
|
||||
void RakuraiExecutionGateway::start() { me_->start(); }
|
||||
|
||||
void RakuraiExecutionGateway::dispatch_incoming_fix_messages() {
|
||||
std::deque<PImpl::BufferedReport> reports;
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(me_->mutex);
|
||||
reports.swap(me_->buffered_reports);
|
||||
}
|
||||
|
||||
if (!reports.empty()) {
|
||||
LOG_FN(
|
||||
Info,
|
||||
"dispatch_incoming_fix_messages",
|
||||
"dispatching buffered reports. count:{} has_callback:{}",
|
||||
reports.size(),
|
||||
me_->exe_report != nullptr);
|
||||
}
|
||||
if (!me_->exe_report) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (auto& buffered : reports) {
|
||||
LOG_FN(
|
||||
Info,
|
||||
"dispatch_incoming_fix_messages",
|
||||
"publishing execution report. order_key:{} exec_type:{} order_status:{} msg:{}",
|
||||
buffered.report->order_key.value(),
|
||||
api::orderman::ExecType_Name(buffered.report->exec_type),
|
||||
api::orderman::NativeOrderStatus_Name(buffered.report->order_status),
|
||||
buffered.report->msg);
|
||||
me_->exe_report(std::move(buffered.report), buffered.cor_id);
|
||||
}
|
||||
}
|
||||
|
||||
void RakuraiExecutionGateway::on_securities(
|
||||
const mercury::Update<NewSecuritiesEvent>& update) {
|
||||
LOG_FN(
|
||||
Info,
|
||||
"on_securities",
|
||||
"received securities update. type:{} has_events:{} count:{}",
|
||||
static_cast<int>(update.event_type),
|
||||
update.events != nullptr,
|
||||
update.events != nullptr ? update.events->size() : 0);
|
||||
}
|
||||
|
||||
void RakuraiExecutionGateway::on_admin_data(
|
||||
const mercury::Update<NewAdminDataEvent>& update) {
|
||||
LOG_FN(
|
||||
Info,
|
||||
"on_admin_data",
|
||||
"received admin data update. type:{} has_events:{} count:{}",
|
||||
static_cast<int>(update.event_type),
|
||||
update.events != nullptr,
|
||||
update.events != nullptr ? update.events->size() : 0);
|
||||
}
|
||||
|
||||
void RakuraiExecutionGateway::send_fix(const jetfix::OutFIXPtr&, trace::CorID) {
|
||||
WARN_FN("send_fix", "FIX transport is not supported for Rakurai");
|
||||
}
|
||||
|
||||
void RakuraiExecutionGateway::send_msgs(
|
||||
std::vector<OrderMsg> const& msgs,
|
||||
trace::CorID cor_id) {
|
||||
LOG_FN(
|
||||
Info,
|
||||
"send_msgs",
|
||||
"received outbound message batch. count:{}",
|
||||
msgs.size());
|
||||
for (const auto& msg : msgs) {
|
||||
std::visit(
|
||||
[&](const auto& order_msg) {
|
||||
using T = std::decay_t<decltype(order_msg)>;
|
||||
if constexpr (std::is_same_v<T, CreateOrderMsg>) {
|
||||
me_->send_create(order_msg, cor_id);
|
||||
} else if constexpr (std::is_same_v<T, CancelOrderMsg>) {
|
||||
me_->send_cancel_reject(
|
||||
order_msg.key,
|
||||
"Rakurai DEX gateway does not support order cancel");
|
||||
} else if constexpr (std::is_same_v<T, CancelReplaceOrderMsg>) {
|
||||
me_->send_cancel_reject(
|
||||
order_msg.create_fields.key,
|
||||
"Rakurai DEX gateway does not support order replace");
|
||||
} else if constexpr (std::is_same_v<T, RFQMsg>) {
|
||||
WARN_FN("send_msgs", "RFQ is not supported for Rakurai");
|
||||
}
|
||||
},
|
||||
msg);
|
||||
}
|
||||
}
|
||||
|
||||
OnExeReport& RakuraiExecutionGateway::on_exe_report() { return me_->exe_report; }
|
||||
|
||||
OnGwStatusChange& RakuraiExecutionGateway::on_gw_status_change() {
|
||||
return me_->gw_status_change;
|
||||
}
|
||||
|
||||
OnCancelReject& RakuraiExecutionGateway::on_cancel_reject() {
|
||||
return me_->cancel_reject;
|
||||
}
|
||||
|
||||
OnExchangePositionState& RakuraiExecutionGateway::on_exchange_position_state() {
|
||||
return me_->exchange_position_state;
|
||||
}
|
||||
|
||||
OnExchangeOpenOrdersState& RakuraiExecutionGateway::on_exchange_open_orders() {
|
||||
return me_->exchange_open_orders;
|
||||
}
|
||||
|
||||
} // namespace bts::orderman::rakurai
|
||||
@@ -1,76 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include "spark/io/JsonReader.h"
|
||||
#include "spark/orderman/gateways/IExecutionGateway.h"
|
||||
#include "spark/os/os.h"
|
||||
|
||||
#include <memory>
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace bts::config {
|
||||
class JetFIX_Connection;
|
||||
}
|
||||
|
||||
namespace bts::md {
|
||||
class SecurityOracle;
|
||||
}
|
||||
|
||||
namespace bts::orderman::rakurai {
|
||||
|
||||
struct GatewayConfig {
|
||||
std::string account;
|
||||
std::string operator_id;
|
||||
std::string json_conf_file_name;
|
||||
std::uint64_t default_priority_fee = 1000;
|
||||
nlohmann::json connector_config;
|
||||
|
||||
void expand_env() {
|
||||
bts::os::expand_env_in_place(account);
|
||||
bts::os::expand_env_in_place(operator_id);
|
||||
bts::os::expand_env_in_place(json_conf_file_name);
|
||||
}
|
||||
};
|
||||
|
||||
inline void from_json(const nlohmann::json& j, GatewayConfig& cfg) {
|
||||
cfg.connector_config = j;
|
||||
|
||||
const bts::io::JsonReader jr{&j};
|
||||
jr.read_or("account", cfg.account);
|
||||
jr.read_or("operator_id", cfg.operator_id);
|
||||
jr.read_or("json_conf_file_name", cfg.json_conf_file_name);
|
||||
jr.read_or("default_priority_fee", cfg.default_priority_fee);
|
||||
}
|
||||
|
||||
class RakuraiExecutionGateway : public IExecutionGateway {
|
||||
public:
|
||||
explicit RakuraiExecutionGateway(
|
||||
bts::md::SecurityOracle sec_oracle,
|
||||
GatewayConfig config,
|
||||
bts::config::JetFIX_Connection jetfix_connection);
|
||||
~RakuraiExecutionGateway() override;
|
||||
|
||||
void start() override;
|
||||
void dispatch_incoming_fix_messages() override;
|
||||
void on_securities(
|
||||
const bts::mercury::Update<bts::orderman::NewSecuritiesEvent>&)
|
||||
override;
|
||||
void on_admin_data(
|
||||
const bts::mercury::Update<bts::orderman::NewAdminDataEvent>&) override;
|
||||
void send_fix(const bts::jetfix::OutFIXPtr&, bts::trace::CorID) override;
|
||||
void send_msgs(
|
||||
std::vector<bts::orderman::OrderMsg> const&,
|
||||
bts::trace::CorID) override;
|
||||
|
||||
bts::orderman::OnExeReport& on_exe_report() override;
|
||||
bts::orderman::OnGwStatusChange& on_gw_status_change() override;
|
||||
bts::orderman::OnCancelReject& on_cancel_reject() override;
|
||||
bts::orderman::OnExchangePositionState&
|
||||
on_exchange_position_state() override;
|
||||
bts::orderman::OnExchangeOpenOrdersState& on_exchange_open_orders() override;
|
||||
|
||||
private:
|
||||
struct PImpl;
|
||||
std::unique_ptr<PImpl> me_;
|
||||
};
|
||||
|
||||
} // namespace bts::orderman::rakurai
|
||||
@@ -4,7 +4,7 @@
|
||||
#include "spark/api/md/MarketDataConfig.pb.h"
|
||||
#include "spark/api/md/MarketDataConfigUtils.h"
|
||||
#include "spark/core/FMT.h"
|
||||
#include "spark/core/time_span.h"
|
||||
#include "spark/core/TimeSpan.h"
|
||||
#include "spark/domain/DomainSecurityUtils.h"
|
||||
#include "spark/domain/SecurityID.h"
|
||||
#include "spark/domain_model/Venue.h"
|
||||
@@ -25,7 +25,6 @@
|
||||
#include "sparkd/md/ccapi/CCApiMarketDataProvider.h"
|
||||
#include "sparkd/md/databento/DatabentoMarketDataProvider.h"
|
||||
#include "sparkd/md/emumba/EmumbaMarketDataProvider.h"
|
||||
#include "sparkd/md/rakurai/RakuraiMarketDataProvider.h"
|
||||
#include "sparkd/md/taifex/TaifexMarketDataProvider.h"
|
||||
|
||||
#include <bits/stdint-uintn.h>
|
||||
@@ -82,8 +81,6 @@ struct MarketDataGateway::PImpl {
|
||||
std::make_unique<OnixsMarketDataProvider>(this->config));
|
||||
providers.push_back(std::make_unique<DatabentoMarketDataProvider>(
|
||||
this->config, this->sec_id_store));
|
||||
providers.push_back(std::make_unique<RakuraiMarketDataProvider>(
|
||||
this->config, this->sec_id_store));
|
||||
providers.push_back(std::make_unique<TaifexMarketDataProvider>(
|
||||
this->config, this->sec_id_store));
|
||||
providers.push_back(std::make_unique<PyMarketDataProvider>(
|
||||
@@ -144,6 +141,5 @@ MarketDataGateway::MarketDataGateway(
|
||||
MarketDataGateway::~MarketDataGateway() = default;
|
||||
|
||||
void MarketDataGateway::start() { me_->start(); }
|
||||
void MarketDataGateway::stop() { me_->stop(); }
|
||||
|
||||
} // namespace bts::md
|
||||
@@ -29,7 +29,6 @@ class MarketDataGateway {
|
||||
~MarketDataGateway();
|
||||
|
||||
void start();
|
||||
void stop();
|
||||
|
||||
private:
|
||||
struct PImpl;
|
||||
|
||||
@@ -15,8 +15,8 @@
|
||||
#include "spark/api/domain/SecurityType.h"
|
||||
#include "spark/api/domain/Venue.h"
|
||||
#include "spark/api/md/PyMarketDataProviderConfig.pb.h"
|
||||
#include "spark/core/time_span.h"
|
||||
#include "spark/core/time_stamp.h"
|
||||
#include "spark/core/TimeSpan.h"
|
||||
#include "spark/core/TimeStamp.h"
|
||||
#include "spark/domain/AggressorSide.h"
|
||||
#include "spark/domain/FutureSecurity.h"
|
||||
#include "spark/domain/Security.h"
|
||||
|
||||
Loaded 100 of 596 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user