Compare commits
60
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ba664b95b8 | ||
|
|
d6f370f862 | ||
|
|
079826f4de | ||
|
|
2eee3cdaab | ||
|
|
5e3215ee52 | ||
|
|
7739cc2f3d | ||
|
|
8347a7c126 | ||
|
|
fbfc960df6 | ||
|
|
451f721030 | ||
|
|
bb992c0410 | ||
|
|
db18aebf92 | ||
|
|
8127c9c9fe | ||
|
|
1825b339b1 | ||
|
|
03ab3fbe90 | ||
|
|
b61bf16a42 | ||
|
|
d969489bf4 | ||
|
|
9cd4974f49 | ||
|
|
793634c8cb | ||
|
|
eae20b6c94 | ||
|
|
2848bb6a5d | ||
|
|
687324998b | ||
|
|
97cebd62d4 | ||
|
|
492ca0c28f | ||
|
|
baa5019039 | ||
|
|
ac2d31e1fc | ||
|
|
9821134bc3 | ||
|
|
9cf892322d | ||
|
|
77bf91be1e | ||
|
|
63b064ac79 | ||
|
|
b617775c29 | ||
|
|
4f7d77420c | ||
|
|
3ba6a91ca7 | ||
|
|
0abde9eb7c | ||
|
|
1531a57bc5 | ||
|
|
a62ea64c5e | ||
|
|
450773f588 | ||
|
|
f09d6badaf | ||
|
|
6b054e0718 | ||
|
|
5e2789d16e | ||
|
|
9ebdb2e460 | ||
|
|
d7bf82722b | ||
|
|
e1526e25a7 | ||
|
|
089ba9b369 | ||
|
|
567976043f | ||
|
|
2fd3382939 | ||
|
|
a96f1c8f17 | ||
|
|
6f623f9a37 | ||
|
|
827189cdb4 | ||
|
|
123f12ab88 | ||
|
|
f51ba8fb56 | ||
|
|
2506fb4f2b | ||
|
|
f92964114e | ||
|
|
baa0f1430b | ||
|
|
7104bd787a | ||
|
|
5db92d1f7d | ||
|
|
eb0fcf0b84 | ||
|
|
a52b187313 | ||
|
|
669b059eb2 | ||
|
|
36ab8492d3 | ||
|
|
54f37b2401 |
No files matched your search
@@ -9,7 +9,7 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
"fable": {
|
"fable": {
|
||||||
"version": "4.24.0",
|
"version": "4.0.6",
|
||||||
"commands": [
|
"commands": [
|
||||||
"fable"
|
"fable"
|
||||||
]
|
]
|
||||||
@@ -27,4 +27,4 @@
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -66,6 +66,7 @@ RUN \
|
|||||||
|
|
||||||
# Dotnet
|
# Dotnet
|
||||||
RUN apt-get update && apt-get install -y dotnet7
|
RUN apt-get update && apt-get install -y dotnet7
|
||||||
|
RUN apt-get update && apt-get install -y dotnet8
|
||||||
|
|
||||||
# Node
|
# Node
|
||||||
ARG node_major=18
|
ARG node_major=18
|
||||||
@@ -87,9 +88,6 @@ RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
|
|||||||
# Pcap
|
# Pcap
|
||||||
RUN apt-get update && apt-get install -y libpcap0.8-dev
|
RUN apt-get update && apt-get install -y libpcap0.8-dev
|
||||||
|
|
||||||
# Databento (deps)
|
|
||||||
RUN apt-get update && apt-get -y install libzstd-dev
|
|
||||||
|
|
||||||
# Set up default executables (update-alternatives)
|
# Set up default executables (update-alternatives)
|
||||||
RUN \
|
RUN \
|
||||||
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
|
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
|
||||||
|
|||||||
@@ -1,10 +1,15 @@
|
|||||||
{
|
{
|
||||||
"dockerFile": "Dockerfile",
|
"dockerFile": "Dockerfile",
|
||||||
|
// Creates local dir to mount to HOME in docker
|
||||||
|
"initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
|
||||||
"hostRequirements": {
|
"hostRequirements": {
|
||||||
"memory": "8gb",
|
"memory": "4gb",
|
||||||
"storage": "10gb"
|
"storage": "6gb"
|
||||||
},
|
},
|
||||||
|
"postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
|
||||||
"containerUser": "bts",
|
"containerUser": "bts",
|
||||||
|
"workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
|
||||||
|
"workspaceFolder": "/home/bts/spark",
|
||||||
"updateRemoteUserUID": true,
|
"updateRemoteUserUID": true,
|
||||||
"shutdownAction": "none",
|
"shutdownAction": "none",
|
||||||
// Note: If running multiple containers on the same host, each must use a
|
// Note: If running multiple containers on the same host, each must use a
|
||||||
@@ -12,7 +17,15 @@
|
|||||||
// example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
|
// example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
|
||||||
"runArgs": [
|
"runArgs": [
|
||||||
"--shm-size",
|
"--shm-size",
|
||||||
"8gb",
|
"2gb",
|
||||||
|
"-p",
|
||||||
|
"5000:5000",
|
||||||
|
"-p",
|
||||||
|
"5001:5001",
|
||||||
|
"-p",
|
||||||
|
"5173:5173",
|
||||||
|
"-p",
|
||||||
|
"5174:5174",
|
||||||
"-p",
|
"-p",
|
||||||
"8000:8000",
|
"8000:8000",
|
||||||
"-p",
|
"-p",
|
||||||
@@ -22,16 +35,23 @@
|
|||||||
"-p",
|
"-p",
|
||||||
"8003:8003",
|
"8003:8003",
|
||||||
"-p",
|
"-p",
|
||||||
"12345:12345"
|
"12345:12345",
|
||||||
|
],
|
||||||
|
"mounts": [
|
||||||
|
"source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
|
||||||
],
|
],
|
||||||
"customizations": {
|
"customizations": {
|
||||||
"vscode": {
|
"vscode": {
|
||||||
|
"settings": {
|
||||||
|
"clangd.arguments": [
|
||||||
|
"--compile-commands-dir=${workspaceFolder}/build/clang/debug"
|
||||||
|
]
|
||||||
|
},
|
||||||
"extensions": [
|
"extensions": [
|
||||||
"eamodio.gitlens",
|
"eamodio.gitlens",
|
||||||
"llvm-vs-code-extensions.vscode-clangd",
|
"llvm-vs-code-extensions.vscode-clangd",
|
||||||
"ms-vscode.cmake-tools",
|
"ms-vscode.cmake-tools",
|
||||||
"ms-vscode.cpptools",
|
"ms-vscode.cpptools"
|
||||||
"xaver.clang-format"
|
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+30
-16
@@ -207,17 +207,23 @@ dotnet_naming_symbols.parameters.applicable_accessibilities = public,internal,pr
|
|||||||
dotnet_naming_symbols.private_symbols.applicable_kinds = property,method,field,event,delegate
|
dotnet_naming_symbols.private_symbols.applicable_kinds = property,method,field,event,delegate
|
||||||
dotnet_naming_symbols.private_symbols.applicable_accessibilities = protected,private
|
dotnet_naming_symbols.private_symbols.applicable_accessibilities = protected,private
|
||||||
|
|
||||||
|
# local symbols
|
||||||
|
dotnet_naming_symbols.local_symbols.applicable_kinds = local
|
||||||
|
|
||||||
|
# parameter symbols
|
||||||
|
dotnet_naming_symbols.parameter_symbols.applicable_kinds = parameter
|
||||||
|
|
||||||
# Naming styles
|
# Naming styles
|
||||||
|
|
||||||
dotnet_naming_style.snake_case.required_prefix =
|
dotnet_naming_style.pascal_case.required_prefix =
|
||||||
dotnet_naming_style.snake_case.required_suffix =
|
dotnet_naming_style.pascal_case.required_suffix =
|
||||||
dotnet_naming_style.snake_case.word_separator = _
|
dotnet_naming_style.pascal_case.word_separator =
|
||||||
dotnet_naming_style.snake_case.capitalization = all_lower
|
dotnet_naming_style.pascal_case.capitalization = pascal_case
|
||||||
|
|
||||||
dotnet_naming_style.private_snake_case.required_prefix = _
|
dotnet_naming_style.camel_case.required_prefix =
|
||||||
dotnet_naming_style.private_snake_case.required_suffix =
|
dotnet_naming_style.camel_case.required_suffix =
|
||||||
dotnet_naming_style.private_snake_case.word_separator = _
|
dotnet_naming_style.camel_case.word_separator =
|
||||||
dotnet_naming_style.private_snake_case.capitalization = all_lower
|
dotnet_naming_style.camel_case.capitalization = camel_case
|
||||||
|
|
||||||
dotnet_naming_style.user_types.required_prefix =
|
dotnet_naming_style.user_types.required_prefix =
|
||||||
dotnet_naming_style.user_types.required_suffix =
|
dotnet_naming_style.user_types.required_suffix =
|
||||||
@@ -229,6 +235,10 @@ dotnet_naming_style.interface.required_suffix =
|
|||||||
dotnet_naming_style.interface.word_separator =
|
dotnet_naming_style.interface.word_separator =
|
||||||
dotnet_naming_style.interface.capitalization = pascal_case
|
dotnet_naming_style.interface.capitalization = pascal_case
|
||||||
|
|
||||||
|
dotnet_naming_style.local.capitalization = camel_case
|
||||||
|
|
||||||
|
dotnet_naming_style.parameter.capitalization = camel_case
|
||||||
|
|
||||||
# rules
|
# rules
|
||||||
dotnet_naming_rule.user_types.symbols = user_types
|
dotnet_naming_rule.user_types.symbols = user_types
|
||||||
dotnet_naming_rule.user_types.style = user_types
|
dotnet_naming_rule.user_types.style = user_types
|
||||||
@@ -238,14 +248,18 @@ dotnet_naming_rule.interfaces.symbols = interface
|
|||||||
dotnet_naming_rule.interfaces.style = interface_style
|
dotnet_naming_rule.interfaces.style = interface_style
|
||||||
dotnet_naming_rule.interfaces.severity = suggestion
|
dotnet_naming_rule.interfaces.severity = suggestion
|
||||||
|
|
||||||
dotnet_naming_rule.public_members_snake_case.symbols = public_symbols
|
dotnet_naming_rule.public_members.symbols = public_symbols
|
||||||
dotnet_naming_rule.public_members_snake_case.style = snake_case
|
dotnet_naming_rule.public_members.style = pascal_case
|
||||||
dotnet_naming_rule.public_members_snake_case.severity = suggestion
|
dotnet_naming_rule.public_members.severity = suggestion
|
||||||
|
|
||||||
dotnet_naming_rule.private_members_snake_case.symbols = private_symbols
|
dotnet_naming_rule.private_members.symbols = private_symbols
|
||||||
dotnet_naming_rule.private_members_snake_case.style = private_snake_case
|
dotnet_naming_rule.private_members.style = camel_case
|
||||||
dotnet_naming_rule.private_members_snake_case.severity = suggestion
|
dotnet_naming_rule.private_members.severity = suggestion
|
||||||
|
|
||||||
dotnet_naming_rule.parameters.symbols = parameters
|
dotnet_naming_rule.variables_are_camel_case.severity = suggestion
|
||||||
dotnet_naming_rule.parameters.style = snake_case
|
dotnet_naming_rule.variables_are_camel_case.symbols = local_symbols
|
||||||
|
dotnet_naming_rule.variables_are_camel_case.style = local
|
||||||
|
|
||||||
|
dotnet_naming_rule.parameters.symbols = parameter_symbols
|
||||||
|
dotnet_naming_rule.parameters.style = parameter
|
||||||
dotnet_naming_rule.parameters.severity = suggestion
|
dotnet_naming_rule.parameters.severity = suggestion
|
||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[submodule "externals"]
|
[submodule "externals"]
|
||||||
path = externals
|
path = externals
|
||||||
url = git@bitbucket.org:bluetradesys/externals.git
|
url = https://bitbucket.org/bluetradesys/externals.git
|
||||||
[submodule "vcpkg"]
|
[submodule "vcpkg"]
|
||||||
path = vcpkg
|
path = vcpkg
|
||||||
url = https://github.com/microsoft/vcpkg.git
|
url = https://github.com/microsoft/vcpkg.git
|
||||||
Vendored
+74
-190
@@ -6,6 +6,35 @@
|
|||||||
"type": "coreclr",
|
"type": "coreclr",
|
||||||
"request": "attach"
|
"request": "attach"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "Bts.DataStreamWebApp.Server",
|
||||||
|
"type": "coreclr",
|
||||||
|
"request": "launch",
|
||||||
|
"preLaunchTask": "Build DataStream WebApp (Debug)",
|
||||||
|
"program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
|
||||||
|
],
|
||||||
|
"env": {
|
||||||
|
"ASPNETCORE_ENVIRONMENT": "Development",
|
||||||
|
"ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Bts.DataStreamWebApp.Server (Skip Build)",
|
||||||
|
"type": "coreclr",
|
||||||
|
"request": "launch",
|
||||||
|
"program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
|
||||||
|
],
|
||||||
|
"env": {
|
||||||
|
"ASPNETCORE_ENVIRONMENT": "Development",
|
||||||
|
"ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
|
||||||
|
}
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "DataStreamPublisher",
|
"name": "DataStreamPublisher",
|
||||||
"type": "coreclr",
|
"type": "coreclr",
|
||||||
@@ -22,59 +51,34 @@
|
|||||||
"--config",
|
"--config",
|
||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
|
||||||
],
|
],
|
||||||
"cwd": "build/Debug/bin/net7.0",
|
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
|
||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
"console": "integratedTerminal"
|
"console": "integratedTerminal"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "(gdb) Launch",
|
"name": "DataStreamPublisher (Skip Build)",
|
||||||
"type": "cppdbg",
|
"type": "coreclr",
|
||||||
"request": "launch",
|
"request": "launch",
|
||||||
"program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
|
"program": "",
|
||||||
|
"linux": {
|
||||||
|
"program": "DataStreamPublisher",
|
||||||
|
},
|
||||||
|
"windows": {
|
||||||
|
"program": "DataStreamPublisher.exe",
|
||||||
|
},
|
||||||
"args": [
|
"args": [
|
||||||
"run"
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
|
||||||
],
|
],
|
||||||
|
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
|
||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
"cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
|
"console": "integratedTerminal"
|
||||||
"environment": [],
|
|
||||||
"externalConsole": false,
|
|
||||||
"MIMode": "gdb",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "spreader tests2",
|
|
||||||
"type": "cppdbg",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
|
|
||||||
"args": [
|
|
||||||
"test",
|
|
||||||
"[spreader2]"
|
|
||||||
],
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
|
|
||||||
"environment": [],
|
|
||||||
"externalConsole": false,
|
|
||||||
"MIMode": "gdb",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "spreader tests",
|
|
||||||
"type": "cppdbg",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
|
|
||||||
"args": [
|
|
||||||
"test",
|
|
||||||
"[spreader]"
|
|
||||||
],
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
|
|
||||||
"environment": [],
|
|
||||||
"externalConsole": false,
|
|
||||||
"MIMode": "gdb",
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "DataStreamQueryAnalyzer",
|
"name": "DataStreamQueryAnalyzer",
|
||||||
"type": "coreclr",
|
"type": "coreclr",
|
||||||
"request": "launch",
|
"request": "launch",
|
||||||
//"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
|
"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
|
||||||
"program": "",
|
"program": "",
|
||||||
"windows": {
|
"windows": {
|
||||||
"program": "DataStreamQueryAnalyzer.exe",
|
"program": "DataStreamQueryAnalyzer.exe",
|
||||||
@@ -110,6 +114,25 @@
|
|||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
"console": "integratedTerminal"
|
"console": "integratedTerminal"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "PubSubServer (Skip Build)",
|
||||||
|
"type": "coreclr",
|
||||||
|
"request": "launch",
|
||||||
|
"program": "",
|
||||||
|
"linux": {
|
||||||
|
"program": "PubSubServer",
|
||||||
|
},
|
||||||
|
"windows": {
|
||||||
|
"program": "PubSubServer.exe",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
|
||||||
|
],
|
||||||
|
"cwd": "build/Debug/bin/net7.0",
|
||||||
|
"stopAtEntry": false,
|
||||||
|
"console": "integratedTerminal"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "Spark",
|
"name": "Spark",
|
||||||
"type": "coreclr",
|
"type": "coreclr",
|
||||||
@@ -123,18 +146,6 @@
|
|||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
"console": "integratedTerminal"
|
"console": "integratedTerminal"
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"name": "Spark (Skip Build)",
|
|
||||||
"type": "coreclr",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "",
|
|
||||||
"windows": {
|
|
||||||
"program": "Spark.exe",
|
|
||||||
"cwd": "build/Debug/bin/net7.0-windows",
|
|
||||||
},
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"console": "integratedTerminal"
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"name": "Spark (Parameterized)",
|
"name": "Spark (Parameterized)",
|
||||||
"type": "coreclr",
|
"type": "coreclr",
|
||||||
@@ -152,139 +163,22 @@
|
|||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
"console": "integratedTerminal"
|
"console": "integratedTerminal"
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"name": "Spark Test",
|
|
||||||
"type": "coreclr",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "",
|
|
||||||
"windows": {
|
|
||||||
"preLaunchTask": "Build Spark (Debug)",
|
|
||||||
"program": "Spark.Console.exe",
|
|
||||||
"cwd": "build/Debug/bin/net7.0",
|
|
||||||
"args": [
|
|
||||||
"test",
|
|
||||||
"${input:sparkTest}"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"console": "integratedTerminal"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "Spark Test (Skip Build)",
|
|
||||||
"type": "coreclr",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "",
|
|
||||||
"windows": {
|
|
||||||
"program": "Spark.Console.exe",
|
|
||||||
"cwd": "build/Debug/bin/net7.0",
|
|
||||||
"args": [
|
|
||||||
"test",
|
|
||||||
"${input:sparkTest}"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"console": "integratedTerminal"
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"name": "Sparkd",
|
"name": "Sparkd",
|
||||||
"type": "cppdbg",
|
"type": "coreclr",
|
||||||
"request": "launch",
|
"request": "launch",
|
||||||
|
"preLaunchTask": "Build Sparkd (Debug | Clang)",
|
||||||
"program": "",
|
"program": "",
|
||||||
"linux": {
|
"linux": {
|
||||||
"preLaunchTask": "Build Sparkd (Debug)",
|
"program": "sparkd",
|
||||||
"program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
|
},
|
||||||
"args": [
|
"args": [
|
||||||
"run"
|
"run"
|
||||||
],
|
],
|
||||||
"stopAtEntry": false,
|
"cwd": "build/sparkd-debug-clang/server",
|
||||||
"cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
|
"stopAtEntry": false,
|
||||||
"environment": [],
|
"console": "integratedTerminal"
|
||||||
"externalConsole": false,
|
}
|
||||||
"MIMode": "gdb",
|
|
||||||
"setupCommands": [
|
|
||||||
{
|
|
||||||
"description": "Enable pretty-printing for gdb",
|
|
||||||
"text": "-enable-pretty-printing",
|
|
||||||
"ignoreFailures": true
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "Sparkd (Skip Build)",
|
|
||||||
"type": "cppdbg",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "",
|
|
||||||
"linux": {
|
|
||||||
"program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
|
|
||||||
"args": [
|
|
||||||
"run"
|
|
||||||
],
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
|
|
||||||
"environment": [],
|
|
||||||
"externalConsole": false,
|
|
||||||
"MIMode": "gdb",
|
|
||||||
"setupCommands": [
|
|
||||||
{
|
|
||||||
"description": "Enable pretty-printing for gdb",
|
|
||||||
"text": "-enable-pretty-printing",
|
|
||||||
"ignoreFailures": true
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "Sparkd Test",
|
|
||||||
"type": "cppdbg",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "",
|
|
||||||
"linux": {
|
|
||||||
"preLaunchTask": "Build Sparkd (Debug)",
|
|
||||||
"program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
|
|
||||||
"args": [
|
|
||||||
"test",
|
|
||||||
"${input:sparkdTest}",
|
|
||||||
],
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
|
|
||||||
"environment": [],
|
|
||||||
"externalConsole": false,
|
|
||||||
"MIMode": "gdb",
|
|
||||||
"setupCommands": [
|
|
||||||
{
|
|
||||||
"description": "Enable pretty-printing for gdb",
|
|
||||||
"text": "-enable-pretty-printing",
|
|
||||||
"ignoreFailures": true
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "Sparkd Test (Skip Build)",
|
|
||||||
"type": "cppdbg",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "",
|
|
||||||
"linux": {
|
|
||||||
"program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
|
|
||||||
"args": [
|
|
||||||
"test",
|
|
||||||
"${input:sparkdTest}",
|
|
||||||
],
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
|
|
||||||
"environment": [],
|
|
||||||
"externalConsole": false,
|
|
||||||
"MIMode": "gdb",
|
|
||||||
"setupCommands": [
|
|
||||||
{
|
|
||||||
"description": "Enable pretty-printing for gdb",
|
|
||||||
"text": "-enable-pretty-printing",
|
|
||||||
"ignoreFailures": true
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
},
|
|
||||||
],
|
],
|
||||||
"inputs": [
|
"inputs": [
|
||||||
{
|
{
|
||||||
@@ -315,15 +209,5 @@
|
|||||||
"description": "Config Server URL",
|
"description": "Config Server URL",
|
||||||
"default": "http://localhost:8000/config",
|
"default": "http://localhost:8000/config",
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"id": "sparkTest",
|
|
||||||
"type": "promptString",
|
|
||||||
"description": "Test to run",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "sparkdTest",
|
|
||||||
"type": "promptString",
|
|
||||||
"description": "Test to run",
|
|
||||||
},
|
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
Vendored
+111
-106
@@ -121,6 +121,97 @@
|
|||||||
"color": "terminal.ansiCyan"
|
"color": "terminal.ansiCyan"
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"label": "Build DataStreamQueryAnalyzer",
|
||||||
|
"group": "build",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"windows": {
|
||||||
|
"command": "tools/build_data_stream_query_analyzer.cmd",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"--configuration",
|
||||||
|
"${input:buildConfiguration}",
|
||||||
|
],
|
||||||
|
"icon": {
|
||||||
|
"id": "terminal",
|
||||||
|
"color": "terminal.ansiCyan"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Build DataStreamQueryAnalyzer (Debug)",
|
||||||
|
"group": "build",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"windows": {
|
||||||
|
"command": "tools/build_data_stream_query_analyzer.cmd",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"--configuration",
|
||||||
|
"Debug",
|
||||||
|
],
|
||||||
|
"icon": {
|
||||||
|
"id": "terminal",
|
||||||
|
"color": "terminal.ansiCyan"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Build DataStreamQueryAnalyzer (Release)",
|
||||||
|
"group": "build",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"windows": {
|
||||||
|
"command": "tools/build_data_stream_query_analyzer.cmd",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"--configuration",
|
||||||
|
"Release",
|
||||||
|
],
|
||||||
|
"icon": {
|
||||||
|
"id": "terminal",
|
||||||
|
"color": "terminal.ansiCyan"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Build DataStream WebApp",
|
||||||
|
"group": "build",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "tools/build_data_stream_web_app.sh",
|
||||||
|
},
|
||||||
|
"windows": {
|
||||||
|
"command": "tools/build_data_stream_web_app.cmd",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"--configuration",
|
||||||
|
"${input:buildConfiguration}",
|
||||||
|
],
|
||||||
|
"icon": {
|
||||||
|
"id": "terminal",
|
||||||
|
"color": "terminal.ansiCyan"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"label": "Build DataStream WebApp (Debug)",
|
||||||
|
"group": "build",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "tools/build_data_stream_web_app.sh",
|
||||||
|
},
|
||||||
|
"windows": {
|
||||||
|
"command": "tools/build_data_stream_web_app.cmd",
|
||||||
|
},
|
||||||
|
"args": [
|
||||||
|
"--configuration",
|
||||||
|
"Debug",
|
||||||
|
],
|
||||||
|
"icon": {
|
||||||
|
"id": "terminal",
|
||||||
|
"color": "terminal.ansiCyan"
|
||||||
|
},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"label": "Build Spark",
|
"label": "Build Spark",
|
||||||
"group": "build",
|
"group": "build",
|
||||||
@@ -303,6 +394,22 @@
|
|||||||
"color": "terminal.ansiCyan"
|
"color": "terminal.ansiCyan"
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"label": "Publish DataStreamWebApp",
|
||||||
|
"group": "build",
|
||||||
|
"type": "process",
|
||||||
|
"command": "",
|
||||||
|
"linux": {
|
||||||
|
"command": "tools/publish_data_stream_web_app.sh",
|
||||||
|
},
|
||||||
|
"windows": {
|
||||||
|
"command": "tools/publish_data_stream_web_app.cmd",
|
||||||
|
},
|
||||||
|
"icon": {
|
||||||
|
"id": "terminal",
|
||||||
|
"color": "terminal.ansiCyan"
|
||||||
|
},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"label": "Edit DataStream Config",
|
"label": "Edit DataStream Config",
|
||||||
"type": "process",
|
"type": "process",
|
||||||
@@ -353,7 +460,7 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"label": "Run DataStreamQueryAnalyzer",
|
"label": "Run DataStreamQueryAnalyzer",
|
||||||
"dependsOn": "Build DataStream",
|
"dependsOn": "Build DataStreamQueryAnalyzer",
|
||||||
"type": "process",
|
"type": "process",
|
||||||
"command": "",
|
"command": "",
|
||||||
"windows": {
|
"windows": {
|
||||||
@@ -373,7 +480,7 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"label": "Run DataStreamQueryAnalyzer (Debug)",
|
"label": "Run DataStreamQueryAnalyzer (Debug)",
|
||||||
"dependsOn": "Build DataStream (Debug)",
|
"dependsOn": "Build DataStreamQueryAnalyzer (Debug)",
|
||||||
"type": "process",
|
"type": "process",
|
||||||
"command": "",
|
"command": "",
|
||||||
"windows": {
|
"windows": {
|
||||||
@@ -393,7 +500,7 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"label": "Run DataStreamQueryAnalyzer (Release)",
|
"label": "Run DataStreamQueryAnalyzer (Release)",
|
||||||
"dependsOn": "Build DataStream (Release)",
|
"dependsOn": "Build DataStreamQueryAnalyzer (Release)",
|
||||||
"type": "process",
|
"type": "process",
|
||||||
"command": "",
|
"command": "",
|
||||||
"windows": {
|
"windows": {
|
||||||
@@ -641,21 +748,6 @@
|
|||||||
"color": "terminal.ansiGreen",
|
"color": "terminal.ansiGreen",
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"label": "Run Spark (Skip Build)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "Spark.exe",
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"label": "Run Spark Config Query",
|
"label": "Run Spark Config Query",
|
||||||
"dependsOn": "Build Spark",
|
"dependsOn": "Build Spark",
|
||||||
@@ -699,24 +791,6 @@
|
|||||||
"color": "terminal.ansiGreen",
|
"color": "terminal.ansiGreen",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"label": "Run Sparkd (Skip Build)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "sparkd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"run"
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"label": "Run Sparkd Config Query",
|
"label": "Run Sparkd Config Query",
|
||||||
"dependsOn": "Build Sparkd",
|
"dependsOn": "Build Sparkd",
|
||||||
@@ -738,65 +812,6 @@
|
|||||||
"color": "terminal.ansiGreen",
|
"color": "terminal.ansiGreen",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"label": "Test Spark (Debug)",
|
|
||||||
"dependsOn": "Build Spark (Debug)",
|
|
||||||
"type": "process",
|
|
||||||
"icon": {
|
|
||||||
"id": "coverage",
|
|
||||||
"color": "terminal.ansiYellow",
|
|
||||||
},
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "Spark.Console.exe",
|
|
||||||
"args": [
|
|
||||||
"test",
|
|
||||||
"${input:sparkTest}"
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
|
|
||||||
}
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Test Sparkd",
|
|
||||||
"dependsOn": "Build Sparkd",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "sparkd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"test",
|
|
||||||
"${input:sparkdTest}",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "coverage",
|
|
||||||
"color": "terminal.ansiYellow",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Test Sparkd (Skip Build)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"linux": {
|
|
||||||
"command": "sparkd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"test",
|
|
||||||
"${input:sparkdTest}",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "coverage",
|
|
||||||
"color": "terminal.ansiYellow",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"label": "View DataStreamQueryAnalyzer Logs (Last Run)",
|
"label": "View DataStreamQueryAnalyzer Logs (Last Run)",
|
||||||
"type": "process",
|
"type": "process",
|
||||||
@@ -915,16 +930,6 @@
|
|||||||
"id": "sparkdConfigQuery",
|
"id": "sparkdConfigQuery",
|
||||||
"type": "promptString",
|
"type": "promptString",
|
||||||
"description": "Config Type(s)"
|
"description": "Config Type(s)"
|
||||||
},
|
}
|
||||||
{
|
|
||||||
"id": "sparkTest",
|
|
||||||
"type": "promptString",
|
|
||||||
"description": "Test to run",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "sparkdTest",
|
|
||||||
"type": "promptString",
|
|
||||||
"description": "Test to run",
|
|
||||||
},
|
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
+7
-18
@@ -1,6 +1,5 @@
|
|||||||
cmake_minimum_required(VERSION 3.12)
|
cmake_minimum_required(VERSION 3.12)
|
||||||
project(spark)
|
project(spark)
|
||||||
include(GNUInstallDirs)
|
|
||||||
set(CMAKE_CXX_STANDARD 20)
|
set(CMAKE_CXX_STANDARD 20)
|
||||||
|
|
||||||
if(WIN32)
|
if(WIN32)
|
||||||
@@ -50,6 +49,8 @@ else()
|
|||||||
# make sparkd look for libspark.so in its directory
|
# make sparkd look for libspark.so in its directory
|
||||||
set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
|
set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
|
||||||
set(CMAKE_INSTALL_RPATH ".")
|
set(CMAKE_INSTALL_RPATH ".")
|
||||||
|
set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
|
||||||
|
set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")
|
||||||
|
|
||||||
if(BUILD_SPARKD)
|
if(BUILD_SPARKD)
|
||||||
# Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
|
# Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
|
||||||
@@ -88,15 +89,9 @@ else()
|
|||||||
add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
|
add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
|
||||||
|
|
||||||
# ** Binance USDS Futures **
|
# ** Binance USDS Futures **
|
||||||
add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://fapi.binance.com")
|
add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
|
||||||
add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/fapi/v1/exchangeInfo")
|
add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
|
||||||
add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/fapi/v1/ticker/24hr")
|
add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
|
||||||
add_compile_definitions(BINANCE_USDS_FUTURES_USER_DATA_PATH="/api/v3/userDataStream")
|
|
||||||
|
|
||||||
# ** Binance Coin Futures **
|
|
||||||
add_compile_definitions(CCAPI_BINANCE_COIN_FUTURES_URL_REST_BASE="https://dapi.binance.com")
|
|
||||||
add_compile_definitions(BINANCE_COIN_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
|
|
||||||
add_compile_definitions(BINANCE_COIN_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
|
|
||||||
|
|
||||||
# ** Deribit **
|
# ** Deribit **
|
||||||
add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
|
add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
|
||||||
@@ -116,7 +111,6 @@ else()
|
|||||||
|
|
||||||
# subdirs
|
# subdirs
|
||||||
if (BUILD_SPARKD)
|
if (BUILD_SPARKD)
|
||||||
add_subdirectory(src/lib_binance_execution)
|
|
||||||
add_subdirectory(src/server/sparkd)
|
add_subdirectory(src/server/sparkd)
|
||||||
add_subdirectory(src/data_files)
|
add_subdirectory(src/data_files)
|
||||||
endif()
|
endif()
|
||||||
@@ -138,8 +132,6 @@ else()
|
|||||||
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
|
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
|
||||||
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
|
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
|
||||||
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
|
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
|
||||||
COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
|
|
||||||
|
|
||||||
COMMAND
|
COMMAND
|
||||||
ln ARGS -sf
|
ln ARGS -sf
|
||||||
${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
|
${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
|
||||||
@@ -153,14 +145,11 @@ else()
|
|||||||
|
|
||||||
install(
|
install(
|
||||||
DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
|
DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
|
||||||
DESTINATION "${CMAKE_INSTALL_BINDIR}")
|
DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
|
||||||
install(
|
install(
|
||||||
FILES
|
FILES
|
||||||
"${SPARK_LIB}/libACE-5.7.5.so"
|
"${SPARK_LIB}/libACE-5.7.5.so"
|
||||||
"${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
|
"${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
|
||||||
"${SPARK_LIB}/libonixs-fix-engine.so.4.10.1"
|
DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
|
||||||
"${SPARK_LIB}/libonixs-fix-engine.so.4"
|
|
||||||
"${SPARK_LIB}/libonixs-fix-engine.so"
|
|
||||||
DESTINATION "${CMAKE_INSTALL_BINDIR}")
|
|
||||||
endif()
|
endif()
|
||||||
endif()
|
endif()
|
||||||
+14
-6
@@ -26,14 +26,12 @@
|
|||||||
"CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
|
"CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
|
||||||
"CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
|
"CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
|
||||||
"CMAKE_CXX_STANDARD": "20",
|
"CMAKE_CXX_STANDARD": "20",
|
||||||
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
|
|
||||||
"CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
|
"CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
|
||||||
"CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
|
"CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
|
||||||
"CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
|
"CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
|
||||||
"EXTERNALS_ROOT": "${sourceDir}/externals/windows",
|
"EXTERNALS_ROOT": "${sourceDir}/externals/windows",
|
||||||
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
|
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
|
||||||
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
|
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
|
||||||
"DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows",
|
|
||||||
"EXPORT_MD_CONSUMER": {
|
"EXPORT_MD_CONSUMER": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "ON"
|
"value": "ON"
|
||||||
@@ -62,7 +60,7 @@
|
|||||||
},
|
},
|
||||||
"CMAKE_CXX_STANDARD": "20",
|
"CMAKE_CXX_STANDARD": "20",
|
||||||
"CMAKE_INSTALL_RPATH": ".",
|
"CMAKE_INSTALL_RPATH": ".",
|
||||||
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
|
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}/sparkd",
|
||||||
"CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
|
"CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
|
||||||
"EXTERNALS_ROOT": "${sourceDir}/externals/linux",
|
"EXTERNALS_ROOT": "${sourceDir}/externals/linux",
|
||||||
"BUILD_SDK": {
|
"BUILD_SDK": {
|
||||||
@@ -223,6 +221,7 @@
|
|||||||
"APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
|
"APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
|
||||||
"APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
|
"APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
|
||||||
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
|
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
|
||||||
|
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows;${sourceDir}/build/$<CONFIG>/bin/net8.0",
|
||||||
"SHARED_MEMORY_PREFIX": "data_stream"
|
"SHARED_MEMORY_PREFIX": "data_stream"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -259,7 +258,7 @@
|
|||||||
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
||||||
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
|
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
|
||||||
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
|
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
|
||||||
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
|
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net8.0",
|
||||||
"EXPORT_MD_CONSUMER": {
|
"EXPORT_MD_CONSUMER": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "ON"
|
"value": "ON"
|
||||||
@@ -303,7 +302,16 @@
|
|||||||
"lhs": "${hostSystemName}",
|
"lhs": "${hostSystemName}",
|
||||||
"rhs": "Linux"
|
"rhs": "Linux"
|
||||||
},
|
},
|
||||||
"hidden": true
|
"hidden": true,
|
||||||
|
"targets": [
|
||||||
|
"data_files_dir",
|
||||||
|
"admin_web_app_bundle",
|
||||||
|
"install",
|
||||||
|
"libspark",
|
||||||
|
"rapidjson_project",
|
||||||
|
"server_layout",
|
||||||
|
"sparkd"
|
||||||
|
]
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -14,34 +14,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
|
|||||||
EndProject
|
EndProject
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}"
|
|
||||||
EndProject
|
|
||||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}"
|
|
||||||
EndProject
|
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{EF26A475-6101-4C9C-A717-D60BB92A2302}"
|
|
||||||
EndProject
|
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{911B08A3-515A-441D-84C8-8A43740B5FC1}"
|
|
||||||
EndProject
|
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{14140310-2233-430A-821B-F3058A25234B}"
|
|
||||||
EndProject
|
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}"
|
|
||||||
EndProject
|
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExpressionCore", "src\DataStreamSuite\ExpressionCore\ExpressionCore.csproj", "{6F22DB96-6EAB-4034-99D9-287021C36F01}"
|
|
||||||
EndProject
|
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{A5C33956-A41F-49B3-A41D-1082CBA59D02}"
|
|
||||||
EndProject
|
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}"
|
|
||||||
EndProject
|
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}"
|
|
||||||
EndProject
|
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ColumnMapExpressions", "src\DataStreamSuite\ColumnMapExpressions\ColumnMapExpressions.csproj", "{B945A183-2174-4ACB-B289-8A5C546656FE}"
|
|
||||||
EndProject
|
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{9F6D7DE8-185B-4C1D-B809-3314D092DF28}"
|
|
||||||
EndProject
|
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}"
|
|
||||||
EndProject
|
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
|
|
||||||
EndProject
|
|
||||||
Global
|
Global
|
||||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||||
Debug|x64 = Debug|x64
|
Debug|x64 = Debug|x64
|
||||||
@@ -98,102 +70,6 @@ Global
|
|||||||
{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
|
{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
|
||||||
{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
|
{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
|
||||||
{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
|
{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
|
||||||
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
|
||||||
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
|
||||||
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
|
||||||
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
|
||||||
{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
|
||||||
{14140310-2233-430A-821B-F3058A25234B}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{14140310-2233-430A-821B-F3058A25234B}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
|
||||||
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
|
||||||
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
|
||||||
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
|
||||||
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
|
||||||
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
|
||||||
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
|
||||||
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
|
||||||
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
|
||||||
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
|
||||||
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
|
||||||
{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
|
||||||
{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
|
||||||
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
|
||||||
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
|
||||||
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
|
||||||
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
|
||||||
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
|
||||||
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
|
||||||
EndGlobalSection
|
EndGlobalSection
|
||||||
GlobalSection(SolutionProperties) = preSolution
|
GlobalSection(SolutionProperties) = preSolution
|
||||||
HideSolutionNode = FALSE
|
HideSolutionNode = FALSE
|
||||||
@@ -204,19 +80,4 @@ Global
|
|||||||
GlobalSection(Performance) = preSolution
|
GlobalSection(Performance) = preSolution
|
||||||
HasPerformanceSessions = true
|
HasPerformanceSessions = true
|
||||||
EndGlobalSection
|
EndGlobalSection
|
||||||
GlobalSection(NestedProjects) = preSolution
|
|
||||||
{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
|
|
||||||
{EF26A475-6101-4C9C-A717-D60BB92A2302} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
|
||||||
{911B08A3-515A-441D-84C8-8A43740B5FC1} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
|
||||||
{14140310-2233-430A-821B-F3058A25234B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
|
||||||
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
|
||||||
{6F22DB96-6EAB-4034-99D9-287021C36F01} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
|
||||||
{A5C33956-A41F-49B3-A41D-1082CBA59D02} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
|
||||||
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
|
||||||
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
|
||||||
{B945A183-2174-4ACB-B289-8A5C546656FE} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
|
||||||
{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
|
||||||
{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
|
||||||
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
|
||||||
EndGlobalSection
|
|
||||||
EndGlobal
|
EndGlobal
|
||||||
@@ -1,49 +0,0 @@
|
|||||||
include_guard(DIRECTORY)
|
|
||||||
|
|
||||||
function(protobuf_generate_fs)
|
|
||||||
set(_options)
|
|
||||||
set(_one_value_args TARGET_NAME OUT_DIR PROTOC)
|
|
||||||
set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
|
|
||||||
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
|
||||||
|
|
||||||
if (NOT DEFINED _ARGS_TARGET_NAME)
|
|
||||||
set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_fs_protos")
|
|
||||||
endif()
|
|
||||||
if (NOT DEFINED _ARGS_PROTOC)
|
|
||||||
set(_ARGS_PROTOC "${SPARK_BIN}/protoc")
|
|
||||||
endif()
|
|
||||||
|
|
||||||
set(_proto_include_cli_args)
|
|
||||||
foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
|
|
||||||
list(APPEND _proto_include_cli_args -I ${_dir})
|
|
||||||
endforeach()
|
|
||||||
|
|
||||||
set(_outputs)
|
|
||||||
foreach(_proto_source ${_ARGS_PROTO_SOURCES})
|
|
||||||
# get_filename_component(_abs_proto_source ${_proto_source} ABSOLUTE)
|
|
||||||
get_filename_component(_proto_source_we ${_proto_source} NAME_WE)
|
|
||||||
# get_filename_component(_proto_source_dir ${_proto_source} DIRECTORY)
|
|
||||||
list(APPEND _outputs ${_ARGS_OUT_DIR}/${_proto_source_we}.fable.fs)
|
|
||||||
endforeach()
|
|
||||||
|
|
||||||
set(_venv_exec ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh)
|
|
||||||
set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py)
|
|
||||||
set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/templates/fs/fable.fs)
|
|
||||||
add_custom_command(
|
|
||||||
OUTPUT ${_outputs}
|
|
||||||
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
|
|
||||||
COMMAND ${_venv_exec}
|
|
||||||
${_ARGS_PROTOC}
|
|
||||||
--plugin=protoc-gen-service=${_plugin_py}
|
|
||||||
--service_out=lang=fs,strip_out_path=1:${_ARGS_OUT_DIR}
|
|
||||||
${_proto_include_cli_args}
|
|
||||||
${_ARGS_PROTO_SOURCES}
|
|
||||||
DEPENDS
|
|
||||||
${_venv_exec}
|
|
||||||
${_plugin_py}
|
|
||||||
${_template_fs}
|
|
||||||
${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
|
|
||||||
${_ARGS_PROTO_SOURCES}
|
|
||||||
)
|
|
||||||
add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS "${_outputs}")
|
|
||||||
endfunction()
|
|
||||||
@@ -1,33 +0,0 @@
|
|||||||
include_guard(DIRECTORY)
|
|
||||||
|
|
||||||
function(protobuf_generate_js)
|
|
||||||
set(_options)
|
|
||||||
set(_one_value_args TARGET_NAME OUTPUT)
|
|
||||||
set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
|
|
||||||
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
|
||||||
|
|
||||||
if (NOT DEFINED _ARGS_TARGET_NAME)
|
|
||||||
set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_js_protos")
|
|
||||||
endif()
|
|
||||||
|
|
||||||
set(_proto_include_cli_args)
|
|
||||||
foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
|
|
||||||
list(APPEND _proto_include_cli_args -p ${_dir})
|
|
||||||
endforeach()
|
|
||||||
|
|
||||||
add_custom_command(
|
|
||||||
OUTPUT ${_ARGS_OUTPUT}
|
|
||||||
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
|
|
||||||
COMMAND npx
|
|
||||||
--no-install pbjs
|
|
||||||
-t json-module
|
|
||||||
-w commonjs
|
|
||||||
-o ${_ARGS_OUTPUT}
|
|
||||||
${_proto_include_cli_args}
|
|
||||||
${_ARGS_PROTO_SOURCES}
|
|
||||||
DEPENDS
|
|
||||||
${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
|
|
||||||
${_ARGS_PROTO_SOURCES}
|
|
||||||
)
|
|
||||||
add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS ${_ARGS_OUTPUT})
|
|
||||||
endfunction()
|
|
||||||
@@ -1,49 +0,0 @@
|
|||||||
include_guard(DIRECTORY)
|
|
||||||
include(${CMAKE_SOURCE_DIR}/cmake/TernaryOperator.cmake)
|
|
||||||
|
|
||||||
function(sym_link_dir)
|
|
||||||
set(_options)
|
|
||||||
set(_one_value_args TARGET_NAME SOURCE DEST COMMENT)
|
|
||||||
set(_multi_value_args)
|
|
||||||
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
|
||||||
add_custom_command(
|
|
||||||
OUTPUT ${_ARGS_DEST}
|
|
||||||
COMMAND ln ARGS -sf ${_ARGS_SOURCE} ${_ARGS_DEST}
|
|
||||||
)
|
|
||||||
set_ternary(_comment _ARGS_COMMENT "${_ARGS_COMMENT}" "linking dir ${_ARGS_SOURCE} -> ${_ARGS_DEST}")
|
|
||||||
add_custom_target(
|
|
||||||
${_ARGS_TARGET_NAME} ALL
|
|
||||||
DEPENDS ${_ARGS_DEST}
|
|
||||||
COMMENT "${_comment}"
|
|
||||||
)
|
|
||||||
endfunction()
|
|
||||||
|
|
||||||
function(sym_link_file)
|
|
||||||
set(_options)
|
|
||||||
set(_one_value_args SOURCE DEST INSTALL_DEST)
|
|
||||||
set(_multi_value_args)
|
|
||||||
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
|
||||||
file(
|
|
||||||
CREATE_LINK
|
|
||||||
${_ARGS_SOURCE}
|
|
||||||
${_ARGS_DEST}
|
|
||||||
SYMBOLIC
|
|
||||||
)
|
|
||||||
if (DEFINED _ARGS_INSTALL_DEST)
|
|
||||||
install(
|
|
||||||
FILES
|
|
||||||
${_ARGS_SOURCE}
|
|
||||||
DESTINATION ${_ARGS_INSTALL_DEST}
|
|
||||||
)
|
|
||||||
endif()
|
|
||||||
endfunction()
|
|
||||||
|
|
||||||
function(sym_link_local_lib LIB_NAME)
|
|
||||||
set(_lib_file_name "${CMAKE_SHARED_LIBRARY_PREFIX}${LIB_NAME}.so")
|
|
||||||
file(
|
|
||||||
CREATE_LINK
|
|
||||||
"${CMAKE_BINARY_DIR}/src/lib_${LIB_NAME}/${_lib_file_name}"
|
|
||||||
"${CMAKE_CURRENT_BINARY_DIR}/${_lib_file_name}"
|
|
||||||
SYMBOLIC
|
|
||||||
)
|
|
||||||
endfunction()
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
include_guard(GLOBAL)
|
|
||||||
|
|
||||||
macro(set_ternary RESULT CONDITION TRUE_VALUE FALSE_VALUE)
|
|
||||||
if(${CONDITION})
|
|
||||||
set(${RESULT} ${TRUE_VALUE})
|
|
||||||
else()
|
|
||||||
set(${RESULT} ${FALSE_VALUE})
|
|
||||||
endif()
|
|
||||||
endmacro()
|
|
||||||
+1
-1
Submodule externals updated: 959b63f2f6...cae5fd20da.
@@ -8,7 +8,6 @@
|
|||||||
public int Integer;
|
public int Integer;
|
||||||
public double Real;
|
public double Real;
|
||||||
public string Identifier;
|
public string Identifier;
|
||||||
public DateTime DateTime;
|
|
||||||
public ImmutableList<IArithmeticExpression> Arguments;
|
public ImmutableList<IArithmeticExpression> Arguments;
|
||||||
public ImmutableList<Assignment> Assignments;
|
public ImmutableList<Assignment> Assignments;
|
||||||
public IArithmeticExpression Argument;
|
public IArithmeticExpression Argument;
|
||||||
|
|||||||
@@ -1,4 +1,6 @@
|
|||||||
using System;
|
using System;
|
||||||
|
using System.Runtime.Versioning;
|
||||||
|
|
||||||
using Microsoft.Win32;
|
using Microsoft.Win32;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG
|
||||||
@@ -10,6 +12,7 @@ namespace BCG
|
|||||||
rk.SetValue (name, value.ToString ());
|
rk.SetValue (name, value.ToString ());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[SupportedOSPlatform("windows")]
|
||||||
public static double getValue (RegistryKey rk, string name)
|
public static double getValue (RegistryKey rk, string name)
|
||||||
{
|
{
|
||||||
object v = rk.GetValue (name);
|
object v = rk.GetValue (name);
|
||||||
|
|||||||
@@ -0,0 +1,20 @@
|
|||||||
|
module.exports = {
|
||||||
|
root: true,
|
||||||
|
env: { browser: true, es2020: true },
|
||||||
|
extends: [
|
||||||
|
'eslint:recommended',
|
||||||
|
'plugin:react/recommended',
|
||||||
|
'plugin:react/jsx-runtime',
|
||||||
|
'plugin:react-hooks/recommended',
|
||||||
|
],
|
||||||
|
ignorePatterns: ['dist', '.eslintrc.cjs'],
|
||||||
|
parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
|
||||||
|
settings: { react: { version: '18.3.1' } },
|
||||||
|
plugins: ['react-refresh'],
|
||||||
|
rules: {
|
||||||
|
'react-refresh/only-export-components': [
|
||||||
|
'warn',
|
||||||
|
{ allowConstantExport: true },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
# Logs
|
||||||
|
logs
|
||||||
|
*.log
|
||||||
|
npm-debug.log*
|
||||||
|
yarn-debug.log*
|
||||||
|
yarn-error.log*
|
||||||
|
pnpm-debug.log*
|
||||||
|
lerna-debug.log*
|
||||||
|
|
||||||
|
node_modules
|
||||||
|
dist
|
||||||
|
dist-ssr
|
||||||
|
*.local
|
||||||
|
|
||||||
|
# Editor directories and files
|
||||||
|
.vscode/*
|
||||||
|
!.vscode/extensions.json
|
||||||
|
.idea
|
||||||
|
.DS_Store
|
||||||
|
*.suo
|
||||||
|
*.ntvs*
|
||||||
|
*.njsproj
|
||||||
|
*.sln
|
||||||
|
*.sw?
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
<Project Sdk="Microsoft.VisualStudio.JavaScript.Sdk/0.5.128-alpha">
|
||||||
|
<PropertyGroup>
|
||||||
|
<StartupCommand>npm run dev</StartupCommand>
|
||||||
|
<JavaScriptTestRoot>src\</JavaScriptTestRoot>
|
||||||
|
<JavaScriptTestFramework>Jest</JavaScriptTestFramework>
|
||||||
|
<!-- Allows the build (or compile) script located on package.json to run on Build -->
|
||||||
|
<ShouldRunBuildScript>true</ShouldRunBuildScript>
|
||||||
|
<!-- Folder where production build objects will be placed -->
|
||||||
|
<PublishAssetsDirectory>$(MSBuildProjectDirectory)\distill</PublishAssetsDirectory>
|
||||||
|
</PropertyGroup>
|
||||||
|
</Project>
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
# React + Vite
|
||||||
|
|
||||||
|
This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
|
||||||
|
|
||||||
|
Currently, two official plugins are available:
|
||||||
|
|
||||||
|
- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh
|
||||||
|
- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8" />
|
||||||
|
<link rel="icon" type="image/ico" href="/logo.ico" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<title>BTS Cloud Streams</title>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div id="root"></div>
|
||||||
|
<script type="module" src="/src/main.jsx"></script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
|
<configuration>
|
||||||
|
<packageSources>
|
||||||
|
<clear />
|
||||||
|
<add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
|
||||||
|
</packageSources>
|
||||||
|
<disabledPackageSources>
|
||||||
|
<clear />
|
||||||
|
</disabledPackageSources>
|
||||||
|
</configuration>
|
||||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,38 @@
|
|||||||
|
{
|
||||||
|
"name": "bts.datastreamwebapp.client",
|
||||||
|
"private": true,
|
||||||
|
"version": "0.0.0",
|
||||||
|
"type": "module",
|
||||||
|
"scripts": {
|
||||||
|
"dev": "vite",
|
||||||
|
"build": "vite build",
|
||||||
|
"lint": "eslint . --ext js,jsx --report-unused-disable-directives --max-warnings 0",
|
||||||
|
"preview": "vite preview"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"@angular/material": "^18.0.4",
|
||||||
|
"@emotion/react": "^11.11.4",
|
||||||
|
"@emotion/styled": "^11.11.5",
|
||||||
|
"@fontsource/noto-sans": "^5.0.22",
|
||||||
|
"@material/data-table": "^15.0.0-canary.7f224ddd4.0",
|
||||||
|
"@mui/icons-material": "^5.15.20",
|
||||||
|
"@mui/material": "^5.15.20",
|
||||||
|
"@mui/x-data-grid": "^7.7.1",
|
||||||
|
"bulma": "^1.0.1",
|
||||||
|
"react": "^18.3.1",
|
||||||
|
"react-data-table-component": "^7.6.2",
|
||||||
|
"react-dom": "^18.3.1",
|
||||||
|
"react-use-websocket": "^4.8.1",
|
||||||
|
"rpc-websockets": "^7.11.2"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/react": "^18.3.3",
|
||||||
|
"@types/react-dom": "^18.3.0",
|
||||||
|
"@vitejs/plugin-react": "^4.3.1",
|
||||||
|
"eslint": "^8.57.0",
|
||||||
|
"eslint-plugin-react": "^7.34.2",
|
||||||
|
"eslint-plugin-react-hooks": "^4.6.2",
|
||||||
|
"eslint-plugin-react-refresh": "^0.4.7",
|
||||||
|
"vite": "^5.2.13"
|
||||||
|
}
|
||||||
|
}
|
||||||
Binary file not shown.
|
After Width: | Height: | Size: 8.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 107 KiB |
@@ -0,0 +1 @@
|
|||||||
|
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>
|
||||||
|
After Width: | Height: | Size: 1.5 KiB |
@@ -0,0 +1,28 @@
|
|||||||
|
@use "@material/checkbox";
|
||||||
|
@use "@material/icon-button";
|
||||||
|
@use "@material/data-table/data-table";
|
||||||
|
|
||||||
|
h1#main-header {
|
||||||
|
padding: 4px 8px 4px 8px;
|
||||||
|
text-align: center;
|
||||||
|
}
|
||||||
|
|
||||||
|
td:hover {
|
||||||
|
background-color: cadetblue;
|
||||||
|
}
|
||||||
|
|
||||||
|
td#cell-decimal {
|
||||||
|
text-align: right;
|
||||||
|
}
|
||||||
|
|
||||||
|
td#cell-integer {
|
||||||
|
text-align: right;
|
||||||
|
}
|
||||||
|
|
||||||
|
td#cell-string {
|
||||||
|
text-align: center;
|
||||||
|
}
|
||||||
|
|
||||||
|
td#cell-date-time {
|
||||||
|
text-align: center;
|
||||||
|
}
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
import { createTheme, ThemeProvider, CssBaseline } from '@mui/material';
|
||||||
|
import useMediaQuery from '@mui/material/useMediaQuery';
|
||||||
|
import QueryControl from './QueryControl';
|
||||||
|
import getTheme from './Theme';
|
||||||
|
import '@fontsource/noto-sans';
|
||||||
|
import './App.css';
|
||||||
|
|
||||||
|
const App = () => {
|
||||||
|
const prefersDarkMode = useMediaQuery('(prefers-color-scheme: dark)');
|
||||||
|
const theme = getTheme({ mode: prefersDarkMode ? 'dark' : 'light' });
|
||||||
|
|
||||||
|
return (
|
||||||
|
<ThemeProvider theme={theme}>
|
||||||
|
<CssBaseline />
|
||||||
|
<h1 id='main-header' style={{
|
||||||
|
color: theme.palette.text.primary,
|
||||||
|
fontFamily: 'Noto Sans',
|
||||||
|
}}>BTS Cloud Streams</h1>
|
||||||
|
<QueryControl websocketUrl="ws://localhost:5000/ws" theme={theme} />
|
||||||
|
</ThemeProvider>
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
export default App;
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
@use "@material/checkbox";
|
||||||
|
@use "@material/icon-button";
|
||||||
|
@use "@material/data-table/data-table";
|
||||||
|
|
||||||
|
th {
|
||||||
|
font-size: large;
|
||||||
|
padding: 15px;
|
||||||
|
}
|
||||||
|
|
||||||
|
td {
|
||||||
|
padding: 15px;
|
||||||
|
}
|
||||||
|
|
||||||
|
td:hover {
|
||||||
|
background-color: cadetblue;
|
||||||
|
}
|
||||||
|
|
||||||
|
td#cell-decimal {
|
||||||
|
text-align: right;
|
||||||
|
}
|
||||||
|
|
||||||
|
td#cell-integer {
|
||||||
|
text-align: right;
|
||||||
|
}
|
||||||
|
|
||||||
|
td#cell-string {
|
||||||
|
text-align: center;
|
||||||
|
}
|
||||||
|
|
||||||
|
td#cell-date-time {
|
||||||
|
text-align: center;
|
||||||
|
}
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
import PropTypes from 'prop-types';
|
||||||
|
import { DataGrid } from '@mui/x-data-grid';
|
||||||
|
|
||||||
|
function getGridColDef(col) {
|
||||||
|
return {
|
||||||
|
field: col.name,
|
||||||
|
headerName: col.name,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export default function DataTable({ columns, rows }) {
|
||||||
|
return (
|
||||||
|
<DataGrid
|
||||||
|
columns={columns.map(getGridColDef)}
|
||||||
|
rows={rows}
|
||||||
|
sx={{
|
||||||
|
borderColor: 'primary.main',
|
||||||
|
backgroundColor: 'primary.dark',
|
||||||
|
color: 'text.primary',
|
||||||
|
'& .MuiDataGrid-columnHeader': {
|
||||||
|
color: 'primary.main',
|
||||||
|
fontWeight: 700,
|
||||||
|
backgroundColor: 'primary.dark'
|
||||||
|
},
|
||||||
|
'& .MuiDataGrid-filler': {
|
||||||
|
color: 'primary.main',
|
||||||
|
fontWeight: 700,
|
||||||
|
backgroundColor: 'primary.dark'
|
||||||
|
},
|
||||||
|
'& .MuiDataGrid-cell': {
|
||||||
|
color: 'text.primary',
|
||||||
|
},
|
||||||
|
'& .MuiDataGrid-cell:hover': {
|
||||||
|
color: 'primary.main',
|
||||||
|
},
|
||||||
|
'& .MuiDataGrid-row': {
|
||||||
|
'--DataGrid-rowBorderColor': 'secondary.dark'
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
DataTable.propTypes = {
|
||||||
|
columns: PropTypes.arrayOf(
|
||||||
|
PropTypes.shape({
|
||||||
|
name: PropTypes.string.isRequired,
|
||||||
|
type: PropTypes.string.isRequired,
|
||||||
|
})
|
||||||
|
).isRequired,
|
||||||
|
rows: PropTypes.arrayOf(PropTypes.array).isRequired,
|
||||||
|
};
|
||||||
@@ -0,0 +1,100 @@
|
|||||||
|
import PropTypes from 'prop-types';
|
||||||
|
import DataTable from './DataTable';
|
||||||
|
import QueryInputForm from './QueryInputForm';
|
||||||
|
import Container from '@mui/material/Container';
|
||||||
|
import React from 'react';
|
||||||
|
|
||||||
|
var rpcReqIdCounter = 1;
|
||||||
|
|
||||||
|
function subscribeReq(query) {
|
||||||
|
return {
|
||||||
|
jsonrpc: "2.0",
|
||||||
|
id: rpcReqIdCounter++,
|
||||||
|
method: "subscribe",
|
||||||
|
params: {
|
||||||
|
subscription_id: "1",
|
||||||
|
query: query
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function unsubscribeReq() {
|
||||||
|
return {
|
||||||
|
jsonrpc: "2.0",
|
||||||
|
id: rpcReqIdCounter++,
|
||||||
|
method: "unsubscribe",
|
||||||
|
params: {
|
||||||
|
subscription_id: "1",
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
class QueryControl extends React.Component {
|
||||||
|
constructor({ websocketUrl, theme }) {
|
||||||
|
super(websocketUrl);
|
||||||
|
console.log('Connecting to WebSocket:', websocketUrl);
|
||||||
|
this.state = {
|
||||||
|
rows: null,
|
||||||
|
ws: new WebSocket(websocketUrl),
|
||||||
|
columns: null,
|
||||||
|
isRunning: false,
|
||||||
|
theme: theme,
|
||||||
|
};
|
||||||
|
this.state.ws.onopen = () => {
|
||||||
|
console.info('WebSocket open');
|
||||||
|
};
|
||||||
|
this.state.ws.onclose = () => {
|
||||||
|
console.info('WebSocket closed');
|
||||||
|
};
|
||||||
|
this.state.ws.onmessage = e => {
|
||||||
|
console.info(`Received message: ${e.data}`);
|
||||||
|
const msg = JSON.parse(e.data);
|
||||||
|
if (msg.id != null) {
|
||||||
|
let newState = { ...this.state };
|
||||||
|
newState.isRunning = msg.result.status == 'Subscribed';
|
||||||
|
this.setState(newState);
|
||||||
|
} else {
|
||||||
|
let newState = { ...this.state };
|
||||||
|
newState.rows = msg.params.all;
|
||||||
|
console.info(`Received rows: ${newState.rows}`);
|
||||||
|
newState.columns = msg.params.columns;
|
||||||
|
this.setState(newState);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
this.start = (query) => {
|
||||||
|
this.state.ws.send(JSON.stringify(subscribeReq(query)));
|
||||||
|
};
|
||||||
|
|
||||||
|
this.stop = () => {
|
||||||
|
this.state.ws.send(JSON.stringify(unsubscribeReq()));
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
componentWillUnmount() {
|
||||||
|
if (this.state.ws) {
|
||||||
|
console.warn('Closing websocket because component is unmounting');
|
||||||
|
this.state.ws.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
render() {
|
||||||
|
console.info('Rendering');
|
||||||
|
return (
|
||||||
|
<Container>
|
||||||
|
<QueryInputForm isRunning={this.state.isRunning} onStart={this.start} onStop={this.stop} />
|
||||||
|
{
|
||||||
|
this.state.rows == null ? <p style={{ color: this.state.theme.palette.warning.main }}>No data received</p> :
|
||||||
|
<DataTable columns={this.state.columns} rows={this.state.rows} />
|
||||||
|
}
|
||||||
|
</Container>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
QueryControl.propTypes = {
|
||||||
|
websocketUrl: PropTypes.string.isRequired,
|
||||||
|
theme: PropTypes.object.isRequired,
|
||||||
|
};
|
||||||
|
|
||||||
|
export default QueryControl;
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
import { useRef } from 'react';
|
||||||
|
import Box from '@mui/material/Box';
|
||||||
|
import Button from '@mui/material/Button';
|
||||||
|
import Grid from '@mui/material/Grid';
|
||||||
|
import Paper from '@mui/material/Paper';
|
||||||
|
import PlayArrowIcon from '@mui/icons-material/PlayArrow';
|
||||||
|
import PropTypes from 'prop-types';
|
||||||
|
import StopIcon from '@mui/icons-material/Stop';
|
||||||
|
import TextField from '@mui/material/TextField';
|
||||||
|
import Tooltip from '@mui/material/Tooltip';
|
||||||
|
|
||||||
|
const QueryInputForm = ({ isRunning, onStart, onStop }) => {
|
||||||
|
let textInput = useRef(null);
|
||||||
|
|
||||||
|
const onClearClick = () => {
|
||||||
|
textInput.current.value = '';
|
||||||
|
};
|
||||||
|
|
||||||
|
const onRunClick = () => {
|
||||||
|
onStart(textInput.current.value);
|
||||||
|
};
|
||||||
|
|
||||||
|
const onStopClick = () => {
|
||||||
|
onStop();
|
||||||
|
};
|
||||||
|
|
||||||
|
const onKeyDown = (e) => {
|
||||||
|
// Detect keyboard shortcuts.
|
||||||
|
// Ctrl+Enter : Run
|
||||||
|
if (e.ctrlKey && e.key === 'Enter') {
|
||||||
|
onRunClick();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Box sx={{ p: 2 }}>
|
||||||
|
<Paper elevation={3} sx={{ color: 'primary.main' }}>
|
||||||
|
<Box sx={{ p: 2 }}>
|
||||||
|
<TextField
|
||||||
|
fullWidth
|
||||||
|
multiline
|
||||||
|
rows={10}
|
||||||
|
variant="outlined"
|
||||||
|
onKeyDown={onKeyDown}
|
||||||
|
inputRef={textInput}
|
||||||
|
placeholder="Enter Query..."
|
||||||
|
/>
|
||||||
|
</Box>
|
||||||
|
<Box sx={{ p: 2, borderTop: 1 }}>
|
||||||
|
<Grid container spacing={2} justifyContent="flex-start">
|
||||||
|
<Grid item xs>
|
||||||
|
<Tooltip title="Clear Query Text">
|
||||||
|
<Button onClick={onClearClick} sx={{ padding: '11px', color: 'text.dark' }}>
|
||||||
|
Clear
|
||||||
|
</Button>
|
||||||
|
</Tooltip>
|
||||||
|
</Grid>
|
||||||
|
<Grid item>
|
||||||
|
<Tooltip title="Run (Ctrl+Enter)">
|
||||||
|
<Button onClick={onRunClick} sx={{
|
||||||
|
color: `query_start_button.${isRunning ? 'active' : 'inactive'}.color`,
|
||||||
|
bgcolor: `query_start_button.${isRunning ? 'active' : 'inactive'}.background_color`,
|
||||||
|
padding: '11px'
|
||||||
|
}}>
|
||||||
|
<PlayArrowIcon />
|
||||||
|
</Button>
|
||||||
|
</Tooltip>
|
||||||
|
</Grid>
|
||||||
|
<Grid item>
|
||||||
|
<Tooltip title="Stop">
|
||||||
|
<Button onClick={onStopClick} sx={{
|
||||||
|
color: `query_stop_button.${isRunning ? 'inactive' : 'active'}.color`,
|
||||||
|
bgcolor: `query_stop_button.${isRunning ? 'inactive' : 'active'}.background_color`,
|
||||||
|
padding: '11px'
|
||||||
|
}}>
|
||||||
|
<StopIcon />
|
||||||
|
</Button>
|
||||||
|
</Tooltip>
|
||||||
|
</Grid>
|
||||||
|
</Grid>
|
||||||
|
</Box>
|
||||||
|
</Paper>
|
||||||
|
</Box>
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
QueryInputForm.propTypes = {
|
||||||
|
isRunning: PropTypes.bool.isRequired,
|
||||||
|
onStart: PropTypes.func.isRequired,
|
||||||
|
onStop: PropTypes.func.isRequired,
|
||||||
|
};
|
||||||
|
|
||||||
|
export default QueryInputForm;
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
import { createTheme } from '@mui/material/styles';
|
||||||
|
import PropTypes from 'prop-types';
|
||||||
|
import '@fontsource/noto-sans';
|
||||||
|
import './App.css';
|
||||||
|
|
||||||
|
const getLightPalette = () => {
|
||||||
|
return {
|
||||||
|
mode: 'light',
|
||||||
|
primary: {
|
||||||
|
light: '#6573c3',
|
||||||
|
main: '#3f51b5',
|
||||||
|
dark: '#2c387e',
|
||||||
|
contrastText: '#ffffff',
|
||||||
|
},
|
||||||
|
secondary: {
|
||||||
|
light: '#f73378',
|
||||||
|
main: '#f50057',
|
||||||
|
dark: '#ab003c',
|
||||||
|
contrastText: '#ffffff',
|
||||||
|
},
|
||||||
|
text: {
|
||||||
|
primary: '#000000c4',
|
||||||
|
secondary: '#00000086',
|
||||||
|
disabled: '#0000005b',
|
||||||
|
hint: '#0000005b',
|
||||||
|
}
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const getDarkPalette = () => {
|
||||||
|
return {
|
||||||
|
mode: 'dark',
|
||||||
|
primary: {
|
||||||
|
light: '#bcc5ff',
|
||||||
|
main: '#33baf9',
|
||||||
|
dark: '#202334',
|
||||||
|
contrastText: '#78a2ac',
|
||||||
|
},
|
||||||
|
secondary: {
|
||||||
|
light: '#9fd0b8',
|
||||||
|
main: '#5eb289',
|
||||||
|
dark: '#ffffff12',
|
||||||
|
contrastText: '#c8d4f7',
|
||||||
|
},
|
||||||
|
text: {
|
||||||
|
primary: '#eaf8f8',
|
||||||
|
dark: '#eaf8f8a0'
|
||||||
|
},
|
||||||
|
query_start_button: {
|
||||||
|
active: {
|
||||||
|
color: '#33baf9',
|
||||||
|
background_color: '#ffffff40',
|
||||||
|
},
|
||||||
|
inactive: {
|
||||||
|
color: '#ffffff40',
|
||||||
|
background_color: '#ffffff00',
|
||||||
|
}
|
||||||
|
},
|
||||||
|
warning: {
|
||||||
|
main: '#c4823f'
|
||||||
|
},
|
||||||
|
query_stop_button: {
|
||||||
|
active: {
|
||||||
|
color: '#ff4444a0',
|
||||||
|
background_color: '#ffffff40',
|
||||||
|
},
|
||||||
|
inactive: {
|
||||||
|
color: '#ff444450',
|
||||||
|
background_color: '#ffffff00',
|
||||||
|
}
|
||||||
|
},
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const getTheme = ({ mode }) => {
|
||||||
|
return createTheme({
|
||||||
|
typography: {
|
||||||
|
fontFamily: '"Helvetica"'
|
||||||
|
},
|
||||||
|
palette: mode == 'dark' ? getDarkPalette() : getLightPalette()
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
export default getTheme;
|
||||||
|
|
||||||
|
getTheme.propTypes = {
|
||||||
|
mode: PropTypes.string.isRequired,
|
||||||
|
};
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="35.93" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 228"><path fill="#00D8FF" d="M210.483 73.824a171.49 171.49 0 0 0-8.24-2.597c.465-1.9.893-3.777 1.273-5.621c6.238-30.281 2.16-54.676-11.769-62.708c-13.355-7.7-35.196.329-57.254 19.526a171.23 171.23 0 0 0-6.375 5.848a155.866 155.866 0 0 0-4.241-3.917C100.759 3.829 77.587-4.822 63.673 3.233C50.33 10.957 46.379 33.89 51.995 62.588a170.974 170.974 0 0 0 1.892 8.48c-3.28.932-6.445 1.924-9.474 2.98C17.309 83.498 0 98.307 0 113.668c0 15.865 18.582 31.778 46.812 41.427a145.52 145.52 0 0 0 6.921 2.165a167.467 167.467 0 0 0-2.01 9.138c-5.354 28.2-1.173 50.591 12.134 58.266c13.744 7.926 36.812-.22 59.273-19.855a145.567 145.567 0 0 0 5.342-4.923a168.064 168.064 0 0 0 6.92 6.314c21.758 18.722 43.246 26.282 56.54 18.586c13.731-7.949 18.194-32.003 12.4-61.268a145.016 145.016 0 0 0-1.535-6.842c1.62-.48 3.21-.974 4.76-1.488c29.348-9.723 48.443-25.443 48.443-41.52c0-15.417-17.868-30.326-45.517-39.844Zm-6.365 70.984c-1.4.463-2.836.91-4.3 1.345c-3.24-10.257-7.612-21.163-12.963-32.432c5.106-11 9.31-21.767 12.459-31.957c2.619.758 5.16 1.557 7.61 2.4c23.69 8.156 38.14 20.213 38.14 29.504c0 9.896-15.606 22.743-40.946 31.14Zm-10.514 20.834c2.562 12.94 2.927 24.64 1.23 33.787c-1.524 8.219-4.59 13.698-8.382 15.893c-8.067 4.67-25.32-1.4-43.927-17.412a156.726 156.726 0 0 1-6.437-5.87c7.214-7.889 14.423-17.06 21.459-27.246c12.376-1.098 24.068-2.894 34.671-5.345a134.17 134.17 0 0 1 1.386 6.193ZM87.276 214.515c-7.882 2.783-14.16 2.863-17.955.675c-8.075-4.657-11.432-22.636-6.853-46.752a156.923 156.923 0 0 1 1.869-8.499c10.486 2.32 22.093 3.988 34.498 4.994c7.084 9.967 14.501 19.128 21.976 27.15a134.668 134.668 0 0 1-4.877 4.492c-9.933 8.682-19.886 14.842-28.658 17.94ZM50.35 144.747c-12.483-4.267-22.792-9.812-29.858-15.863c-6.35-5.437-9.555-10.836-9.555-15.216c0-9.322 13.897-21.212 37.076-29.293c2.813-.98 5.757-1.905 8.812-2.773c3.204 10.42 7.406 21.315 12.477 32.332c-5.137 11.18-9.399 22.249-12.634 32.792a134.718 134.718 0 0 1-6.318-1.979Zm12.378-84.26c-4.811-24.587-1.616-43.134 6.425-47.789c8.564-4.958 27.502 2.111 47.463 19.835a144.318 144.318 0 0 1 3.841 3.545c-7.438 7.987-14.787 17.08-21.808 26.988c-12.04 1.116-23.565 2.908-34.161 5.309a160.342 160.342 0 0 1-1.76-7.887Zm110.427 27.268a347.8 347.8 0 0 0-7.785-12.803c8.168 1.033 15.994 2.404 23.343 4.08c-2.206 7.072-4.956 14.465-8.193 22.045a381.151 381.151 0 0 0-7.365-13.322Zm-45.032-43.861c5.044 5.465 10.096 11.566 15.065 18.186a322.04 322.04 0 0 0-30.257-.006c4.974-6.559 10.069-12.652 15.192-18.18ZM82.802 87.83a323.167 323.167 0 0 0-7.227 13.238c-3.184-7.553-5.909-14.98-8.134-22.152c7.304-1.634 15.093-2.97 23.209-3.984a321.524 321.524 0 0 0-7.848 12.897Zm8.081 65.352c-8.385-.936-16.291-2.203-23.593-3.793c2.26-7.3 5.045-14.885 8.298-22.6a321.187 321.187 0 0 0 7.257 13.246c2.594 4.48 5.28 8.868 8.038 13.147Zm37.542 31.03c-5.184-5.592-10.354-11.779-15.403-18.433c4.902.192 9.899.29 14.978.29c5.218 0 10.376-.117 15.453-.343c-4.985 6.774-10.018 12.97-15.028 18.486Zm52.198-57.817c3.422 7.8 6.306 15.345 8.596 22.52c-7.422 1.694-15.436 3.058-23.88 4.071a382.417 382.417 0 0 0 7.859-13.026a347.403 347.403 0 0 0 7.425-13.565Zm-16.898 8.101a358.557 358.557 0 0 1-12.281 19.815a329.4 329.4 0 0 1-23.444.823c-7.967 0-15.716-.248-23.178-.732a310.202 310.202 0 0 1-12.513-19.846h.001a307.41 307.41 0 0 1-10.923-20.627a310.278 310.278 0 0 1 10.89-20.637l-.001.001a307.318 307.318 0 0 1 12.413-19.761c7.613-.576 15.42-.876 23.31-.876H128c7.926 0 15.743.303 23.354.883a329.357 329.357 0 0 1 12.335 19.695a358.489 358.489 0 0 1 11.036 20.54a329.472 329.472 0 0 1-11 20.722Zm22.56-122.124c8.572 4.944 11.906 24.881 6.52 51.026c-.344 1.668-.73 3.367-1.15 5.09c-10.622-2.452-22.155-4.275-34.23-5.408c-7.034-10.017-14.323-19.124-21.64-27.008a160.789 160.789 0 0 1 5.888-5.4c18.9-16.447 36.564-22.941 44.612-18.3ZM128 90.808c12.625 0 22.86 10.235 22.86 22.86s-10.235 22.86-22.86 22.86s-22.86-10.235-22.86-22.86s10.235-22.86 22.86-22.86Z"></path></svg>
|
||||||
|
After Width: | Height: | Size: 4.0 KiB |
@@ -0,0 +1,51 @@
|
|||||||
|
.dark-high-contrast {
|
||||||
|
--md-sys-color-primary: rgb(251 250 255);
|
||||||
|
--md-sys-color-surface-tint: rgb(165 200 255);
|
||||||
|
--md-sys-color-on-primary: rgb(0 0 0);
|
||||||
|
--md-sys-color-primary-container: rgb(173 204 255);
|
||||||
|
--md-sys-color-on-primary-container: rgb(0 0 0);
|
||||||
|
--md-sys-color-secondary: rgb(250 250 255);
|
||||||
|
--md-sys-color-on-secondary: rgb(0 0 0);
|
||||||
|
--md-sys-color-secondary-container: rgb(159 208 255);
|
||||||
|
--md-sys-color-on-secondary-container: rgb(0 0 0);
|
||||||
|
--md-sys-color-tertiary: rgb(255 249 251);
|
||||||
|
--md-sys-color-on-tertiary: rgb(0 0 0);
|
||||||
|
--md-sys-color-tertiary-container: rgb(222 193 231);
|
||||||
|
--md-sys-color-on-tertiary-container: rgb(0 0 0);
|
||||||
|
--md-sys-color-error: rgb(255 249 249);
|
||||||
|
--md-sys-color-on-error: rgb(0 0 0);
|
||||||
|
--md-sys-color-error-container: rgb(255 186 177);
|
||||||
|
--md-sys-color-on-error-container: rgb(0 0 0);
|
||||||
|
--md-sys-color-background: rgb(17 19 24);
|
||||||
|
--md-sys-color-on-background: rgb(225 226 233);
|
||||||
|
--md-sys-color-surface: rgb(17 19 24);
|
||||||
|
--md-sys-color-on-surface: rgb(255 255 255);
|
||||||
|
--md-sys-color-surface-variant: rgb(67 71 78);
|
||||||
|
--md-sys-color-on-surface-variant: rgb(251 250 255);
|
||||||
|
--md-sys-color-outline: rgb(200 202 212);
|
||||||
|
--md-sys-color-outline-variant: rgb(200 202 212);
|
||||||
|
--md-sys-color-shadow: rgb(0 0 0);
|
||||||
|
--md-sys-color-scrim: rgb(0 0 0);
|
||||||
|
--md-sys-color-inverse-surface: rgb(225 226 233);
|
||||||
|
--md-sys-color-inverse-on-surface: rgb(0 0 0);
|
||||||
|
--md-sys-color-inverse-primary: rgb(0 42 83);
|
||||||
|
--md-sys-color-primary-fixed: rgb(219 231 255);
|
||||||
|
--md-sys-color-on-primary-fixed: rgb(0 0 0);
|
||||||
|
--md-sys-color-primary-fixed-dim: rgb(173 204 255);
|
||||||
|
--md-sys-color-on-primary-fixed-variant: rgb(0 22 49);
|
||||||
|
--md-sys-color-secondary-fixed: rgb(214 233 255);
|
||||||
|
--md-sys-color-on-secondary-fixed: rgb(0 0 0);
|
||||||
|
--md-sys-color-secondary-fixed-dim: rgb(159 208 255);
|
||||||
|
--md-sys-color-on-secondary-fixed-variant: rgb(0 24 42);
|
||||||
|
--md-sys-color-tertiary-fixed: rgb(249 222 255);
|
||||||
|
--md-sys-color-on-tertiary-fixed: rgb(0 0 0);
|
||||||
|
--md-sys-color-tertiary-fixed-dim: rgb(222 193 231);
|
||||||
|
--md-sys-color-on-tertiary-fixed-variant: rgb(33 14 42);
|
||||||
|
--md-sys-color-surface-dim: rgb(17 19 24);
|
||||||
|
--md-sys-color-surface-bright: rgb(55 57 62);
|
||||||
|
--md-sys-color-surface-container-lowest: rgb(12 14 19);
|
||||||
|
--md-sys-color-surface-container-low: rgb(25 28 32);
|
||||||
|
--md-sys-color-surface-container: rgb(29 32 36);
|
||||||
|
--md-sys-color-surface-container-high: rgb(40 42 47);
|
||||||
|
--md-sys-color-surface-container-highest: rgb(50 53 58);
|
||||||
|
}
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
.dark-medium-contrast {
|
||||||
|
--md-sys-color-primary: rgb(173 204 255);
|
||||||
|
--md-sys-color-surface-tint: rgb(165 200 255);
|
||||||
|
--md-sys-color-on-primary: rgb(0 22 49);
|
||||||
|
--md-sys-color-primary-container: rgb(112 146 198);
|
||||||
|
--md-sys-color-on-primary-container: rgb(0 0 0);
|
||||||
|
--md-sys-color-secondary: rgb(159 208 255);
|
||||||
|
--md-sys-color-on-secondary: rgb(0 24 42);
|
||||||
|
--md-sys-color-secondary-container: rgb(100 149 193);
|
||||||
|
--md-sys-color-on-secondary-container: rgb(0 0 0);
|
||||||
|
--md-sys-color-tertiary: rgb(222 193 231);
|
||||||
|
--md-sys-color-on-tertiary: rgb(33 14 42);
|
||||||
|
--md-sys-color-tertiary-container: rgb(162 136 171);
|
||||||
|
--md-sys-color-on-tertiary-container: rgb(0 0 0);
|
||||||
|
--md-sys-color-error: rgb(255 186 177);
|
||||||
|
--md-sys-color-on-error: rgb(55 0 1);
|
||||||
|
--md-sys-color-error-container: rgb(255 84 73);
|
||||||
|
--md-sys-color-on-error-container: rgb(0 0 0);
|
||||||
|
--md-sys-color-background: rgb(17 19 24);
|
||||||
|
--md-sys-color-on-background: rgb(225 226 233);
|
||||||
|
--md-sys-color-surface: rgb(17 19 24);
|
||||||
|
--md-sys-color-on-surface: rgb(251 250 255);
|
||||||
|
--md-sys-color-surface-variant: rgb(67 71 78);
|
||||||
|
--md-sys-color-on-surface-variant: rgb(200 202 212);
|
||||||
|
--md-sys-color-outline: rgb(160 163 171);
|
||||||
|
--md-sys-color-outline-variant: rgb(128 131 139);
|
||||||
|
--md-sys-color-shadow: rgb(0 0 0);
|
||||||
|
--md-sys-color-scrim: rgb(0 0 0);
|
||||||
|
--md-sys-color-inverse-surface: rgb(225 226 233);
|
||||||
|
--md-sys-color-inverse-on-surface: rgb(40 42 47);
|
||||||
|
--md-sys-color-inverse-primary: rgb(35 73 120);
|
||||||
|
--md-sys-color-primary-fixed: rgb(212 227 255);
|
||||||
|
--md-sys-color-on-primary-fixed: rgb(0 17 40);
|
||||||
|
--md-sys-color-primary-fixed-dim: rgb(165 200 255);
|
||||||
|
--md-sys-color-on-primary-fixed-variant: rgb(10 55 100);
|
||||||
|
--md-sys-color-secondary-fixed: rgb(206 229 255);
|
||||||
|
--md-sys-color-on-secondary-fixed: rgb(0 18 34);
|
||||||
|
--md-sys-color-secondary-fixed-dim: rgb(154 203 250);
|
||||||
|
--md-sys-color-on-secondary-fixed-variant: rgb(0 57 91);
|
||||||
|
--md-sys-color-tertiary-fixed: rgb(247 216 255);
|
||||||
|
--md-sys-color-on-tertiary-fixed: rgb(28 9 37);
|
||||||
|
--md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
|
||||||
|
--md-sys-color-on-tertiary-fixed-variant: rgb(67 46 76);
|
||||||
|
--md-sys-color-surface-dim: rgb(17 19 24);
|
||||||
|
--md-sys-color-surface-bright: rgb(55 57 62);
|
||||||
|
--md-sys-color-surface-container-lowest: rgb(12 14 19);
|
||||||
|
--md-sys-color-surface-container-low: rgb(25 28 32);
|
||||||
|
--md-sys-color-surface-container: rgb(29 32 36);
|
||||||
|
--md-sys-color-surface-container-high: rgb(40 42 47);
|
||||||
|
--md-sys-color-surface-container-highest: rgb(50 53 58);
|
||||||
|
}
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
.dark {
|
||||||
|
--md-sys-color-primary: rgb(165 200 255);
|
||||||
|
--md-sys-color-surface-tint: rgb(165 200 255);
|
||||||
|
--md-sys-color-on-primary: rgb(0 49 94);
|
||||||
|
--md-sys-color-primary-container: rgb(34 72 118);
|
||||||
|
--md-sys-color-on-primary-container: rgb(212 227 255);
|
||||||
|
--md-sys-color-secondary: rgb(154 203 250);
|
||||||
|
--md-sys-color-on-secondary: rgb(0 51 83);
|
||||||
|
--md-sys-color-secondary-container: rgb(12 74 114);
|
||||||
|
--md-sys-color-on-secondary-container: rgb(206 229 255);
|
||||||
|
--md-sys-color-tertiary: rgb(218 189 226);
|
||||||
|
--md-sys-color-on-tertiary: rgb(61 41 70);
|
||||||
|
--md-sys-color-tertiary-container: rgb(85 63 93);
|
||||||
|
--md-sys-color-on-tertiary-container: rgb(247 216 255);
|
||||||
|
--md-sys-color-error: rgb(255 180 171);
|
||||||
|
--md-sys-color-on-error: rgb(105 0 5);
|
||||||
|
--md-sys-color-error-container: rgb(147 0 10);
|
||||||
|
--md-sys-color-on-error-container: rgb(255 218 214);
|
||||||
|
--md-sys-color-background: rgb(17 19 24);
|
||||||
|
--md-sys-color-on-background: rgb(225 226 233);
|
||||||
|
--md-sys-color-surface: rgb(17 19 24);
|
||||||
|
--md-sys-color-on-surface: rgb(225 226 233);
|
||||||
|
--md-sys-color-surface-variant: rgb(67 71 78);
|
||||||
|
--md-sys-color-on-surface-variant: rgb(195 198 207);
|
||||||
|
--md-sys-color-outline: rgb(141 145 153);
|
||||||
|
--md-sys-color-outline-variant: rgb(67 71 78);
|
||||||
|
--md-sys-color-shadow: rgb(0 0 0);
|
||||||
|
--md-sys-color-scrim: rgb(0 0 0);
|
||||||
|
--md-sys-color-inverse-surface: rgb(225 226 233);
|
||||||
|
--md-sys-color-inverse-on-surface: rgb(46 48 53);
|
||||||
|
--md-sys-color-inverse-primary: rgb(60 96 144);
|
||||||
|
--md-sys-color-primary-fixed: rgb(212 227 255);
|
||||||
|
--md-sys-color-on-primary-fixed: rgb(0 28 58);
|
||||||
|
--md-sys-color-primary-fixed-dim: rgb(165 200 255);
|
||||||
|
--md-sys-color-on-primary-fixed-variant: rgb(34 72 118);
|
||||||
|
--md-sys-color-secondary-fixed: rgb(206 229 255);
|
||||||
|
--md-sys-color-on-secondary-fixed: rgb(0 29 50);
|
||||||
|
--md-sys-color-secondary-fixed-dim: rgb(154 203 250);
|
||||||
|
--md-sys-color-on-secondary-fixed-variant: rgb(12 74 114);
|
||||||
|
--md-sys-color-tertiary-fixed: rgb(247 216 255);
|
||||||
|
--md-sys-color-on-tertiary-fixed: rgb(39 20 48);
|
||||||
|
--md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
|
||||||
|
--md-sys-color-on-tertiary-fixed-variant: rgb(85 63 93);
|
||||||
|
--md-sys-color-surface-dim: rgb(17 19 24);
|
||||||
|
--md-sys-color-surface-bright: rgb(55 57 62);
|
||||||
|
--md-sys-color-surface-container-lowest: rgb(12 14 19);
|
||||||
|
--md-sys-color-surface-container-low: rgb(25 28 32);
|
||||||
|
--md-sys-color-surface-container: rgb(29 32 36);
|
||||||
|
--md-sys-color-surface-container-high: rgb(40 42 47);
|
||||||
|
--md-sys-color-surface-container-highest: rgb(50 53 58);
|
||||||
|
}
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
.light-high-contrast {
|
||||||
|
--md-sys-color-primary: rgb(0 34 70);
|
||||||
|
--md-sys-color-surface-tint: rgb(60 96 144);
|
||||||
|
--md-sys-color-on-primary: rgb(255 255 255);
|
||||||
|
--md-sys-color-primary-container: rgb(29 68 114);
|
||||||
|
--md-sys-color-on-primary-container: rgb(255 255 255);
|
||||||
|
--md-sys-color-secondary: rgb(0 36 61);
|
||||||
|
--md-sys-color-on-secondary: rgb(255 255 255);
|
||||||
|
--md-sys-color-secondary-container: rgb(3 70 110);
|
||||||
|
--md-sys-color-on-secondary-container: rgb(255 255 255);
|
||||||
|
--md-sys-color-tertiary: rgb(46 26 55);
|
||||||
|
--md-sys-color-on-tertiary: rgb(255 255 255);
|
||||||
|
--md-sys-color-tertiary-container: rgb(81 59 89);
|
||||||
|
--md-sys-color-on-tertiary-container: rgb(255 255 255);
|
||||||
|
--md-sys-color-error: rgb(78 0 2);
|
||||||
|
--md-sys-color-on-error: rgb(255 255 255);
|
||||||
|
--md-sys-color-error-container: rgb(140 0 9);
|
||||||
|
--md-sys-color-on-error-container: rgb(255 255 255);
|
||||||
|
--md-sys-color-background: rgb(249 249 255);
|
||||||
|
--md-sys-color-on-background: rgb(25 28 32);
|
||||||
|
--md-sys-color-surface: rgb(249 249 255);
|
||||||
|
--md-sys-color-on-surface: rgb(0 0 0);
|
||||||
|
--md-sys-color-surface-variant: rgb(224 226 236);
|
||||||
|
--md-sys-color-on-surface-variant: rgb(32 36 43);
|
||||||
|
--md-sys-color-outline: rgb(63 67 74);
|
||||||
|
--md-sys-color-outline-variant: rgb(63 67 74);
|
||||||
|
--md-sys-color-shadow: rgb(0 0 0);
|
||||||
|
--md-sys-color-scrim: rgb(0 0 0);
|
||||||
|
--md-sys-color-inverse-surface: rgb(46 48 53);
|
||||||
|
--md-sys-color-inverse-on-surface: rgb(255 255 255);
|
||||||
|
--md-sys-color-inverse-primary: rgb(228 236 255);
|
||||||
|
--md-sys-color-primary-fixed: rgb(29 68 114);
|
||||||
|
--md-sys-color-on-primary-fixed: rgb(255 255 255);
|
||||||
|
--md-sys-color-primary-fixed-dim: rgb(0 45 88);
|
||||||
|
--md-sys-color-on-primary-fixed-variant: rgb(255 255 255);
|
||||||
|
--md-sys-color-secondary-fixed: rgb(3 70 110);
|
||||||
|
--md-sys-color-on-secondary-fixed: rgb(255 255 255);
|
||||||
|
--md-sys-color-secondary-fixed-dim: rgb(0 47 77);
|
||||||
|
--md-sys-color-on-secondary-fixed-variant: rgb(255 255 255);
|
||||||
|
--md-sys-color-tertiary-fixed: rgb(81 59 89);
|
||||||
|
--md-sys-color-on-tertiary-fixed: rgb(255 255 255);
|
||||||
|
--md-sys-color-tertiary-fixed-dim: rgb(57 37 66);
|
||||||
|
--md-sys-color-on-tertiary-fixed-variant: rgb(255 255 255);
|
||||||
|
--md-sys-color-surface-dim: rgb(217 218 224);
|
||||||
|
--md-sys-color-surface-bright: rgb(249 249 255);
|
||||||
|
--md-sys-color-surface-container-lowest: rgb(255 255 255);
|
||||||
|
--md-sys-color-surface-container-low: rgb(243 243 250);
|
||||||
|
--md-sys-color-surface-container: rgb(237 237 244);
|
||||||
|
--md-sys-color-surface-container-high: rgb(231 232 238);
|
||||||
|
--md-sys-color-surface-container-highest: rgb(225 226 233);
|
||||||
|
}
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
.light-medium-contrast {
|
||||||
|
--md-sys-color-primary: rgb(29 68 114);
|
||||||
|
--md-sys-color-surface-tint: rgb(60 96 144);
|
||||||
|
--md-sys-color-on-primary: rgb(255 255 255);
|
||||||
|
--md-sys-color-primary-container: rgb(83 118 167);
|
||||||
|
--md-sys-color-on-primary-container: rgb(255 255 255);
|
||||||
|
--md-sys-color-secondary: rgb(3 70 110);
|
||||||
|
--md-sys-color-on-secondary: rgb(255 255 255);
|
||||||
|
--md-sys-color-secondary-container: rgb(71 121 163);
|
||||||
|
--md-sys-color-on-secondary-container: rgb(255 255 255);
|
||||||
|
--md-sys-color-tertiary: rgb(81 59 89);
|
||||||
|
--md-sys-color-on-tertiary: rgb(255 255 255);
|
||||||
|
--md-sys-color-tertiary-container: rgb(133 108 141);
|
||||||
|
--md-sys-color-on-tertiary-container: rgb(255 255 255);
|
||||||
|
--md-sys-color-error: rgb(140 0 9);
|
||||||
|
--md-sys-color-on-error: rgb(255 255 255);
|
||||||
|
--md-sys-color-error-container: rgb(218 52 46);
|
||||||
|
--md-sys-color-on-error-container: rgb(255 255 255);
|
||||||
|
--md-sys-color-background: rgb(249 249 255);
|
||||||
|
--md-sys-color-on-background: rgb(25 28 32);
|
||||||
|
--md-sys-color-surface: rgb(249 249 255);
|
||||||
|
--md-sys-color-on-surface: rgb(25 28 32);
|
||||||
|
--md-sys-color-surface-variant: rgb(224 226 236);
|
||||||
|
--md-sys-color-on-surface-variant: rgb(63 67 74);
|
||||||
|
--md-sys-color-outline: rgb(91 95 103);
|
||||||
|
--md-sys-color-outline-variant: rgb(119 123 131);
|
||||||
|
--md-sys-color-shadow: rgb(0 0 0);
|
||||||
|
--md-sys-color-scrim: rgb(0 0 0);
|
||||||
|
--md-sys-color-inverse-surface: rgb(46 48 53);
|
||||||
|
--md-sys-color-inverse-on-surface: rgb(240 240 247);
|
||||||
|
--md-sys-color-inverse-primary: rgb(165 200 255);
|
||||||
|
--md-sys-color-primary-fixed: rgb(83 118 167);
|
||||||
|
--md-sys-color-on-primary-fixed: rgb(255 255 255);
|
||||||
|
--md-sys-color-primary-fixed-dim: rgb(58 93 141);
|
||||||
|
--md-sys-color-on-primary-fixed-variant: rgb(255 255 255);
|
||||||
|
--md-sys-color-secondary-fixed: rgb(71 121 163);
|
||||||
|
--md-sys-color-on-secondary-fixed: rgb(255 255 255);
|
||||||
|
--md-sys-color-secondary-fixed-dim: rgb(43 96 137);
|
||||||
|
--md-sys-color-on-secondary-fixed-variant: rgb(255 255 255);
|
||||||
|
--md-sys-color-tertiary-fixed: rgb(133 108 141);
|
||||||
|
--md-sys-color-on-tertiary-fixed: rgb(255 255 255);
|
||||||
|
--md-sys-color-tertiary-fixed-dim: rgb(107 84 116);
|
||||||
|
--md-sys-color-on-tertiary-fixed-variant: rgb(255 255 255);
|
||||||
|
--md-sys-color-surface-dim: rgb(217 218 224);
|
||||||
|
--md-sys-color-surface-bright: rgb(249 249 255);
|
||||||
|
--md-sys-color-surface-container-lowest: rgb(255 255 255);
|
||||||
|
--md-sys-color-surface-container-low: rgb(243 243 250);
|
||||||
|
--md-sys-color-surface-container: rgb(237 237 244);
|
||||||
|
--md-sys-color-surface-container-high: rgb(231 232 238);
|
||||||
|
--md-sys-color-surface-container-highest: rgb(225 226 233);
|
||||||
|
}
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
.light {
|
||||||
|
--md-sys-color-primary: rgb(60 96 144);
|
||||||
|
--md-sys-color-surface-tint: rgb(60 96 144);
|
||||||
|
--md-sys-color-on-primary: rgb(255 255 255);
|
||||||
|
--md-sys-color-primary-container: rgb(212 227 255);
|
||||||
|
--md-sys-color-on-primary-container: rgb(0 28 58);
|
||||||
|
--md-sys-color-secondary: rgb(46 98 140);
|
||||||
|
--md-sys-color-on-secondary: rgb(255 255 255);
|
||||||
|
--md-sys-color-secondary-container: rgb(206 229 255);
|
||||||
|
--md-sys-color-on-secondary-container: rgb(0 29 50);
|
||||||
|
--md-sys-color-tertiary: rgb(110 86 118);
|
||||||
|
--md-sys-color-on-tertiary: rgb(255 255 255);
|
||||||
|
--md-sys-color-tertiary-container: rgb(247 216 255);
|
||||||
|
--md-sys-color-on-tertiary-container: rgb(39 20 48);
|
||||||
|
--md-sys-color-error: rgb(186 26 26);
|
||||||
|
--md-sys-color-on-error: rgb(255 255 255);
|
||||||
|
--md-sys-color-error-container: rgb(255 218 214);
|
||||||
|
--md-sys-color-on-error-container: rgb(65 0 2);
|
||||||
|
--md-sys-color-background: rgb(249 249 255);
|
||||||
|
--md-sys-color-on-background: rgb(25 28 32);
|
||||||
|
--md-sys-color-surface: rgb(249 249 255);
|
||||||
|
--md-sys-color-on-surface: rgb(25 28 32);
|
||||||
|
--md-sys-color-surface-variant: rgb(224 226 236);
|
||||||
|
--md-sys-color-on-surface-variant: rgb(67 71 78);
|
||||||
|
--md-sys-color-outline: rgb(116 119 127);
|
||||||
|
--md-sys-color-outline-variant: rgb(195 198 207);
|
||||||
|
--md-sys-color-shadow: rgb(0 0 0);
|
||||||
|
--md-sys-color-scrim: rgb(0 0 0);
|
||||||
|
--md-sys-color-inverse-surface: rgb(46 48 53);
|
||||||
|
--md-sys-color-inverse-on-surface: rgb(240 240 247);
|
||||||
|
--md-sys-color-inverse-primary: rgb(165 200 255);
|
||||||
|
--md-sys-color-primary-fixed: rgb(212 227 255);
|
||||||
|
--md-sys-color-on-primary-fixed: rgb(0 28 58);
|
||||||
|
--md-sys-color-primary-fixed-dim: rgb(165 200 255);
|
||||||
|
--md-sys-color-on-primary-fixed-variant: rgb(34 72 118);
|
||||||
|
--md-sys-color-secondary-fixed: rgb(206 229 255);
|
||||||
|
--md-sys-color-on-secondary-fixed: rgb(0 29 50);
|
||||||
|
--md-sys-color-secondary-fixed-dim: rgb(154 203 250);
|
||||||
|
--md-sys-color-on-secondary-fixed-variant: rgb(12 74 114);
|
||||||
|
--md-sys-color-tertiary-fixed: rgb(247 216 255);
|
||||||
|
--md-sys-color-on-tertiary-fixed: rgb(39 20 48);
|
||||||
|
--md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
|
||||||
|
--md-sys-color-on-tertiary-fixed-variant: rgb(85 63 93);
|
||||||
|
--md-sys-color-surface-dim: rgb(217 218 224);
|
||||||
|
--md-sys-color-surface-bright: rgb(249 249 255);
|
||||||
|
--md-sys-color-surface-container-lowest: rgb(255 255 255);
|
||||||
|
--md-sys-color-surface-container-low: rgb(243 243 250);
|
||||||
|
--md-sys-color-surface-container: rgb(237 237 244);
|
||||||
|
--md-sys-color-surface-container-high: rgb(231 232 238);
|
||||||
|
--md-sys-color-surface-container-highest: rgb(225 226 233);
|
||||||
|
}
|
||||||
@@ -0,0 +1,107 @@
|
|||||||
|
:root {
|
||||||
|
font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
|
||||||
|
line-height: 1.5;
|
||||||
|
font-weight: 400;
|
||||||
|
|
||||||
|
color-scheme: light dark;
|
||||||
|
color: rgba(255, 255, 255, 0.87);
|
||||||
|
background-color: #242424;
|
||||||
|
|
||||||
|
font-synthesis: none;
|
||||||
|
text-rendering: optimizeLegibility;
|
||||||
|
-webkit-font-smoothing: antialiased;
|
||||||
|
-moz-osx-font-smoothing: grayscale;
|
||||||
|
}
|
||||||
|
|
||||||
|
a {
|
||||||
|
font-weight: 500;
|
||||||
|
color: #646cff;
|
||||||
|
text-decoration: inherit;
|
||||||
|
}
|
||||||
|
a:hover {
|
||||||
|
color: #535bf2;
|
||||||
|
}
|
||||||
|
|
||||||
|
body {
|
||||||
|
margin: 0;
|
||||||
|
display: flex;
|
||||||
|
place-items: center;
|
||||||
|
min-width: 320px;
|
||||||
|
min-height: 100vh;
|
||||||
|
}
|
||||||
|
|
||||||
|
button {
|
||||||
|
border-radius: 8px;
|
||||||
|
border: 1px solid transparent;
|
||||||
|
padding: 0.6em 1.2em;
|
||||||
|
font-size: 1em;
|
||||||
|
font-weight: 500;
|
||||||
|
font-family: inherit;
|
||||||
|
background-color: #1a1a1a;
|
||||||
|
cursor: pointer;
|
||||||
|
transition: border-color 0.25s;
|
||||||
|
}
|
||||||
|
button:hover {
|
||||||
|
border-color: #646cff;
|
||||||
|
}
|
||||||
|
button:focus,
|
||||||
|
button:focus-visible {
|
||||||
|
outline: 4px auto -webkit-focus-ring-color;
|
||||||
|
}
|
||||||
|
|
||||||
|
button#query-button {
|
||||||
|
height: 5em;
|
||||||
|
vertical-align: top;
|
||||||
|
}
|
||||||
|
|
||||||
|
button#play-button {
|
||||||
|
padding: '10px 20px';
|
||||||
|
font-size: '16px';
|
||||||
|
cursor: 'pointer';
|
||||||
|
background-color: '#28a745';
|
||||||
|
color: '#fff';
|
||||||
|
border: 'none';
|
||||||
|
border-radius: '5px';
|
||||||
|
};
|
||||||
|
|
||||||
|
button#pause-button {
|
||||||
|
background-color: #ffc107;
|
||||||
|
}
|
||||||
|
|
||||||
|
button#stop-button {
|
||||||
|
background-color: #dc3545;
|
||||||
|
}
|
||||||
|
|
||||||
|
h1 {
|
||||||
|
font-size: 3.2em;
|
||||||
|
line-height: 1.1;
|
||||||
|
}
|
||||||
|
|
||||||
|
p#footer {
|
||||||
|
float: none;
|
||||||
|
clear: both;
|
||||||
|
vertical-align: bottom;
|
||||||
|
}
|
||||||
|
|
||||||
|
table#query-output {
|
||||||
|
float: none;
|
||||||
|
clear: both;
|
||||||
|
}
|
||||||
|
|
||||||
|
textarea#query-input {
|
||||||
|
height: 5em;
|
||||||
|
width: 50em;
|
||||||
|
}
|
||||||
|
|
||||||
|
@media (prefers-color-scheme: light) {
|
||||||
|
:root {
|
||||||
|
color: #213547;
|
||||||
|
background-color: #ffffff;
|
||||||
|
}
|
||||||
|
a:hover {
|
||||||
|
color: #747bff;
|
||||||
|
}
|
||||||
|
button {
|
||||||
|
background-color: #f9f9f9;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
import React from 'react';
|
||||||
|
import ReactDOM from 'react-dom/client';
|
||||||
|
import App from './App';
|
||||||
|
|
||||||
|
ReactDOM.createRoot(document.getElementById('root')).render(
|
||||||
|
<React.StrictMode>
|
||||||
|
<App />
|
||||||
|
</React.StrictMode>
|
||||||
|
)
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
import { fileURLToPath, URL } from 'node:url';
|
||||||
|
import { defineConfig } from 'vite';
|
||||||
|
import plugin from '@vitejs/plugin-react';
|
||||||
|
import fs from 'fs';
|
||||||
|
import path from 'path';
|
||||||
|
import child_process from 'child_process';
|
||||||
|
import process, { env } from 'process';
|
||||||
|
|
||||||
|
const baseFolder =
|
||||||
|
process.env.APPDATA !== undefined && process.env.APPDATA !== ''
|
||||||
|
? `${process.env.APPDATA}/ASP.NET/https`
|
||||||
|
: `${process.env.HOME}/.aspnet/https`;
|
||||||
|
|
||||||
|
const certificateName = "bts.datastreamwebapp.client";
|
||||||
|
const certFilePath = path.join(baseFolder, `${certificateName}.pem`);
|
||||||
|
const keyFilePath = path.join(baseFolder, `${certificateName}.key`);
|
||||||
|
|
||||||
|
if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) {
|
||||||
|
if (0 !== child_process.spawnSync('dotnet', [
|
||||||
|
'dev-certs',
|
||||||
|
'https',
|
||||||
|
'--export-path',
|
||||||
|
certFilePath,
|
||||||
|
'--format',
|
||||||
|
'Pem',
|
||||||
|
'--no-password',
|
||||||
|
], { stdio: 'inherit', }).status) {
|
||||||
|
throw new Error("Could not create certificate.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const target = env.ASPNETCORE_HTTPS_PORT
|
||||||
|
? `https://localhost:${env.ASPNETCORE_HTTPS_PORT}`
|
||||||
|
: env.ASPNETCORE_URLS
|
||||||
|
? env.ASPNETCORE_URLS.split(';')[0]
|
||||||
|
: 'https://localhost:5001';
|
||||||
|
|
||||||
|
export default defineConfig(({ $$command, mode, $$isSsrBuild, $$isPreview }) => {
|
||||||
|
return {
|
||||||
|
plugins: [plugin()],
|
||||||
|
resolve: {
|
||||||
|
alias: {
|
||||||
|
'@': fileURLToPath(new URL('./src', import.meta.url))
|
||||||
|
}
|
||||||
|
},
|
||||||
|
build: {
|
||||||
|
minify: false,
|
||||||
|
sourcemap: true,
|
||||||
|
outDir: `../../../build/Bts.DataStreamWebApp.Client/${mode}/dist`
|
||||||
|
},
|
||||||
|
server: {
|
||||||
|
proxy: {
|
||||||
|
'^/api': {
|
||||||
|
target,
|
||||||
|
secure: false
|
||||||
|
},
|
||||||
|
'^/ws': {
|
||||||
|
target,
|
||||||
|
secure: false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
port: 5173,
|
||||||
|
https: {
|
||||||
|
key: fs.readFileSync(keyFilePath),
|
||||||
|
cert: fs.readFileSync(certFilePath),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
});
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
<Project Sdk="Microsoft.NET.Sdk.Web">
|
||||||
|
|
||||||
|
<PropertyGroup>
|
||||||
|
<TargetFramework>net7.0</TargetFramework>
|
||||||
|
<Nullable>enable</Nullable>
|
||||||
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
|
<InvariantGlobalization>true</InvariantGlobalization>
|
||||||
|
<SpaRoot>..\Bts.DataStreamWebApp.Client</SpaRoot>
|
||||||
|
<SpaProxyLaunchCommand>npm run dev</SpaProxyLaunchCommand>
|
||||||
|
<SpaProxyServerUrl>https://localhost:5173</SpaProxyServerUrl>
|
||||||
|
</PropertyGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="CommandLineParser" Version="2.9.1" />
|
||||||
|
<PackageReference Include="Microsoft.AspNetCore.SpaProxy" Version="7.0.20" />
|
||||||
|
<PackageReference Include="Microsoft.AspNetCore.SpaServices.Extensions" Version="7.0.20" />
|
||||||
|
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
|
||||||
|
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<ProjectReference Include="..\Bts.DataStreamWebApp.Client\Bts.DataStreamWebApp.Client.esproj">
|
||||||
|
<ReferenceOutputAssembly>false</ReferenceOutputAssembly>
|
||||||
|
</ProjectReference>
|
||||||
|
<ProjectReference Include="..\DataStream\DataStream.csproj" />
|
||||||
|
<ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
|
||||||
|
<ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
|
||||||
|
<ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
|
||||||
|
<ProjectReference Include="..\..\client\spark_websocket\spark_websocket.csproj" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
<!-- <ItemGroup>
|
||||||
|
<Content Update="wwwroot\alt-index.html" CopyToPublishDirectory="Never" />
|
||||||
|
</ItemGroup> -->
|
||||||
|
</Project>
|
||||||
@@ -0,0 +1,106 @@
|
|||||||
|
{
|
||||||
|
"MasterWorker": {
|
||||||
|
"Interval": "00:00:00.1"
|
||||||
|
},
|
||||||
|
"ViewStreamDatabase": {
|
||||||
|
"Interval": "00:00:00.1"
|
||||||
|
},
|
||||||
|
"Sparkd": {
|
||||||
|
"Config": {
|
||||||
|
"ConfigServer": "http://localhost:8000/config"
|
||||||
|
},
|
||||||
|
"Credentials": {
|
||||||
|
"User": "bts",
|
||||||
|
"Password": ""
|
||||||
|
},
|
||||||
|
"EnableAdmin": true,
|
||||||
|
"EnableOrders": true,
|
||||||
|
"EnablePosition": true
|
||||||
|
},
|
||||||
|
"PubSub": {
|
||||||
|
"MillisecondsBetweenSubInfoUpdates": 1000,
|
||||||
|
"MillisecondsBetweenTimeSliceUpdates": 5000,
|
||||||
|
"MillisecondsBetweenNoBufferSpaceRetries": 100,
|
||||||
|
"RetriesOnNoBufferSpace": 1,
|
||||||
|
"DoItAllSecondsBetweenLogs": 0,
|
||||||
|
"HeartbeatInterval": 15000,
|
||||||
|
"MissingHeartbeatsBeforeGc": 50,
|
||||||
|
"UseManagedSockets": true,
|
||||||
|
"StateUpdatePeriod": 5000,
|
||||||
|
"StatisticsUpdatePeriod": 5000,
|
||||||
|
"DisposedConnectionCheckPeriod": 1000,
|
||||||
|
"MessageProcessingWarningThreshold": 1000,
|
||||||
|
"ServerPort": 12345,
|
||||||
|
"Routing": {
|
||||||
|
"PublisherRoutes": [
|
||||||
|
{
|
||||||
|
"Name": "default",
|
||||||
|
"ServerAddress": "bts-cloud://127.0.0.1:12345",
|
||||||
|
"TagRegExp": ".*"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"SubscriberRoutes": [
|
||||||
|
{
|
||||||
|
"Name": "default",
|
||||||
|
"ServerAddress": "bts-cloud://127.0.0.1:12345",
|
||||||
|
"TagRegExp": ".*"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"MaxPacketSize": 500000000,
|
||||||
|
"BlueCommMustDie": true,
|
||||||
|
"CrushableTags": [],
|
||||||
|
"TagTypeToAssemblyMap": {
|
||||||
|
"Entries": [
|
||||||
|
{
|
||||||
|
"HandlerClass": "BCG.PubSubServerTagHandler",
|
||||||
|
"Assembly": "DataStreamCore.dll"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"ServerQueue": "PubSubQueue",
|
||||||
|
"PortRange": {
|
||||||
|
"First": 30400,
|
||||||
|
"Last": 31700
|
||||||
|
},
|
||||||
|
"BindToRandomPort": false,
|
||||||
|
"DebugTags": "^NOTAGS$",
|
||||||
|
"LatencyReportThresholdMs": 1500,
|
||||||
|
"UseSubstations": false,
|
||||||
|
"UseDynamicClients": false,
|
||||||
|
"IsSubstation": false,
|
||||||
|
"InternallyPublishSubInfo": true,
|
||||||
|
"UseManagedClientsInNativeCode": false,
|
||||||
|
"CompressedTags": "^NOTAGS$",
|
||||||
|
"TagsToReportCompressionRatio": "^NOTAGS$",
|
||||||
|
"DisableMsmq": false,
|
||||||
|
"UsePrepublishedServerState": false,
|
||||||
|
"IpTranslation": null,
|
||||||
|
"GcActivationThreshold": 0,
|
||||||
|
"WaitPeriodAfterGc": 1000,
|
||||||
|
"TagHandlerFactory": {
|
||||||
|
"Handlers": [
|
||||||
|
{
|
||||||
|
"TagType": "DataStream",
|
||||||
|
"Assembly": "DataStreamCore",
|
||||||
|
"Class": "bts.PubsubServerStreamTagHandler"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"TagType": "System_Sub",
|
||||||
|
"Assembly": "PubSub",
|
||||||
|
"Class": "BCG.SystemSubscriptionMapPublisher"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"TagType": "System_Stats",
|
||||||
|
"Assembly": "PubSub",
|
||||||
|
"Class": "BCG.SystemStatsPublisher"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"TagType": ".*",
|
||||||
|
"Assembly": "DataStreamCore",
|
||||||
|
"Class": "bts.PubsubServerGenericDataHandler"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
{
|
||||||
|
"content_root_path": "build/Bts.DataStreamWebApp.Client/production",
|
||||||
|
"web_root_path": "dist"
|
||||||
|
}
|
||||||
+30
@@ -0,0 +1,30 @@
|
|||||||
|
namespace Bts.DataStreamWebApp.Server.Controllers;
|
||||||
|
|
||||||
|
using Microsoft.AspNetCore.Mvc;
|
||||||
|
|
||||||
|
[Route("/ws")]
|
||||||
|
public class DataStreamSubscriptionController : ControllerBase {
|
||||||
|
readonly ILogger logger;
|
||||||
|
|
||||||
|
public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger) {
|
||||||
|
this.logger = logger;
|
||||||
|
logger.LogInformation("Creating {ControllerType}", GetType().Name);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Route("/ws")]
|
||||||
|
public async Task Get(CancellationToken cancellationToken) {
|
||||||
|
if (!HttpContext.WebSockets.IsWebSocketRequest) {
|
||||||
|
logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
logger.LogInformation("Establishing websocket connection...");
|
||||||
|
try {
|
||||||
|
using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
|
||||||
|
logger.LogInformation("Websocket connection established.");
|
||||||
|
using var server = new DataStreamWebSocketServer(webSocket, logger);
|
||||||
|
await server.Run(cancellationToken);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
logger.LogError("DataStreamSubscriptionController.Run threw exception: {Exception}", ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,125 @@
|
|||||||
|
namespace Bts.DataStreamWebApp.Server.Controllers;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using bts.websocket_core.json_rpc;
|
||||||
|
|
||||||
|
using Bts.DataStreamCore;
|
||||||
|
using Bts.DataStreamWebApp.Server.Messages;
|
||||||
|
|
||||||
|
using System.Net.WebSockets;
|
||||||
|
using System.Text;
|
||||||
|
using System.Text.Json;
|
||||||
|
using System.Text.Json.Serialization;
|
||||||
|
|
||||||
|
class DataStreamSubscription : IDisposable, IHostedService {
|
||||||
|
public readonly ViewStream ViewStream;
|
||||||
|
public DataPacket? LastFired;
|
||||||
|
|
||||||
|
readonly ILogger logger;
|
||||||
|
readonly WebSocket webSocket;
|
||||||
|
|
||||||
|
public DataStreamSubscription(string id, JsonRpcId requestId, ViewStream viewStream, WebSocket webSocket, ILogger logger) {
|
||||||
|
ViewStream = viewStream;
|
||||||
|
RequestId = requestId;
|
||||||
|
Id = id;
|
||||||
|
this.webSocket = webSocket;
|
||||||
|
this.logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
public string Id { get; }
|
||||||
|
public JsonRpcId RequestId { get; }
|
||||||
|
public SubscriptionStatus Status { get; private set; }
|
||||||
|
|
||||||
|
public void Start() {
|
||||||
|
if (Status == SubscriptionStatus.Subscribed) {
|
||||||
|
logger.LogWarning("Ignoring redundant call to 'Start'. Id = {Id} Status = {Status}", Id, Status);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
logger.LogInformation("Subscribing to ViewStream. SubscriptionId = {SubscriptionId}", Id);
|
||||||
|
Status = SubscriptionStatus.Subscribed;
|
||||||
|
ViewStream.onChange += onChange;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Task StartAsync(CancellationToken cancellationToken) {
|
||||||
|
cancellationToken.ThrowIfCancellationRequested();
|
||||||
|
Start();
|
||||||
|
return Task.CompletedTask;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Task Clear() {
|
||||||
|
if (LastFired is null) {
|
||||||
|
return Task.CompletedTask;
|
||||||
|
}
|
||||||
|
logger.LogInformation("Clearing subscription (firing DataPacket to delete all rows). Id = {Id} Status = {Status}", Id, Status);
|
||||||
|
return sendDataPacket("clear", new(new BlueTable(LastFired.schema), new BlueTable(LastFired.schema), LastFired.newRows));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Stop() {
|
||||||
|
if (Status == SubscriptionStatus.Unsubscribed) {
|
||||||
|
logger.LogWarning("Ignoring redundant call to 'Stop'. Id = {Id} Status = {Status}", Id, Status);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
logger.LogInformation("Unsubscribing from ViewStream. SubscriptionId = {SubscriptionId}", Id);
|
||||||
|
Status = SubscriptionStatus.Unsubscribed;
|
||||||
|
ViewStream.onChange -= onChange;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Task StopAsync(CancellationToken cancellationToken) {
|
||||||
|
cancellationToken.ThrowIfCancellationRequested();
|
||||||
|
Stop();
|
||||||
|
return Task.CompletedTask;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Dispose() =>
|
||||||
|
Stop();
|
||||||
|
|
||||||
|
async void onChange(object newValue) {
|
||||||
|
if (newValue is not DataPacket dp) {
|
||||||
|
logger.LogError("Expected DataPacket in stream callback (actually received {}).", newValue?.GetType());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (webSocket.State != WebSocketState.Open) {
|
||||||
|
logger.LogError(
|
||||||
|
"Unsubscribing from stream early because WebSocket state = {WebSocketState}", webSocket.State);
|
||||||
|
Dispose();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await sendDataPacket("subscribe", dp);
|
||||||
|
}
|
||||||
|
|
||||||
|
async Task sendDataPacket(string method, DataPacket dp) {
|
||||||
|
var serializerOpts = new JsonSerializerOptions {
|
||||||
|
NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals
|
||||||
|
};
|
||||||
|
serializerOpts.Converters.Add(new DataPacketJsonConverter() {
|
||||||
|
IncludeCurrent = true,
|
||||||
|
UseObjectRows = true,
|
||||||
|
});
|
||||||
|
var res = JsonRpcRequest.CreateNotification(method, dp);
|
||||||
|
var str = serialize(res, serializerOpts);
|
||||||
|
if (str is null) {
|
||||||
|
logger.LogError("Unsubscribing from stream early because we failed to serialize an outgoing DataPacket");
|
||||||
|
Dispose();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
logger.LogInformation(
|
||||||
|
"Sending DataPacket. NumRows = {NumRows} NumCols = {NumCols} MessageLength = {MessageLength}",
|
||||||
|
dp.newRows.Count,
|
||||||
|
dp.schema.columnNames.Length,
|
||||||
|
str.Length);
|
||||||
|
var bytes = Encoding.UTF8.GetBytes(str);
|
||||||
|
await webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, CancellationToken.None);
|
||||||
|
LastFired = dp;
|
||||||
|
}
|
||||||
|
|
||||||
|
string? serialize<T>(T message, JsonSerializerOptions? opts) {
|
||||||
|
opts ??= JsonSerializerOptions.Default;
|
||||||
|
try {
|
||||||
|
return JsonSerializer.Serialize(message, opts);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
logger.LogError("Failed to serialize message of type {MessageType}: {Exception}", typeof(T), ex);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,89 @@
|
|||||||
|
namespace Bts.DataStreamWebApp.Server;
|
||||||
|
|
||||||
|
using Bts.PubSub;
|
||||||
|
|
||||||
|
using CommandLine;
|
||||||
|
|
||||||
|
using System.Text.Json;
|
||||||
|
using System.Text.Json.Serialization;
|
||||||
|
|
||||||
|
class DataStreamWebAppServerConfig {
|
||||||
|
/// <summary>
|
||||||
|
/// The login user for sparkd
|
||||||
|
/// </summary>
|
||||||
|
[JsonPropertyName("user")]
|
||||||
|
[Option("user", HelpText = "The sparkd login user")]
|
||||||
|
public string? User { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The login password for sparkd
|
||||||
|
/// </summary>
|
||||||
|
[JsonPropertyName("password")]
|
||||||
|
[Option("pass", HelpText = "The sparkd login password")]
|
||||||
|
public string? Password { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The config server url for sparkd
|
||||||
|
/// </summary>
|
||||||
|
[JsonPropertyName("sparkd")]
|
||||||
|
[Option("sparkd", HelpText = "The sparkd config server")]
|
||||||
|
public string? Sparkd { get; set; }
|
||||||
|
|
||||||
|
[JsonPropertyName("pubsub")]
|
||||||
|
public PubSubConfig? PubSub { get; set; }
|
||||||
|
|
||||||
|
[Option("content-root-path")]
|
||||||
|
[JsonPropertyName("content_root_path")]
|
||||||
|
public string? ContentRootPath { get; set; }
|
||||||
|
|
||||||
|
[Option("web-root-path")]
|
||||||
|
[JsonPropertyName("web_root_path")]
|
||||||
|
public string? WebRootPath { get; set; }
|
||||||
|
|
||||||
|
[Option("environment-name")]
|
||||||
|
[JsonPropertyName("environment_name")]
|
||||||
|
public string? EnvironmentName { get; set; }
|
||||||
|
|
||||||
|
public DirectoryBrowserOptions? DirectoryBrowser { get; set; }
|
||||||
|
|
||||||
|
[Option(
|
||||||
|
"config",
|
||||||
|
MetaValue = "PATH",
|
||||||
|
HelpText = "Path to json file representing these options (DataStreamPublisherConfig)")]
|
||||||
|
[JsonIgnore]
|
||||||
|
public string? ConfigFile { get; set; }
|
||||||
|
|
||||||
|
public static DataStreamWebAppServerConfig Default() =>
|
||||||
|
new() {
|
||||||
|
User = "bts",
|
||||||
|
PubSub = PubSubConfig.Default(),
|
||||||
|
};
|
||||||
|
|
||||||
|
public void MergeWith(DataStreamWebAppServerConfig other) {
|
||||||
|
User ??= other.User;
|
||||||
|
Password ??= other.Password;
|
||||||
|
PubSub ??= other.PubSub;
|
||||||
|
Sparkd ??= other.Sparkd;
|
||||||
|
ContentRootPath ??= other.ContentRootPath;
|
||||||
|
WebRootPath ??= other.WebRootPath;
|
||||||
|
ConfigFile ??= other.ConfigFile;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Merges options with those defined in `ConfigFile` (does not overwrite
|
||||||
|
/// non-null existing values).
|
||||||
|
/// </summary>
|
||||||
|
/// <exception cref="InvalidOperationException">
|
||||||
|
/// JsonSerializer.Deserialize returns null
|
||||||
|
/// </exception>
|
||||||
|
public void MergeWithConfigFile() {
|
||||||
|
if (ConfigFile is null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
var cfgString = File.ReadAllText(ConfigFile);
|
||||||
|
var parsedCfg = JsonSerializer.Deserialize<DataStreamWebAppServerConfig>(cfgString) ??
|
||||||
|
throw new InvalidOperationException($"Failed to parse json from {ConfigFile}");
|
||||||
|
MergeWith(parsedCfg);
|
||||||
|
ConfigFile = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,121 @@
|
|||||||
|
namespace Bts.DataStreamWebApp.Server.Controllers;
|
||||||
|
|
||||||
|
using bts.websocket_core.json_rpc;
|
||||||
|
|
||||||
|
using Bts.DataStreamWebApp.Server.Messages;
|
||||||
|
|
||||||
|
using System.Net.WebSockets;
|
||||||
|
using System.Text;
|
||||||
|
using System.Text.Json;
|
||||||
|
using System.Text.Json.Nodes;
|
||||||
|
|
||||||
|
public class DataStreamWebSocketServer : IDisposable, IAsyncDisposable {
|
||||||
|
readonly ILogger logger;
|
||||||
|
readonly WebSocket webSocket;
|
||||||
|
readonly Dictionary<string, DataStreamSubscription> subscriptions = new();
|
||||||
|
|
||||||
|
public DataStreamWebSocketServer(WebSocket webSocket, ILogger logger) {
|
||||||
|
this.webSocket = webSocket;
|
||||||
|
this.logger = logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<int> Run(CancellationToken cancellationToken) {
|
||||||
|
logger.LogInformation("DataStreamWebSocketServer running");
|
||||||
|
await foreach (var req in JsonRpcRequestAsyncEnumerable.Create(webSocket, logger, cancellationToken)) {
|
||||||
|
await (req.Method switch {
|
||||||
|
"subscribe" => onSubscribe(req, cancellationToken),
|
||||||
|
"unsubscribe" => onUnsubscribe(req.Deserialize<UnsubscribeRequest>(), cancellationToken),
|
||||||
|
_ => Task.CompletedTask,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
logger.LogInformation("DataStreamWebSocketServer stopped");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Disposes internal subscriptions. To additionally notify the client of the subscription status
|
||||||
|
/// change(s), use <see cref="DisposeAsync"/>.
|
||||||
|
/// </summary>
|
||||||
|
public void Dispose() {
|
||||||
|
GC.SuppressFinalize(this);
|
||||||
|
disposeSubscriptions();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Disposes internal subscriptions (like <see cref="Dispose"/>), then sends 'subscription status change'
|
||||||
|
/// notifications to the client (unlike <see cref="Dispose"/>).
|
||||||
|
/// </summary>
|
||||||
|
public async ValueTask DisposeAsync() {
|
||||||
|
GC.SuppressFinalize(this);
|
||||||
|
var finalSubs = subscriptions.Values.Select(s => (s.RequestId, s.Id)).ToList();
|
||||||
|
disposeSubscriptions();
|
||||||
|
// Send a 'subscription status change' msg to the client for each one.
|
||||||
|
foreach (var (requestId, id) in finalSubs) {
|
||||||
|
await sendSubscriptionStatus(requestId, id, "unsubscribe", SubscriptionStatus.Unsubscribed, CancellationToken.None);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async Task onSubscribe(JsonRpcRequest<JsonNode> req, CancellationToken cancellationToken) {
|
||||||
|
ArgumentNullException.ThrowIfNull(req.Id);
|
||||||
|
ArgumentException.ThrowIfNullOrEmpty(req.Method);
|
||||||
|
if (!SubscribeRequest.ParseQuery(req, out var vs, out var subId, out var errMsg, out var errCode)) {
|
||||||
|
logger.LogError("Invalid 'subscribe' request: {Reason}", errMsg);
|
||||||
|
var rpcError = new JsonRpcError<JsonNode>() {
|
||||||
|
Code = (long)errCode,
|
||||||
|
Message = $"{errCode}: {errMsg}",
|
||||||
|
};
|
||||||
|
await send(rpcError, cancellationToken);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (subscriptions.TryGetValue(subId, out var oldSub)) {
|
||||||
|
oldSub.Dispose();
|
||||||
|
}
|
||||||
|
var sub = new DataStreamSubscription(subId, req.Id.Value, vs, webSocket, logger);
|
||||||
|
subscriptions[subId] = sub;
|
||||||
|
await sendSubscriptionStatus(req.Id.Value, subId, req.Method, SubscriptionStatus.Subscribed, cancellationToken);
|
||||||
|
await sub.StartAsync(cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
async Task onUnsubscribe(JsonRpcRequest<UnsubscribeRequest> req, CancellationToken cancellationToken) {
|
||||||
|
ArgumentNullException.ThrowIfNull(req.Id);
|
||||||
|
ArgumentException.ThrowIfNullOrEmpty(req.Method);
|
||||||
|
var subId = req.Params?.SubscriptionId ?? string.Empty;
|
||||||
|
if (!subscriptions.TryGetValue(subId, out var sub)) {
|
||||||
|
logger.LogInformation("Ignoring '{Method}' request because no subscription was found with id {Id}.", req.Method, subId);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
logger.LogInformation("Disposing subscription");
|
||||||
|
sub.Dispose();
|
||||||
|
subscriptions.Remove(subId);
|
||||||
|
await sendSubscriptionStatus(req.Id.Value, subId, req.Method, SubscriptionStatus.Unsubscribed, cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
async Task unsubscribe(JsonRpcId requestId, string subscriptionId, CancellationToken cancellationToken) {
|
||||||
|
await sendSubscriptionStatus(requestId, subscriptionId, "unsubscribe", SubscriptionStatus.Unsubscribed, cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
Task sendSubscriptionStatus(JsonRpcId reqId, string subId, string method, SubscriptionStatus status, CancellationToken cancellationToken) {
|
||||||
|
logger.LogInformation("Sending {Method} ack. SubscriptionId = {SubscriptionId}", method, subId);
|
||||||
|
var response = JsonRpcResponse.Create(reqId, new SubscriptionStatusMessage() {
|
||||||
|
SubscriptionId = subId,
|
||||||
|
Status = status,
|
||||||
|
});
|
||||||
|
return send(response, cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
Task send<T>(T message, CancellationToken cancellationToken) {
|
||||||
|
if (message is null) {
|
||||||
|
return Task.CompletedTask;
|
||||||
|
}
|
||||||
|
var str = JsonSerializer.Serialize(message);
|
||||||
|
var bytes = Encoding.UTF8.GetBytes(str);
|
||||||
|
logger.LogTrace("Sending data. Length = {Length}", bytes.Length);
|
||||||
|
return webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
void disposeSubscriptions() {
|
||||||
|
logger.LogInformation("Disposing DataStreamWebSocketServer subscriptions. Count: {Count}", subscriptions.Count);
|
||||||
|
subscriptions.Values.DisposeAll();
|
||||||
|
subscriptions.Clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
namespace Bts.DataStreamWebApp.Server.Messages;
|
||||||
|
|
||||||
|
enum ErrorCode {
|
||||||
|
None = 0,
|
||||||
|
InvalidSubscriptionRequest,
|
||||||
|
}
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
namespace Bts.DataStreamWebApp.Server.Messages;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using bts.websocket_core.json_rpc;
|
||||||
|
|
||||||
|
using Bts.DataStreamExpressions;
|
||||||
|
|
||||||
|
using System.Diagnostics.CodeAnalysis;
|
||||||
|
using System.Text.Json;
|
||||||
|
using System.Text.Json.Nodes;
|
||||||
|
using System.Text.Json.Serialization;
|
||||||
|
|
||||||
|
public class SubscribeRequest {
|
||||||
|
[JsonPropertyName("query")]
|
||||||
|
public string? Query { get; set; }
|
||||||
|
|
||||||
|
[JsonPropertyName("subscription_id")]
|
||||||
|
public string? SubscriptionId { get; set; }
|
||||||
|
|
||||||
|
internal static bool ParseQuery(JsonRpcRequest<JsonNode> req, [NotNullWhen(true)] out ViewStream? stream, [NotNullWhen(true)] out string? subscriptionId, [NotNullWhen(false)] out string? errorMessage, out ErrorCode errorCode) {
|
||||||
|
stream = null;
|
||||||
|
subscriptionId = null;
|
||||||
|
errorCode = ErrorCode.None;
|
||||||
|
var paramsJsonObj = req.Params?.AsObject();
|
||||||
|
if (paramsJsonObj is null) {
|
||||||
|
errorCode = ErrorCode.InvalidSubscriptionRequest;
|
||||||
|
errorMessage = "'params' object not found";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
var subReq = paramsJsonObj.Deserialize<SubscribeRequest>();
|
||||||
|
if (subReq is null) {
|
||||||
|
errorCode = ErrorCode.InvalidSubscriptionRequest;
|
||||||
|
errorMessage = "'params' object invalid";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
subscriptionId = subReq.SubscriptionId ?? string.Empty;
|
||||||
|
var query = subReq.Query;
|
||||||
|
if (query is null) {
|
||||||
|
errorCode = ErrorCode.InvalidSubscriptionRequest;
|
||||||
|
errorMessage = "'params.query' field not found (expected a string)";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
errorMessage = null;
|
||||||
|
var exp = DataStreamExpressionsInterpreter.Evaluate(query);
|
||||||
|
if (exp is null) {
|
||||||
|
errorCode = ErrorCode.InvalidSubscriptionRequest;
|
||||||
|
errorMessage = "'params.query' expression has invalid syntax";
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
string? exceptionMsg = null;
|
||||||
|
try {
|
||||||
|
stream = exp.getView(new Dictionary<string, StatementExpression>(0));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
exceptionMsg = ex.ToString();
|
||||||
|
}
|
||||||
|
if (stream is null) {
|
||||||
|
errorCode = ErrorCode.InvalidSubscriptionRequest;
|
||||||
|
errorMessage = "Failed to create ViewStream from 'params.query' expression (check for typos in arguments/typenames)";
|
||||||
|
if (exceptionMsg is not null) {
|
||||||
|
errorMessage += ". Error" + exceptionMsg;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
namespace Bts.DataStreamWebApp.Server.Messages;
|
||||||
|
|
||||||
|
using System.Text.Json.Serialization;
|
||||||
|
|
||||||
|
enum SubscriptionStatus {
|
||||||
|
None,
|
||||||
|
Subscribed,
|
||||||
|
Unsubscribed,
|
||||||
|
}
|
||||||
|
|
||||||
|
class SubscriptionStatusMessage {
|
||||||
|
[JsonPropertyName("subscription_id")]
|
||||||
|
public string? SubscriptionId { get; set; }
|
||||||
|
|
||||||
|
[JsonPropertyName("status")]
|
||||||
|
[JsonConverter(typeof(JsonStringEnumConverter))]
|
||||||
|
public SubscriptionStatus Status { get; set; } = SubscriptionStatus.None;
|
||||||
|
}
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
namespace Bts.DataStreamWebApp.Server.Messages;
|
||||||
|
|
||||||
|
using System.Text.Json.Serialization;
|
||||||
|
|
||||||
|
public class UnsubscribeRequest {
|
||||||
|
[JsonPropertyName("subscription_id")]
|
||||||
|
public string? SubscriptionId { get; set; }
|
||||||
|
}
|
||||||
@@ -0,0 +1,59 @@
|
|||||||
|
using Bts.DataStreamCore.Services;
|
||||||
|
using Bts.PubSub.Services;
|
||||||
|
|
||||||
|
using Spark.Api.Services;
|
||||||
|
|
||||||
|
using System.Reactive.Linq;
|
||||||
|
|
||||||
|
var builder = WebApplication.CreateBuilder(new WebApplicationOptions {
|
||||||
|
Args = args,
|
||||||
|
ApplicationName = "Bts.DataStreamWebApp.Server",
|
||||||
|
});
|
||||||
|
ConfigureApp(builder.Configuration);
|
||||||
|
ConfigureServices(builder.Services, builder.Configuration);
|
||||||
|
|
||||||
|
var app = builder.Build();
|
||||||
|
if (app.Environment.IsDevelopment()) {
|
||||||
|
app.UseDirectoryBrowser();
|
||||||
|
}
|
||||||
|
var wsOpts = new WebSocketOptions {
|
||||||
|
KeepAliveInterval = TimeSpan.FromMinutes(2)
|
||||||
|
};
|
||||||
|
app.UseWebSockets(wsOpts);
|
||||||
|
app.UseDefaultFiles();
|
||||||
|
app.UseStaticFiles();
|
||||||
|
if (app.Environment.IsDevelopment()) {
|
||||||
|
app.UseSwagger();
|
||||||
|
app.UseSwaggerUI();
|
||||||
|
}
|
||||||
|
app.UseHttpsRedirection();
|
||||||
|
app.UseAuthorization();
|
||||||
|
app.MapControllers();
|
||||||
|
try {
|
||||||
|
Console.Write($"Starting Application");
|
||||||
|
app.Run();
|
||||||
|
Console.Write($"Application exited gracefully.");
|
||||||
|
} catch (Exception ex) {
|
||||||
|
Console.Error.Write($"Application crashed: {ex}");
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
static void ConfigureApp(ConfigurationManager configBuilder) {
|
||||||
|
configBuilder.AddCommandLine(Environment.GetCommandLineArgs());
|
||||||
|
var cfgRoot = configBuilder.GetSection("config");
|
||||||
|
if (cfgRoot?.Value is not null) {
|
||||||
|
configBuilder.AddJsonFile(cfgRoot.Value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void ConfigureServices(IServiceCollection services, IConfiguration config) {
|
||||||
|
services.AddDirectoryBrowser();
|
||||||
|
services.AddControllers();
|
||||||
|
services.AddEndpointsApiExplorer();
|
||||||
|
services.AddSwaggerGen();
|
||||||
|
services.AddMasterWorker(config.GetSection("MasterWorker"));
|
||||||
|
services.AddViewStreamDatabase(config.GetSection("ViewStreamDatabase"));
|
||||||
|
services.AddPubSubClient(config.GetSection("PubSub"));
|
||||||
|
services.AddSparkd(config.GetSection("Sparkd"));
|
||||||
|
}
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
{
|
||||||
|
"$schema": "http://json.schemastore.org/launchsettings.json",
|
||||||
|
"profiles": {
|
||||||
|
"http": {
|
||||||
|
"commandName": "Project",
|
||||||
|
"dotnetRunMessages": true,
|
||||||
|
"launchBrowser": true,
|
||||||
|
"launchUrl": "swagger",
|
||||||
|
"applicationUrl": "http://localhost:5000",
|
||||||
|
"environmentVariables": {
|
||||||
|
"ASPNETCORE_ENVIRONMENT": "Development",
|
||||||
|
"ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"https": {
|
||||||
|
"commandName": "Project",
|
||||||
|
"dotnetRunMessages": true,
|
||||||
|
"launchBrowser": true,
|
||||||
|
"applicationUrl": "http://localhost:5000;https://localhost:5001",
|
||||||
|
"environmentVariables": {
|
||||||
|
"ASPNETCORE_ENVIRONMENT": "Development",
|
||||||
|
"ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
{
|
||||||
|
"Logging": {
|
||||||
|
"LogLevel": {
|
||||||
|
"Default": "Information",
|
||||||
|
"Microsoft.AspNetCore": "Warning"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"AllowedHosts": "*"
|
||||||
|
}
|
||||||
@@ -0,0 +1,168 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<title></title>
|
||||||
|
<style>
|
||||||
|
table {
|
||||||
|
border: 0
|
||||||
|
}
|
||||||
|
|
||||||
|
.commslog-data {
|
||||||
|
font-family: Consolas, Courier New, Courier, monospace;
|
||||||
|
}
|
||||||
|
|
||||||
|
.commslog-server {
|
||||||
|
background-color: red;
|
||||||
|
color: white
|
||||||
|
}
|
||||||
|
|
||||||
|
.commslog-client {
|
||||||
|
background-color: green;
|
||||||
|
color: white
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<h1>ALT WebSocket Sample Application</h1>
|
||||||
|
<p id="stateLabel">Ready to connect...</p>
|
||||||
|
<div>
|
||||||
|
<label for="connectionUrl">WebSocket Server URL:</label>
|
||||||
|
<input id="connectionUrl" />
|
||||||
|
<button id="connectButton" type="submit">Connect</button>
|
||||||
|
</div>
|
||||||
|
<p></p>
|
||||||
|
<div>
|
||||||
|
<label for="sendMessage">Message to send:</label>
|
||||||
|
<input id="sendMessage" disabled />
|
||||||
|
<button id="sendButton" type="submit" disabled>Send</button>
|
||||||
|
<button id="closeButton" disabled>Close Socket</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<h2>Communication Log</h2>
|
||||||
|
<table style="width: 800px">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<td style="width: 100px">From</td>
|
||||||
|
<td style="width: 100px">To</td>
|
||||||
|
<td>Data</td>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody id="commsLog">
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
var connectionUrl = document.getElementById("connectionUrl");
|
||||||
|
var connectButton = document.getElementById("connectButton");
|
||||||
|
var stateLabel = document.getElementById("stateLabel");
|
||||||
|
var sendMessage = document.getElementById("sendMessage");
|
||||||
|
var sendButton = document.getElementById("sendButton");
|
||||||
|
var commsLog = document.getElementById("commsLog");
|
||||||
|
var closeButton = document.getElementById("closeButton");
|
||||||
|
var socket;
|
||||||
|
|
||||||
|
var scheme = document.location.protocol === "https:" ? "wss" : "ws";
|
||||||
|
var port = document.location.port ? (":" + document.location.port) : "";
|
||||||
|
|
||||||
|
connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
|
||||||
|
|
||||||
|
function updateState() {
|
||||||
|
function disable() {
|
||||||
|
sendMessage.disabled = true;
|
||||||
|
sendButton.disabled = true;
|
||||||
|
closeButton.disabled = true;
|
||||||
|
}
|
||||||
|
function enable() {
|
||||||
|
sendMessage.disabled = false;
|
||||||
|
sendButton.disabled = false;
|
||||||
|
closeButton.disabled = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
connectionUrl.disabled = true;
|
||||||
|
connectButton.disabled = true;
|
||||||
|
|
||||||
|
if (!socket) {
|
||||||
|
disable();
|
||||||
|
} else {
|
||||||
|
switch (socket.readyState) {
|
||||||
|
case WebSocket.CLOSED:
|
||||||
|
stateLabel.innerHTML = "Closed";
|
||||||
|
disable();
|
||||||
|
connectionUrl.disabled = false;
|
||||||
|
connectButton.disabled = false;
|
||||||
|
break;
|
||||||
|
case WebSocket.CLOSING:
|
||||||
|
stateLabel.innerHTML = "Closing...";
|
||||||
|
disable();
|
||||||
|
break;
|
||||||
|
case WebSocket.CONNECTING:
|
||||||
|
stateLabel.innerHTML = "Connecting...";
|
||||||
|
disable();
|
||||||
|
break;
|
||||||
|
case WebSocket.OPEN:
|
||||||
|
stateLabel.innerHTML = "Open";
|
||||||
|
enable();
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
|
||||||
|
disable();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
closeButton.onclick = function () {
|
||||||
|
if (!socket || socket.readyState !== WebSocket.OPEN) {
|
||||||
|
alert("socket not connected");
|
||||||
|
}
|
||||||
|
socket.close(1000, "Closing from client");
|
||||||
|
};
|
||||||
|
|
||||||
|
sendButton.onclick = function () {
|
||||||
|
if (!socket || socket.readyState !== WebSocket.OPEN) {
|
||||||
|
alert("socket not connected");
|
||||||
|
}
|
||||||
|
var data = sendMessage.value;
|
||||||
|
socket.send(data);
|
||||||
|
commsLog.innerHTML += '<tr>' +
|
||||||
|
'<td class="commslog-client">Client</td>' +
|
||||||
|
'<td class="commslog-server">Server</td>' +
|
||||||
|
'<td class="commslog-data">' + htmlEscape(data) + '</td></tr>';
|
||||||
|
};
|
||||||
|
|
||||||
|
connectButton.onclick = function() {
|
||||||
|
stateLabel.innerHTML = "Connecting";
|
||||||
|
socket = new WebSocket(connectionUrl.value);
|
||||||
|
socket.onopen = function (event) {
|
||||||
|
updateState();
|
||||||
|
commsLog.innerHTML += '<tr>' +
|
||||||
|
'<td colspan="3" class="commslog-data">Connection opened</td>' +
|
||||||
|
'</tr>';
|
||||||
|
};
|
||||||
|
socket.onclose = function (event) {
|
||||||
|
updateState();
|
||||||
|
commsLog.innerHTML += '<tr>' +
|
||||||
|
'<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
|
||||||
|
'</tr>';
|
||||||
|
};
|
||||||
|
socket.onerror = updateState;
|
||||||
|
socket.onmessage = function (event) {
|
||||||
|
commsLog.innerHTML += '<tr>' +
|
||||||
|
'<td class="commslog-server">Server</td>' +
|
||||||
|
'<td class="commslog-client">Client</td>' +
|
||||||
|
'<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
function htmlEscape(str) {
|
||||||
|
return str.toString()
|
||||||
|
.replace(/&/g, '&')
|
||||||
|
.replace(/"/g, '"')
|
||||||
|
.replace(/'/g, ''')
|
||||||
|
.replace(/</g, '<')
|
||||||
|
.replace(/>/g, '>');
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
Binary file not shown.
|
After Width: | Height: | Size: 8.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.1 KiB |
@@ -0,0 +1,176 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<title></title>
|
||||||
|
<style>
|
||||||
|
table {
|
||||||
|
border: 0
|
||||||
|
}
|
||||||
|
|
||||||
|
.commslog-data {
|
||||||
|
font-family: Consolas, Courier New, Courier, monospace;
|
||||||
|
}
|
||||||
|
|
||||||
|
.commslog-server {
|
||||||
|
background-color: red;
|
||||||
|
color: white
|
||||||
|
}
|
||||||
|
|
||||||
|
.commslog-client {
|
||||||
|
background-color: green;
|
||||||
|
color: white
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<h1>DataStreams via WebSocket</h1>
|
||||||
|
<p id="stateLabel">Ready to connect...</p>
|
||||||
|
<div>
|
||||||
|
<label for="connectionUrl">WebSocket Server URL:</label>
|
||||||
|
<input id="connectionUrl" />
|
||||||
|
<button id="connectButton" type="submit">Connect</button>
|
||||||
|
</div>
|
||||||
|
<p></p>
|
||||||
|
<div>
|
||||||
|
<label for="sendMessage">Query:</label>
|
||||||
|
<input id="sendMessage" disabled />
|
||||||
|
<button id="sendButton" type="submit" disabled>Send</button>
|
||||||
|
<button id="closeButton" disabled>Close Socket</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<h2>Communication Log</h2>
|
||||||
|
<table style="width: 800px">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<td style="width: 100px">From</td>
|
||||||
|
<td style="width: 100px">To</td>
|
||||||
|
<td>Data</td>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody id="commsLog">
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
var connectionUrl = document.getElementById("connectionUrl");
|
||||||
|
var connectButton = document.getElementById("connectButton");
|
||||||
|
var stateLabel = document.getElementById("stateLabel");
|
||||||
|
var sendMessage = document.getElementById("sendMessage");
|
||||||
|
var sendButton = document.getElementById("sendButton");
|
||||||
|
var commsLog = document.getElementById("commsLog");
|
||||||
|
var closeButton = document.getElementById("closeButton");
|
||||||
|
var socket;
|
||||||
|
|
||||||
|
var scheme = document.location.protocol === "https:" ? "wss" : "ws";
|
||||||
|
var port = document.location.port ? (":" + document.location.port) : "";
|
||||||
|
|
||||||
|
connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
|
||||||
|
|
||||||
|
function updateState() {
|
||||||
|
function disable() {
|
||||||
|
sendMessage.disabled = true;
|
||||||
|
sendButton.disabled = true;
|
||||||
|
closeButton.disabled = true;
|
||||||
|
}
|
||||||
|
function enable() {
|
||||||
|
sendMessage.disabled = false;
|
||||||
|
sendButton.disabled = false;
|
||||||
|
closeButton.disabled = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
connectionUrl.disabled = true;
|
||||||
|
connectButton.disabled = true;
|
||||||
|
|
||||||
|
if (!socket) {
|
||||||
|
disable();
|
||||||
|
} else {
|
||||||
|
switch (socket.readyState) {
|
||||||
|
case WebSocket.CLOSED:
|
||||||
|
stateLabel.innerHTML = "Closed";
|
||||||
|
disable();
|
||||||
|
connectionUrl.disabled = false;
|
||||||
|
connectButton.disabled = false;
|
||||||
|
break;
|
||||||
|
case WebSocket.CLOSING:
|
||||||
|
stateLabel.innerHTML = "Closing...";
|
||||||
|
disable();
|
||||||
|
break;
|
||||||
|
case WebSocket.CONNECTING:
|
||||||
|
stateLabel.innerHTML = "Connecting...";
|
||||||
|
disable();
|
||||||
|
break;
|
||||||
|
case WebSocket.OPEN:
|
||||||
|
stateLabel.innerHTML = "Open";
|
||||||
|
enable();
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
|
||||||
|
disable();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
closeButton.onclick = function () {
|
||||||
|
if (!socket || socket.readyState !== WebSocket.OPEN) {
|
||||||
|
alert("socket not connected");
|
||||||
|
}
|
||||||
|
socket.close(1000, "Closing from client");
|
||||||
|
};
|
||||||
|
|
||||||
|
sendButton.onclick = function () {
|
||||||
|
if (!socket || socket.readyState !== WebSocket.OPEN) {
|
||||||
|
alert("socket not connected");
|
||||||
|
}
|
||||||
|
const rpcReq = {
|
||||||
|
jsonrpc: "2.0",
|
||||||
|
method: "subscribe",
|
||||||
|
params: {
|
||||||
|
query: sendMessage.value,
|
||||||
|
},
|
||||||
|
id: 1,
|
||||||
|
};
|
||||||
|
const rpcReqStr = JSON.stringify(rpcReq);
|
||||||
|
socket.send(rpcReqStr);
|
||||||
|
commsLog.innerHTML += '<tr>' +
|
||||||
|
'<td class="commslog-client">Client</td>' +
|
||||||
|
'<td class="commslog-server">Server</td>' +
|
||||||
|
'<td class="commslog-data">' + htmlEscape(rpcReqStr) + '</td></tr>';
|
||||||
|
};
|
||||||
|
|
||||||
|
connectButton.onclick = function() {
|
||||||
|
stateLabel.innerHTML = "Connecting";
|
||||||
|
socket = new WebSocket(connectionUrl.value);
|
||||||
|
socket.onopen = function (event) {
|
||||||
|
updateState();
|
||||||
|
commsLog.innerHTML += '<tr>' +
|
||||||
|
'<td colspan="3" class="commslog-data">Connection opened</td>' +
|
||||||
|
'</tr>';
|
||||||
|
};
|
||||||
|
socket.onclose = function (event) {
|
||||||
|
updateState();
|
||||||
|
commsLog.innerHTML += '<tr>' +
|
||||||
|
'<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
|
||||||
|
'</tr>';
|
||||||
|
};
|
||||||
|
socket.onerror = updateState;
|
||||||
|
socket.onmessage = function (event) {
|
||||||
|
commsLog.innerHTML += '<tr>' +
|
||||||
|
'<td class="commslog-server">Server</td>' +
|
||||||
|
'<td class="commslog-client">Client</td>' +
|
||||||
|
'<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
function htmlEscape(str) {
|
||||||
|
return str.toString()
|
||||||
|
.replace(/&/g, '&')
|
||||||
|
.replace(/"/g, '"')
|
||||||
|
.replace(/'/g, ''')
|
||||||
|
.replace(/</g, '<')
|
||||||
|
.replace(/>/g, '>');
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@@ -18,11 +18,8 @@
|
|||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
|
<PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
|
||||||
<PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
|
|
||||||
<PackageReference Include="MongoDB.Bson" Version="2.23.1" />
|
<PackageReference Include="MongoDB.Bson" Version="2.23.1" />
|
||||||
<PackageReference Include="MongoDB.Driver" Version="2.23.1" />
|
<PackageReference Include="MongoDB.Driver" Version="2.23.1" />
|
||||||
<PackageReference Include="Parquet.Net" Version="5.0.1" />
|
|
||||||
<PackageReference Include="System.Data.SQLite" Version="1.0.118" />
|
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
@@ -1,32 +0,0 @@
|
|||||||
namespace bts.SilverPine {
|
|
||||||
using System;
|
|
||||||
using BCG;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using Bts.DataStreamExpressions;
|
|
||||||
|
|
||||||
// All this does is take in a dsl expression, parse it and fire the data that come out of running it.
|
|
||||||
// So for example, if an expresison is passed
|
|
||||||
[DataStreamUserRespresentation("dsl")]
|
|
||||||
public class DSLStream : ViewStream {
|
|
||||||
private readonly string _dsl;
|
|
||||||
public DSLStream(string dsl) {
|
|
||||||
Log.Gen.info($"DSLStream.cons dsl: {dsl}");
|
|
||||||
_dsl = dsl;
|
|
||||||
ViewStream rs = build_stream(dsl);
|
|
||||||
rs = rs ?? new LiteralStream("error", $"msg:invalid dsl \"{dsl}\"");
|
|
||||||
mySetRealSource(rs);
|
|
||||||
}
|
|
||||||
|
|
||||||
static ViewStream? build_stream(string stream_expression) {
|
|
||||||
try {
|
|
||||||
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
|
|
||||||
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
|
|
||||||
return rv;
|
|
||||||
|
|
||||||
} catch (Exception ex) {
|
|
||||||
Log.Gen.error($"Could not build stream: {stream_expression} ex: {ex}");
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+8
-4
@@ -4,6 +4,8 @@ using System.Collections.Generic;
|
|||||||
using Bts.DataStreamExpressions;
|
using Bts.DataStreamExpressions;
|
||||||
using BCG.Utilities.General;
|
using BCG.Utilities.General;
|
||||||
using bts;
|
using bts;
|
||||||
|
using Bts.PubSub;
|
||||||
|
using Bts.DataStreamCore;
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG {
|
||||||
// listens to the requests posted on "cloud_streams" request table, and
|
// listens to the requests posted on "cloud_streams" request table, and
|
||||||
@@ -40,13 +42,15 @@ namespace BCG {
|
|||||||
private IDictionary<string, StreamPubUnit>
|
private IDictionary<string, StreamPubUnit>
|
||||||
_stream_pub_map = new Dictionary<string, StreamPubUnit>();
|
_stream_pub_map = new Dictionary<string, StreamPubUnit>();
|
||||||
private ViewStream _stream = new TableStream("cloud_streams");
|
private ViewStream _stream = new TableStream("cloud_streams");
|
||||||
private PubTableBroker _broker = new PubTableBroker();
|
readonly IPubTableBroker _broker;
|
||||||
private ManagedSubscriber _sub = ManagedSubscriber.instance;
|
readonly IManagedSubscriber _sub;
|
||||||
|
|
||||||
|
|
||||||
public RemoteQueryManager() {
|
public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker) {
|
||||||
|
_sub = managedSubscriber;
|
||||||
|
_broker = pubTableBroker;
|
||||||
_stream.onChange += _on_str;
|
_stream.onChange += _on_str;
|
||||||
_sub.subscribe("RemoteRequest", (string tag, byte[] data) => {
|
_sub.Subscribe("RemoteRequest", (string tag, byte[] data) => {
|
||||||
RemoteRequest req = RemoteRequest.construct_from_bytes(data);
|
RemoteRequest req = RemoteRequest.construct_from_bytes(data);
|
||||||
_spawn_remote_stream(req.published_table_name, req.stream_expression, false);
|
_spawn_remote_stream(req.published_table_name, req.stream_expression, false);
|
||||||
});
|
});
|
||||||
@@ -1,24 +1,23 @@
|
|||||||
using bts;
|
|
||||||
|
|
||||||
using System;
|
using System;
|
||||||
using System.Collections;
|
using System.Collections;
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG
|
||||||
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Provides a stream that with distinct rows with respect to the columns
|
/// Provides a stream that with distinct rows with respect to the columns
|
||||||
/// the user specifies in the argument
|
/// the user specifies in the argument
|
||||||
/// </summary>
|
/// </summary>
|
||||||
[DataStreamUserRespresentation("distinct")]
|
[DataStreamUserRespresentation ("distinct")]
|
||||||
public class DistinctRowStream : ViewStream {
|
public class DistinctRowStream : ViewStream
|
||||||
public DistinctRowStream(ViewStream stream, params string[] cols) {
|
{
|
||||||
var g_by_eval = new DistinctRowEvaluator(cols);
|
public DistinctRowStream (ViewStream stream, params string[] cols)
|
||||||
ViewStream r = new GroupByStream(stream, g_by_eval);
|
{
|
||||||
var proj_exp = String.Empty;
|
ArrayList groupByCols = new ArrayList ();
|
||||||
foreach (var c in cols)
|
foreach (string col in cols)
|
||||||
proj_exp += $"{c} ";
|
foreach (string c in col.Split (' '))
|
||||||
proj_exp = proj_exp.Trim();
|
groupByCols.Add (c);
|
||||||
r = new NewProjectStream(r, proj_exp);
|
mySetRealSource (new GroupByStream (stream,
|
||||||
mySetRealSource(r);
|
new DistinctRowEvaluator (groupByCols.ToArray (typeof(string)) as string[]), false));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
// using System;
|
||||||
|
// using System.Collections.Generic;
|
||||||
|
// using System.Text;
|
||||||
|
// using System.IO;
|
||||||
|
|
||||||
|
// using System.Runtime.Serialization.Formatters.Binary;
|
||||||
|
|
||||||
|
// namespace BCG
|
||||||
|
// {
|
||||||
|
// public class DumpLoaderStream : ViewStream, ISideEffectStream
|
||||||
|
// {
|
||||||
|
// //wtf
|
||||||
|
// //public class Exception : System.Exception
|
||||||
|
// //{public Exception(string msg) : base(msg) { }}
|
||||||
|
|
||||||
|
// public DumpLoaderStream(string fileName, int secondsBetweenPolls)
|
||||||
|
// {
|
||||||
|
// mySetRealSource(
|
||||||
|
// new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls,
|
||||||
|
// ViewStreamDatabase.instance));
|
||||||
|
// }
|
||||||
|
|
||||||
|
// public DumpLoaderStream(string fileName)
|
||||||
|
// {
|
||||||
|
// if (fileName == null)
|
||||||
|
// throw new LoaderStream.Exception("File name cannot be null");
|
||||||
|
|
||||||
|
// for (int i = 0; i < 5; i++)
|
||||||
|
// try
|
||||||
|
// {
|
||||||
|
// using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
|
||||||
|
// {
|
||||||
|
// IClientDeserializer deser = new ClientUniversalDeserializer();
|
||||||
|
// byte[] the_ops;
|
||||||
|
// Array the_rows;
|
||||||
|
// deser.deserialize(new Stream[] { fs }, out the_ops, out the_rows);
|
||||||
|
|
||||||
|
// mySetRealSource(new BlueRowCollectionStaticStream((IBlueRowCollection)the_rows.GetValue(0)));
|
||||||
|
// }
|
||||||
|
// break;
|
||||||
|
// }
|
||||||
|
// catch (IOException)
|
||||||
|
// {
|
||||||
|
// System.Threading.Thread.Sleep(50);
|
||||||
|
// }
|
||||||
|
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
@@ -1,7 +1,6 @@
|
|||||||
using BCG;
|
using BCG;
|
||||||
|
|
||||||
namespace bts {
|
namespace bts {
|
||||||
// This is the other side of the SaverStream, its "loads" the stream and fires once
|
|
||||||
[DataStreamUserRespresentation("loader")]
|
[DataStreamUserRespresentation("loader")]
|
||||||
public class LoaderStream : ViewStream {
|
public class LoaderStream : ViewStream {
|
||||||
|
|
||||||
|
|||||||
@@ -5,11 +5,10 @@ using System.Text;
|
|||||||
using System.IO;
|
using System.IO;
|
||||||
using Google.Protobuf;
|
using Google.Protobuf;
|
||||||
using System.Globalization;
|
using System.Globalization;
|
||||||
using BCG;
|
using bts;
|
||||||
|
|
||||||
namespace bts {
|
namespace BCG {
|
||||||
// Writes all the udpates to a stream in a file or multiples files dependong on the rollover size (as opposed to Saver
|
// Writes all the udpates to a stream in a file.
|
||||||
// which just writes one update, as a create)
|
|
||||||
// File naming:
|
// File naming:
|
||||||
// token_epoch_yyyyMMdd_HHmmss
|
// token_epoch_yyyyMMdd_HHmmss
|
||||||
// File location: directory(passed)\token\
|
// File location: directory(passed)\token\
|
||||||
@@ -95,10 +94,10 @@ namespace bts {
|
|||||||
void _on_change(object new_value) {
|
void _on_change(object new_value) {
|
||||||
DataPacket dp = new_value as DataPacket;
|
DataPacket dp = new_value as DataPacket;
|
||||||
byte[] serialized_update = null;
|
byte[] serialized_update = null;
|
||||||
bool first_write = _serializer == null || _n_updates_in_current_file == 0;
|
bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
|
||||||
_serializer = _serializer ?? new UniversalStreamIOSerializer();
|
_serializer = _serializer ?? new UniversalStreamIOSerializer();
|
||||||
if (first_write) {
|
if (_write_create) {
|
||||||
// first file always starts with create
|
// new files always starts with create
|
||||||
serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
|
serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
|
||||||
} else
|
} else
|
||||||
serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
|
serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
|
||||||
@@ -106,8 +105,8 @@ namespace bts {
|
|||||||
long utc_epoch = epoch_now();
|
long utc_epoch = epoch_now();
|
||||||
|
|
||||||
//time, length of data, data
|
//time, length of data, data
|
||||||
//bw.Write(utc_epoch);
|
bw.Write(utc_epoch);
|
||||||
//bw.Write(serialized_update.Length);
|
bw.Write(serialized_update.Length);
|
||||||
bw.Write(serialized_update);
|
bw.Write(serialized_update);
|
||||||
_out_fs.Flush();
|
_out_fs.Flush();
|
||||||
myFireEvent(dp);
|
myFireEvent(dp);
|
||||||
|
|||||||
@@ -0,0 +1,190 @@
|
|||||||
|
// using System;
|
||||||
|
// using System.Collections.Generic;
|
||||||
|
// using System.Linq;
|
||||||
|
// using System.Text;
|
||||||
|
// using System.IO;
|
||||||
|
// using System.Runtime.Serialization.Formatters.Binary;
|
||||||
|
// using BCG.Utilities.General;
|
||||||
|
// using bts.collections;
|
||||||
|
// using System.Reflection.Metadata;
|
||||||
|
|
||||||
|
|
||||||
|
// namespace BCG {
|
||||||
|
|
||||||
|
// public class PlaybackStream : ViewStream, IWorker {
|
||||||
|
// private readonly PlaybackType _type;
|
||||||
|
// private BlueTable _current_table;
|
||||||
|
// private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
|
||||||
|
// // these are the packets already downloaded from the current file
|
||||||
|
// private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
|
||||||
|
// //private IEnumerable<FileStream> _file_streams = new List<FileStream>();
|
||||||
|
// private IEnumerable<string> _file_names;
|
||||||
|
// private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
|
||||||
|
// private DateTime _playback_t0, _playback_t1;
|
||||||
|
// private bool _all_files_read = false;
|
||||||
|
// SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
|
||||||
|
|
||||||
|
// public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
|
||||||
|
// _type = PlaybackType.Streaming;
|
||||||
|
// _playback_t0 = PersistorStream.parse_datetime(datetime_start);
|
||||||
|
// _playback_t1 = PersistorStream.parse_datetime(datetime_end);
|
||||||
|
// string token_directory_name = $"{directory_name}\\{token}";
|
||||||
|
// bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
|
||||||
|
// List<string> files_in_range = new List<string>();
|
||||||
|
// if (Directory.Exists(token_directory_name)) {
|
||||||
|
// _file_names =
|
||||||
|
// Directory.GetFiles(token_directory_name).Where(f => {
|
||||||
|
// DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
|
||||||
|
// return file_time > _playback_t0 && file_time < _playback_t1;
|
||||||
|
// });
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
// protected override void myOnFirstSubscriber() {
|
||||||
|
// FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
|
||||||
|
// _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
|
||||||
|
// PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
|
||||||
|
// if (snapshot_packet != null) {
|
||||||
|
// foreach (PlaybackPacket p in _snapshot_data.packets) {
|
||||||
|
// byte[] ops;
|
||||||
|
// Array row_array;
|
||||||
|
// _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
|
||||||
|
// for (int i = 0; i < ops.Length; i++) {
|
||||||
|
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
||||||
|
// byte op = ops[i];
|
||||||
|
// if (op == StreamOpCode.create) {
|
||||||
|
// IBlueRowCollection full_table = rows_args[0];
|
||||||
|
// _current_table = new BlueTable(full_table);
|
||||||
|
// } else if (op == StreamOpCode.multipleRemoveAdd) {
|
||||||
|
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
||||||
|
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
||||||
|
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
||||||
|
// BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
||||||
|
// } else {
|
||||||
|
// bts.Log.assert(false, $"Oprator: {op} is not expected");
|
||||||
|
// }
|
||||||
|
// if (p.idx == snapshot_packet.idx) {
|
||||||
|
// myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
|
||||||
|
// break;
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
|
||||||
|
// private void _load_packets_from_current_file() {
|
||||||
|
// int curr_idx = _current_fs.Item1;
|
||||||
|
// FileStream curr_fs = _current_fs.Item2;
|
||||||
|
// _packets = new List<PlaybackPacket>();
|
||||||
|
// _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
|
||||||
|
// }
|
||||||
|
|
||||||
|
// private bool current_file_all_replayed {
|
||||||
|
// get {
|
||||||
|
// bool rv = true;
|
||||||
|
// _packets.ForEach(p => rv = rv && p.replayed);
|
||||||
|
// return rv;
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
// protected override void myOnLastSubscriber() {
|
||||||
|
// var fs = _current_fs.Item2;
|
||||||
|
// if (fs != null) {
|
||||||
|
// fs.Close();
|
||||||
|
// }
|
||||||
|
// ViewStreamDatabase.instance.removeWorker(this);
|
||||||
|
// }
|
||||||
|
|
||||||
|
// // public event PlaybackStreamProgress onProgress;
|
||||||
|
// // public event Action onFileExhausted;
|
||||||
|
// private PlaybackPacket _locate_packet_to_replay() {
|
||||||
|
// foreach (var p in _packets) {
|
||||||
|
// // returns the first non replayed packet
|
||||||
|
// if (!p.replayed)
|
||||||
|
// return p;
|
||||||
|
// }
|
||||||
|
// //else
|
||||||
|
// {
|
||||||
|
// double secondsElapsed = new BinaryReader(myFileReader).ReadDouble();
|
||||||
|
// myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed));
|
||||||
|
// if (onProgress != null)
|
||||||
|
// onProgress(myFileReader.Position, myFileReader.Length);
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
// #region IWorker Members
|
||||||
|
|
||||||
|
// public void doWork() {
|
||||||
|
// if (_all_files_read)
|
||||||
|
// return;
|
||||||
|
// if (current_file_all_replayed) {
|
||||||
|
// _load_next_file();
|
||||||
|
// }
|
||||||
|
// PlaybackPacket curr_packet = _locate_packet_to_replay();
|
||||||
|
// if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway
|
||||||
|
// byte[] ops;
|
||||||
|
// Array row_array;
|
||||||
|
// // what a stupid interface!
|
||||||
|
// _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
|
||||||
|
// _fire_packet(ops, row_array);
|
||||||
|
// curr_packet.replay_timestamp = PersistorStream.epoch_now();
|
||||||
|
// } else {
|
||||||
|
// PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
|
||||||
|
// // this is the time we need to fire (replay) this packet
|
||||||
|
// long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
|
||||||
|
// long now_epoch = PersistorStream.epoch_now();
|
||||||
|
// // fire when we are past the
|
||||||
|
// if (now_epoch >= time_to_fire) {
|
||||||
|
// byte[] ops;
|
||||||
|
// Array row_array;
|
||||||
|
// _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
|
||||||
|
// _fire_packet(ops, row_array);
|
||||||
|
// curr_packet.replay_timestamp = PersistorStream.epoch_now();
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
// private void _fire_packet(byte[] ops, Array row_array) {
|
||||||
|
// for (int i = 0; i < ops.Length; i++) {
|
||||||
|
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
|
||||||
|
// byte op = ops[i];
|
||||||
|
// if (op == StreamOpCode.create) {
|
||||||
|
// IBlueRowCollection full_table = rows_args[0];
|
||||||
|
// _current_table = new BlueTable(full_table);
|
||||||
|
// //_current_table.addRows(full_table);
|
||||||
|
// myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
|
||||||
|
// (myLastValue as DataPacket).newRows));
|
||||||
|
// } else if (op == StreamOpCode.multipleRemoveAdd) {
|
||||||
|
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
|
||||||
|
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
|
||||||
|
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
|
||||||
|
// BlueDataUtility.absorbChanges(_current_table, dels, adds);
|
||||||
|
// myFireEvent(new DataPacket(_current_table, adds, dels));
|
||||||
|
// } else {
|
||||||
|
// bts.Log.assert(false, $"Oprator: {op} is not expected");
|
||||||
|
// }
|
||||||
|
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
// private void _load_next_file() {
|
||||||
|
// var new_idx = _current_fs.Item1 + 1;
|
||||||
|
// if (new_idx >= _file_names.Count()) {
|
||||||
|
// _all_files_read = true;
|
||||||
|
// bts.Log.Gen.info("All files read");
|
||||||
|
// return;
|
||||||
|
// }
|
||||||
|
// var prev_fs = _current_fs.Item2;
|
||||||
|
// if (prev_fs != null)
|
||||||
|
// prev_fs.Close();
|
||||||
|
// var new_filename = _file_names.ElementAt(new_idx);
|
||||||
|
// bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
|
||||||
|
// var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
|
||||||
|
// _current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
|
||||||
|
// _load_packets_from_current_file();
|
||||||
|
// }
|
||||||
|
|
||||||
|
// #endregion
|
||||||
|
// }
|
||||||
|
// }
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
// using System;
|
||||||
|
// using System.Collections.Generic;
|
||||||
|
// using System.Linq;
|
||||||
|
// using System.Text;
|
||||||
|
// using System.IO;
|
||||||
|
|
||||||
|
// namespace BCG
|
||||||
|
// {
|
||||||
|
// public class RecorderStream : ViewStream
|
||||||
|
// {
|
||||||
|
// private ViewStream mySource;
|
||||||
|
// private DateTime myStartTime;
|
||||||
|
// private FileStream myOutFile;
|
||||||
|
// private string myFileName;
|
||||||
|
// private IBlueDataSerializer mySerializer;
|
||||||
|
|
||||||
|
// public RecorderStream(ViewStream src, string recordFile)
|
||||||
|
// {
|
||||||
|
// mySource = src;
|
||||||
|
// myFileName = recordFile;
|
||||||
|
|
||||||
|
// myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss"));
|
||||||
|
// }
|
||||||
|
|
||||||
|
// protected override void myOnFirstSubscriber()
|
||||||
|
// {
|
||||||
|
// myStartTime = DateTime.Now;
|
||||||
|
// mySource.onChange += new ChangeDelegate(mySource_onChange);
|
||||||
|
// myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
|
||||||
|
// }
|
||||||
|
|
||||||
|
// protected override void myOnLastSubscriber()
|
||||||
|
// {
|
||||||
|
// mySource.onChange -= mySource_onChange;
|
||||||
|
// myOutFile.Close();
|
||||||
|
// }
|
||||||
|
|
||||||
|
// void mySource_onChange(object newValue)
|
||||||
|
// {
|
||||||
|
// DataPacket dp = (DataPacket)newValue;
|
||||||
|
|
||||||
|
// BlueTable toWrite = new BlueTable(dp.schema);
|
||||||
|
|
||||||
|
// if (mySerializer == null)
|
||||||
|
// {
|
||||||
|
// mySerializer = new UniversalSerializer(dp.schema);
|
||||||
|
// }
|
||||||
|
// foreach (IBlueRow r in dp.newRows)
|
||||||
|
// toWrite.addRow(r);
|
||||||
|
|
||||||
|
// byte[] result;
|
||||||
|
|
||||||
|
// if (myLastValue == null)
|
||||||
|
// mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
|
||||||
|
// else
|
||||||
|
// mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
|
||||||
|
|
||||||
|
// BinaryWriter bw = new BinaryWriter(myOutFile);
|
||||||
|
// //write down what time this happened
|
||||||
|
// bw.Write((DateTime.Now - myStartTime).TotalSeconds);
|
||||||
|
// bw.Write(result.Length);
|
||||||
|
// bw.Write(result);
|
||||||
|
// myOutFile.Flush();
|
||||||
|
|
||||||
|
// myFireEvent(dp);
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
@@ -5,8 +5,7 @@ using System.IO;
|
|||||||
using BCG;
|
using BCG;
|
||||||
|
|
||||||
namespace bts {
|
namespace bts {
|
||||||
// this saves the updates as a Create on the same file, each update calls -save, which
|
// this saves the total state in the filr provided in the cons
|
||||||
// reqrites the whole data to the packet. The other side of this stream is the Loader
|
|
||||||
[DataStreamUserRespresentation("saver")]
|
[DataStreamUserRespresentation("saver")]
|
||||||
public class SaverStream : ViewStream {
|
public class SaverStream : ViewStream {
|
||||||
private ViewStream _stream;
|
private ViewStream _stream;
|
||||||
@@ -32,6 +31,7 @@ namespace bts {
|
|||||||
StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
|
StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
|
||||||
byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
|
byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
|
||||||
StreamSerializationUtils.write_to_file(str_bytes, _filename);
|
StreamSerializationUtils.write_to_file(str_bytes, _filename);
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -3,7 +3,6 @@ using System.Collections.Generic;
|
|||||||
using System.Linq;
|
using System.Linq;
|
||||||
using System.Globalization;
|
using System.Globalization;
|
||||||
using System.IO;
|
using System.IO;
|
||||||
using bts;
|
|
||||||
|
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG {
|
||||||
|
|||||||
@@ -1,24 +0,0 @@
|
|||||||
namespace bts.SilverPine {
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
|
|
||||||
public class NullFilterStream : ViewStream {
|
|
||||||
public NullFilterStream(ViewStream parent_str, string col_name)
|
|
||||||
: this(parent_str, col_name, true) { }
|
|
||||||
public NullFilterStream(ViewStream parent_str, string col_name, bool filter_out_doublenans = false) {
|
|
||||||
ViewStream rs = new FilterStream(parent_str,
|
|
||||||
r => {
|
|
||||||
bool is_dbnull = r.is_null(col_name);
|
|
||||||
if (is_dbnull) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (r.schema[col_name].type_code() == TypeCode.Double) {
|
|
||||||
return !Double.IsNaN(r.get<double>(col_name));
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
});
|
|
||||||
mySetRealSource(rs);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,102 +1,102 @@
|
|||||||
// using System;
|
using System;
|
||||||
// using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
// using System.Linq;
|
using System.Linq;
|
||||||
// using System.Text;
|
using System.Text;
|
||||||
|
|
||||||
// namespace BCG
|
namespace BCG
|
||||||
// {
|
{
|
||||||
|
|
||||||
// public class AverageGroupsStream : ViewStream
|
public class AverageGroupsStream : ViewStream
|
||||||
// {
|
{
|
||||||
// public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
|
public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
|
||||||
// {
|
{
|
||||||
|
|
||||||
// myColToAverage = colToAverage;
|
myColToAverage = colToAverage;
|
||||||
// myInput = input;
|
myInput = input;
|
||||||
// myKeyColumn = keyColumn;
|
myKeyColumn = keyColumn;
|
||||||
// }
|
}
|
||||||
|
|
||||||
// private ViewStream myInput;
|
private ViewStream myInput;
|
||||||
// private string myColToAverage;
|
private string myColToAverage;
|
||||||
// private BlueTableSchema mySchema;
|
private BlueTableSchema mySchema;
|
||||||
// private string myKeyColumn;
|
private string myKeyColumn;
|
||||||
|
|
||||||
// protected override void myOnFirstSubscriber()
|
protected override void myOnFirstSubscriber()
|
||||||
// {
|
{
|
||||||
// myInput.onChange += new ChangeDelegate(myInput_onChange);
|
myInput.onChange += new ChangeDelegate(myInput_onChange);
|
||||||
// }
|
}
|
||||||
// protected override void myOnLastSubscriber()
|
protected override void myOnLastSubscriber()
|
||||||
// {
|
{
|
||||||
// myInput.onChange -= myInput_onChange;
|
myInput.onChange -= myInput_onChange;
|
||||||
// }
|
}
|
||||||
// private void myInput_onChange(object newValue)
|
private void myInput_onChange(object newValue)
|
||||||
// {
|
{
|
||||||
// if (mySchema == null)
|
if (mySchema == null)
|
||||||
// myBuildSchema(((DataPacket)newValue).schema);
|
myBuildSchema(((DataPacket)newValue).schema);
|
||||||
|
|
||||||
// IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);
|
IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);
|
||||||
|
|
||||||
// if (myLastValue == null)
|
if (myLastValue == null)
|
||||||
// myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
|
myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
|
||||||
// else
|
else
|
||||||
// {
|
{
|
||||||
// IBlueRowCollection added, deleted;
|
IBlueRowCollection added, deleted;
|
||||||
// BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
|
BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
|
||||||
// myFireEvent(new DataPacket(newRows, added, deleted));
|
myFireEvent(new DataPacket(newRows, added, deleted));
|
||||||
// }
|
}
|
||||||
// }
|
}
|
||||||
|
|
||||||
// private void myBuildSchema(IBlueTableSchema schema)
|
private void myBuildSchema(IBlueTableSchema schema)
|
||||||
// {
|
{
|
||||||
// List<ColumnInfo> columns = new List<ColumnInfo>();
|
List<ColumnInfo> columns = new List<ColumnInfo>();
|
||||||
|
|
||||||
// foreach (ColumnInfo ci in schema.columnInfo)
|
foreach (ColumnInfo ci in schema.columnInfo)
|
||||||
// columns.Add(ci);
|
columns.Add(ci);
|
||||||
|
|
||||||
// columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
|
columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
|
||||||
// mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
|
mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
|
||||||
// }
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
// private IBlueRowCollection myEvaluate(DataPacket packet)
|
private IBlueRowCollection myEvaluate(DataPacket packet)
|
||||||
// {
|
{
|
||||||
|
|
||||||
// IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>();
|
IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>();
|
||||||
// IDictionary<string, int> colToCount = new Dictionary<string, int>();
|
IDictionary<string, int> colToCount = new Dictionary<string, int>();
|
||||||
// IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>();
|
IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>();
|
||||||
|
|
||||||
// foreach (IBlueRow r in packet.newRows)
|
foreach (IBlueRow r in packet.newRows)
|
||||||
// {
|
{
|
||||||
// double thisVal = Convert.ToDouble(r[myColToAverage]);
|
double thisVal = Convert.ToDouble(r[myColToAverage]);
|
||||||
|
|
||||||
// string keyVal = r[myKeyColumn].ToString();
|
string keyVal = r[myKeyColumn].ToString();
|
||||||
// if (!colToCount.ContainsKey(keyVal))
|
if (!colToCount.ContainsKey(keyVal))
|
||||||
// {
|
{
|
||||||
// colToRunningTotal[keyVal] = thisVal;
|
colToRunningTotal[keyVal] = thisVal;
|
||||||
// colToCount[keyVal] = 1;
|
colToCount[keyVal] = 1;
|
||||||
// }
|
}
|
||||||
// else
|
else
|
||||||
// {
|
{
|
||||||
// colToRunningTotal[keyVal] += thisVal;
|
colToRunningTotal[keyVal] += thisVal;
|
||||||
// colToCount[keyVal] += 1;
|
colToCount[keyVal] += 1;
|
||||||
// }
|
}
|
||||||
// colToLastRow[keyVal] = r;
|
colToLastRow[keyVal] = r;
|
||||||
// }
|
}
|
||||||
|
|
||||||
// BlueTable result = new BlueTable(mySchema);
|
BlueTable result = new BlueTable(mySchema);
|
||||||
// foreach (string key in colToCount.Keys)
|
foreach (string key in colToCount.Keys)
|
||||||
// {
|
{
|
||||||
// object[] values = new object[mySchema.columnInfo.Length];
|
object[] values = new object[mySchema.columnInfo.Length];
|
||||||
|
|
||||||
// colToLastRow[key].CopyTo(values, 0);
|
colToLastRow[key].CopyTo(values, 0);
|
||||||
// values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
|
values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
|
||||||
// result.addRow(new BlueRow(mySchema, values));
|
result.addRow(new BlueRow(mySchema, values));
|
||||||
// }
|
}
|
||||||
|
|
||||||
// return result;
|
return result;
|
||||||
// }
|
}
|
||||||
|
|
||||||
|
|
||||||
// }
|
}
|
||||||
// }
|
}
|
||||||
@@ -1,31 +0,0 @@
|
|||||||
namespace bts.SilverPine {
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
|
|
||||||
[DataStreamUserRespresentation("Average")]
|
|
||||||
public class AverageNonIncGroupsStream : ViewStream {
|
|
||||||
public AverageNonIncGroupsStream(ViewStream parent_stream, string inp_col_name, params string[] group_by_cols) {
|
|
||||||
NonIncGroupByStream rs = new NonIncGroupByStream(parent_stream, () => group_by_cols,
|
|
||||||
(g_rows) => _find_mean(inp_col_name, g_rows));
|
|
||||||
mySetRealSource(rs);
|
|
||||||
}
|
|
||||||
|
|
||||||
private List<IBlueRow> _find_mean(string inp_col_name, List<IBlueRow> g_rows) {
|
|
||||||
IBlueRow inp_row = g_rows.ElementAt(0);
|
|
||||||
IEnumerable<double> inp_vals = BlueDataUtility.get_colvalues<double>(g_rows, inp_col_name);
|
|
||||||
double sum = 0.0;
|
|
||||||
inp_vals.ToList().ForEach(d => sum += d);
|
|
||||||
string out_col_name = $"average_{inp_col_name}";
|
|
||||||
double average = sum / inp_vals.Count();
|
|
||||||
IBlueTableSchema schema_out = inp_row.schema.add_column(TypeCode.Double, out_col_name);
|
|
||||||
MutableBlueRow out_row = new MutableBlueRow(schema_out, inp_row);
|
|
||||||
out_row.setValue(out_col_name, average);
|
|
||||||
List<IBlueRow> rv = new List<IBlueRow>();
|
|
||||||
rv.Add(out_row);
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,91 +0,0 @@
|
|||||||
namespace bts.SilverPine {
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
|
|
||||||
using ZstdSharp.Unsafe;
|
|
||||||
|
|
||||||
public enum GroupByBundleOp {
|
|
||||||
Mean,
|
|
||||||
Median,
|
|
||||||
Mode,
|
|
||||||
StdDev,
|
|
||||||
Max,
|
|
||||||
Min,
|
|
||||||
First,
|
|
||||||
Last,
|
|
||||||
Sum
|
|
||||||
}
|
|
||||||
public class GroupByBundle {
|
|
||||||
public readonly List<Tuple<string, GroupByBundleOp>> operations;
|
|
||||||
|
|
||||||
public static GroupByBundle parse(string text) {
|
|
||||||
// text is inp_col: op, inp_col: op, inp_col:op
|
|
||||||
List<Tuple<string, GroupByBundleOp>> rv = new List<Tuple<string, GroupByBundleOp>>();
|
|
||||||
string[] calcs = text.Split(',');
|
|
||||||
foreach (string c in calcs) {
|
|
||||||
string[] toks = c.Split(':');
|
|
||||||
string inp_col = toks[0].Trim();
|
|
||||||
string op_txt = toks[1].Trim();
|
|
||||||
GroupByBundleOp op = Enum.Parse<GroupByBundleOp>(op_txt, true);
|
|
||||||
rv.Add(Tuple.Create(inp_col, op));
|
|
||||||
}
|
|
||||||
return new GroupByBundle(rv);
|
|
||||||
}
|
|
||||||
|
|
||||||
public GroupByBundle(List<Tuple<string, GroupByBundleOp>> operations_) {
|
|
||||||
operations = operations_;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static IBlueRow group_by(List<IBlueRow> rows, GroupByBundle parameters) {
|
|
||||||
if (rows.Count() == 0) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
IBlueRow ir = rows.ElementAt(0);
|
|
||||||
IBlueTableSchema in_schema = ir.schema;
|
|
||||||
IBlueTableSchema out_schema = in_schema;
|
|
||||||
IDictionary<string, Tuple<string, GroupByBundleOp>> out_col_map =
|
|
||||||
new Dictionary<string, Tuple<string, GroupByBundleOp>>();
|
|
||||||
foreach (var t in parameters.operations) {
|
|
||||||
string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
|
|
||||||
out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
|
|
||||||
out_col_map[out_col_name] = t;
|
|
||||||
}
|
|
||||||
MutableBlueRow or = new MutableBlueRow(out_schema, ir);
|
|
||||||
foreach (var kv in out_col_map) {
|
|
||||||
string out_col_name = kv.Key;
|
|
||||||
string in_col_name = kv.Value.Item1;
|
|
||||||
GroupByBundleOp op = kv.Value.Item2;
|
|
||||||
IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
|
|
||||||
double comp_res = compute_op(col_vals, op);
|
|
||||||
or.setValue(out_col_name, comp_res);
|
|
||||||
}
|
|
||||||
return or;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static double compute_op(IEnumerable<double> col_vals, GroupByBundleOp op) {
|
|
||||||
switch (op) {
|
|
||||||
case GroupByBundleOp.Max:
|
|
||||||
return col_vals.Max<double>();
|
|
||||||
case GroupByBundleOp.Min:
|
|
||||||
return col_vals.Min<double>();
|
|
||||||
case GroupByBundleOp.Mean:
|
|
||||||
return col_vals.Sum() / col_vals.Count();
|
|
||||||
case GroupByBundleOp.Sum:
|
|
||||||
return col_vals.Sum();
|
|
||||||
case GroupByBundleOp.First:
|
|
||||||
return col_vals.First();
|
|
||||||
case GroupByBundleOp.Last:
|
|
||||||
return col_vals.Last();
|
|
||||||
default:
|
|
||||||
// TODO
|
|
||||||
Log.assert(false);
|
|
||||||
return Double.NaN;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -10,6 +10,6 @@ namespace BCG {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public MaxStream(ViewStream stream, string outputCol)
|
public MaxStream(ViewStream stream, string outputCol)
|
||||||
: this(new AddColumnsStream(stream, "fake_col: 0"), outputCol, new string[] { "fake_col" }) { }
|
: this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
+15
-66
@@ -1,40 +1,13 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
using System.Linq;
|
|
||||||
using System.Text.Json;
|
|
||||||
|
|
||||||
using bts;
|
using bts;
|
||||||
using bts.SilverPine;
|
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG
|
||||||
[DataStreamUserRespresentation("aggregate")]
|
{
|
||||||
public class NonIncGroupByStream : ViewStream {
|
public class NonIncGroupByStream : ViewStream
|
||||||
private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
|
{
|
||||||
Func<string[]> _groupby_cols_func;
|
|
||||||
Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
|
|
||||||
public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
|
|
||||||
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
|
|
||||||
_groupby_cols_func = groupby_cols_func;
|
|
||||||
_evaluate_func = evaluate_func;
|
|
||||||
}
|
|
||||||
public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
|
|
||||||
BlueTable rv = null;
|
|
||||||
foreach (var kv in grouped_rows) {
|
|
||||||
List<IBlueRow> g_rows = kv.Value;
|
|
||||||
List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
|
|
||||||
if (out_rows != null && out_rows.Count > 0) {
|
|
||||||
rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
|
|
||||||
}
|
|
||||||
rv.addRows(out_rows);
|
|
||||||
}
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
public string[] get_group_by_cols() {
|
|
||||||
return _groupby_cols_func();
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
private ViewStream _main_stream;
|
private ViewStream _main_stream;
|
||||||
private INonIncGroupByEvaluator _groupby_func;
|
private INonIncGroupByEvaluator _groupby_func;
|
||||||
private BlueRowHasher _row_hasher;
|
private BlueRowHasher _row_hasher;
|
||||||
@@ -42,47 +15,24 @@ namespace BCG {
|
|||||||
private Stopwatch _timer = new();
|
private Stopwatch _timer = new();
|
||||||
private int _n_total_fires = 0, _n_expensive_fires = 0;
|
private int _n_total_fires = 0, _n_expensive_fires = 0;
|
||||||
|
|
||||||
public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
|
public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
|
||||||
|
{
|
||||||
_main_stream = main_stream;
|
_main_stream = main_stream;
|
||||||
_groupby_func = groupby_func;
|
_groupby_func = groupby_func;
|
||||||
_groupby_cols = _groupby_func.get_group_by_cols();
|
_groupby_cols = _groupby_func.get_group_by_cols();
|
||||||
_row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
|
_row_hasher = new BlueRowHasher(_groupby_cols.Length);
|
||||||
}
|
}
|
||||||
|
|
||||||
public NonIncGroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
|
|
||||||
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
|
|
||||||
: this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
|
|
||||||
|
|
||||||
public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col = null) {
|
|
||||||
GroupByBundle std_group_by = GroupByBundle.parse(standard_group_by_text);
|
|
||||||
Func<string[]> groupby_cols_func = () => {
|
|
||||||
return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
|
|
||||||
};
|
|
||||||
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
|
|
||||||
if (!String.IsNullOrEmpty(optional_sort_col))
|
|
||||||
group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
|
|
||||||
IBlueRow rv = GroupByBundle.group_by(group_rows, std_group_by);
|
|
||||||
return new IBlueRow[] { rv }.ToList();
|
|
||||||
};
|
|
||||||
ViewStream rs = new NonIncGroupByStream(parent_str, groupby_cols_func, evaluate_func);
|
|
||||||
mySetRealSource(rs);
|
|
||||||
}
|
|
||||||
public NonIncGroupByStream(ViewStream parent_str, string standard_group_by_text) :
|
|
||||||
this(parent_str, String.Empty, standard_group_by_text) {
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
|
protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
|
protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
|
||||||
|
|
||||||
private void _main_stream_onchange(object nv) {
|
private void _main_stream_onchange(object nv)
|
||||||
|
{
|
||||||
DataPacket dp = nv as DataPacket;
|
DataPacket dp = nv as DataPacket;
|
||||||
///grouping together
|
///grouping together
|
||||||
_timer.Restart();
|
_timer.Restart();
|
||||||
IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
|
IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
|
||||||
new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
|
|
||||||
BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
|
|
||||||
_timer.Stop();
|
_timer.Stop();
|
||||||
var groupElapsedMs = _timer.ElapsedMilliseconds;
|
var groupElapsedMs = _timer.ElapsedMilliseconds;
|
||||||
if (groupElapsedMs >= 50) {
|
if (groupElapsedMs >= 50) {
|
||||||
@@ -94,15 +44,14 @@ namespace BCG {
|
|||||||
IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
|
IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
|
||||||
_timer.Stop();
|
_timer.Stop();
|
||||||
var applyElapsedMs = _timer.ElapsedMilliseconds;
|
var applyElapsedMs = _timer.ElapsedMilliseconds;
|
||||||
_n_total_fires++;
|
_n_total_fires++;
|
||||||
if (applyElapsedMs >= 50) {
|
if (applyElapsedMs >= 50)
|
||||||
|
{
|
||||||
_n_expensive_fires++;
|
_n_expensive_fires++;
|
||||||
Log.DataFlow.info(
|
Log.DataFlow.info(
|
||||||
$"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
|
$"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
|
||||||
}
|
}
|
||||||
// TODO,
|
myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
|
||||||
if (total_state != null)
|
|
||||||
myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
+32
-56
@@ -1,16 +1,15 @@
|
|||||||
using bts;
|
using bts;
|
||||||
using bts.collections;
|
|
||||||
|
|
||||||
using mbts;
|
using mbts;
|
||||||
|
|
||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
using System.Linq;
|
|
||||||
using System.Security.Cryptography;
|
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG
|
||||||
public class NonIncJoinBookkeeper : IDisposable {
|
{
|
||||||
|
public class NonIncJoinBookkeeper : IDisposable
|
||||||
|
{
|
||||||
public Action on_new_data;
|
public Action on_new_data;
|
||||||
private readonly NonIncJoinSpec _join_spec;
|
private readonly NonIncJoinSpec _join_spec;
|
||||||
private readonly int _join_position;
|
private readonly int _join_position;
|
||||||
@@ -27,71 +26,41 @@ namespace BCG {
|
|||||||
|
|
||||||
private int[] _join_col_indices = null,
|
private int[] _join_col_indices = null,
|
||||||
_data_col_indices = null;
|
_data_col_indices = null;
|
||||||
|
|
||||||
public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) {
|
public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher)
|
||||||
|
{
|
||||||
_join_spec = the_join_spec;
|
_join_spec = the_join_spec;
|
||||||
_join_position = join_position;
|
_join_position = join_position;
|
||||||
_start_index_in_output = start_index_in_output;
|
_start_index_in_output = start_index_in_output;
|
||||||
_row_hasher = row_hasher;
|
_row_hasher = row_hasher;
|
||||||
_is_left_most = _join_position == 0;
|
_is_left_most = _join_position == 0;
|
||||||
_is_tag_along = _join_spec.join_col_names.Length == 0;
|
_is_tag_along = _join_spec.join_col_names.Length == 0;
|
||||||
}
|
|
||||||
|
|
||||||
private bool _filled_missing_data = false;
|
|
||||||
private void _fill_missing_data(DataPacket dp) {
|
|
||||||
if (_filled_missing_data)
|
|
||||||
return;
|
|
||||||
Func<string[], bool> null_or_empty = (str_arr) => {
|
|
||||||
if (str_arr == null || str_arr.Length == 0)
|
|
||||||
return true;
|
|
||||||
foreach (string s in str_arr) {
|
|
||||||
if (!String.IsNullOrEmpty(s)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
};
|
|
||||||
Func<string[], string[]> remove_bad_cols = (col_names) => {
|
|
||||||
HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
|
|
||||||
List<string> cols_list = new List<string>(col_names);
|
|
||||||
cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
|
|
||||||
return cols_list.ToArray();
|
|
||||||
};
|
|
||||||
if (null_or_empty(_join_spec.data_col_names)) {
|
|
||||||
_join_spec.data_col_names = dp.schema.columnNames;
|
|
||||||
}
|
|
||||||
_join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names);
|
|
||||||
if (null_or_empty(_join_spec.data_col_names)) {
|
|
||||||
_join_spec.data_col_names = dp.schema.columnNames;
|
|
||||||
}
|
|
||||||
if (null_or_empty(_join_spec.projected_col_names)) {
|
|
||||||
_join_spec.projected_col_names = _join_spec.data_col_names;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public void init() {
|
|
||||||
_join_spec.stream.onChange += _on_stream_cb;
|
_join_spec.stream.onChange += _on_stream_cb;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void _on_stream_cb(object dp_obj) {
|
public void _on_stream_cb(object dp_obj)
|
||||||
Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}");
|
{
|
||||||
var timer = Stopwatch.StartNew();
|
var timer = Stopwatch.StartNew();
|
||||||
DataPacket dp = (DataPacket)dp_obj;
|
DataPacket dp = (DataPacket)dp_obj;
|
||||||
_fill_missing_data(dp);
|
if (_schema == null)
|
||||||
if (_schema == null) {
|
{
|
||||||
_schema = dp.schema;
|
_schema = dp.schema;
|
||||||
_join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
|
_join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
|
||||||
_data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
|
_data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (_is_left_most) {
|
if (_is_left_most)
|
||||||
|
{
|
||||||
_row_list.Clear();
|
_row_list.Clear();
|
||||||
foreach (IBlueRow r in dp.newRows)
|
foreach (IBlueRow r in dp.newRows)
|
||||||
_row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));
|
_row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));
|
||||||
} else {
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
if (_is_tag_along)
|
if (_is_tag_along)
|
||||||
_solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
|
_solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
|
||||||
else {
|
else
|
||||||
|
{
|
||||||
_row_map.Clear();
|
_row_map.Clear();
|
||||||
foreach (IBlueRow r in dp.newRows)
|
foreach (IBlueRow r in dp.newRows)
|
||||||
_row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
|
_row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
|
||||||
@@ -110,25 +79,31 @@ namespace BCG {
|
|||||||
|
|
||||||
public bool is_ready { get { return _schema != null; } }
|
public bool is_ready { get { return _schema != null; } }
|
||||||
|
|
||||||
public void constribute_cols_to_schema(List<ColumnInfo> cis) {
|
public void constribute_cols_to_schema(List<ColumnInfo> cis)
|
||||||
|
{
|
||||||
for (int i = 0; i < _join_spec.data_col_names.Length; i++)
|
for (int i = 0; i < _join_spec.data_col_names.Length; i++)
|
||||||
cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
|
cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
|
||||||
}
|
}
|
||||||
|
|
||||||
public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
|
public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
|
||||||
|
|
||||||
public void copy_master_row_to_output_row(int row_index, object[] output_row) {
|
public void copy_master_row_to_output_row(int row_index, object[] output_row)
|
||||||
|
{
|
||||||
IBlueRow this_row = _row_list[row_index].second;
|
IBlueRow this_row = _row_list[row_index].second;
|
||||||
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
|
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void copy_matching_row_to_output_row(int row_hash, object[] output_row) {
|
public void copy_matching_row_to_output_row(int row_hash, object[] output_row)
|
||||||
if (_is_tag_along) {
|
{
|
||||||
|
if (_is_tag_along)
|
||||||
|
{
|
||||||
if (_solitary_row != null)
|
if (_solitary_row != null)
|
||||||
BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
|
BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
|
||||||
else
|
else
|
||||||
copy_dbnull_to_output_row(output_row);
|
copy_dbnull_to_output_row(output_row);
|
||||||
} else {
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
IBlueRow this_row = null;
|
IBlueRow this_row = null;
|
||||||
if (_row_map.TryGetValue(row_hash, out this_row))
|
if (_row_map.TryGetValue(row_hash, out this_row))
|
||||||
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
|
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
|
||||||
@@ -137,7 +112,8 @@ namespace BCG {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void copy_dbnull_to_output_row(object[] output_row) {
|
private void copy_dbnull_to_output_row(object[] output_row)
|
||||||
|
{
|
||||||
for (int i = 0; i < _data_col_indices.Length; i++)
|
for (int i = 0; i < _data_col_indices.Length; i++)
|
||||||
output_row[i + _start_index_in_output] = DBNull.Value;
|
output_row[i + _start_index_in_output] = DBNull.Value;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,63 +3,63 @@ using System.Collections.Generic;
|
|||||||
using System.Linq;
|
using System.Linq;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
using System.Threading.Tasks;
|
using System.Threading.Tasks;
|
||||||
|
|
||||||
using mbts;
|
using mbts;
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG
|
||||||
// this a collection of data, theres no joining logic in here
|
{
|
||||||
public class NonIncJoinSpec {
|
public class NonIncJoinSpec
|
||||||
|
{
|
||||||
public ViewStream stream;
|
public ViewStream stream;
|
||||||
// join col names are the names on which the streams are joined (for equality),
|
|
||||||
// data col names are the names in the original stream
|
|
||||||
// projected col names are their new names, ie if its x:l_x, then x is in the
|
|
||||||
// data_col_names and l_x in projected cols
|
|
||||||
public string[] join_col_names, data_col_names, projected_col_names;
|
public string[] join_col_names, data_col_names, projected_col_names;
|
||||||
|
|
||||||
public NonIncJoinSpec() { }
|
public NonIncJoinSpec() { }
|
||||||
|
|
||||||
public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
|
public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
|
||||||
: this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { }
|
: this (the_stream, the_join_col_names, the_data_col_names, the_data_col_names)
|
||||||
|
{ }
|
||||||
|
|
||||||
public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names) {
|
public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names)
|
||||||
|
{
|
||||||
stream = the_stream;
|
stream = the_stream;
|
||||||
join_col_names = the_join_col_names;
|
join_col_names = the_join_col_names;
|
||||||
data_col_names = the_data_col_names;
|
data_col_names = the_data_col_names;
|
||||||
projected_col_names = the_projected_col_names;
|
projected_col_names = the_projected_col_names;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs) {
|
public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs)
|
||||||
|
{
|
||||||
List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
|
List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
|
||||||
foreach (Triplet<ViewStream, string, string> r in join_specs) {
|
foreach (Triplet<ViewStream, string, string> r in join_specs)
|
||||||
string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' ');
|
{
|
||||||
|
string[] keys = String.IsNullOrEmpty (r.second) ? new string[] {} : r.second.Split(' ');
|
||||||
string[] visible_cols = r.third.Split(' ');
|
string[] visible_cols = r.third.Split(' ');
|
||||||
List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
|
List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
|
||||||
foreach (string s in visible_cols) {
|
foreach (string s in visible_cols)
|
||||||
|
{
|
||||||
string[] toks = s.Split(':');
|
string[] toks = s.Split(':');
|
||||||
if (toks.Length == 1) {
|
if (toks.Length == 1)
|
||||||
|
{
|
||||||
data_col_names.Add(toks[0]);
|
data_col_names.Add(toks[0]);
|
||||||
mapped_col_names.Add(toks[0]);
|
mapped_col_names.Add(toks[0]);
|
||||||
} else if (toks.Length == 2) {
|
}
|
||||||
|
else if (toks.Length == 2)
|
||||||
|
{
|
||||||
data_col_names.Add(toks[0]);
|
data_col_names.Add(toks[0]);
|
||||||
mapped_col_names.Add(toks[1]);
|
mapped_col_names.Add(toks[1]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray()));
|
rv.Add (new NonIncJoinSpec (r.first, keys, data_col_names.ToArray (), mapped_col_names.ToArray()));
|
||||||
}
|
}
|
||||||
return rv.ToArray();
|
return rv.ToArray ();
|
||||||
}
|
|
||||||
|
|
||||||
public override string ToString() {
|
|
||||||
Func<string[], string> arr_txt = (a) => {
|
|
||||||
string rv = "";
|
|
||||||
foreach (string s in a) {
|
|
||||||
rv += "{s} ";
|
|
||||||
}
|
|
||||||
rv = rv.Trim();
|
|
||||||
return rv;
|
|
||||||
};
|
|
||||||
string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}";
|
|
||||||
return rv;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
//[DataStreamUserRespresentation("ninctest")]
|
||||||
|
//public class NonIncJoinTestStream : ViewStream
|
||||||
|
//{
|
||||||
|
// public NonIncJoinTestStream (string key_cols, string data_cols)
|
||||||
|
// {
|
||||||
|
// NonIncJoinSpec.from_triplet(new Triplet<ViewStream, string, string>(null, key_cols, data_cols));
|
||||||
|
// }
|
||||||
|
//}
|
||||||
}
|
}
|
||||||
@@ -6,9 +6,11 @@ using System;
|
|||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG
|
||||||
|
{
|
||||||
[DataStreamUserRespresentation("fastJoin")]
|
[DataStreamUserRespresentation("fastJoin")]
|
||||||
public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker {
|
public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker
|
||||||
|
{
|
||||||
private readonly NonIncJoinSpec[] _join_specs;
|
private readonly NonIncJoinSpec[] _join_specs;
|
||||||
private readonly INonIncJoinInlineMapper _output_mapper;
|
private readonly INonIncJoinInlineMapper _output_mapper;
|
||||||
private readonly BlueRowHasher _row_hasher;
|
private readonly BlueRowHasher _row_hasher;
|
||||||
@@ -26,19 +28,24 @@ namespace BCG {
|
|||||||
{
|
{
|
||||||
new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
|
new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
|
||||||
new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
|
new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
|
||||||
}) { }
|
})
|
||||||
|
{ }
|
||||||
|
|
||||||
public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
|
public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
|
||||||
: this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { }
|
: this(NonIncJoinSpec.from_triplet(join_specs), null, 0)
|
||||||
|
{ }
|
||||||
|
|
||||||
public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
|
public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
|
||||||
params Triplet<ViewStream, string, string>[] join_specs)
|
params Triplet<ViewStream, string, string>[] join_specs)
|
||||||
: this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { }
|
: this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms)
|
||||||
|
{ }
|
||||||
|
|
||||||
public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
|
public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
|
||||||
: this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { }
|
: this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms)
|
||||||
|
{ }
|
||||||
|
|
||||||
public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) {
|
public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
|
||||||
|
{
|
||||||
_join_specs = join_specs;
|
_join_specs = join_specs;
|
||||||
_output_mapper = output_mapper;
|
_output_mapper = output_mapper;
|
||||||
_row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
|
_row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
|
||||||
@@ -46,11 +53,12 @@ namespace BCG {
|
|||||||
_vsd = vsd;
|
_vsd = vsd;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
protected override void myOnFirstSubscriber()
|
||||||
|
{
|
||||||
int data_col_no = 1;
|
int data_col_no = 1;
|
||||||
for (int jp = 0; jp < _join_specs.Length; jp++) {
|
for (int jp = 0; jp < _join_specs.Length; jp++)
|
||||||
|
{
|
||||||
NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
|
NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
|
||||||
Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
|
|
||||||
data_col_no += _join_specs[jp].data_col_names.Length;
|
data_col_no += _join_specs[jp].data_col_names.Length;
|
||||||
s.on_new_data += _stream_callback;
|
s.on_new_data += _stream_callback;
|
||||||
_book_keeping_data.Add(s);
|
_book_keeping_data.Add(s);
|
||||||
@@ -58,11 +66,10 @@ namespace BCG {
|
|||||||
if (_output_mapper != null)
|
if (_output_mapper != null)
|
||||||
_output_mapper.onMapperChange += _on_mapper_change;
|
_output_mapper.onMapperChange += _on_mapper_change;
|
||||||
_vsd.addWorker(this);
|
_vsd.addWorker(this);
|
||||||
foreach (var bkd in _book_keeping_data)
|
|
||||||
bkd.init();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
protected override void myOnLastSubscriber()
|
||||||
|
{
|
||||||
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
|
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
|
||||||
d.Dispose();
|
d.Dispose();
|
||||||
if (_output_mapper != null)
|
if (_output_mapper != null)
|
||||||
@@ -70,26 +77,30 @@ namespace BCG {
|
|||||||
_vsd.removeWorker(this);
|
_vsd.removeWorker(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void _stream_callback() {
|
private void _stream_callback()
|
||||||
Log.Gen.info("NonIncJoinStream._stream_callback received callback");
|
{
|
||||||
_is_dirty = true;
|
_is_dirty = true;
|
||||||
doWork();
|
doWork();
|
||||||
}
|
}
|
||||||
|
|
||||||
private void _on_mapper_change() {
|
private void _on_mapper_change()
|
||||||
|
{
|
||||||
_is_dirty = true;
|
_is_dirty = true;
|
||||||
doWork();
|
doWork();
|
||||||
}
|
}
|
||||||
|
|
||||||
private bool _is_ready() {
|
private bool _is_ready()
|
||||||
|
{
|
||||||
bool rv = true;
|
bool rv = true;
|
||||||
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
|
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
|
||||||
rv &= d.is_ready;
|
rv &= d.is_ready;
|
||||||
return rv;
|
return rv;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void _construct_schema() {
|
private void _construct_schema()
|
||||||
if (_output_schema == null) {
|
{
|
||||||
|
if (_output_schema == null)
|
||||||
|
{
|
||||||
List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
|
List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
|
||||||
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
|
foreach (NonIncJoinBookkeeper d in _book_keeping_data)
|
||||||
d.constribute_cols_to_schema(cis);
|
d.constribute_cols_to_schema(cis);
|
||||||
@@ -98,23 +109,27 @@ namespace BCG {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private int _n_total_fires = 0, _n_expensive_fires = 0;
|
private int _n_total_fires = 0, _n_expensive_fires = 0;
|
||||||
private void _form_and_fire() {
|
private void _form_and_fire()
|
||||||
Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}");
|
{
|
||||||
if (_is_ready()) {
|
if (_is_ready())
|
||||||
|
{
|
||||||
_construct_schema();
|
_construct_schema();
|
||||||
var timer = Stopwatch.StartNew();
|
var timer = Stopwatch.StartNew();
|
||||||
IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
|
IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
|
||||||
List<object[]> output_rows = new List<object[]>();
|
List<object[]> output_rows = new List<object[]>();
|
||||||
|
|
||||||
for (int i = 0; i < left_row_list.Count; i++) {
|
for (int i = 0; i < left_row_list.Count; i++)
|
||||||
|
{
|
||||||
object[] or = new object[_output_schema.columnInfo.Length];
|
object[] or = new object[_output_schema.columnInfo.Length];
|
||||||
or[0] = i;
|
or[0] = i;
|
||||||
output_rows.Add(or);
|
output_rows.Add(or);
|
||||||
}
|
}
|
||||||
|
|
||||||
for (int ji = 0; ji < _book_keeping_data.Count; ji++) {
|
for (int ji = 0; ji < _book_keeping_data.Count; ji++)
|
||||||
|
{
|
||||||
NonIncJoinBookkeeper bk_data = _book_keeping_data[ji];
|
NonIncJoinBookkeeper bk_data = _book_keeping_data[ji];
|
||||||
for (int i = 0; i < left_row_list.Count; i++) {
|
for (int i = 0; i < left_row_list.Count; i++)
|
||||||
|
{
|
||||||
if (ji == 0)
|
if (ji == 0)
|
||||||
bk_data.copy_master_row_to_output_row(i, output_rows[i]);
|
bk_data.copy_master_row_to_output_row(i, output_rows[i]);
|
||||||
else
|
else
|
||||||
@@ -124,20 +139,19 @@ namespace BCG {
|
|||||||
IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
|
IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
|
||||||
timer.Stop();
|
timer.Stop();
|
||||||
_n_total_fires++;
|
_n_total_fires++;
|
||||||
//if (timer.ElapsedMilliseconds >= 250) {
|
if (timer.ElapsedMilliseconds >= 250) {
|
||||||
_n_expensive_fires++;
|
_n_expensive_fires++;
|
||||||
string log_msg = $"Join and map time for {_book_keeping_data.Count} "
|
string log_msg = $"Join and map time for {_book_keeping_data.Count} "
|
||||||
+ $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}";
|
+ $"tables {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}";
|
||||||
Log.Gen.info(log_msg);
|
}
|
||||||
//}
|
myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
|
||||||
var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket);
|
|
||||||
myFireEvent(dp_out);
|
|
||||||
//myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) {
|
private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows)
|
||||||
if (_output_mapper != null) {
|
{
|
||||||
|
if (_output_mapper != null)
|
||||||
|
{
|
||||||
IBlueRowCollection output_rows;
|
IBlueRowCollection output_rows;
|
||||||
var timer = Stopwatch.StartNew();
|
var timer = Stopwatch.StartNew();
|
||||||
output_rows = _output_mapper.map_rows(unmapped_rows);
|
output_rows = _output_mapper.map_rows(unmapped_rows);
|
||||||
@@ -146,14 +160,17 @@ namespace BCG {
|
|||||||
Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
|
Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
|
||||||
}
|
}
|
||||||
return output_rows;
|
return output_rows;
|
||||||
} else {
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
return unmapped_rows;
|
return unmapped_rows;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void doWork() {
|
public void doWork()
|
||||||
Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}");
|
{
|
||||||
if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) {
|
if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty)
|
||||||
|
{
|
||||||
_is_dirty = false;
|
_is_dirty = false;
|
||||||
_last_fire_time = DateTime.Now;
|
_last_fire_time = DateTime.Now;
|
||||||
_form_and_fire();
|
_form_and_fire();
|
||||||
|
|||||||
@@ -1,9 +1,7 @@
|
|||||||
using System;
|
#nullable disable
|
||||||
|
using System;
|
||||||
using System.Collections;
|
using System.Collections;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Text;
|
|
||||||
using BCG.Utilities.General;
|
|
||||||
using BCG.Utilities;
|
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -1,68 +0,0 @@
|
|||||||
namespace bts {
|
|
||||||
using BCG;
|
|
||||||
using BCG.Utilities.General;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Diagnostics;
|
|
||||||
|
|
||||||
using ZstdSharp.Unsafe;
|
|
||||||
|
|
||||||
// functon that joins the 2 rows and returns the outputput row. The function
|
|
||||||
// can return a null which means the 2 rows dont need to be joined
|
|
||||||
using JoinFunc = System.Func<BCG.IBlueRow, BCG.IBlueRow, BCG.IBlueRow>;
|
|
||||||
public class ProximityJoinStream : ViewStream, IWorker {
|
|
||||||
private ViewStream _lvs, _rvs;
|
|
||||||
private JoinFunc _join_func;
|
|
||||||
private bool _dirty = false;
|
|
||||||
private Stopwatch _stopwatch = new Stopwatch();
|
|
||||||
private ViewStreamDatabase _vsd = ViewStreamDatabase.instance;
|
|
||||||
public ProximityJoinStream(ViewStream lvs, ViewStream rvs, JoinFunc join_func) {
|
|
||||||
_lvs = lvs;
|
|
||||||
_rvs = rvs;
|
|
||||||
_join_func = join_func;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
_lvs.onChange += _handle_data;
|
|
||||||
_rvs.onChange += _handle_data;
|
|
||||||
_vsd.addWorker(this);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
|
||||||
_lvs.onChange -= _handle_data;
|
|
||||||
_rvs.onChange -= _handle_data;
|
|
||||||
_vsd.removeWorker(this);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _handle_data(object dp_obj) {
|
|
||||||
_dirty = true;
|
|
||||||
doWork();
|
|
||||||
}
|
|
||||||
|
|
||||||
public void doWork() {
|
|
||||||
Func<bool> all_data_revcd = () => {
|
|
||||||
return _lvs.rows != null && _rvs.rows != null;
|
|
||||||
};
|
|
||||||
_stopwatch.Start();
|
|
||||||
if (_dirty && all_data_revcd()) {
|
|
||||||
_dirty = false;
|
|
||||||
BlueTable rv = null;
|
|
||||||
foreach (IBlueRow lr in _lvs.rows) {
|
|
||||||
foreach (IBlueRow rr in _rvs.rows) {
|
|
||||||
IBlueRow resr = _join_func(lr, rr);
|
|
||||||
if (resr == null)
|
|
||||||
continue;
|
|
||||||
rv = rv ?? new BlueTable(resr.schema);
|
|
||||||
rv.addRow(resr);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_stopwatch.Stop();
|
|
||||||
Log.Gen.info($"ProximityJoinStream left_count: {_lvs.rows.Count} right_count: {_rvs.rows.Count} result_count: {rv.Count} time: {_stopwatch.ElapsedMilliseconds} msecs");
|
|
||||||
var dp = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket);
|
|
||||||
myFireEvent(dp);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,47 +0,0 @@
|
|||||||
namespace bts.SilverPine {
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.CodeDom;
|
|
||||||
using System.Configuration;
|
|
||||||
using System.Reactive.Disposables;
|
|
||||||
|
|
||||||
public class TimeProximityJoinStream : ViewStream {
|
|
||||||
private int _id = 0;
|
|
||||||
|
|
||||||
public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
|
|
||||||
string pre_join_period, string post_join_period) :
|
|
||||||
this(ls, l_time_col, rs, r_time_col, TimeBasedFilter.parse_duration(pre_join_period),
|
|
||||||
TimeBasedFilter.parse_duration(post_join_period)) { }
|
|
||||||
|
|
||||||
|
|
||||||
public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
|
|
||||||
TimeSpan pre_join_period, TimeSpan post_join_period) {
|
|
||||||
Func<IBlueRow, IBlueRow, IBlueRow> create_row = (lr, rr) => {
|
|
||||||
var id = 0;
|
|
||||||
lock (this) {
|
|
||||||
id = _id++;
|
|
||||||
}
|
|
||||||
IBlueRow rv = BlueDataUtility.create_composite_row(lr, rr, id);
|
|
||||||
return rv;
|
|
||||||
};
|
|
||||||
Func<IBlueRow, IBlueRow, IBlueRow> join_func = (lr, rr) => {
|
|
||||||
DateTime ldt = lr.get_datetime(l_time_col);
|
|
||||||
DateTime rdt = rr.get_datetime(r_time_col);
|
|
||||||
TimeSpan diff = ldt - rdt;
|
|
||||||
if (ldt < rdt) {
|
|
||||||
if (diff.Duration() > pre_join_period) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if (diff > post_join_period)
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return create_row(lr, rr);
|
|
||||||
};
|
|
||||||
|
|
||||||
ViewStream rstr = new ProximityJoinStream(ls, rs, join_func);
|
|
||||||
mySetRealSource(rstr);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -2,13 +2,9 @@ using System;
|
|||||||
using System.Collections;
|
using System.Collections;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
|
|
||||||
using Amazon.Runtime.Internal.Util;
|
|
||||||
|
|
||||||
using BCG;
|
using BCG;
|
||||||
using BCG.Utilities;
|
using BCG.Utilities;
|
||||||
|
|
||||||
using bts;
|
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG {
|
||||||
public class LiteralStream : ViewStream {
|
public class LiteralStream : ViewStream {
|
||||||
|
|
||||||
@@ -26,34 +22,6 @@ namespace BCG {
|
|||||||
return rv;
|
return rv;
|
||||||
}
|
}
|
||||||
|
|
||||||
//literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
|
|
||||||
public LiteralStream(string table_name, string rows) {
|
|
||||||
Func<string, int, IBlueRow> get_row = (json_row, id) => {
|
|
||||||
Dictionary<string, object> map = new Dictionary<string, object>();
|
|
||||||
string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
|
|
||||||
//Logger.assert(col_vals.Length == 2);
|
|
||||||
foreach (string t in toks) {
|
|
||||||
var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
|
|
||||||
string col = col_vals[0];
|
|
||||||
string val = col_vals[1];
|
|
||||||
map.Add(col, val);
|
|
||||||
}
|
|
||||||
map["id"] = id;
|
|
||||||
return new BlueRow(map);
|
|
||||||
};
|
|
||||||
string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
|
|
||||||
BlueTable table = null;
|
|
||||||
foreach (string json_string in row_arr) {
|
|
||||||
IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
|
|
||||||
get_row(json_string, table == null ? 0 : table.Count);
|
|
||||||
table = table ?? new BlueTable(r.schema);
|
|
||||||
table.addRow(r);
|
|
||||||
}
|
|
||||||
ViewStream str = new BlueTableStream(table);
|
|
||||||
str = new SchemaGuesserStream(str);
|
|
||||||
mySetRealSource(str);
|
|
||||||
}
|
|
||||||
|
|
||||||
public LiteralStream(string tableName, string schema, string value) {
|
public LiteralStream(string tableName, string schema, string value) {
|
||||||
string[] columns = schema.Split(' ');
|
string[] columns = schema.Split(' ');
|
||||||
|
|
||||||
@@ -87,7 +55,7 @@ namespace BCG {
|
|||||||
mySetRealSource(new BlueRowCollectionStaticStream(t));
|
mySetRealSource(new BlueRowCollectionStaticStream(t));
|
||||||
}
|
}
|
||||||
|
|
||||||
// public LiteralStream(string schema, string value) : this("literal", schema, value) { }
|
public LiteralStream(string schema, string value) : this("literal", schema, value) { }
|
||||||
|
|
||||||
// public LiteralStream(string value) {
|
// public LiteralStream(string value) {
|
||||||
|
|
||||||
|
|||||||
@@ -1,72 +0,0 @@
|
|||||||
using BCG;
|
|
||||||
|
|
||||||
using MongoDB.Driver.Core.WireProtocol.Messages;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
|
|
||||||
// the first arg is all the rows, the second arg is the index of the row that needs mapping, and
|
|
||||||
// the func returns the mapped row (ie mapped version of the index row)
|
|
||||||
using GlobalMap = System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow>;
|
|
||||||
|
|
||||||
namespace bts {
|
|
||||||
// In GlobalMap, each row is mapped with the full view of all rows, ie the map functions for a row
|
|
||||||
// can use the state fo all rows.
|
|
||||||
public class GlobalMapStream : ViewStream {
|
|
||||||
private ViewStream _base_stream;
|
|
||||||
private readonly string _sort_col_name, _group_by_col;
|
|
||||||
private GlobalMap _map_func;
|
|
||||||
public GlobalMapStream(ViewStream base_stream, string sort_col_name, string group_by_col, GlobalMap map_func) {
|
|
||||||
_base_stream = base_stream;
|
|
||||||
_sort_col_name = sort_col_name;
|
|
||||||
_group_by_col = group_by_col;
|
|
||||||
_map_func = map_func;
|
|
||||||
}
|
|
||||||
|
|
||||||
public GlobalMapStream(ViewStream base_stream, string sort_col_name, GlobalMap map_func) :
|
|
||||||
this(base_stream, sort_col_name, null, map_func) {
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
_base_stream.onChange += _handle_stream_update;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
|
||||||
_base_stream.onChange -= _handle_stream_update;
|
|
||||||
}
|
|
||||||
|
|
||||||
void _handle_stream_update(object dp_obj) {
|
|
||||||
DataPacket dp = dp_obj as DataPacket;
|
|
||||||
IDictionary<int, List<IBlueRow>> groups = !String.IsNullOrEmpty(_group_by_col) ?
|
|
||||||
BlueDataUtility.group_together(dp.newRows, new string[] { _group_by_col })
|
|
||||||
:
|
|
||||||
new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } };
|
|
||||||
BlueTable out_table = null;
|
|
||||||
foreach (var kv in groups) {
|
|
||||||
List<IBlueRow> group_rows = kv.Value;
|
|
||||||
group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, _sort_col_name));
|
|
||||||
uint i = 0;
|
|
||||||
foreach (IBlueRow r in group_rows) {
|
|
||||||
IBlueRow nr = _map_func(group_rows, i++);
|
|
||||||
out_table = out_table ?? new BlueTable(nr.schema);
|
|
||||||
out_table.addRow(nr);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
// List<IBlueRow> rows = dp.newRows.to_ienum().ToList();
|
|
||||||
// rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r1, r2, _sort_col_name));
|
|
||||||
// uint i = 0;
|
|
||||||
|
|
||||||
// BlueTable out_table = null;
|
|
||||||
// foreach (IBlueRow r in rows) {
|
|
||||||
// IBlueRow nr = _map_func(rows, i++);
|
|
||||||
// out_table = out_table ?? new BlueTable(nr.schema);
|
|
||||||
// out_table.addRow(nr);
|
|
||||||
// }
|
|
||||||
DataPacket dp_out = BlueDataUtility.build_dp_inc(out_table, myLastValue as DataPacket);
|
|
||||||
myFireEvent(dp_out);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,6 +1,10 @@
|
|||||||
|
using bts;
|
||||||
|
|
||||||
using Bts.DataStream;
|
using Bts.DataStream;
|
||||||
|
using Bts.DataStreamCore;
|
||||||
|
|
||||||
using mbts.config;
|
using mbts.config;
|
||||||
|
using mbts.utils;
|
||||||
|
|
||||||
using System;
|
using System;
|
||||||
|
|
||||||
@@ -16,8 +20,8 @@ namespace BCG {
|
|||||||
[DataStreamUserRespresentation (false)]
|
[DataStreamUserRespresentation (false)]
|
||||||
public class MapStream : MethodDrivenStream
|
public class MapStream : MethodDrivenStream
|
||||||
{
|
{
|
||||||
protected IStreamMapper myMapper;
|
protected IStreamMapper? myMapper;
|
||||||
protected IChangeableStreamMapper myChangeableMapper = null;
|
protected IChangeableStreamMapper? myChangeableMapper;
|
||||||
private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;
|
private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;
|
||||||
|
|
||||||
public class FuncWrapper : IStreamMapper {
|
public class FuncWrapper : IStreamMapper {
|
||||||
@@ -72,7 +76,7 @@ namespace BCG {
|
|||||||
|
|
||||||
|
|
||||||
protected override void myHandleFirstCall() {
|
protected override void myHandleFirstCall() {
|
||||||
|
ArgumentNullException.ThrowIfNull(myMapper);
|
||||||
IBlueTableSchema outputSchema = myMapper.getOutputSchema(myLastArrivedPacket.newRows.schema);
|
IBlueTableSchema outputSchema = myMapper.getOutputSchema(myLastArrivedPacket.newRows.schema);
|
||||||
myLastFiredPacket = new DataPacket();
|
myLastFiredPacket = new DataPacket();
|
||||||
myLastFiredPacket.newRows = new BlueTable(outputSchema);
|
myLastFiredPacket.newRows = new BlueTable(outputSchema);
|
||||||
@@ -83,28 +87,34 @@ namespace BCG {
|
|||||||
}
|
}
|
||||||
|
|
||||||
protected override void myHandleIncrementalAdd() {
|
protected override void myHandleIncrementalAdd() {
|
||||||
//t.start ();
|
if (myLastArrivedPacket.addedRows.Count == 0) {
|
||||||
myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows.schema);
|
Log.Gen.warn("MapStream.myHandleIncrementalAdd called with 0 addedRows");
|
||||||
foreach (IBlueRow row in myLastArrivedPacket.addedRows) {
|
return;
|
||||||
IBlueRow newRow = myMapper.map(row);
|
|
||||||
(myLastFiredPacket.newRows as BlueTable).addRow(newRow);
|
|
||||||
(myLastFiredPacket.addedRows as BlueTable).addRow(newRow);
|
|
||||||
}
|
}
|
||||||
//t.stop ();
|
ArgumentNullException.ThrowIfNull(myMapper);
|
||||||
//System.Diagnostics.Trace.WriteLine ("adding " + myLastFiredPacket.addedRows.Count.ToString () + " : " + t.lastIntervalTime);
|
var addedRows = new BlueTable(myLastFiredPacket.newRows.schema);
|
||||||
//System.Diagnostics.Trace.WriteLine (" ");
|
var newRows = ExceptionAssert.Cast(myLastFiredPacket.newRows).To<BlueTable>();
|
||||||
|
foreach (IBlueRow row in myLastArrivedPacket.addedRows) {
|
||||||
|
var newRow = myMapper.map(row);
|
||||||
|
newRows.addRow(newRow);
|
||||||
|
addedRows.addRow(newRow);
|
||||||
|
}
|
||||||
|
myLastFiredPacket.addedRows = addedRows;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myHandleIncrementalDelete() {
|
protected override void myHandleIncrementalDelete() {
|
||||||
//t.start ();
|
if (myLastArrivedPacket.deletedRows.Count == 0) {
|
||||||
myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows.schema);
|
Log.Gen.warn("MapStream.myHandleIncrementalDelete called with 0 deletedRows");
|
||||||
foreach (IBlueRow row in myLastArrivedPacket.deletedRows) {
|
return;
|
||||||
IBlueRow delRow = (myLastFiredPacket.newRows as BlueTable)[(int)row["id"]];
|
|
||||||
(myLastFiredPacket.deletedRows as BlueTable).addRow(delRow);
|
|
||||||
(myLastFiredPacket.newRows as BlueTable).deleteRow(row);
|
|
||||||
}
|
}
|
||||||
//t.stop ();
|
var deletedRows = new BlueTable(myLastFiredPacket.newRows.schema);
|
||||||
//System.Diagnostics.Trace.WriteLine ("deleting " + myLastFiredPacket.deletedRows.Count.ToString () + " : " + t.lastIntervalTime);
|
var newRows = ExceptionAssert.Cast(myLastFiredPacket.newRows).To<BlueTable>();
|
||||||
|
foreach (IBlueRow row in myLastArrivedPacket.deletedRows) {
|
||||||
|
var delRow = newRows[row.Id()];
|
||||||
|
deletedRows.addRow(delRow);
|
||||||
|
newRows.deleteRow(row);
|
||||||
|
}
|
||||||
|
myLastFiredPacket.deletedRows = deletedRows;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,11 +1,9 @@
|
|||||||
|
using Bts.DataStreamCore;
|
||||||
|
|
||||||
|
using mbts.utils;
|
||||||
|
|
||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Text;
|
|
||||||
|
|
||||||
using BCG.Utilities;
|
|
||||||
using BCG.Utilities.General;
|
|
||||||
|
|
||||||
using mbts;
|
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG {
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@@ -56,15 +54,15 @@ namespace BCG {
|
|||||||
return rows == null ? new IBlueRow[] { } : new List<IBlueRow>(rows).ToArray();
|
return rows == null ? new IBlueRow[] { } : new List<IBlueRow>(rows).ToArray();
|
||||||
}
|
}
|
||||||
|
|
||||||
public event MapperChangeDelegate onMapperChange;
|
public event MapperChangeDelegate? onMapperChange;
|
||||||
private void _fake() { onMapperChange(); }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
//maps the id of the underlying stream to the
|
//maps the id of the underlying stream to the
|
||||||
//rows of the resulting stream
|
//rows of the resulting stream
|
||||||
private Dictionary<int, List<IBlueRow>> myMap = new Dictionary<int, List<IBlueRow>>();
|
private Dictionary<int, List<IBlueRow>> myMap = new Dictionary<int, List<IBlueRow>>();
|
||||||
private int myID = 0;
|
private int myID = 0;
|
||||||
private BlueTable myRows, myBaseRows;
|
private BlueTable? myRows;
|
||||||
|
private BlueTable? myBaseRows;
|
||||||
private ViewStream myStream;
|
private ViewStream myStream;
|
||||||
private ISuperStreamMapper myMapper;
|
private ISuperStreamMapper myMapper;
|
||||||
public bool generate_ids = true;
|
public bool generate_ids = true;
|
||||||
@@ -92,49 +90,40 @@ namespace BCG {
|
|||||||
myRows = null;
|
myRows = null;
|
||||||
myBaseRows = null;
|
myBaseRows = null;
|
||||||
myMap.Clear();
|
myMap.Clear();
|
||||||
//myMapper.onMapperChange -= new MapperChangeDelegate(myHandleMapChange);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void myHandleData(object newValue) {
|
private void myHandleData(object newValue) {
|
||||||
DataPacket idp = newValue as DataPacket;
|
var dp = ExceptionAssert.Cast(newValue).To<DataPacket>();
|
||||||
//HighPerformanceTimer timer = new HighPerformanceTimer();
|
myRows ??= new BlueTable(myMapper.getOutputSchema(dp.schema));
|
||||||
//timer.start();
|
myBaseRows ??= new BlueTable(dp.schema);
|
||||||
DataPacket dp = newValue as DataPacket;
|
|
||||||
myRows = myRows ?? new BlueTable(myMapper.getOutputSchema(dp.schema));
|
|
||||||
myBaseRows = myBaseRows ?? new BlueTable(dp.schema);
|
|
||||||
BlueDataUtility.absorbChanges(myBaseRows, dp.deletedRows, dp.addedRows);
|
BlueDataUtility.absorbChanges(myBaseRows, dp.deletedRows, dp.addedRows);
|
||||||
BlueTable dels = new BlueTable(myRows.schema),
|
var dels = new BlueTable(myRows.schema);
|
||||||
adds = new BlueTable(myRows.schema);
|
var adds = new BlueTable(myRows.schema);
|
||||||
|
|
||||||
foreach (IBlueRow r in dp.deletedRows) {
|
foreach (IBlueRow r in dp.deletedRows) {
|
||||||
List<IBlueRow> mappedRows = myMap[(int)r["id"]];
|
var mappedRows = myMap[r.Id()];
|
||||||
foreach (IBlueRow mr in mappedRows) {
|
foreach (IBlueRow mr in mappedRows) {
|
||||||
dels.addRow(mr);
|
dels.addRow(mr);
|
||||||
myRows.deleteRow(mr);
|
myRows.deleteRow(mr);
|
||||||
}
|
}
|
||||||
myMap.Remove((int)r["id"]);
|
myMap.Remove(r.Id());
|
||||||
}
|
}
|
||||||
foreach (IBlueRow r in dp.addedRows) {
|
foreach (IBlueRow r in dp.addedRows) {
|
||||||
IBlueRow[] mappedRows = myMapper.map(r);
|
IBlueRow[] mappedRows = myMapper.map(r);
|
||||||
foreach (IBlueRow mr in mappedRows) {
|
foreach (IBlueRow mr in mappedRows) {
|
||||||
IBlueRow idChangedMapRow = generate_ids ? BlueDataUtility.getNewReplacedRow(mr, myID++, "id") : mr;
|
IBlueRow idChangedMapRow = generate_ids ? BlueDataUtility.getNewReplacedRow(mr, myID++, "id") : mr;
|
||||||
if (!myMap.ContainsKey((int)r["id"])) {
|
myMap.GetOrAdd(r.Id(), _ => new()).Add(idChangedMapRow);
|
||||||
myMap.Add((int)r["id"], new List<IBlueRow>());
|
|
||||||
}
|
|
||||||
myMap[(int)r["id"]].Add(idChangedMapRow);
|
|
||||||
myRows.addRow(idChangedMapRow);
|
myRows.addRow(idChangedMapRow);
|
||||||
adds.addRow(idChangedMapRow);
|
adds.addRow(idChangedMapRow);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
//timer.stop();
|
|
||||||
// if (timer.lastIntervalTime > 20)
|
|
||||||
// {}//mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("SuperMapper:myHandlerData for mapper {0} took {1} msecs", myMapper.GetType().ToString(), timer.lastIntervalTime));
|
|
||||||
myFireEvent(new DataPacket(myRows, adds, dels));
|
myFireEvent(new DataPacket(myRows, adds, dels));
|
||||||
}
|
}
|
||||||
|
|
||||||
private void myHandleMapChange() {
|
private void myHandleMapChange() {
|
||||||
if (myBaseRows != null)
|
if (myBaseRows is not null) {
|
||||||
myHandleData(new DataPacket(myBaseRows, new BlueTable(myBaseRows), new BlueTable(myBaseRows)));
|
myHandleData(new DataPacket(myBaseRows, new BlueTable(myBaseRows), new BlueTable(myBaseRows)));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -8,33 +8,26 @@ using System;
|
|||||||
using System.IO;
|
using System.IO;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
|
|
||||||
namespace bts
|
namespace bts {
|
||||||
{
|
public class PubSubTestingStream : ViewStream, IWorker {
|
||||||
public class PubSubTestingStream : ViewStream, IWorker
|
public class PayloadMessage {
|
||||||
{
|
|
||||||
public class PayloadMessage
|
|
||||||
{
|
|
||||||
public int msg_id = 0;
|
public int msg_id = 0;
|
||||||
public string guid;
|
public string guid;
|
||||||
public Int64 travel_epoch_ms = 0;
|
public Int64 travel_epoch_ms = 0;
|
||||||
public byte[] payload = null;
|
public byte[] payload = null;
|
||||||
public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte)
|
public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte) {
|
||||||
{
|
|
||||||
msg_id = msg_id_args;
|
msg_id = msg_id_args;
|
||||||
travel_epoch_ms = send_epoch_args;
|
travel_epoch_ms = send_epoch_args;
|
||||||
payload = new byte[payload_size];
|
payload = new byte[payload_size];
|
||||||
for (int i = 0; i < payload_size; i++)
|
for (int i = 0; i < payload_size; i++) {
|
||||||
{
|
|
||||||
payload[i] = payload_byte;
|
payload[i] = payload_byte;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
private PayloadMessage() { }
|
private PayloadMessage() { }
|
||||||
public PayloadMessage(int msg_id, int payload_size, byte payload_byte) :
|
public PayloadMessage(int msg_id, int payload_size, byte payload_byte) :
|
||||||
this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte)
|
this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte) { }
|
||||||
{ }
|
|
||||||
|
|
||||||
public byte[] serialize()
|
public byte[] serialize() {
|
||||||
{
|
|
||||||
var str = new MemoryStream();
|
var str = new MemoryStream();
|
||||||
BinaryWriter bw = new BinaryWriter(str);
|
BinaryWriter bw = new BinaryWriter(str);
|
||||||
bw.Write(msg_id);
|
bw.Write(msg_id);
|
||||||
@@ -47,8 +40,7 @@ namespace bts
|
|||||||
return bytes;
|
return bytes;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static PayloadMessage deserialize(byte[] ser_bytes)
|
public static PayloadMessage deserialize(byte[] ser_bytes) {
|
||||||
{
|
|
||||||
var str = new MemoryStream(ser_bytes);
|
var str = new MemoryStream(ser_bytes);
|
||||||
BinaryReader br = new BinaryReader(str);
|
BinaryReader br = new BinaryReader(str);
|
||||||
int msg_id = br.ReadInt32();
|
int msg_id = br.ReadInt32();
|
||||||
@@ -56,8 +48,7 @@ namespace bts
|
|||||||
Int64 epoch = br.ReadInt64();
|
Int64 epoch = br.ReadInt64();
|
||||||
int len = br.ReadInt32();
|
int len = br.ReadInt32();
|
||||||
var payload = br.ReadBytes(len);
|
var payload = br.ReadBytes(len);
|
||||||
return new PayloadMessage()
|
return new PayloadMessage() {
|
||||||
{
|
|
||||||
msg_id = msg_id,
|
msg_id = msg_id,
|
||||||
guid = guid,
|
guid = guid,
|
||||||
travel_epoch_ms = epoch,
|
travel_epoch_ms = epoch,
|
||||||
@@ -82,8 +73,7 @@ namespace bts
|
|||||||
private readonly int _payload_sz;
|
private readonly int _payload_sz;
|
||||||
private const byte _payload_byte = (byte)'A';
|
private const byte _payload_byte = (byte)'A';
|
||||||
private readonly string _tag_name;
|
private readonly string _tag_name;
|
||||||
public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd)
|
public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd) {
|
||||||
{
|
|
||||||
_vsd = vsd;
|
_vsd = vsd;
|
||||||
_payload_sz = payload_size;
|
_payload_sz = payload_size;
|
||||||
var schema = BlueDataUtility.form_schema("PubSubTest",
|
var schema = BlueDataUtility.form_schema("PubSubTest",
|
||||||
@@ -106,53 +96,43 @@ namespace bts
|
|||||||
public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size, string tag_name)
|
public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size, string tag_name)
|
||||||
: this(publish_period_seconds, max_pubs, msg_size, tag_name, ViewStreamDatabase.instance) { }
|
: this(publish_period_seconds, max_pubs, msg_size, tag_name, ViewStreamDatabase.instance) { }
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber()
|
protected override void myOnFirstSubscriber() {
|
||||||
{
|
|
||||||
_vsd.addWorker(this);
|
_vsd.addWorker(this);
|
||||||
System.Threading.Thread th = new System.Threading.Thread(_subscribe_thread_func);
|
System.Threading.Thread th = new System.Threading.Thread(_subscribe_thread_func);
|
||||||
th.IsBackground = true;
|
th.IsBackground = true;
|
||||||
th.Start();
|
th.Start();
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber()
|
protected override void myOnLastSubscriber() {
|
||||||
{
|
|
||||||
_vsd.removeWorker(this);
|
_vsd.removeWorker(this);
|
||||||
_continue_sub_thread = false;
|
_continue_sub_thread = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
void _log_updates(IEnumerable<PubSubServerMessage> updates)
|
void _log_updates(IEnumerable<PubSubServerMessage> updates) {
|
||||||
{
|
|
||||||
var idx = 0;
|
var idx = 0;
|
||||||
foreach (var msg in updates)
|
foreach (var msg in updates) {
|
||||||
{
|
|
||||||
bts.Log.Gen.info($"PubSubTesStream._subscribe_thread_func idx: {idx} msg: {msg}");
|
bts.Log.Gen.info($"PubSubTesStream._subscribe_thread_func idx: {idx} msg: {msg}");
|
||||||
idx++;
|
idx++;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void _subscribe_thread_func()
|
private void _subscribe_thread_func() {
|
||||||
{
|
|
||||||
// subscription is done in another thread
|
// subscription is done in another thread
|
||||||
SubscriberCore sub_core = new SubscriberCore();
|
SubscriberCore sub_core = new SubscriberCore();
|
||||||
sub_core.subscribe(_tag_name);
|
sub_core.subscribe(_tag_name);
|
||||||
while (_continue_sub_thread)
|
while (_continue_sub_thread) {
|
||||||
{
|
|
||||||
// TODO recv_id
|
// TODO recv_id
|
||||||
var updates = sub_core.get_updates(TimeSpan.FromMilliseconds(250));
|
var updates = sub_core.get_updates(TimeSpan.FromMilliseconds(250));
|
||||||
if (updates == null || updates.Count() == 0)
|
if (updates == null || updates.Count() == 0)
|
||||||
continue;
|
continue;
|
||||||
_log_updates(updates);
|
_log_updates(updates);
|
||||||
_n_updates += updates.Count();
|
_n_updates += updates.Count();
|
||||||
if (updates.Count() > 0)
|
if (updates.Count() > 0) {
|
||||||
{
|
|
||||||
var server_msg = updates.Last();
|
var server_msg = updates.Last();
|
||||||
if (server_msg.data != null && server_msg.data.Length > 0)
|
if (server_msg.data != null && server_msg.data.Length > 0) {
|
||||||
{
|
|
||||||
_last_recvd_msg = PayloadMessage.deserialize(server_msg.data);
|
_last_recvd_msg = PayloadMessage.deserialize(server_msg.data);
|
||||||
_last_recvd_msg.travel_epoch_ms = epoch_now();
|
_last_recvd_msg.travel_epoch_ms = epoch_now();
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
// why are we getting empty or null data ? (server is sending data regardless?)
|
// why are we getting empty or null data ? (server is sending data regardless?)
|
||||||
}
|
}
|
||||||
_fire_dirty = true;
|
_fire_dirty = true;
|
||||||
@@ -160,36 +140,29 @@ namespace bts
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void _publish_next_packet()
|
private void _publish_next_packet() {
|
||||||
{
|
if (_last_sent_msg == null) {
|
||||||
if (_last_sent_msg == null)
|
_last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte) {
|
||||||
{
|
|
||||||
_last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte)
|
|
||||||
{
|
|
||||||
guid = Guid.NewGuid().ToString()
|
guid = Guid.NewGuid().ToString()
|
||||||
};
|
};
|
||||||
}
|
} else {
|
||||||
else
|
|
||||||
{
|
|
||||||
_last_sent_msg.msg_id++;
|
_last_sent_msg.msg_id++;
|
||||||
_last_sent_msg.guid = Guid.NewGuid().ToString();
|
_last_sent_msg.guid = Guid.NewGuid().ToString();
|
||||||
_last_sent_msg.travel_epoch_ms = epoch_now();
|
_last_sent_msg.travel_epoch_ms = epoch_now();
|
||||||
}
|
}
|
||||||
var ser_bytes = _last_sent_msg.serialize();
|
var ser_bytes = _last_sent_msg.serialize();
|
||||||
_n_pubs++;
|
_n_pubs++;
|
||||||
_pub_core.publish("Blob", _tag_name, ser_bytes);
|
_pub_core.publish(String.Empty, _tag_name, ser_bytes);
|
||||||
_fire_dirty = true;
|
_fire_dirty = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void _fire_dp()
|
private void _fire_dp() {
|
||||||
{
|
|
||||||
object last_sent_id = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.msg_id;
|
object last_sent_id = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.msg_id;
|
||||||
object last_recv_id = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.msg_id;
|
object last_recv_id = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.msg_id;
|
||||||
object echo_time = DBNull.Value;
|
object echo_time = DBNull.Value;
|
||||||
object last_sent_guid = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.guid;
|
object last_sent_guid = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.guid;
|
||||||
object last_recv_guid = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.guid;
|
object last_recv_guid = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.guid;
|
||||||
if (_last_sent_msg != null && _last_recvd_msg != null)
|
if (_last_sent_msg != null && _last_recvd_msg != null) {
|
||||||
{
|
|
||||||
echo_time = _last_recvd_msg.travel_epoch_ms - _last_sent_msg.travel_epoch_ms;
|
echo_time = _last_recvd_msg.travel_epoch_ms - _last_sent_msg.travel_epoch_ms;
|
||||||
}
|
}
|
||||||
var match = last_sent_id.Equals(last_recv_id) && last_sent_guid.Equals(last_recv_guid);
|
var match = last_sent_id.Equals(last_recv_id) && last_sent_guid.Equals(last_recv_guid);
|
||||||
@@ -201,25 +174,21 @@ namespace bts
|
|||||||
myFireEvent(dp);
|
myFireEvent(dp);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void doWork()
|
public void doWork() {
|
||||||
{
|
|
||||||
if (DateTime.Now - _last_publish_time > _publish_period)
|
if (DateTime.Now - _last_publish_time > _publish_period)
|
||||||
if (_n_pubs <= _max_pubs)
|
if (_n_pubs <= _max_pubs) {
|
||||||
{
|
|
||||||
_publish_next_packet();
|
_publish_next_packet();
|
||||||
_last_publish_time = DateTime.Now;
|
_last_publish_time = DateTime.Now;
|
||||||
}
|
}
|
||||||
// publish
|
// publish
|
||||||
// TODO: send id
|
// TODO: send id
|
||||||
if (_fire_dirty)
|
if (_fire_dirty) {
|
||||||
{
|
|
||||||
_fire_dirty = false;
|
_fire_dirty = false;
|
||||||
_fire_dp();
|
_fire_dp();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static long epoch_now()
|
public static long epoch_now() {
|
||||||
{
|
|
||||||
return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
|
return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -8,23 +8,19 @@ using System.IO;
|
|||||||
using System.Linq;
|
using System.Linq;
|
||||||
using System.Runtime.CompilerServices;
|
using System.Runtime.CompilerServices;
|
||||||
using bts.api;
|
using bts.api;
|
||||||
|
using Bts.PubSub.Routing;
|
||||||
|
|
||||||
namespace bts
|
namespace bts {
|
||||||
{
|
|
||||||
[DataStreamUserRespresentation("pss_echo")]
|
[DataStreamUserRespresentation("pss_echo")]
|
||||||
public class PubsubEchoTestStream : ViewStream, IWorker
|
public class PubsubEchoTestStream : ViewStream, IWorker {
|
||||||
{
|
public class PayloadMessage {
|
||||||
public class PayloadMessage
|
|
||||||
{
|
|
||||||
public readonly string msg_id;
|
public readonly string msg_id;
|
||||||
public readonly Int64 timestamp = 0;
|
public readonly Int64 timestamp = 0;
|
||||||
public readonly byte[] payload = null;
|
public readonly byte[] payload = null;
|
||||||
|
|
||||||
public static byte[] create_data(int payload_size, byte payload_byte)
|
public static byte[] create_data(int payload_size, byte payload_byte) {
|
||||||
{
|
|
||||||
var rv = new byte[payload_size];
|
var rv = new byte[payload_size];
|
||||||
for (int i = 0; i < payload_size; i++)
|
for (int i = 0; i < payload_size; i++) {
|
||||||
{
|
|
||||||
rv[i] = payload_byte;
|
rv[i] = payload_byte;
|
||||||
}
|
}
|
||||||
return rv;
|
return rv;
|
||||||
@@ -40,16 +36,14 @@ namespace bts
|
|||||||
// payload[i] = payload_byte;
|
// payload[i] = payload_byte;
|
||||||
// }
|
// }
|
||||||
// }
|
// }
|
||||||
public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_)
|
public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_) {
|
||||||
{
|
|
||||||
msg_id = guid_;
|
msg_id = guid_;
|
||||||
timestamp = timestamp_;
|
timestamp = timestamp_;
|
||||||
payload = payload_;
|
payload = payload_;
|
||||||
}
|
}
|
||||||
|
|
||||||
private PayloadMessage() { }
|
private PayloadMessage() { }
|
||||||
public byte[] serialize()
|
public byte[] serialize() {
|
||||||
{
|
|
||||||
var str = new MemoryStream();
|
var str = new MemoryStream();
|
||||||
BinaryWriter bw = new BinaryWriter(str);
|
BinaryWriter bw = new BinaryWriter(str);
|
||||||
bw.Write(msg_id);
|
bw.Write(msg_id);
|
||||||
@@ -61,8 +55,7 @@ namespace bts
|
|||||||
return bytes;
|
return bytes;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static PayloadMessage deserialize(byte[] ser_bytes)
|
public static PayloadMessage deserialize(byte[] ser_bytes) {
|
||||||
{
|
|
||||||
var str = new MemoryStream(ser_bytes);
|
var str = new MemoryStream(ser_bytes);
|
||||||
BinaryReader br = new BinaryReader(str);
|
BinaryReader br = new BinaryReader(str);
|
||||||
string guid = br.ReadString();
|
string guid = br.ReadString();
|
||||||
@@ -88,17 +81,17 @@ namespace bts
|
|||||||
private readonly string _tag_name;
|
private readonly string _tag_name;
|
||||||
private readonly int _payload_sz;
|
private readonly int _payload_sz;
|
||||||
private readonly IPubSubRouter _router;
|
private readonly IPubSubRouter _router;
|
||||||
private enum IOState
|
private enum IOState {
|
||||||
{
|
|
||||||
SEND = 0,
|
SEND = 0,
|
||||||
RECV = 1
|
RECV = 1
|
||||||
};
|
};
|
||||||
IOState _state = IOState.SEND;
|
IOState _state = IOState.SEND;
|
||||||
|
|
||||||
public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd)
|
public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd) {
|
||||||
{
|
_router = new PubSubRouter(new PubSubRouterConfig() {
|
||||||
var route = new PubSubRoute(server_url, ".*");
|
PublisherRoutes = new() { new(server_url, ".*") },
|
||||||
_router = new PubSubRouter(route);
|
SubscriberRoutes = new() { new(server_url, ".*") },
|
||||||
|
});
|
||||||
_vsd = vsd;
|
_vsd = vsd;
|
||||||
_payload_sz = payload_size;
|
_payload_sz = payload_size;
|
||||||
var schema = BlueDataUtility.form_schema("PubSubTest",
|
var schema = BlueDataUtility.form_schema("PubSubTest",
|
||||||
@@ -117,8 +110,7 @@ namespace bts
|
|||||||
public PubsubEchoTestStream(string server_url, int publish_period_msecs, int payload_size, string tag_name)
|
public PubsubEchoTestStream(string server_url, int publish_period_msecs, int payload_size, string tag_name)
|
||||||
: this(server_url, TimeSpan.FromMilliseconds(publish_period_msecs), payload_size, tag_name, ViewStreamDatabase.instance) { }
|
: this(server_url, TimeSpan.FromMilliseconds(publish_period_msecs), payload_size, tag_name, ViewStreamDatabase.instance) { }
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber()
|
protected override void myOnFirstSubscriber() {
|
||||||
{
|
|
||||||
_pub_core = new PublisherCore(_router);
|
_pub_core = new PublisherCore(_router);
|
||||||
_sub_core = new SubscriberCore(_router);
|
_sub_core = new SubscriberCore(_router);
|
||||||
_sub_core.subscribe(_tag_name);
|
_sub_core.subscribe(_tag_name);
|
||||||
@@ -126,14 +118,12 @@ namespace bts
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber()
|
protected override void myOnLastSubscriber() {
|
||||||
{
|
|
||||||
_vsd.removeWorker(this);
|
_vsd.removeWorker(this);
|
||||||
_sub_core.unsubscribe(_tag_name);
|
_sub_core.unsubscribe(_tag_name);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void _fire_dp_after_send()
|
private void _fire_dp_after_send() {
|
||||||
{
|
|
||||||
// as soon as the data is published, its reflected in the stream state
|
// as soon as the data is published, its reflected in the stream state
|
||||||
Log.assert(_last_sent_msg != null);
|
Log.assert(_last_sent_msg != null);
|
||||||
BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value,
|
BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value,
|
||||||
@@ -143,8 +133,7 @@ namespace bts
|
|||||||
myFireEvent(dp_out);
|
myFireEvent(dp_out);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void _fire_dp_after_recv()
|
private void _fire_dp_after_recv() {
|
||||||
{
|
|
||||||
Log.assert(_last_recvd_msg != null);
|
Log.assert(_last_recvd_msg != null);
|
||||||
// update the last publish row
|
// update the last publish row
|
||||||
int id = _table.Count - 1;
|
int id = _table.Count - 1;
|
||||||
@@ -158,13 +147,11 @@ namespace bts
|
|||||||
myFireEvent(dp_out);
|
myFireEvent(dp_out);
|
||||||
}
|
}
|
||||||
|
|
||||||
private string _get_next_id()
|
private string _get_next_id() {
|
||||||
{
|
|
||||||
return _table.Count.ToString();
|
return _table.Count.ToString();
|
||||||
}
|
}
|
||||||
|
|
||||||
private void _publish_next_packet()
|
private void _publish_next_packet() {
|
||||||
{
|
|
||||||
var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte);
|
var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte);
|
||||||
_last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes);
|
_last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes);
|
||||||
var ser_bytes = _last_sent_msg.serialize();
|
var ser_bytes = _last_sent_msg.serialize();
|
||||||
@@ -173,25 +160,19 @@ namespace bts
|
|||||||
_last_publish_time = DateTime.Now;
|
_last_publish_time = DateTime.Now;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void doWork()
|
public void doWork() {
|
||||||
{
|
Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) => {
|
||||||
Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) =>
|
foreach (var u in updates) {
|
||||||
{
|
|
||||||
foreach (var u in updates)
|
|
||||||
{
|
|
||||||
var recv_msg = PayloadMessage.deserialize(u.data);
|
var recv_msg = PayloadMessage.deserialize(u.data);
|
||||||
if (recv_msg.msg_id == _last_sent_msg.msg_id)
|
if (recv_msg.msg_id == _last_sent_msg.msg_id) {
|
||||||
{
|
|
||||||
return recv_msg;
|
return recv_msg;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
};
|
};
|
||||||
switch (_state)
|
switch (_state) {
|
||||||
{
|
|
||||||
case IOState.SEND:
|
case IOState.SEND:
|
||||||
if (DateTime.Now - _last_publish_time >= _publish_period)
|
if (DateTime.Now - _last_publish_time >= _publish_period) {
|
||||||
{
|
|
||||||
_publish_next_packet();
|
_publish_next_packet();
|
||||||
_last_publish_time = DateTime.Now;
|
_last_publish_time = DateTime.Now;
|
||||||
_fire_dp_after_send();
|
_fire_dp_after_send();
|
||||||
@@ -203,8 +184,7 @@ namespace bts
|
|||||||
var updates = _sub_core.get_updates(TimeSpan.Zero);
|
var updates = _sub_core.get_updates(TimeSpan.Zero);
|
||||||
updates = updates.Where(u => u.tag == _tag_name);
|
updates = updates.Where(u => u.tag == _tag_name);
|
||||||
var recv_msg = find_packet(updates);
|
var recv_msg = find_packet(updates);
|
||||||
if (recv_msg != null)
|
if (recv_msg != null) {
|
||||||
{
|
|
||||||
_last_recvd_msg = recv_msg;
|
_last_recvd_msg = recv_msg;
|
||||||
_fire_dp_after_recv();
|
_fire_dp_after_recv();
|
||||||
_state = IOState.SEND;
|
_state = IOState.SEND;
|
||||||
@@ -214,10 +194,9 @@ namespace bts
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static long epoch_now()
|
public static long epoch_now() {
|
||||||
{
|
|
||||||
return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
|
return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -95,7 +95,7 @@ namespace BCG {
|
|||||||
sub_msg.serialize_into_str(str);
|
sub_msg.serialize_into_str(str);
|
||||||
str.Position = 0;
|
str.Position = 0;
|
||||||
bts.SubscribeMsg sub_msg_ = new bts.SubscribeMsg();
|
bts.SubscribeMsg sub_msg_ = new bts.SubscribeMsg();
|
||||||
sub_msg_.consruct_deserialize(str);
|
sub_msg_.construct_deserialize(str);
|
||||||
bool res = sub_msg.Equals(sub_msg_);
|
bool res = sub_msg.Equals(sub_msg_);
|
||||||
return new TestResult {
|
return new TestResult {
|
||||||
test_name = "SubscribeMsg",
|
test_name = "SubscribeMsg",
|
||||||
@@ -117,7 +117,7 @@ namespace BCG {
|
|||||||
pub_msg.serialize_into_str(str);
|
pub_msg.serialize_into_str(str);
|
||||||
str.Position = 0;
|
str.Position = 0;
|
||||||
var pub_msg_ = new bts.PublishMsg();
|
var pub_msg_ = new bts.PublishMsg();
|
||||||
pub_msg_.consruct_deserialize(str);
|
pub_msg_.construct_deserialize(str);
|
||||||
bool res = pub_msg.Equals(pub_msg_);
|
bool res = pub_msg.Equals(pub_msg_);
|
||||||
return new TestResult {
|
return new TestResult {
|
||||||
test_name = "PublishMsg",
|
test_name = "PublishMsg",
|
||||||
@@ -135,7 +135,7 @@ namespace BCG {
|
|||||||
msg.serialize_into_str(str);
|
msg.serialize_into_str(str);
|
||||||
str.Position = 0;
|
str.Position = 0;
|
||||||
var msg_ = new bts.PubSubServerMessage();
|
var msg_ = new bts.PubSubServerMessage();
|
||||||
msg_.consruct_deserialize(str);
|
msg_.construct_deserialize(str);
|
||||||
bool res = msg.Equals(msg_);
|
bool res = msg.Equals(msg_);
|
||||||
return new TestResult {
|
return new TestResult {
|
||||||
test_name = "PubSubServerMessage",
|
test_name = "PubSubServerMessage",
|
||||||
@@ -156,7 +156,7 @@ namespace BCG {
|
|||||||
msg.serialize_into_str(str);
|
msg.serialize_into_str(str);
|
||||||
str.Position = 0;
|
str.Position = 0;
|
||||||
var msg_ = new PubSubClientMessage();
|
var msg_ = new PubSubClientMessage();
|
||||||
msg_.consruct_deserialize(str);
|
msg_.construct_deserialize(str);
|
||||||
var res = msg.Equals(msg_);
|
var res = msg.Equals(msg_);
|
||||||
return new TestResult {
|
return new TestResult {
|
||||||
test_name = "PubSubClientMessage",
|
test_name = "PubSubClientMessage",
|
||||||
@@ -617,14 +617,15 @@ namespace BCG {
|
|||||||
|
|
||||||
|
|
||||||
private static TestResult _run_single_tag_data_test_clean() {
|
private static TestResult _run_single_tag_data_test_clean() {
|
||||||
|
// var tagHandlerFactory
|
||||||
// create message
|
// create message
|
||||||
var test_name = MethodBase.GetCurrentMethod().Name;
|
var test_name = MethodBase.GetCurrentMethod()?.Name;
|
||||||
var schema = create_schema_with_all_types();
|
var schema = create_schema_with_all_types();
|
||||||
var tag = "DS_" + schema.tableName;
|
var tag = "DS_" + schema.tableName;
|
||||||
BlueTable table = new BlueTable(schema);
|
BlueTable table = new BlueTable(schema);
|
||||||
IStreamIOMsgSerializer serializer = new StreamIOSerializationController();
|
IStreamIOMsgSerializer serializer = new StreamIOSerializationController();
|
||||||
IStreamMsgDeserilizer deser = new StreamIODeserializationController();
|
IStreamMsgDeserilizer deser = new StreamIODeserializationController();
|
||||||
bts.SingleTagData st_data = new bts.SingleTagData(tag, DSTagManager.DataStreamTagType);
|
bts.SingleTagData st_data = new bts.SingleTagData(tag, DSTagManager.DataStreamTagType, new PubsubServerStreamTagHandler());
|
||||||
// create with 100 rows
|
// create with 100 rows
|
||||||
int n_rows = 100;
|
int n_rows = 100;
|
||||||
bool is_succ = true;
|
bool is_succ = true;
|
||||||
|
|||||||
@@ -1,8 +1,9 @@
|
|||||||
|
using Google.Protobuf.WellKnownTypes;
|
||||||
|
|
||||||
using System;
|
using System;
|
||||||
|
using BCG.Utilities.General;
|
||||||
|
|
||||||
using BCG;
|
namespace BCG {
|
||||||
|
|
||||||
namespace bts.SilverPine {
|
|
||||||
// Keeps adding changing rows to the stream, hence preserving the history
|
// Keeps adding changing rows to the stream, hence preserving the history
|
||||||
// and creating a time series, for example this can be used to make bbo updates
|
// and creating a time series, for example this can be used to make bbo updates
|
||||||
// into a time series of prices/qty
|
// into a time series of prices/qty
|
||||||
@@ -40,6 +41,7 @@ namespace bts.SilverPine {
|
|||||||
}
|
}
|
||||||
_table.addRows(added_rows);
|
_table.addRows(added_rows);
|
||||||
myFireEvent(new DataPacket(_table, added_rows, new BlueTable(_table.schema)));
|
myFireEvent(new DataPacket(_table, added_rows, new BlueTable(_table.schema)));
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+8
-29
@@ -7,45 +7,26 @@ namespace bts {
|
|||||||
using System.Text.Json;
|
using System.Text.Json;
|
||||||
using System.Text.Json.Nodes;
|
using System.Text.Json.Nodes;
|
||||||
[DataStreamUserRespresentation("benzingaNews")]
|
[DataStreamUserRespresentation("benzingaNews")]
|
||||||
public class BenzingaNewsStream : ViewStream {
|
public class BenzingaNewStream : ViewStream {
|
||||||
private IBlueTableSchema _schema = null;
|
private IBlueTableSchema _schema = null;
|
||||||
private int _id = 0;
|
private int _id = 0;
|
||||||
|
// constructor takes in the raw mongo db stream, and the time to live (ttl), "7d" for example
|
||||||
|
public BenzingaNewStream(ViewStream mongodb_raw_stream, string ttl) {
|
||||||
private static ViewStream _pre_process_raw_kvstream(ViewStream mongo_db_str) {
|
|
||||||
ViewStream r = new KeyValueToRelationalStream(mongo_db_str, "doc_id", "key", "value");
|
|
||||||
r = new DateStandardizationStream(r, "created_at", "ISO8601");
|
|
||||||
return r;
|
|
||||||
}
|
|
||||||
|
|
||||||
public BenzingaNewsStream(string connection_string, string db_name, string collection_name, string poll_period, string ttl)
|
|
||||||
: this(new MongoDBStream(connection_string, db_name, collection_name, poll_period), ttl) {
|
|
||||||
}
|
|
||||||
|
|
||||||
// public BenzingaNewsStream()
|
|
||||||
// : this(new MongoDBStream(DataStreamPub connection_string, db_name, collection_name, poll_period), ttl) {
|
|
||||||
// }
|
|
||||||
|
|
||||||
// assumes it gets a raw mongodb_raw_kv_stream with key value pairs
|
|
||||||
public BenzingaNewsStream(ViewStream mongodb_raw_kv_stream, string ttl) {
|
|
||||||
ViewStream output_stream = _pre_process_raw_kvstream(mongodb_raw_kv_stream);
|
|
||||||
// Hiding body (with project, columns can be hidden, or renamed)
|
// Hiding body (with project, columns can be hidden, or renamed)
|
||||||
output_stream = new ProjectStream(output_stream, "!body");
|
ViewStream output_stream = new ProjectStream(mongodb_raw_stream, "!body");
|
||||||
// add column news_source = 'Benzinga', so we can append other news sources
|
// add column news_source = 'Benzinga', so we can append other news sources
|
||||||
output_stream = new AddColumnsStream(output_stream, @"news_source : Benzinga");
|
output_stream = new AddColumnsStream(output_stream, @"news_source : Benzinga");
|
||||||
// now SuperMapStream is used to map one row into multiple, one per symbol,
|
// now SuperMapStream is used to map one row into multiple, one per symbol,
|
||||||
// (SuperMapStream takes 2 lambdas, one to get the enw schema and another to
|
// (SuperMapStream takes 2 lambdas, one to get the enw schema and another to
|
||||||
// return an array of rows, per one row)
|
// return an array of rows, per one row)
|
||||||
output_stream = new SuperMapStream(output_stream,
|
output_stream = new SuperMapStream(mongodb_raw_stream,
|
||||||
|
|
||||||
(IBlueTableSchema s) => {
|
(IBlueTableSchema s) => {
|
||||||
// this lambda returns the new schema (ie with extra cols symbol, exchange, is_primary_exchange)
|
// this lambda returns the new schema (ie with extra cols symbol, exchange, is_primary_exchange)
|
||||||
if (_schema == null) {
|
if (_schema == null) {
|
||||||
var out_schema = s;
|
var out_schema = s;
|
||||||
out_schema = BlueDataUtility.remove_columns(out_schema, "securities");
|
out_schema = BlueDataUtility.remove_columns(out_schema, "securities");
|
||||||
out_schema = BlueDataUtility.remove_columns(out_schema, "body");
|
out_schema = BlueDataUtility.add_column(s, "doc_id_internal", typeof(string));
|
||||||
out_schema = BlueDataUtility.remove_columns(out_schema, "tags");
|
|
||||||
out_schema = BlueDataUtility.add_column(out_schema, "doc_id_internal", typeof(string));
|
|
||||||
out_schema = BlueDataUtility.add_column(out_schema, "symbol", typeof(string));
|
out_schema = BlueDataUtility.add_column(out_schema, "symbol", typeof(string));
|
||||||
out_schema = BlueDataUtility.add_column(out_schema, "exchange", typeof(string));
|
out_schema = BlueDataUtility.add_column(out_schema, "exchange", typeof(string));
|
||||||
out_schema = BlueDataUtility.add_column(out_schema, "is_primary_exchange", typeof(bool));
|
out_schema = BlueDataUtility.add_column(out_schema, "is_primary_exchange", typeof(bool));
|
||||||
@@ -63,13 +44,11 @@ namespace bts {
|
|||||||
}
|
}
|
||||||
foreach (JsonNode jnode in json_objs) {
|
foreach (JsonNode jnode in json_objs) {
|
||||||
if (jnode is JsonObject json_object) {
|
if (jnode is JsonObject json_object) {
|
||||||
string symbol = jnode["symbol"] == null ? String.Empty : jnode["symbol"].ToString();
|
|
||||||
string exchange = jnode["exchange"] == null ? String.Empty : jnode["exchange"].ToString();
|
|
||||||
MutableBlueRow nr = new MutableBlueRow(_schema, r);
|
MutableBlueRow nr = new MutableBlueRow(_schema, r);
|
||||||
nr.setValue("id", _id++);
|
nr.setValue("id", _id++);
|
||||||
nr.setValue("doc_id_internal", $"{r["doc_id"]}_{jnode["symbol"]}");
|
nr.setValue("doc_id_internal", $"{r["doc_id"]}_{jnode["symbol"]}");
|
||||||
nr.setValue("symbol", symbol);
|
nr.setValue("symbol", jnode["symbol"]);
|
||||||
nr.setValue("exchange", exchange);
|
nr.setValue("exchange", jnode["exchange"]);
|
||||||
nr.setValue("is_primary_exchange", Boolean.Parse(jnode["primary"].ToString()));
|
nr.setValue("is_primary_exchange", Boolean.Parse(jnode["primary"].ToString()));
|
||||||
expanded_rows.Add(nr);
|
expanded_rows.Add(nr);
|
||||||
}
|
}
|
||||||
@@ -7,127 +7,27 @@ using MongoDB.Driver;
|
|||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
using System.Threading.Tasks;
|
|
||||||
using System.Threading;
|
|
||||||
|
|
||||||
namespace bts {
|
namespace bts {
|
||||||
// [DataStreamUserRespresentation("mongodb_sql")]
|
[DataStreamUserRespresentation("mongodb_sql")]
|
||||||
// public class MongoDBSQLStream : ViewStream, IWorker {
|
public class MongoDBSQLStream : ViewStream, IWorker {
|
||||||
// private ViewStreamDatabase _vsd;
|
|
||||||
// private readonly string _connection_string, _db_name, _collection_name;
|
|
||||||
// private MongoClient _client;
|
|
||||||
// // make this a sorted table
|
|
||||||
// private BlueTable _table;
|
|
||||||
|
|
||||||
|
|
||||||
// // Access the database
|
|
||||||
// private IMongoDatabase _database;
|
|
||||||
// private IDictionary<string, int> _id_map = new Dictionary<string, int>();
|
|
||||||
// // mongodb://192.168.1.80:27017
|
|
||||||
|
|
||||||
// public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
|
|
||||||
// : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
|
|
||||||
|
|
||||||
// }
|
|
||||||
// public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
|
|
||||||
// bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
|
|
||||||
// $"db_name: {db_name}");
|
|
||||||
// _connection_string = connection_string;
|
|
||||||
// _db_name = db_name;
|
|
||||||
// _collection_name = collection_name;
|
|
||||||
// _vsd = vsd;
|
|
||||||
// _client = new MongoClient(_connection_string);
|
|
||||||
// _database = _client.GetDatabase(_db_name);
|
|
||||||
// }
|
|
||||||
|
|
||||||
// protected override void myOnFirstSubscriber() {
|
|
||||||
// MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
|
|
||||||
// }
|
|
||||||
|
|
||||||
// protected override void myOnLastSubscriber() {
|
|
||||||
// MasterWorker.getInstance(false).removeWorker(this);
|
|
||||||
// }
|
|
||||||
|
|
||||||
// public void doWork() {
|
|
||||||
// _read_mogodb();
|
|
||||||
// }
|
|
||||||
|
|
||||||
// private void _read_mogodb() {
|
|
||||||
// var timer = Stopwatch.StartNew();
|
|
||||||
// // Access the collection
|
|
||||||
// IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
|
|
||||||
|
|
||||||
// // Fetch all documents from the collection
|
|
||||||
// var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
|
|
||||||
// _table = _table == null ? null : new BlueTable(_table.schema);
|
|
||||||
// // Iterate through the documents and print values
|
|
||||||
// foreach (var document in documents.ToEnumerable()) {
|
|
||||||
// bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
|
|
||||||
// if (_table == null) {
|
|
||||||
// _init_table(document);
|
|
||||||
// }
|
|
||||||
// _add_row(document);
|
|
||||||
// }
|
|
||||||
// timer.Stop();
|
|
||||||
// Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms");
|
|
||||||
// timer.Restart();
|
|
||||||
// var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
|
|
||||||
// timer.Stop();
|
|
||||||
// Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms");
|
|
||||||
// if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
|
|
||||||
// return;
|
|
||||||
// myFireEvent(dp);
|
|
||||||
// }
|
|
||||||
|
|
||||||
// private void _init_table(BsonDocument doc_row) {
|
|
||||||
// bts.Log.assert(_table == null);
|
|
||||||
// List<ColumnInfo> cis = new List<ColumnInfo>();
|
|
||||||
// cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
|
|
||||||
// foreach (var n in doc_row.Names) {
|
|
||||||
// cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
|
|
||||||
// }
|
|
||||||
// IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
|
|
||||||
// _table = new BlueTable(schema);
|
|
||||||
// }
|
|
||||||
|
|
||||||
// private void _add_row(BsonDocument doc) {
|
|
||||||
// var key_val = doc["_id"];
|
|
||||||
// var key = key_val.ToString();
|
|
||||||
// int id;
|
|
||||||
// if (!_id_map.TryGetValue(key, out id)) {
|
|
||||||
// id = _id_map.Count;
|
|
||||||
// _id_map.Add(key, id);
|
|
||||||
// }
|
|
||||||
// MutableBlueRow nr = new MutableBlueRow(_table.schema);
|
|
||||||
// nr.setValue("id", id);
|
|
||||||
// foreach (var n in doc.Names) {
|
|
||||||
// var val = doc[n].ToString();
|
|
||||||
// nr.setValue(n, val);
|
|
||||||
// }
|
|
||||||
// _table.addRow(nr);
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
[DataStreamUserRespresentation("mongodb")]
|
|
||||||
public class MongoDBStream : ViewStream, IWorker {
|
|
||||||
private ViewStreamDatabase _vsd;
|
private ViewStreamDatabase _vsd;
|
||||||
private readonly string _connection_string, _db_name, _collection_name;
|
private readonly string _connection_string, _db_name, _collection_name;
|
||||||
private readonly TimeSpan _poll_period;
|
|
||||||
private MongoClient _client;
|
private MongoClient _client;
|
||||||
|
// make this a sorted table
|
||||||
private BlueTable _table;
|
private BlueTable _table;
|
||||||
private SyncQueue<DataPacket> _table_queue = new SyncQueue<DataPacket>();
|
|
||||||
private BlockingQueue _work_queue = new BlockingQueue();
|
|
||||||
|
|
||||||
// Access the database
|
// Access the database
|
||||||
private IMongoDatabase _database;
|
private IMongoDatabase _database;
|
||||||
private IDictionary<string, int> _id_map = new Dictionary<string, int>();
|
private IDictionary<string, int> _id_map = new Dictionary<string, int>();
|
||||||
// mongodb://192.168.1.80:27017
|
// mongodb://192.168.1.80:27017
|
||||||
|
|
||||||
public MongoDBStream(string connection_string, string db_name, string collection_name, string poll_period)
|
public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
|
||||||
: this(connection_string, db_name, collection_name, TimeBasedFilter.parse_duration(poll_period), ViewStreamDatabase.instance) {
|
: this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
|
||||||
|
|
||||||
}
|
}
|
||||||
public MongoDBStream(string connection_string, string db_name, string collection_name, TimeSpan poll_period, ViewStreamDatabase vsd) {
|
public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
|
||||||
bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
|
bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
|
||||||
$"db_name: {db_name}");
|
$"db_name: {db_name}");
|
||||||
_connection_string = connection_string;
|
_connection_string = connection_string;
|
||||||
@@ -135,30 +35,114 @@ namespace bts {
|
|||||||
_collection_name = collection_name;
|
_collection_name = collection_name;
|
||||||
_vsd = vsd;
|
_vsd = vsd;
|
||||||
_client = new MongoClient(_connection_string);
|
_client = new MongoClient(_connection_string);
|
||||||
_poll_period = poll_period;
|
|
||||||
_database = _client.GetDatabase(_db_name);
|
_database = _client.GetDatabase(_db_name);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
protected override void myOnFirstSubscriber() {
|
||||||
_vsd.addWorker(this);
|
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
|
||||||
//MasterWorker.getInstance(false).addWorker(this, _poll_period);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
protected override void myOnLastSubscriber() {
|
||||||
_vsd.removeWorker(this);
|
MasterWorker.getInstance(false).removeWorker(this);
|
||||||
//MasterWorker.getInstance(false).removeWorker(this);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private DateTime _last_poll_time = DateTime.MinValue;
|
|
||||||
public void doWork() {
|
public void doWork() {
|
||||||
var now = DateTime.Now;
|
_read_mogodb();
|
||||||
if (now - _last_poll_time > _poll_period) {
|
}
|
||||||
ThreadPool.QueueUserWorkItem(_ => _read_mogodb());
|
|
||||||
_last_poll_time = DateTime.Now;
|
private void _read_mogodb() {
|
||||||
|
var timer = Stopwatch.StartNew();
|
||||||
|
// Access the collection
|
||||||
|
IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
|
||||||
|
|
||||||
|
// Fetch all documents from the collection
|
||||||
|
var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
|
||||||
|
_table = _table == null ? null : new BlueTable(_table.schema);
|
||||||
|
// Iterate through the documents and print values
|
||||||
|
foreach (var document in documents.ToEnumerable()) {
|
||||||
|
bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
|
||||||
|
if (_table == null) {
|
||||||
|
_init_table(document);
|
||||||
|
}
|
||||||
|
_add_row(document);
|
||||||
}
|
}
|
||||||
var dp = _table_queue.dequeue(0);
|
timer.Stop();
|
||||||
if (dp != null)
|
Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms");
|
||||||
myFireEvent(dp);
|
timer.Restart();
|
||||||
|
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
|
||||||
|
timer.Stop();
|
||||||
|
Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms");
|
||||||
|
if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
|
||||||
|
return;
|
||||||
|
myFireEvent(dp);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _init_table(BsonDocument doc_row) {
|
||||||
|
bts.Log.assert(_table == null);
|
||||||
|
List<ColumnInfo> cis = new List<ColumnInfo>();
|
||||||
|
cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
|
||||||
|
foreach (var n in doc_row.Names) {
|
||||||
|
cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
|
||||||
|
}
|
||||||
|
IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
|
||||||
|
_table = new BlueTable(schema);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _add_row(BsonDocument doc) {
|
||||||
|
var key_val = doc["_id"];
|
||||||
|
var key = key_val.ToString();
|
||||||
|
int id;
|
||||||
|
if (!_id_map.TryGetValue(key, out id)) {
|
||||||
|
id = _id_map.Count;
|
||||||
|
_id_map.Add(key, id);
|
||||||
|
}
|
||||||
|
MutableBlueRow nr = new MutableBlueRow(_table.schema);
|
||||||
|
nr.setValue("id", id);
|
||||||
|
foreach (var n in doc.Names) {
|
||||||
|
var val = doc[n].ToString();
|
||||||
|
nr.setValue(n, val);
|
||||||
|
}
|
||||||
|
_table.addRow(nr);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
[DataStreamUserRespresentation("mongodb")]
|
||||||
|
public class MongoDBStream : ViewStream, IWorker {
|
||||||
|
private ViewStreamDatabase _vsd;
|
||||||
|
private readonly string _connection_string, _db_name, _collection_name;
|
||||||
|
private MongoClient _client;
|
||||||
|
private BlueTable _table;
|
||||||
|
|
||||||
|
// Access the database
|
||||||
|
private IMongoDatabase _database;
|
||||||
|
private IDictionary<string, int> _id_map = new Dictionary<string, int>();
|
||||||
|
// mongodb://192.168.1.80:27017
|
||||||
|
|
||||||
|
public MongoDBStream(string connection_string, string db_name, string collection_name)
|
||||||
|
: this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
|
||||||
|
|
||||||
|
}
|
||||||
|
public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
|
||||||
|
bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
|
||||||
|
$"db_name: {db_name}");
|
||||||
|
_connection_string = connection_string;
|
||||||
|
_db_name = db_name;
|
||||||
|
_collection_name = collection_name;
|
||||||
|
_vsd = vsd;
|
||||||
|
_client = new MongoClient(_connection_string);
|
||||||
|
_database = _client.GetDatabase(_db_name);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() {
|
||||||
|
MasterWorker.getInstance(false).removeWorker(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void doWork() {
|
||||||
|
_read_mogodb();
|
||||||
}
|
}
|
||||||
|
|
||||||
private void _read_mogodb() {
|
private void _read_mogodb() {
|
||||||
@@ -182,11 +166,10 @@ namespace bts {
|
|||||||
timer.Restart();
|
timer.Restart();
|
||||||
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
|
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
|
||||||
timer.Stop();
|
timer.Stop();
|
||||||
Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms"); ;
|
Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms");;
|
||||||
if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
|
if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
|
||||||
return;
|
return;
|
||||||
_table_queue.enqueue(dp);
|
myFireEvent(dp);
|
||||||
//myFireEvent(dp);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void _init_table(BsonDocument doc_row) {
|
private void _init_table(BsonDocument doc_row) {
|
||||||
@@ -225,46 +208,46 @@ namespace bts {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// [DataStreamUserRespresentation("mongodb_flat")]
|
[DataStreamUserRespresentation("mongodb_flat")]
|
||||||
// public class MongoDBFlatStream : ViewStream {
|
public class MongoDBFlatStream : ViewStream {
|
||||||
// public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration)
|
public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration)
|
||||||
// :
|
:
|
||||||
// this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { }
|
this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { }
|
||||||
|
|
||||||
// public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) {
|
public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) {
|
||||||
// ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd);
|
ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd);
|
||||||
// rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val");
|
rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val");
|
||||||
// rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601);
|
rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601);
|
||||||
// //rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration);
|
//rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration);
|
||||||
// mySetRealSource(rs);
|
mySetRealSource(rs);
|
||||||
// }
|
}
|
||||||
// }
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
// [DataStreamUserRespresentation("newsreplay")]
|
[DataStreamUserRespresentation("newsreplay")]
|
||||||
// public class NewsReplayStream : ViewStream {
|
public class NewsReplayStream : ViewStream {
|
||||||
// public NewsReplayStream(string connection_string, string db_name, string collection_name,
|
public NewsReplayStream(string connection_string, string db_name, string collection_name,
|
||||||
// int buffer_sz, string fire_period) :
|
int buffer_sz, string fire_period) :
|
||||||
// this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { }
|
this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { }
|
||||||
|
|
||||||
// public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name,
|
public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name,
|
||||||
// int buffer_sz, TimeSpan fire_period) {
|
int buffer_sz, TimeSpan fire_period) {
|
||||||
// ViewStream r;
|
ViewStream r;
|
||||||
|
|
||||||
// r = new MongoDBStream(connection_string, db_name, collection_name, vsd);
|
r = new MongoDBStream(connection_string, db_name, collection_name, vsd);
|
||||||
// r = new KeyValueToRelationalStream(r,
|
r = new KeyValueToRelationalStream(r,
|
||||||
// "doc_id",
|
"doc_id",
|
||||||
// "key",
|
"key",
|
||||||
// "value");
|
"value");
|
||||||
// r = new DateStandardizationStream(r,
|
r = new DateStandardizationStream(r,
|
||||||
// "created_at",
|
"created_at",
|
||||||
// "ISO8601");
|
"ISO8601");
|
||||||
// r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period);
|
r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period);
|
||||||
|
|
||||||
// mySetRealSource(r);
|
mySetRealSource(r);
|
||||||
// }
|
}
|
||||||
// }
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
@@ -6,7 +6,6 @@ namespace bts {
|
|||||||
using System;
|
using System;
|
||||||
using System.Collections;
|
using System.Collections;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Linq;
|
|
||||||
|
|
||||||
public class ProjectData {
|
public class ProjectData {
|
||||||
// schema from the base stream
|
// schema from the base stream
|
||||||
@@ -137,24 +136,10 @@ namespace bts {
|
|||||||
HashSet<string> removed_cols = new HashSet<string>();
|
HashSet<string> removed_cols = new HashSet<string>();
|
||||||
Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>();
|
Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>();
|
||||||
Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
|
Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
|
||||||
// by default every col will show up, so if we have the projection as (a:a1)
|
|
||||||
// it will convert a to a1,
|
|
||||||
// examples: if a schema has cols a b c
|
|
||||||
// project(parent_str, "a:aa ") -> aa b c
|
|
||||||
// project(parent_str, "a:aa !c") -> aa b
|
|
||||||
// project(parent_str, "a:aa !") -> aa
|
|
||||||
// project(parent_str, "a:aa b = (aa*aa)!") -> aa b
|
|
||||||
bool hide_all = false;
|
|
||||||
foreach (string p_tok in project_toks) {
|
foreach (string p_tok in project_toks) {
|
||||||
if (p_tok.StartsWith("!")) {
|
if (p_tok.StartsWith("!")) {
|
||||||
if (p_tok.Length == 1) {
|
string removed_col = p_tok.Substring(1);
|
||||||
// if it is a "blanket "!"=> hide all cols except the one that
|
removed_cols.Add(removed_col);
|
||||||
// are stated explicitly
|
|
||||||
hide_all = true;
|
|
||||||
} else {
|
|
||||||
string removed_col = p_tok.Substring(1);
|
|
||||||
removed_cols.Add(removed_col);
|
|
||||||
}
|
|
||||||
} else if (p_tok.Contains(":")) {
|
} else if (p_tok.Contains(":")) {
|
||||||
string[] map_toks = p_tok.Split(":");
|
string[] map_toks = p_tok.Split(":");
|
||||||
string orig_col = map_toks[0];
|
string orig_col = map_toks[0];
|
||||||
@@ -162,22 +147,12 @@ namespace bts {
|
|||||||
mapped_to_orig_map.Add(mapped_col, orig_col);
|
mapped_to_orig_map.Add(mapped_col, orig_col);
|
||||||
orig_to_mapped.Add(orig_col, mapped_col);
|
orig_to_mapped.Add(orig_col, mapped_col);
|
||||||
} else {
|
} else {
|
||||||
if (input_schema.columnNames.Contains(p_tok)) {
|
Log.Gen.error($"NewProjectStream unexpecetd token: {p_tok}");
|
||||||
string orig_col = p_tok;
|
|
||||||
string mapped_col = orig_col;
|
|
||||||
orig_to_mapped.Add(orig_col, mapped_col);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// make the new schema
|
// make the new schema
|
||||||
List<ColumnInfo> new_cols = new List<ColumnInfo>();
|
List<ColumnInfo> new_cols = new List<ColumnInfo>();
|
||||||
foreach (ColumnInfo ci in input_schema.columnInfo) {
|
foreach (ColumnInfo ci in input_schema.columnInfo) {
|
||||||
if (ci.colName == "id") {
|
|
||||||
ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
|
|
||||||
new_cols.Add(nc);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (removed_cols.Contains(ci.colName)) {
|
if (removed_cols.Contains(ci.colName)) {
|
||||||
continue;
|
continue;
|
||||||
} else if (orig_to_mapped.ContainsKey(ci.colName)) {
|
} else if (orig_to_mapped.ContainsKey(ci.colName)) {
|
||||||
@@ -185,13 +160,8 @@ namespace bts {
|
|||||||
ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
|
ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
|
||||||
new_cols.Add(nc);
|
new_cols.Add(nc);
|
||||||
} else {
|
} else {
|
||||||
if (hide_all)
|
|
||||||
continue;
|
|
||||||
//if (ci.colName == "id" || removed_cols.Count > 0) {
|
|
||||||
// unless columns are explicitely hidden, all columns show up
|
|
||||||
ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
|
ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
|
||||||
new_cols.Add(nc);
|
new_cols.Add(nc);
|
||||||
//}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
|
IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
|
||||||
|
|||||||
@@ -1,122 +0,0 @@
|
|||||||
using System;
|
|
||||||
using System.Data;
|
|
||||||
using System.Data.SQLite;
|
|
||||||
using System.Diagnostics;
|
|
||||||
using System.Collections;
|
|
||||||
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Data.Entity;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Data.SQLite;
|
|
||||||
|
|
||||||
|
|
||||||
namespace bts.SilverPine {
|
|
||||||
|
|
||||||
public static class SQLiteHelper {
|
|
||||||
public static string location = "C:\\Users\\m_tah\\development\\data\\";
|
|
||||||
public static string db_name = "stream_data.db";
|
|
||||||
|
|
||||||
public static DataTable execute_select_query(string connection_string, string query) {
|
|
||||||
// Create a new DataTable to store the results
|
|
||||||
DataTable dataTable = new DataTable();
|
|
||||||
|
|
||||||
// Create and open a SQLite connection using the provided connection string
|
|
||||||
using (SQLiteConnection connection = new SQLiteConnection(connection_string)) {
|
|
||||||
try {
|
|
||||||
connection.Open();
|
|
||||||
|
|
||||||
// Create a SQLite command with the provided query
|
|
||||||
using (SQLiteCommand command = new SQLiteCommand(query, connection)) {
|
|
||||||
// Use SQLiteDataAdapter to fill the DataTable
|
|
||||||
using (SQLiteDataAdapter adapter = new SQLiteDataAdapter(command)) {
|
|
||||||
adapter.Fill(dataTable);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (Exception ex) {
|
|
||||||
Console.WriteLine("An error occurred: " + ex.Message);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Return the populated DataTable
|
|
||||||
return dataTable;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static List<Dictionary<string, object>> ExecuteBulkRead(string connectionString, string query) {
|
|
||||||
var result = new List<Dictionary<string, object>>();
|
|
||||||
|
|
||||||
using (var connection = new SQLiteConnection(connectionString)) {
|
|
||||||
connection.Open();
|
|
||||||
|
|
||||||
// Begin a transaction to optimize read speed
|
|
||||||
using (var transaction = connection.BeginTransaction())
|
|
||||||
using (var command = new SQLiteCommand(query, connection, transaction))
|
|
||||||
using (var reader = command.ExecuteReader()) {
|
|
||||||
while (reader.Read()) {
|
|
||||||
var row = new Dictionary<string, object>();
|
|
||||||
|
|
||||||
for (int i = 0; i < reader.FieldCount; i++) {
|
|
||||||
string columnName = reader.GetName(i);
|
|
||||||
object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
|
|
||||||
row[columnName] = columnValue;
|
|
||||||
}
|
|
||||||
|
|
||||||
result.Add(row);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Commit the transaction
|
|
||||||
transaction.Commit();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static List<Dictionary<string, object>> get_table(string connectionString, string query) {
|
|
||||||
var result = new List<Dictionary<string, object>>();
|
|
||||||
|
|
||||||
using (var connection = new SQLiteConnection(connectionString)) {
|
|
||||||
try {
|
|
||||||
connection.Open();
|
|
||||||
|
|
||||||
using (var command = new SQLiteCommand(query, connection))
|
|
||||||
using (var reader = command.ExecuteReader()) {
|
|
||||||
while (reader.Read()) {
|
|
||||||
var row = new Dictionary<string, object>();
|
|
||||||
|
|
||||||
for (int i = 0; i < reader.FieldCount; i++) {
|
|
||||||
string columnName = reader.GetName(i);
|
|
||||||
object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
|
|
||||||
row[columnName] = columnValue;
|
|
||||||
}
|
|
||||||
|
|
||||||
result.Add(row);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (Exception ex) {
|
|
||||||
Console.WriteLine("An error occurred: " + ex.Message);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static IEnumerable<Dictionary<string, object>> get_table_2(string connectionString, string query) {
|
|
||||||
using (var connection = new SQLiteConnection(connectionString)) {
|
|
||||||
connection.Open();
|
|
||||||
using (var command = new SQLiteCommand(query, connection))
|
|
||||||
using (var reader = command.ExecuteReader()) {
|
|
||||||
while (reader.Read()) {
|
|
||||||
var row = new Dictionary<string, object>();
|
|
||||||
for (int i = 0; i < reader.FieldCount; i++) {
|
|
||||||
string columnName = reader.GetName(i);
|
|
||||||
object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
|
|
||||||
row[columnName] = columnValue;
|
|
||||||
}
|
|
||||||
yield return row;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@@ -1,73 +0,0 @@
|
|||||||
namespace bts.SilverPine {
|
|
||||||
using System;
|
|
||||||
using System.Data;
|
|
||||||
using System.Data.SQLite;
|
|
||||||
using System.Diagnostics;
|
|
||||||
using System.Collections;
|
|
||||||
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
[DataStreamUserRespresentation("SQLite")]
|
|
||||||
public class SQLiteStream : ViewStream {
|
|
||||||
private readonly string _connection_string, _query;
|
|
||||||
private Stopwatch _stopwatch = new Stopwatch();
|
|
||||||
|
|
||||||
private SQLiteStream(string connection_string, string query, bool noop) {
|
|
||||||
_connection_string = connection_string;
|
|
||||||
_query = query;
|
|
||||||
}
|
|
||||||
|
|
||||||
public SQLiteStream(string db_name, string query)
|
|
||||||
:
|
|
||||||
this($"Data Source={SQLiteHelper.location}{db_name};Version=3", query, true) { }
|
|
||||||
|
|
||||||
public SQLiteStream(string query)
|
|
||||||
:
|
|
||||||
this(SQLiteHelper.db_name, query) {
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private void _read_from_sqlite_and_fire() {
|
|
||||||
_stopwatch.Start();
|
|
||||||
DataTable dt = SQLiteHelper.execute_select_query(_connection_string, _query);
|
|
||||||
_stopwatch.Stop();
|
|
||||||
Log.Gen.info($"SQLiteStream.ExecuteSelectQuery time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
|
|
||||||
_stopwatch.Restart();
|
|
||||||
#region perf measures
|
|
||||||
// SQLiteHelper.get_table(_connection_string, _query);
|
|
||||||
// _stopwatch.Stop();
|
|
||||||
// Log.Gen.info($"SQLiteStream.get_table time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
|
|
||||||
// _stopwatch.Restart();
|
|
||||||
// SQLiteHelper.ExecuteBulkRead(_connection_string, _query);
|
|
||||||
// _stopwatch.Stop();
|
|
||||||
// Log.Gen.info($"SQLiteStream.ExecuteBulkRead time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
|
|
||||||
// _stopwatch.Restart();
|
|
||||||
// SQLiteHelper.get_table_2(_connection_string, _query);
|
|
||||||
// _stopwatch.Stop();
|
|
||||||
// Log.Gen.info($"SQLiteStream.get_table_2 time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
|
|
||||||
// _stopwatch.Restart();
|
|
||||||
#endregion
|
|
||||||
IBlueRowCollection rows = BCG.BlueDataUtility.toIBlueRowCollection(dt);
|
|
||||||
//IBlueRowCollection rows = new DataTableWrapperTable(dt);
|
|
||||||
_stopwatch.Stop();
|
|
||||||
Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to load into IBlueRowCollection: {_stopwatch.ElapsedMilliseconds} msecs");
|
|
||||||
DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
|
|
||||||
myFireEvent(dp);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
_read_from_sqlite_and_fire();
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
|
||||||
// this value will be cleared by the base class
|
|
||||||
// but in this particular stream this behaviour
|
|
||||||
// is not usefull since any new subscribers
|
|
||||||
// would force the query to be recreated
|
|
||||||
//myLastDP = myLastValue;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -3,15 +3,12 @@ using System.Collections.Generic;
|
|||||||
using System.Text;
|
using System.Text;
|
||||||
using System.IO;
|
using System.IO;
|
||||||
using BCG.Utilities;
|
using BCG.Utilities;
|
||||||
using System.Diagnostics;
|
|
||||||
using Microsoft.Data.Analysis;
|
|
||||||
using Parquet;
|
|
||||||
using Parquet.Data;
|
|
||||||
|
|
||||||
|
namespace BCG
|
||||||
namespace BCG {
|
{
|
||||||
[DataStreamUserRespresentation("CSVSaver")]
|
[DataStreamUserRespresentation("CSVSaver")]
|
||||||
public class CSVSaverStream : ViewStream, ISideEffectStream {
|
public class CSVSaverStream : ViewStream, ISideEffectStream
|
||||||
|
{
|
||||||
private IBlueTableSchema myResultSchema = new BlueTableSchema("CSVResult", new ColumnInfo("id", 0, typeof(int)),
|
private IBlueTableSchema myResultSchema = new BlueTableSchema("CSVResult", new ColumnInfo("id", 0, typeof(int)),
|
||||||
new ColumnInfo("fileName", 1, typeof(string)),
|
new ColumnInfo("fileName", 1, typeof(string)),
|
||||||
new ColumnInfo("nWrites", 2, typeof(int)),
|
new ColumnInfo("nWrites", 2, typeof(int)),
|
||||||
@@ -23,9 +20,11 @@ namespace BCG {
|
|||||||
private int myNWrites = 0, myNMaxWrites = Int32.MaxValue;
|
private int myNWrites = 0, myNMaxWrites = Int32.MaxValue;
|
||||||
|
|
||||||
public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName)
|
public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName)
|
||||||
: this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false) { }
|
: this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false)
|
||||||
|
{ }
|
||||||
|
|
||||||
public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot) {
|
public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot)
|
||||||
|
{
|
||||||
myStream = vs;
|
myStream = vs;
|
||||||
myFullyQualifiedFileName = fullyQualifiedFileName;
|
myFullyQualifiedFileName = fullyQualifiedFileName;
|
||||||
myOverwrite = overwrite;
|
myOverwrite = overwrite;
|
||||||
@@ -37,14 +36,18 @@ namespace BCG {
|
|||||||
|
|
||||||
protected override void myOnLastSubscriber() { myStream.onChange -= new ChangeDelegate(myHandleChanges); }
|
protected override void myOnLastSubscriber() { myStream.onChange -= new ChangeDelegate(myHandleChanges); }
|
||||||
|
|
||||||
private void myHandleChanges(object newValue) {
|
private void myHandleChanges(object newValue)
|
||||||
|
{
|
||||||
IBlueRowCollection totalState = (newValue as DataPacket).newRows;
|
IBlueRowCollection totalState = (newValue as DataPacket).newRows;
|
||||||
if (totalState.Count > 0 && myNWrites < myNMaxWrites) {
|
if (totalState.Count > 0 && myNWrites < myNMaxWrites)
|
||||||
|
{
|
||||||
myNWrites++;
|
myNWrites++;
|
||||||
int i = 0;
|
int i = 0;
|
||||||
StreamWriter writer = new StreamWriter(new FileStream(myGetNewFileName(), FileMode.Create, FileAccess.Write));
|
StreamWriter writer = new StreamWriter(new FileStream(myGetNewFileName(), FileMode.Create, FileAccess.Write));
|
||||||
foreach (IBlueRow r in totalState) {
|
foreach (IBlueRow r in totalState)
|
||||||
if (i++ == 0) {
|
{
|
||||||
|
if (i++ == 0)
|
||||||
|
{
|
||||||
string header = String.Empty;
|
string header = String.Empty;
|
||||||
foreach (string col in r.schema.columnNames)
|
foreach (string col in r.schema.columnNames)
|
||||||
if (col != "id")
|
if (col != "id")
|
||||||
@@ -56,17 +59,20 @@ namespace BCG {
|
|||||||
writer.Flush();
|
writer.Flush();
|
||||||
}
|
}
|
||||||
writer.Close();
|
writer.Close();
|
||||||
if (myFireResultSnapshot) {
|
if (myFireResultSnapshot)
|
||||||
|
{
|
||||||
IBlueRow nr = new BlueRow(myResultSchema, 0, myFullyQualifiedFileName, myNWrites, totalState.Count);
|
IBlueRow nr = new BlueRow(myResultSchema, 0, myFullyQualifiedFileName, myNWrites, totalState.Count);
|
||||||
BlueTable nRows = new BlueTable(myResultSchema);
|
BlueTable nRows = new BlueTable(myResultSchema);
|
||||||
nRows.addRow(nr);
|
nRows.addRow(nr);
|
||||||
myFireEvent(new DataPacket(nRows, nRows, new BlueTable(myLastValue == null ? new BlueTable(myResultSchema) : (myLastValue as DataPacket).newRows)));
|
myFireEvent(new DataPacket(nRows, nRows, new BlueTable(myLastValue == null ? new BlueTable(myResultSchema) : (myLastValue as DataPacket).newRows)));
|
||||||
} else
|
}
|
||||||
|
else
|
||||||
myFireEvent(newValue);
|
myFireEvent(newValue);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private string myGetNewFileName() {
|
private string myGetNewFileName()
|
||||||
|
{
|
||||||
string[] tags = myFullyQualifiedFileName.Split('.');
|
string[] tags = myFullyQualifiedFileName.Split('.');
|
||||||
if (myOverwrite)
|
if (myOverwrite)
|
||||||
return tags[0] + ".csv";
|
return tags[0] + ".csv";
|
||||||
@@ -74,11 +80,14 @@ namespace BCG {
|
|||||||
return tags[0] + "_" + myFormateDateTime(DateTime.Now) + ".csv";
|
return tags[0] + "_" + myFormateDateTime(DateTime.Now) + ".csv";
|
||||||
}
|
}
|
||||||
|
|
||||||
private static string myGetCSVRow(IBlueRow r) {
|
private static string myGetCSVRow(IBlueRow r)
|
||||||
|
{
|
||||||
string rv = String.Empty;
|
string rv = String.Empty;
|
||||||
int i = 0;
|
int i = 0;
|
||||||
foreach (string col in r.schema.columnNames) {
|
foreach (string col in r.schema.columnNames)
|
||||||
if (col != "id") {
|
{
|
||||||
|
if (col != "id")
|
||||||
|
{
|
||||||
string colText = String.Empty;
|
string colText = String.Empty;
|
||||||
if (r[col] != null && r[col] != DBNull.Value)
|
if (r[col] != null && r[col] != DBNull.Value)
|
||||||
colText = r[col].ToString();
|
colText = r[col].ToString();
|
||||||
@@ -90,7 +99,8 @@ namespace BCG {
|
|||||||
return rv;
|
return rv;
|
||||||
}
|
}
|
||||||
|
|
||||||
private string myFormateDateTime(DateTime dt) {
|
private string myFormateDateTime(DateTime dt)
|
||||||
|
{
|
||||||
return
|
return
|
||||||
string.Format("{0}-{1}-{2}-{3}-{4}-{5}-{6}",
|
string.Format("{0}-{1}-{2}-{3}-{4}-{5}-{6}",
|
||||||
dt.Year.ToString("0000"),
|
dt.Year.ToString("0000"),
|
||||||
@@ -104,49 +114,52 @@ namespace BCG {
|
|||||||
}
|
}
|
||||||
|
|
||||||
[DataStreamUserRespresentation("CSVReader")]
|
[DataStreamUserRespresentation("CSVReader")]
|
||||||
public class CSVReaderStream : ViewStream {
|
public class CSVReaderStream : ViewStream
|
||||||
private readonly string _file_address;
|
{
|
||||||
Stopwatch _stopwatch = new Stopwatch();
|
private IBlueRowCollection myRows;
|
||||||
public CSVReaderStream(string fileAddress) {
|
|
||||||
_file_address = fileAddress;
|
public CSVReaderStream(string fileAddress)
|
||||||
|
{
|
||||||
|
myRows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(fileAddress);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
protected override void myOnFirstSubscriber()
|
||||||
_stopwatch.Start();
|
{
|
||||||
DataFrame df = DataFrame.LoadCsv(_file_address);
|
myFireEvent(new DataPacket(myRows, myRows, new BlueTable(myRows.schema)));
|
||||||
_stopwatch.Stop();
|
|
||||||
bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load col_count: {df.Columns.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
|
|
||||||
_stopwatch.Start();
|
|
||||||
IBlueRowCollection rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(_file_address);
|
|
||||||
_stopwatch.Stop();
|
|
||||||
bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load count: {rows.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
|
|
||||||
DataPacket dp_out = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
|
|
||||||
myFireEvent(dp_out);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
[DataStreamUserRespresentation("CSVReaderStreamMapper")]
|
[DataStreamUserRespresentation ("CSVReaderStreamMapper")]
|
||||||
public class CSVReaderStreamMapperStream : ViewStream {
|
public class CSVReaderStreamMapperStream : ViewStream
|
||||||
public CSVReaderStreamMapperStream(ViewStream stream, string output_schema) {
|
{
|
||||||
|
public CSVReaderStreamMapperStream(ViewStream stream, string output_schema)
|
||||||
|
{
|
||||||
mySetRealSource(new MapStream(stream, new CSVReaderStreamMapper(output_schema)));
|
mySetRealSource(new MapStream(stream, new CSVReaderStreamMapper(output_schema)));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public class CSVReaderStreamMapper : IStreamMapper {
|
public class CSVReaderStreamMapper : IStreamMapper
|
||||||
|
{
|
||||||
private IBlueTableSchema mySchema;
|
private IBlueTableSchema mySchema;
|
||||||
private readonly string[] myOutputSchemaStringArray;
|
private readonly string[] myOutputSchemaStringArray;
|
||||||
|
|
||||||
public CSVReaderStreamMapper(string outputSchema) {
|
public CSVReaderStreamMapper(string outputSchema)
|
||||||
|
{
|
||||||
myOutputSchemaStringArray = outputSchema.Split(' ');
|
myOutputSchemaStringArray = outputSchema.Split(' ');
|
||||||
}
|
}
|
||||||
|
|
||||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
|
public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
|
||||||
if (mySchema == null) {
|
{
|
||||||
|
if (mySchema == null)
|
||||||
|
{
|
||||||
ColumnInfo[] ci = inputSchema.columnInfo;
|
ColumnInfo[] ci = inputSchema.columnInfo;
|
||||||
foreach (string col in myOutputSchemaStringArray) {
|
foreach (string col in myOutputSchemaStringArray)
|
||||||
|
{
|
||||||
string colName = col.Substring(0, col.IndexOf('\\')).ToLower();
|
string colName = col.Substring(0, col.IndexOf('\\')).ToLower();
|
||||||
for (int i = 0; i < ci.Length; i++) {
|
for (int i = 0; i < ci.Length; i++)
|
||||||
if (ci[i].colName.ToLower().Equals(colName)) {
|
{
|
||||||
|
if (ci[i].colName.ToLower().Equals(colName))
|
||||||
|
{
|
||||||
Type t = myGetType(col.Substring(col.IndexOf('\\') + 1));
|
Type t = myGetType(col.Substring(col.IndexOf('\\') + 1));
|
||||||
ci[i] = new ColumnInfo(ci[i].colName, i, t);
|
ci[i] = new ColumnInfo(ci[i].colName, i, t);
|
||||||
break;
|
break;
|
||||||
@@ -158,52 +171,74 @@ namespace BCG {
|
|||||||
return mySchema;
|
return mySchema;
|
||||||
}
|
}
|
||||||
|
|
||||||
public IBlueRow map(IBlueRow row) {
|
public IBlueRow map(IBlueRow row)
|
||||||
|
{
|
||||||
object[] vals = new object[row.schema.columnInfo.Length];
|
object[] vals = new object[row.schema.columnInfo.Length];
|
||||||
row.Values.CopyTo(vals, 0);
|
row.Values.CopyTo(vals, 0);
|
||||||
for (int i = 0; i < mySchema.columnInfo.Length; i++) {
|
for (int i = 0; i < mySchema.columnInfo.Length; i++)
|
||||||
|
{
|
||||||
Type t = mySchema.columnInfo[i].colType;
|
Type t = mySchema.columnInfo[i].colType;
|
||||||
try {
|
try
|
||||||
|
{
|
||||||
vals[i] = System.Convert.ChangeType(vals[i], mySchema.columnInfo[i].colType, System.Globalization.CultureInfo.InvariantCulture);
|
vals[i] = System.Convert.ChangeType(vals[i], mySchema.columnInfo[i].colType, System.Globalization.CultureInfo.InvariantCulture);
|
||||||
} catch {
|
}
|
||||||
if (t.Equals(typeof(Int16))) {
|
catch
|
||||||
|
{
|
||||||
|
if (t.Equals(typeof(Int16)))
|
||||||
|
{
|
||||||
Int16 result;
|
Int16 result;
|
||||||
if (Int16.TryParse((string)vals[i], out result)) {
|
if (Int16.TryParse((string)vals[i], out result))
|
||||||
|
{
|
||||||
vals[i] = result;
|
vals[i] = result;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
double r2;
|
double r2;
|
||||||
if (Double.TryParse((string)vals[i], out r2)) {
|
if (Double.TryParse((string)vals[i], out r2))
|
||||||
try {
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
vals[i] = (Int16)r2;
|
vals[i] = (Int16)r2;
|
||||||
continue;
|
continue;
|
||||||
} catch { }
|
}
|
||||||
|
catch { }
|
||||||
}
|
}
|
||||||
} else if (t.Equals(typeof(Int32))) {
|
}
|
||||||
|
else if (t.Equals(typeof(Int32)))
|
||||||
|
{
|
||||||
Int32 result;
|
Int32 result;
|
||||||
if (Int32.TryParse((string)vals[i], out result)) {
|
if (Int32.TryParse((string)vals[i], out result))
|
||||||
|
{
|
||||||
vals[i] = result;
|
vals[i] = result;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
double r2;
|
double r2;
|
||||||
if (Double.TryParse((string)vals[i], out r2)) {
|
if (Double.TryParse((string)vals[i], out r2))
|
||||||
try {
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
vals[i] = (Int32)r2;
|
vals[i] = (Int32)r2;
|
||||||
continue;
|
continue;
|
||||||
} catch { }
|
}
|
||||||
|
catch { }
|
||||||
}
|
}
|
||||||
} else if (t.Equals(typeof(Int64))) {
|
}
|
||||||
|
else if (t.Equals(typeof(Int64)))
|
||||||
|
{
|
||||||
Int64 result;
|
Int64 result;
|
||||||
if (Int64.TryParse((string)vals[i], out result)) {
|
if (Int64.TryParse((string)vals[i], out result))
|
||||||
|
{
|
||||||
vals[i] = result;
|
vals[i] = result;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
double r2;
|
double r2;
|
||||||
if (Double.TryParse((string)vals[i], out r2)) {
|
if (Double.TryParse((string)vals[i], out r2))
|
||||||
try {
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
vals[i] = (Int64)r2;
|
vals[i] = (Int64)r2;
|
||||||
continue;
|
continue;
|
||||||
} catch { }
|
}
|
||||||
|
catch { }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
vals[i] = DBNull.Value;
|
vals[i] = DBNull.Value;
|
||||||
@@ -212,7 +247,8 @@ namespace BCG {
|
|||||||
return new BlueRow(mySchema, vals);
|
return new BlueRow(mySchema, vals);
|
||||||
}
|
}
|
||||||
|
|
||||||
private Type myGetType(string type) {
|
private Type myGetType(string type)
|
||||||
|
{
|
||||||
type = type.ToLower();
|
type = type.ToLower();
|
||||||
if (type.Contains("int"))
|
if (type.Contains("int"))
|
||||||
return typeof(int);
|
return typeof(int);
|
||||||
|
|||||||
Loaded 100 of 1028 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user