Compare commits
29
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
97b5ec49b4 | ||
|
|
8aaadf1253 | ||
|
|
13aa94863d | ||
|
|
29c951fbca | ||
|
|
ae776cf75f | ||
|
|
f353a8e9fd | ||
|
|
24f4c2bb27 | ||
|
|
2960f19e60 | ||
|
|
653d96675b | ||
|
|
ae2aacdb1c | ||
|
|
11a741eea5 | ||
|
|
56d0217aad | ||
|
|
992b922d68 | ||
|
|
b1103d5edb | ||
|
|
1289992127 | ||
|
|
239548774a | ||
|
|
d0b0ca42ab | ||
|
|
51882570bd | ||
|
|
3ad95cf0ae | ||
|
|
02bcbb900a | ||
|
|
d8efc38eeb | ||
|
|
a51143c94b | ||
|
|
5e401d49ac | ||
|
|
2761a52636 | ||
|
|
b2f6421b1e | ||
|
|
236ab31fbc | ||
|
|
83b77c9fc0 | ||
|
|
9930b65ddc | ||
|
|
33093f3497 |
No files matched your search
@@ -25,12 +25,6 @@
|
|||||||
"commands": [
|
"commands": [
|
||||||
"fake"
|
"fake"
|
||||||
]
|
]
|
||||||
},
|
|
||||||
"dotnet-t4": {
|
|
||||||
"version": "3.0.0",
|
|
||||||
"commands": [
|
|
||||||
"t4"
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -22,7 +22,6 @@
|
|||||||
"extensions": [
|
"extensions": [
|
||||||
"eamodio.gitlens",
|
"eamodio.gitlens",
|
||||||
"llvm-vs-code-extensions.vscode-clangd",
|
"llvm-vs-code-extensions.vscode-clangd",
|
||||||
"ms-python.python",
|
|
||||||
"ms-vscode.cmake-tools",
|
"ms-vscode.cmake-tools",
|
||||||
"ms-vscode.cpptools",
|
"ms-vscode.cpptools",
|
||||||
"xaver.clang-format"
|
"xaver.clang-format"
|
||||||
|
|||||||
+16
-30
@@ -207,23 +207,17 @@ dotnet_naming_symbols.parameters.applicable_accessibilities = public,internal,pr
|
|||||||
dotnet_naming_symbols.private_symbols.applicable_kinds = property,method,field,event,delegate
|
dotnet_naming_symbols.private_symbols.applicable_kinds = property,method,field,event,delegate
|
||||||
dotnet_naming_symbols.private_symbols.applicable_accessibilities = protected,private
|
dotnet_naming_symbols.private_symbols.applicable_accessibilities = protected,private
|
||||||
|
|
||||||
# local symbols
|
|
||||||
dotnet_naming_symbols.local_symbols.applicable_kinds = local
|
|
||||||
|
|
||||||
# parameter symbols
|
|
||||||
dotnet_naming_symbols.parameter_symbols.applicable_kinds = parameter
|
|
||||||
|
|
||||||
# Naming styles
|
# Naming styles
|
||||||
|
|
||||||
dotnet_naming_style.pascal_case.required_prefix =
|
dotnet_naming_style.snake_case.required_prefix =
|
||||||
dotnet_naming_style.pascal_case.required_suffix =
|
dotnet_naming_style.snake_case.required_suffix =
|
||||||
dotnet_naming_style.pascal_case.word_separator =
|
dotnet_naming_style.snake_case.word_separator = _
|
||||||
dotnet_naming_style.pascal_case.capitalization = pascal_case
|
dotnet_naming_style.snake_case.capitalization = all_lower
|
||||||
|
|
||||||
dotnet_naming_style.camel_case.required_prefix =
|
dotnet_naming_style.private_snake_case.required_prefix = _
|
||||||
dotnet_naming_style.camel_case.required_suffix =
|
dotnet_naming_style.private_snake_case.required_suffix =
|
||||||
dotnet_naming_style.camel_case.word_separator =
|
dotnet_naming_style.private_snake_case.word_separator = _
|
||||||
dotnet_naming_style.camel_case.capitalization = camel_case
|
dotnet_naming_style.private_snake_case.capitalization = all_lower
|
||||||
|
|
||||||
dotnet_naming_style.user_types.required_prefix =
|
dotnet_naming_style.user_types.required_prefix =
|
||||||
dotnet_naming_style.user_types.required_suffix =
|
dotnet_naming_style.user_types.required_suffix =
|
||||||
@@ -235,10 +229,6 @@ dotnet_naming_style.interface.required_suffix =
|
|||||||
dotnet_naming_style.interface.word_separator =
|
dotnet_naming_style.interface.word_separator =
|
||||||
dotnet_naming_style.interface.capitalization = pascal_case
|
dotnet_naming_style.interface.capitalization = pascal_case
|
||||||
|
|
||||||
dotnet_naming_style.local.capitalization = camel_case
|
|
||||||
|
|
||||||
dotnet_naming_style.parameter.capitalization = camel_case
|
|
||||||
|
|
||||||
# rules
|
# rules
|
||||||
dotnet_naming_rule.user_types.symbols = user_types
|
dotnet_naming_rule.user_types.symbols = user_types
|
||||||
dotnet_naming_rule.user_types.style = user_types
|
dotnet_naming_rule.user_types.style = user_types
|
||||||
@@ -248,18 +238,14 @@ dotnet_naming_rule.interfaces.symbols = interface
|
|||||||
dotnet_naming_rule.interfaces.style = interface_style
|
dotnet_naming_rule.interfaces.style = interface_style
|
||||||
dotnet_naming_rule.interfaces.severity = suggestion
|
dotnet_naming_rule.interfaces.severity = suggestion
|
||||||
|
|
||||||
dotnet_naming_rule.public_members.symbols = public_symbols
|
dotnet_naming_rule.public_members_snake_case.symbols = public_symbols
|
||||||
dotnet_naming_rule.public_members.style = pascal_case
|
dotnet_naming_rule.public_members_snake_case.style = snake_case
|
||||||
dotnet_naming_rule.public_members.severity = suggestion
|
dotnet_naming_rule.public_members_snake_case.severity = suggestion
|
||||||
|
|
||||||
dotnet_naming_rule.private_members.symbols = private_symbols
|
dotnet_naming_rule.private_members_snake_case.symbols = private_symbols
|
||||||
dotnet_naming_rule.private_members.style = camel_case
|
dotnet_naming_rule.private_members_snake_case.style = private_snake_case
|
||||||
dotnet_naming_rule.private_members.severity = suggestion
|
dotnet_naming_rule.private_members_snake_case.severity = suggestion
|
||||||
|
|
||||||
dotnet_naming_rule.variables_are_camel_case.severity = suggestion
|
dotnet_naming_rule.parameters.symbols = parameters
|
||||||
dotnet_naming_rule.variables_are_camel_case.symbols = local_symbols
|
dotnet_naming_rule.parameters.style = snake_case
|
||||||
dotnet_naming_rule.variables_are_camel_case.style = local
|
|
||||||
|
|
||||||
dotnet_naming_rule.parameters.symbols = parameter_symbols
|
|
||||||
dotnet_naming_rule.parameters.style = parameter
|
|
||||||
dotnet_naming_rule.parameters.severity = suggestion
|
dotnet_naming_rule.parameters.severity = suggestion
|
||||||
@@ -34,7 +34,7 @@
|
|||||||
# src/server/sparkd/Dockerfile
|
# src/server/sparkd/Dockerfile
|
||||||
SPARKD__IMAGE=sparkd
|
SPARKD__IMAGE=sparkd
|
||||||
# The config dir for this service to use
|
# The config dir for this service to use
|
||||||
SPARKD__CONFIG_DIR=src/app_sparkd/configurations/dev
|
SPARKD__CONFIG_DIR=src/server/sparkd/configurations/dev
|
||||||
# Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
|
# Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
|
||||||
# Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
|
# Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
|
||||||
# SPARKD__DATA_DIR=/home/bts/sparkdata/data
|
# SPARKD__DATA_DIR=/home/bts/sparkdata/data
|
||||||
@@ -44,13 +44,5 @@ SPARKD__CONFIG_PORT=8000
|
|||||||
SPARKD__QUICKBUS_PORT=8001
|
SPARKD__QUICKBUS_PORT=8001
|
||||||
SPARKD__SIM_FIX_PORT=8002
|
SPARKD__SIM_FIX_PORT=8002
|
||||||
SPARKD__SHM_SIZE=4gb
|
SPARKD__SHM_SIZE=4gb
|
||||||
SPARKD__PYTHON_REQUIREMENTS=requirements.txt
|
|
||||||
|
|
||||||
### PubSubServer ###
|
|
||||||
PUBSUB__CONFIG_DIR=src/DataStreamSuite/PubSubServer/Configurations/Development
|
|
||||||
PUBSUB__SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
|
|
||||||
|
|
||||||
### MdPublisher ###
|
|
||||||
MD_PUBLISHER__CONFIG_DIR=src/DataStreamSuite/DataStreamPublisher/Configurations/ReplayMarketData
|
|
||||||
MD_PUBLISHER__SOURCE_DIR=build/Release/bin/net8.0/linux-x64/publish
|
|
||||||
# TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)
|
# TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)
|
||||||
@@ -11,7 +11,6 @@ cmake-build-debug/
|
|||||||
**.vscode/*
|
**.vscode/*
|
||||||
!.vscode/launch.json
|
!.vscode/launch.json
|
||||||
!.vscode/tasks.json
|
!.vscode/tasks.json
|
||||||
!.vscode/*.env
|
|
||||||
.vs
|
.vs
|
||||||
obj/
|
obj/
|
||||||
*.csproj.user
|
*.csproj.user
|
||||||
@@ -55,8 +54,6 @@ trace_data/
|
|||||||
.cache/
|
.cache/
|
||||||
/src/Spark.Admin.WebApp/tsconfig.*.g.json
|
/src/Spark.Admin.WebApp/tsconfig.*.g.json
|
||||||
/src/Spark.Admin.WebApp/src/bin/
|
/src/Spark.Admin.WebApp/src/bin/
|
||||||
/src/**/*appsettings.Local.json
|
|
||||||
/src/**/*/Configurations/Local/
|
|
||||||
**/CMakeDeps.cmake
|
**/CMakeDeps.cmake
|
||||||
**/CMakeDeps_win.cmake
|
**/CMakeDeps_win.cmake
|
||||||
.devcontainer/
|
.devcontainer/
|
||||||
|
|||||||
Vendored
+9
-161
@@ -14,49 +14,6 @@
|
|||||||
"type": "coreclr",
|
"type": "coreclr",
|
||||||
"request": "attach"
|
"request": "attach"
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"name": "Api Server",
|
|
||||||
"type": "coreclr",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "",
|
|
||||||
"preLaunchTask": "Build Api Server (Debug)",
|
|
||||||
"windows": {
|
|
||||||
"program": "spark_api_server.exe",
|
|
||||||
},
|
|
||||||
"linux": {
|
|
||||||
"program": "spark_api_server",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"run",
|
|
||||||
],
|
|
||||||
"cwd": "build/Debug/bin/net8.0",
|
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
|
|
||||||
},
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"console": "integratedTerminal"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "Api Server (Skip Build)",
|
|
||||||
"type": "coreclr",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "",
|
|
||||||
"windows": {
|
|
||||||
"program": "spark_api_server.exe",
|
|
||||||
},
|
|
||||||
"linux": {
|
|
||||||
"program": "spark_api_server",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"run",
|
|
||||||
],
|
|
||||||
"cwd": "build/Debug/bin/net8.0",
|
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
|
|
||||||
},
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"console": "integratedTerminal"
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"name": "DataStreamPublisher",
|
"name": "DataStreamPublisher",
|
||||||
"type": "coreclr",
|
"type": "coreclr",
|
||||||
@@ -69,28 +26,11 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"program": "DataStreamPublisher.exe",
|
"program": "DataStreamPublisher.exe",
|
||||||
},
|
},
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
|
||||||
|
],
|
||||||
"cwd": "build/Debug/bin/net8.0",
|
"cwd": "build/Debug/bin/net8.0",
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
|
|
||||||
},
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"console": "integratedTerminal"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "DataStreamPublisher (Skip Build)",
|
|
||||||
"type": "coreclr",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "",
|
|
||||||
"linux": {
|
|
||||||
"program": "DataStreamPublisher",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"program": "DataStreamPublisher.exe",
|
|
||||||
},
|
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
|
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
|
|
||||||
},
|
|
||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
"console": "integratedTerminal"
|
"console": "integratedTerminal"
|
||||||
},
|
},
|
||||||
@@ -142,29 +82,16 @@
|
|||||||
"name": "DataStreamQueryAnalyzer",
|
"name": "DataStreamQueryAnalyzer",
|
||||||
"type": "coreclr",
|
"type": "coreclr",
|
||||||
"request": "launch",
|
"request": "launch",
|
||||||
"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
|
//"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
|
||||||
"program": "",
|
"program": "",
|
||||||
"windows": {
|
"windows": {
|
||||||
"program": "DataStreamQueryAnalyzer.exe",
|
"program": "DataStreamQueryAnalyzer.exe",
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
|
||||||
|
],
|
||||||
"cwd": "build/Debug/bin/net8.0-windows",
|
"cwd": "build/Debug/bin/net8.0-windows",
|
||||||
"env": {
|
"env": {
|
||||||
"DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
|
|
||||||
"PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"console": "integratedTerminal",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "DataStreamQueryAnalyzer (Skip Build)",
|
|
||||||
"type": "coreclr",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "",
|
|
||||||
"windows": {
|
|
||||||
"program": "DataStreamQueryAnalyzer.exe",
|
|
||||||
"cwd": "build/Debug/bin/net8.0-windows",
|
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:dataStreamQueryAnalyzerEnvironment}",
|
|
||||||
"PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
|
"PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -188,27 +115,6 @@
|
|||||||
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
|
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
|
||||||
],
|
],
|
||||||
"cwd": "build/Debug/bin/net8.0",
|
"cwd": "build/Debug/bin/net8.0",
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
|
|
||||||
},
|
|
||||||
"stopAtEntry": false,
|
|
||||||
"console": "integratedTerminal"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "PubSubServer (Skip Build)",
|
|
||||||
"type": "coreclr",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "",
|
|
||||||
"linux": {
|
|
||||||
"program": "PubSubServer",
|
|
||||||
},
|
|
||||||
"windows": {
|
|
||||||
"program": "PubSubServer.exe",
|
|
||||||
},
|
|
||||||
"cwd": "build/Debug/bin/net8.0",
|
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
|
|
||||||
},
|
|
||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
"console": "integratedTerminal"
|
"console": "integratedTerminal"
|
||||||
},
|
},
|
||||||
@@ -221,9 +127,6 @@
|
|||||||
"preLaunchTask": "Build Spark (Debug)",
|
"preLaunchTask": "Build Spark (Debug)",
|
||||||
"program": "Spark.exe",
|
"program": "Spark.exe",
|
||||||
"cwd": "build/Debug/bin/net8.0-windows",
|
"cwd": "build/Debug/bin/net8.0-windows",
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:sparkEnvironment}",
|
|
||||||
}
|
|
||||||
},
|
},
|
||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
"console": "integratedTerminal"
|
"console": "integratedTerminal"
|
||||||
@@ -236,9 +139,6 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"program": "Spark.exe",
|
"program": "Spark.exe",
|
||||||
"cwd": "build/Debug/bin/net8.0-windows",
|
"cwd": "build/Debug/bin/net8.0-windows",
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:sparkEnvironment}",
|
|
||||||
}
|
|
||||||
},
|
},
|
||||||
"stopAtEntry": false,
|
"stopAtEntry": false,
|
||||||
"console": "integratedTerminal"
|
"console": "integratedTerminal"
|
||||||
@@ -351,17 +251,6 @@
|
|||||||
},
|
},
|
||||||
],
|
],
|
||||||
"inputs": [
|
"inputs": [
|
||||||
{
|
|
||||||
"id": "apiServerEnvironment",
|
|
||||||
"type": "pickString",
|
|
||||||
"description": "Api Server Configuration",
|
|
||||||
"options": [
|
|
||||||
"Development",
|
|
||||||
"Local",
|
|
||||||
"Staging",
|
|
||||||
],
|
|
||||||
"default": "Development",
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"id": "dataStreamPublisherConfig",
|
"id": "dataStreamPublisherConfig",
|
||||||
"type": "pickString",
|
"type": "pickString",
|
||||||
@@ -374,16 +263,6 @@
|
|||||||
],
|
],
|
||||||
"default": "MarketDataPublisher",
|
"default": "MarketDataPublisher",
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"id": "dataStreamPublisherEnvironment",
|
|
||||||
"type": "pickString",
|
|
||||||
"description": "DataStreamPublisher Configuration",
|
|
||||||
"options": [
|
|
||||||
"Local",
|
|
||||||
"ReplayMarketData",
|
|
||||||
],
|
|
||||||
"default": "Local",
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"id": "pubSubServerConfig",
|
"id": "pubSubServerConfig",
|
||||||
"type": "pickString",
|
"type": "pickString",
|
||||||
@@ -394,37 +273,6 @@
|
|||||||
],
|
],
|
||||||
"default": "PubSubServer",
|
"default": "PubSubServer",
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"id": "pubSubServerEnvironment",
|
|
||||||
"type": "pickString",
|
|
||||||
"description": "Api Server Configuration",
|
|
||||||
"options": [
|
|
||||||
"Development",
|
|
||||||
"Local",
|
|
||||||
"Staging",
|
|
||||||
],
|
|
||||||
"default": "Development",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "dataStreamQueryAnalyzerEnvironment",
|
|
||||||
"type": "pickString",
|
|
||||||
"description": "DataStreamQueryAnalyzer Configuration",
|
|
||||||
"options": [
|
|
||||||
"Development",
|
|
||||||
"Local",
|
|
||||||
],
|
|
||||||
"default": "Development",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "sparkEnvironment",
|
|
||||||
"type": "pickString",
|
|
||||||
"description": "Spark Configuration",
|
|
||||||
"options": [
|
|
||||||
"Development",
|
|
||||||
"Local",
|
|
||||||
],
|
|
||||||
"default": "Development",
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"id": "sparkConfigServer",
|
"id": "sparkConfigServer",
|
||||||
"type": "promptString",
|
"type": "promptString",
|
||||||
|
|||||||
Vendored
-1
@@ -1 +0,0 @@
|
|||||||
PYTHONPATH=/workspaces/spark/build/sparkd-debug-clang/src/lib_python_interop;/workspaces/spark/src/lib_python_interop/python_interop_plugins;/workspaces/spark/tools/py
|
|
||||||
Vendored
+61
-243
@@ -121,57 +121,6 @@
|
|||||||
"color": "terminal.ansiCyan"
|
"color": "terminal.ansiCyan"
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"label": "Build DataStreamQueryAnalyzer",
|
|
||||||
"group": "build",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "tools/build_data_stream_query_analyzer.cmd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--configuration",
|
|
||||||
"${input:buildConfiguration}",
|
|
||||||
],
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build DataStreamQueryAnalyzer (Debug)",
|
|
||||||
"group": "build",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "tools/build_data_stream_query_analyzer.cmd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--configuration",
|
|
||||||
"Debug",
|
|
||||||
],
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Build DataStreamQueryAnalyzer (Release)",
|
|
||||||
"group": "build",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "tools/build_data_stream_query_analyzer.cmd",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"--configuration",
|
|
||||||
"Release",
|
|
||||||
],
|
|
||||||
"icon": {
|
|
||||||
"id": "terminal",
|
|
||||||
"color": "terminal.ansiCyan"
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"label": "Build Spark",
|
"label": "Build Spark",
|
||||||
"group": "build",
|
"group": "build",
|
||||||
@@ -402,132 +351,9 @@
|
|||||||
"id": "wrench"
|
"id": "wrench"
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"label": "Run Api Server",
|
|
||||||
"dependsOn": "Build Api Server",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "spark_api_server.exe",
|
|
||||||
},
|
|
||||||
"linux": {
|
|
||||||
"command": "spark_api_server",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"run",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
|
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run Api Server (Debug)",
|
|
||||||
"dependsOn": "Build Api Server (Debug)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "spark_api_server.exe",
|
|
||||||
},
|
|
||||||
"linux": {
|
|
||||||
"command": "spark_api_server",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"run",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
|
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run Api Server (Release)",
|
|
||||||
"dependsOn": "Build Api Server (Release)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "spark_api_server.exe",
|
|
||||||
},
|
|
||||||
"linux": {
|
|
||||||
"command": "spark_api_server",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"run",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net8.0",
|
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run Api Server (Debug | Skip Build)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "spark_api_server.exe",
|
|
||||||
},
|
|
||||||
"linux": {
|
|
||||||
"command": "spark_api_server",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"run",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
|
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"label": "Run Api Server (Release | Skip Build)",
|
|
||||||
"type": "process",
|
|
||||||
"command": "",
|
|
||||||
"windows": {
|
|
||||||
"command": "spark_api_server.exe",
|
|
||||||
},
|
|
||||||
"linux": {
|
|
||||||
"command": "spark_api_server",
|
|
||||||
},
|
|
||||||
"args": [
|
|
||||||
"run",
|
|
||||||
],
|
|
||||||
"options": {
|
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net8.0",
|
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:apiServerEnvironment}",
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"icon": {
|
|
||||||
"id": "run",
|
|
||||||
"color": "terminal.ansiGreen",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"label": "Run DataStreamQueryAnalyzer",
|
"label": "Run DataStreamQueryAnalyzer",
|
||||||
"dependsOn": "Build DataStreamQueryAnalyzer",
|
"dependsOn": "Build DataStream",
|
||||||
"type": "process",
|
"type": "process",
|
||||||
"command": "",
|
"command": "",
|
||||||
"windows": {
|
"windows": {
|
||||||
@@ -537,7 +363,7 @@
|
|||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
||||||
],
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
|
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
@@ -547,7 +373,7 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"label": "Run DataStreamQueryAnalyzer (Debug)",
|
"label": "Run DataStreamQueryAnalyzer (Debug)",
|
||||||
"dependsOn": "Build DataStreamQueryAnalyzer (Debug)",
|
"dependsOn": "Build DataStream (Debug)",
|
||||||
"type": "process",
|
"type": "process",
|
||||||
"command": "",
|
"command": "",
|
||||||
"windows": {
|
"windows": {
|
||||||
@@ -557,7 +383,7 @@
|
|||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
||||||
],
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
|
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
@@ -567,7 +393,7 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"label": "Run DataStreamQueryAnalyzer (Release)",
|
"label": "Run DataStreamQueryAnalyzer (Release)",
|
||||||
"dependsOn": "Build DataStreamQueryAnalyzer (Release)",
|
"dependsOn": "Build DataStream (Release)",
|
||||||
"type": "process",
|
"type": "process",
|
||||||
"command": "",
|
"command": "",
|
||||||
"windows": {
|
"windows": {
|
||||||
@@ -577,7 +403,7 @@
|
|||||||
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
|
||||||
],
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
|
"cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
@@ -596,11 +422,12 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "DataStreamPublisher.exe",
|
"command": "DataStreamPublisher.exe",
|
||||||
},
|
},
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
|
||||||
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
|
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -618,11 +445,12 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "DataStreamPublisher.exe",
|
"command": "DataStreamPublisher.exe",
|
||||||
},
|
},
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
|
||||||
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
|
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -640,11 +468,12 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "DataStreamPublisher.exe",
|
"command": "DataStreamPublisher.exe",
|
||||||
},
|
},
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
|
||||||
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net8.0",
|
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:dataStreamPublisherEnvironment}",
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -662,11 +491,12 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "PubSubServer.exe",
|
"command": "PubSubServer.exe",
|
||||||
},
|
},
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
||||||
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0",
|
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -684,11 +514,12 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "PubSubServer.exe",
|
"command": "PubSubServer.exe",
|
||||||
},
|
},
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
||||||
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
|
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -705,11 +536,12 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "PubSubServer.exe",
|
"command": "PubSubServer.exe",
|
||||||
},
|
},
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
||||||
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
|
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -727,11 +559,12 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "PubSubServer.exe",
|
"command": "PubSubServer.exe",
|
||||||
},
|
},
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
||||||
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net8.0",
|
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -748,11 +581,12 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "PubSubServer.exe",
|
"command": "PubSubServer.exe",
|
||||||
},
|
},
|
||||||
|
"args": [
|
||||||
|
"--config",
|
||||||
|
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
|
||||||
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net8.0",
|
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
|
||||||
"env": {
|
|
||||||
"DOTNET_ENVIRONMENT": "${input:pubSubServerEnvironment}",
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -767,7 +601,7 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "Spark.exe",
|
"command": "Spark.exe",
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
|
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
@@ -783,7 +617,7 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "Spark.exe",
|
"command": "Spark.exe",
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0-windows"
|
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
@@ -799,7 +633,7 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "Spark.exe",
|
"command": "Spark.exe",
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Release/bin/net8.0-windows"
|
"cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
@@ -814,7 +648,7 @@
|
|||||||
"windows": {
|
"windows": {
|
||||||
"command": "Spark.exe",
|
"command": "Spark.exe",
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0-windows"
|
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
@@ -839,7 +673,7 @@
|
|||||||
"${input:sparkConfigQuery}",
|
"${input:sparkConfigQuery}",
|
||||||
],
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net8.0"
|
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
|
||||||
},
|
},
|
||||||
"icon": {
|
"icon": {
|
||||||
"id": "run",
|
"id": "run",
|
||||||
@@ -920,7 +754,7 @@
|
|||||||
"${input:sparkTest}"
|
"${input:sparkTest}"
|
||||||
],
|
],
|
||||||
"options": {
|
"options": {
|
||||||
"cwd": "${workspaceFolder}/build/Debug/bin/net8.0",
|
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -1018,17 +852,6 @@
|
|||||||
},
|
},
|
||||||
],
|
],
|
||||||
"inputs": [
|
"inputs": [
|
||||||
{
|
|
||||||
"id": "apiServerEnvironment",
|
|
||||||
"type": "pickString",
|
|
||||||
"description": "Api Server Configuration",
|
|
||||||
"options": [
|
|
||||||
"Development",
|
|
||||||
"Local",
|
|
||||||
"Staging",
|
|
||||||
],
|
|
||||||
"default": "Development",
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"id": "buildConfiguration",
|
"id": "buildConfiguration",
|
||||||
"type": "pickString",
|
"type": "pickString",
|
||||||
@@ -1062,31 +885,26 @@
|
|||||||
"default": "sparkd-debug-clang",
|
"default": "sparkd-debug-clang",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "dataStreamPublisherEnvironment",
|
"id": "dataStreamPublisherConfig",
|
||||||
"type": "pickString",
|
"type": "pickString",
|
||||||
"description": "Publisher Configuration",
|
"description": "Publisher Configuration",
|
||||||
"options": [
|
"options": [
|
||||||
"Development",
|
"MarketDataPublisher",
|
||||||
"MarketData",
|
"SimpleEchoClient",
|
||||||
"Production",
|
"SimpleTestDataPublisher",
|
||||||
"ProductionAws",
|
"TestDataPublisher",
|
||||||
"Staging",
|
|
||||||
],
|
],
|
||||||
"default": "MarketDataPublisher",
|
"default": "MarketDataPublisher",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "pubSubServerEnvironment",
|
"id": "pubSubServerConfig",
|
||||||
"type": "pickString",
|
"type": "pickString",
|
||||||
"description": "PubSub Configuration",
|
"description": "PubSub Configuration",
|
||||||
"options": [
|
"options": [
|
||||||
"Development",
|
"PubSubServer",
|
||||||
"EchoServer",
|
"SimpleEchoClient",
|
||||||
"Local",
|
|
||||||
"Production",
|
|
||||||
"ProductionAws",
|
|
||||||
"Staging",
|
|
||||||
],
|
],
|
||||||
"default": "Development",
|
"default": "PubSubServer",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "sparkConfigQuery",
|
"id": "sparkConfigQuery",
|
||||||
|
|||||||
+7
-6
@@ -21,7 +21,7 @@ set(
|
|||||||
# SHARED_MEMORY_PREFIX serves as a kind of namespace so multiple applications
|
# SHARED_MEMORY_PREFIX serves as a kind of namespace so multiple applications
|
||||||
# can run simultaneously without fighting over the same shm files (e.g., sparkd,
|
# can run simultaneously without fighting over the same shm files (e.g., sparkd,
|
||||||
# api server, etc.).
|
# api server, etc.).
|
||||||
set(SHARED_MEMORY_PREFIX "\${BTS_SHARED_MEMORY_PREFIX}" CACHE STRING "Prefix to use when creating shared memory files")
|
set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
|
||||||
set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
|
set(EXTERNALS_BIN "${EXTERNALS_ROOT}/bin")
|
||||||
set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
|
set(EXTERNALS_INCLUDE "${EXTERNALS_ROOT}/include")
|
||||||
set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
|
set(EXTERNALS_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
|
||||||
@@ -40,8 +40,9 @@ include_directories(BEFORE SYSTEM ${EXTERNALS_INCLUDE} ${EXTERNALS_ROOT}/include
|
|||||||
link_directories(${EXTERNALS_LIB})
|
link_directories(${EXTERNALS_LIB})
|
||||||
|
|
||||||
add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
|
add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}")
|
||||||
if(DEFINED APP_DATA_DIR)
|
add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
|
||||||
add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}")
|
if(DEFINED APP_DATA_DIR_OVERRIDE)
|
||||||
|
add_compile_definitions(APP_DATA_DIR_OVERRIDE="${APP_DATA_DIR_OVERRIDE}")
|
||||||
endif()
|
endif()
|
||||||
add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
|
add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
|
||||||
if(WIN32)
|
if(WIN32)
|
||||||
@@ -112,15 +113,15 @@ else()
|
|||||||
endif()
|
endif()
|
||||||
|
|
||||||
# subdirs
|
# subdirs
|
||||||
add_subdirectory(src/lib_spark)
|
|
||||||
if (BUILD_SPARKD)
|
if (BUILD_SPARKD)
|
||||||
add_subdirectory(src/lib_python_interop)
|
|
||||||
add_subdirectory(src/lib_onixs_fix)
|
add_subdirectory(src/lib_onixs_fix)
|
||||||
add_subdirectory(src/lib_binance)
|
add_subdirectory(src/lib_binance_execution)
|
||||||
add_subdirectory(src/lib_okx_execution)
|
add_subdirectory(src/lib_okx_execution)
|
||||||
|
add_subdirectory(src/lib_python_md)
|
||||||
add_subdirectory(src/app_sparkd)
|
add_subdirectory(src/app_sparkd)
|
||||||
add_subdirectory(src/data_files)
|
add_subdirectory(src/data_files)
|
||||||
endif()
|
endif()
|
||||||
|
add_subdirectory(src/lib_spark)
|
||||||
if (BUILD_ADMIN_WEB_APP)
|
if (BUILD_ADMIN_WEB_APP)
|
||||||
add_subdirectory(src/Spark.Admin.WebApp)
|
add_subdirectory(src/Spark.Admin.WebApp)
|
||||||
endif()
|
endif()
|
||||||
|
|||||||
+22
-6
@@ -20,6 +20,8 @@
|
|||||||
"SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
|
"SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
|
||||||
},
|
},
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
|
"APP_DATA_DIR": "${dollar}{APPDATA}/bts",
|
||||||
|
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_DATA_DIR}",
|
||||||
"CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
|
"CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
|
||||||
"CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
|
"CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
|
||||||
"CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
|
"CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
|
||||||
@@ -34,7 +36,8 @@
|
|||||||
"EXPORT_MD_CONSUMER": {
|
"EXPORT_MD_CONSUMER": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "ON"
|
"value": "ON"
|
||||||
}
|
},
|
||||||
|
"SHARED_MEMORY_PREFIX": "spark"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -50,6 +53,8 @@
|
|||||||
"binaryDir": "${sourceDir}/build/${presetName}",
|
"binaryDir": "${sourceDir}/build/${presetName}",
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
"platform": "",
|
"platform": "",
|
||||||
|
"APP_DATA_DIR": "${dollar}{HOME}/sparkdata",
|
||||||
|
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
|
||||||
"CMAKE_BUILD_WITH_INSTALL_RPATH": {
|
"CMAKE_BUILD_WITH_INSTALL_RPATH": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "OFF"
|
"value": "OFF"
|
||||||
@@ -72,7 +77,8 @@
|
|||||||
"BUILD_ADMIN_WEB_APP": {
|
"BUILD_ADMIN_WEB_APP": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "ON"
|
"value": "ON"
|
||||||
}
|
},
|
||||||
|
"SHARED_MEMORY_PREFIX": "sparkd"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -153,7 +159,11 @@
|
|||||||
{
|
{
|
||||||
"name": "spark-api-server-windows",
|
"name": "spark-api-server-windows",
|
||||||
"inherits": "spark",
|
"inherits": "spark",
|
||||||
"displayName": "Spark Api Server (Windows)"
|
"displayName": "Spark Api Server (Windows)",
|
||||||
|
"cacheVariables": {
|
||||||
|
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
|
||||||
|
"SHARED_MEMORY_PREFIX": "spark_api_server"
|
||||||
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
@@ -162,6 +172,7 @@
|
|||||||
"displayName": "Spark Api Server (Linux)",
|
"displayName": "Spark Api Server (Linux)",
|
||||||
"generator": "Ninja Multi-Config",
|
"generator": "Ninja Multi-Config",
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
|
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
|
||||||
"BUILD_ADMIN_WEB_APP": {
|
"BUILD_ADMIN_WEB_APP": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "OFF"
|
"value": "OFF"
|
||||||
@@ -218,8 +229,10 @@
|
|||||||
"description": "Data Stream utils ported over from Edge",
|
"description": "Data Stream utils ported over from Edge",
|
||||||
"binaryDir": "${sourceDir}/build/${presetName}",
|
"binaryDir": "${sourceDir}/build/${presetName}",
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
|
"APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream",
|
||||||
|
"APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
|
||||||
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
|
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
|
||||||
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0;${sourceDir}/build/$<CONFIG>/bin/net8.0-windows"
|
"SHARED_MEMORY_PREFIX": "data_stream"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -235,6 +248,8 @@
|
|||||||
"inherits": "sparkd",
|
"inherits": "sparkd",
|
||||||
"generator": "Ninja Multi-Config",
|
"generator": "Ninja Multi-Config",
|
||||||
"cacheVariables": {
|
"cacheVariables": {
|
||||||
|
"APP_DATA_DIR": "${dollar}{HOME}/.local/share/DataStream",
|
||||||
|
"APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
|
||||||
"BUILD_ADMIN_WEB_APP": {
|
"BUILD_ADMIN_WEB_APP": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "OFF"
|
"value": "OFF"
|
||||||
@@ -252,11 +267,12 @@
|
|||||||
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
|
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
|
||||||
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
|
||||||
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
|
"CONFIG_DEFAULTS_PATH": "spark/config/profiles/default/*.conf;spark/config/profiles/data_stream/*.conf",
|
||||||
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net8.0",
|
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
|
||||||
"EXPORT_MD_CONSUMER": {
|
"EXPORT_MD_CONSUMER": {
|
||||||
"type": "BOOL",
|
"type": "BOOL",
|
||||||
"value": "ON"
|
"value": "ON"
|
||||||
}
|
},
|
||||||
|
"SHARED_MEMORY_PREFIX": "data_stream"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -42,10 +42,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "sr
|
|||||||
EndProject
|
EndProject
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "sdk", "sdk", "{56ACFCD1-60C7-4E2C-B90D-B1F95ECE33A1}"
|
|
||||||
EndProject
|
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Spark.Sdk", "src\sdk\Bts.Spark.Sdk\Bts.Spark.Sdk.csproj", "{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}"
|
|
||||||
EndProject
|
|
||||||
Global
|
Global
|
||||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||||
Debug|x64 = Debug|x64
|
Debug|x64 = Debug|x64
|
||||||
@@ -198,14 +194,6 @@ Global
|
|||||||
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
|
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
|
||||||
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
||||||
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
||||||
{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
|
||||||
{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
|
||||||
EndGlobalSection
|
EndGlobalSection
|
||||||
GlobalSection(SolutionProperties) = preSolution
|
GlobalSection(SolutionProperties) = preSolution
|
||||||
HideSolutionNode = FALSE
|
HideSolutionNode = FALSE
|
||||||
@@ -230,7 +218,5 @@ Global
|
|||||||
{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
||||||
{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
||||||
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
|
||||||
{56ACFCD1-60C7-4E2C-B90D-B1F95ECE33A1} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
|
|
||||||
{ADA9E802-8EFA-4E72-8E9A-6AA1E4181184} = {56ACFCD1-60C7-4E2C-B90D-B1F95ECE33A1}
|
|
||||||
EndGlobalSection
|
EndGlobalSection
|
||||||
EndGlobal
|
EndGlobal
|
||||||
@@ -1,71 +0,0 @@
|
|||||||
include_guard(DIRECTORY)
|
|
||||||
include(GNUInstallDirs)
|
|
||||||
include(${CMAKE_SOURCE_DIR}/cmake/ExportToParent.cmake)
|
|
||||||
include(${CMAKE_SOURCE_DIR}/cmake/SetFirstDefined.cmake)
|
|
||||||
|
|
||||||
function(init_python_module)
|
|
||||||
set(_options)
|
|
||||||
set(_one_value_args
|
|
||||||
NAME
|
|
||||||
)
|
|
||||||
set(_multi_value_args
|
|
||||||
SOURCES
|
|
||||||
)
|
|
||||||
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
|
||||||
set_first_defined(_name _ARGS_NAME PROJECT_NAME)
|
|
||||||
set_first_defined(_sources _ARGS_SOURCES ${_name}_SOURCES)
|
|
||||||
|
|
||||||
set(module_proj ${_name}_module)
|
|
||||||
set(${module_proj}_SOURCES ${_sources})
|
|
||||||
pybind11_add_module(${module_proj} SHARED ${_sources})
|
|
||||||
target_link_libraries(
|
|
||||||
${module_proj}
|
|
||||||
PUBLIC
|
|
||||||
pybind11::embed
|
|
||||||
spark
|
|
||||||
${_name}
|
|
||||||
)
|
|
||||||
target_include_directories(
|
|
||||||
${module_proj}
|
|
||||||
PRIVATE
|
|
||||||
$<BUILD_INTERFACE:${pybind11_INCLUDE_DIRS}>
|
|
||||||
)
|
|
||||||
target_include_directories(
|
|
||||||
${module_proj}
|
|
||||||
PUBLIC
|
|
||||||
${Python3_INCLUDE_DIRS}
|
|
||||||
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
|
|
||||||
$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
|
|
||||||
)
|
|
||||||
add_custom_command(
|
|
||||||
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
|
|
||||||
DEPENDS ${_name} ${module_proj}
|
|
||||||
COMMAND PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR} pybind11-stubgen --numpy-array-use-type-var --ignore-invalid-expressions '.*' ${_name}_module
|
|
||||||
COMMENT "Generating py stubs"
|
|
||||||
)
|
|
||||||
add_custom_target(
|
|
||||||
"${_name}_py_stubs"
|
|
||||||
ALL
|
|
||||||
DEPENDS
|
|
||||||
${CMAKE_CURRENT_BINARY_DIR}/stubs/${_name}.pyi
|
|
||||||
)
|
|
||||||
install(
|
|
||||||
TARGETS ${module_proj}
|
|
||||||
EXPORT ${module_proj}Targets
|
|
||||||
RUNTIME
|
|
||||||
DESTINATION ${CMAKE_INSTALL_BINDIR}
|
|
||||||
LIBRARY
|
|
||||||
DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
|
||||||
ARCHIVE
|
|
||||||
DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
|
||||||
)
|
|
||||||
foreach(dir ${pybind11_INCLUDE_DIRS})
|
|
||||||
install(DIRECTORY ${dir} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
|
|
||||||
endforeach()
|
|
||||||
install(
|
|
||||||
EXPORT ${module_proj}Targets
|
|
||||||
FILE ${module_proj}Targets.cmake
|
|
||||||
NAMESPACE ${_name}::
|
|
||||||
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
|
|
||||||
)
|
|
||||||
endfunction()
|
|
||||||
@@ -1,65 +0,0 @@
|
|||||||
include_guard(DIRECTORY)
|
|
||||||
include(GNUInstallDirs)
|
|
||||||
|
|
||||||
function(init_python_plugin)
|
|
||||||
set(_options)
|
|
||||||
set(_one_value_args NAME PLUGIN_NAME)
|
|
||||||
set(_multi_value_args SOURCES)
|
|
||||||
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
|
|
||||||
set_first_defined(_name _ARGS_NAME PROJECT_NAME)
|
|
||||||
set(_plugin_name ${_ARGS_PLUGIN_NAME})
|
|
||||||
if (NOT DEFINED _ARGS_PLUGIN_NAME)
|
|
||||||
set(_plugin_name ${_name}_plugins)
|
|
||||||
endif()
|
|
||||||
set(${_plugin_name}_SOURCES ${_ARGS_SOURCES})
|
|
||||||
if (NOT DEFINED _ARGS_SOURCES)
|
|
||||||
file(
|
|
||||||
GLOB_RECURSE ${_plugin_name}_SOURCES
|
|
||||||
RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
|
|
||||||
${_plugin_name}/*.py)
|
|
||||||
endif()
|
|
||||||
set(src_outs)
|
|
||||||
foreach(_src ${${_plugin_name}_SOURCES})
|
|
||||||
set(_src_in ${CMAKE_CURRENT_SOURCE_DIR}/${_src})
|
|
||||||
set(_src_out ${CMAKE_CURRENT_BINARY_DIR}/${_src})
|
|
||||||
add_custom_command(
|
|
||||||
OUTPUT ${_src_out}
|
|
||||||
COMMAND ${CMAKE_COMMAND} -E copy ${_src_in} ${_src_out}
|
|
||||||
DEPENDS ${_src_in}
|
|
||||||
COMMENT "Copying ${_plugin_name} plugin ${_src} to ${_src_out}"
|
|
||||||
)
|
|
||||||
list(APPEND src_outs ${_src_out})
|
|
||||||
endforeach()
|
|
||||||
|
|
||||||
add_custom_target(copy_${_plugin_name} ALL DEPENDS ${src_outs})
|
|
||||||
|
|
||||||
add_library(${_plugin_name} INTERFACE)
|
|
||||||
add_library(${_name}::${_plugin_name} ALIAS ${_plugin_name})
|
|
||||||
|
|
||||||
set_target_properties(
|
|
||||||
${_plugin_name}
|
|
||||||
PROPERTIES
|
|
||||||
INTERFACE_PYTHON_SCRIPTS_DIR
|
|
||||||
$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${_plugin_name}>
|
|
||||||
)
|
|
||||||
|
|
||||||
set_target_properties(
|
|
||||||
${_plugin_name}
|
|
||||||
PROPERTIES
|
|
||||||
INTERFACE_PYTHON_SCRIPTS $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/${${_plugin_name}_SOURCES}>$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/${${_plugin_name}_SOURCES}>
|
|
||||||
)
|
|
||||||
|
|
||||||
install(
|
|
||||||
TARGETS ${_plugin_name}
|
|
||||||
EXPORT ${_plugin_name}Targets
|
|
||||||
DESTINATION ${CMAKE_INSTALL_BINDIR}
|
|
||||||
)
|
|
||||||
|
|
||||||
install(
|
|
||||||
EXPORT ${_plugin_name}Targets
|
|
||||||
FILE ${_plugin_name}Targets.cmake
|
|
||||||
NAMESPACE ${_name}::
|
|
||||||
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${_name}
|
|
||||||
)
|
|
||||||
install(DIRECTORY ${_plugin_name} DESTINATION ${CMAKE_INSTALL_BINDIR})
|
|
||||||
endfunction()
|
|
||||||
+51
-69
@@ -29,43 +29,36 @@ function(Protobuf_Generate_Cpp)
|
|||||||
list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
|
list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h")
|
||||||
|
|
||||||
add_custom_command(
|
add_custom_command(
|
||||||
OUTPUT
|
OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
|
||||||
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
|
||||||
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
|
|
||||||
COMMAND
|
COMMAND
|
||||||
${Protobuf_PROTOC_EXECUTABLE}
|
${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
|
||||||
ARGS
|
|
||||||
--cpp_out=${_ARGS_CXX_OUT_DIR} ${_proto_include_cli_args} ${ABS_FIL}
|
|
||||||
DEPENDS ${ABS_FIL}
|
DEPENDS ${ABS_FIL}
|
||||||
COMMENT "Running C++ protocol buffer compiler on ${FIL}"
|
COMMENT "Running C++ protocol buffer compiler on ${FIL}"
|
||||||
VERBATIM
|
VERBATIM)
|
||||||
)
|
|
||||||
endforeach()
|
endforeach()
|
||||||
|
|
||||||
set_source_files_properties(
|
set_source_files_properties(${${_ARGS_CXX_SOURCES}} ${${_ARGS_CXX_HEADERS}} PROPERTIES GENERATED TRUE)
|
||||||
${${_ARGS_CXX_SOURCES}}
|
|
||||||
${${_ARGS_CXX_HEADERS}}
|
|
||||||
PROPERTIES
|
|
||||||
GENERATED TRUE
|
|
||||||
)
|
|
||||||
|
|
||||||
# remove some warnings from protobuf generated files
|
# remove some warnings from protobuf generated files
|
||||||
if(WIN32)
|
if(WIN32)
|
||||||
set_source_files_properties(
|
set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
|
||||||
${${_ARGS_CXX_SOURCES}}
|
"/wd4309 /wd4838")
|
||||||
PROPERTIES
|
|
||||||
COMPILE_FLAGS "/wd4309 /wd4838"
|
|
||||||
)
|
|
||||||
else()
|
else()
|
||||||
set_source_files_properties(
|
set_source_files_properties(
|
||||||
${${_ARGS_CXX_SOURCES}}
|
${${_ARGS_CXX_SOURCES}}
|
||||||
PROPERTIES
|
PROPERTIES
|
||||||
COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
|
COMPILE_FLAGS
|
||||||
|
"-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
|
||||||
)
|
)
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
|
set(${_ARGS_CXX_SOURCES}
|
||||||
set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
|
${${_ARGS_CXX_SOURCES}}
|
||||||
|
PARENT_SCOPE)
|
||||||
|
set(${_ARGS_CXX_HEADERS}
|
||||||
|
${${_ARGS_CXX_HEADERS}}
|
||||||
|
PARENT_SCOPE)
|
||||||
endfunction()
|
endfunction()
|
||||||
|
|
||||||
function(Service_Compiler_Generate_Cpp)
|
function(Service_Compiler_Generate_Cpp)
|
||||||
@@ -99,51 +92,40 @@ function(Service_Compiler_Generate_Cpp)
|
|||||||
list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
|
list(APPEND ${_ARGS_CXX_HEADERS} "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h")
|
||||||
|
|
||||||
set(TEMPLATES
|
set(TEMPLATES
|
||||||
${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
|
${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.cc
|
||||||
${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
|
${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/client.pb.h
|
||||||
${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
|
${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.cc
|
||||||
${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
|
${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/templates/cpp/server.pb.h
|
||||||
)
|
)
|
||||||
|
|
||||||
set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
|
set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.py)
|
||||||
if(WIN32)
|
if(WIN32)
|
||||||
set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
|
set(_plugin_cmd ${CMAKE_SOURCE_DIR}/src/lib_spark/spark/api/service_compiler/plugin.cmd)
|
||||||
add_custom_command(
|
add_custom_command(
|
||||||
OUTPUT
|
OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
|
||||||
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
|
||||||
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
|
||||||
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
|
||||||
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
|
||||||
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
|
||||||
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
|
|
||||||
COMMAND SET PYTHONPATH=${SPARK_PYTHON}
|
COMMAND SET PYTHONPATH=${SPARK_PYTHON}
|
||||||
COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
|
COMMAND ${CMAKE_SOURCE_DIR}/.venv/Scripts/activate.bat
|
||||||
COMMAND
|
COMMAND
|
||||||
${Protobuf_PROTOC_EXECUTABLE}
|
${Protobuf_PROTOC_EXECUTABLE} ARGS --cpp_out=${_ARGS_CXX_OUT_DIR}
|
||||||
ARGS
|
--plugin=protoc-gen-service=${_plugin_cmd}
|
||||||
--cpp_out=${_ARGS_CXX_OUT_DIR}
|
--service_out=lang=cpp:${_ARGS_CXX_OUT_DIR} -I
|
||||||
--plugin=protoc-gen-service=${_plugin_cmd}
|
${PROTOBUF_INCLUDE_DIRS} -I ${_protobuf_src_dir} ${ABS_FIL}
|
||||||
--service_out=lang=cpp:${_ARGS_CXX_OUT_DIR}
|
DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_cmd} ${_plugin_py}
|
||||||
-I ${PROTOBUF_INCLUDE_DIRS}
|
|
||||||
-I ${_protobuf_src_dir}
|
|
||||||
${ABS_FIL}
|
|
||||||
DEPENDS
|
|
||||||
${ABS_FIL}
|
|
||||||
${TEMPLATES}
|
|
||||||
${_plugin_cmd}
|
|
||||||
${_plugin_py}
|
|
||||||
COMMENT "Running C++ service compiler on ${FIL}"
|
COMMENT "Running C++ service compiler on ${FIL}"
|
||||||
VERBATIM
|
VERBATIM)
|
||||||
)
|
|
||||||
else()
|
else()
|
||||||
add_custom_command(
|
add_custom_command(
|
||||||
OUTPUT
|
OUTPUT "${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
|
||||||
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.cc"
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
|
||||||
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.pb.h"
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
|
||||||
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.cc"
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
|
||||||
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.client.pb.h"
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
|
||||||
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.cc"
|
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
|
||||||
"${_ARGS_CXX_OUT_DIR}/${FIL_WE}.server.pb.h"
|
|
||||||
COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
|
COMMAND ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh
|
||||||
ARGS
|
ARGS
|
||||||
${Protobuf_PROTOC_EXECUTABLE}
|
${Protobuf_PROTOC_EXECUTABLE}
|
||||||
@@ -151,32 +133,32 @@ function(Service_Compiler_Generate_Cpp)
|
|||||||
--plugin=protoc-gen-service=${_plugin_py}
|
--plugin=protoc-gen-service=${_plugin_py}
|
||||||
--service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
|
--service_out=lang=cpp,make_file_name=${ABS_FIL}:${_ARGS_CXX_OUT_DIR}
|
||||||
${_proto_include_cli_args} ${ABS_FIL}
|
${_proto_include_cli_args} ${ABS_FIL}
|
||||||
DEPENDS
|
DEPENDS ${ABS_FIL} ${TEMPLATES} ${_plugin_py}
|
||||||
${ABS_FIL}
|
|
||||||
${TEMPLATES}
|
|
||||||
${_plugin_py}
|
|
||||||
COMMENT "Running C++ service compiler on ${FIL}"
|
COMMENT "Running C++ service compiler on ${FIL}"
|
||||||
VERBATIM
|
VERBATIM)
|
||||||
)
|
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
endforeach()
|
endforeach()
|
||||||
|
|
||||||
# remove some warnings from protobuf generated files
|
# remove some warnings from protobuf generated files
|
||||||
if(WIN32)
|
if(WIN32)
|
||||||
set_source_files_properties(
|
set_source_files_properties(${${_ARGS_CXX_SOURCES}} PROPERTIES COMPILE_FLAGS
|
||||||
${${_ARGS_CXX_SOURCES}}
|
"/wd4309 /wd4838")
|
||||||
PROPERTIES
|
|
||||||
COMPILE_FLAGS "/wd4309 /wd4838"
|
|
||||||
)
|
|
||||||
else()
|
else()
|
||||||
set_source_files_properties(
|
set_source_files_properties(
|
||||||
${${_ARGS_CXX_SOURCES}}
|
${${_ARGS_CXX_SOURCES}}
|
||||||
PROPERTIES
|
PROPERTIES
|
||||||
COMPILE_FLAGS "-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
|
COMPILE_FLAGS
|
||||||
|
"-Wno-unused-parameter -Wno-effc++ -Wno-pedantic -Wno-sign-compare -Wno-uninitialized"
|
||||||
)
|
)
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
set(${_ARGS_CXX_SOURCES} ${${_ARGS_CXX_SOURCES}} PARENT_SCOPE)
|
set(${_ARGS_CXX_SOURCES}
|
||||||
set(${_ARGS_CXX_HEADERS} ${${_ARGS_CXX_HEADERS}} PARENT_SCOPE)
|
${${_ARGS_CXX_SOURCES}}
|
||||||
|
PARENT_SCOPE)
|
||||||
|
set(${_ARGS_CXX_HEADERS}
|
||||||
|
${${_ARGS_CXX_HEADERS}}
|
||||||
|
PARENT_SCOPE)
|
||||||
|
|
||||||
endfunction()
|
endfunction()
|
||||||
|
|
||||||
@@ -1,39 +0,0 @@
|
|||||||
include_guard(DIRECTORY)
|
|
||||||
include(${CMAKE_SOURCE_DIR}/cmake/SymLink.cmake)
|
|
||||||
|
|
||||||
function(sym_link_python_module)
|
|
||||||
set(_options)
|
|
||||||
set(_one_value_args LIB_NAME MODULE_NAME PLUGIN_NAME)
|
|
||||||
set(_multi_value_args)
|
|
||||||
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}"
|
|
||||||
"${_multi_value_args}" ${ARGN})
|
|
||||||
if(NOT DEFINED _ARGS_LIB_NAME)
|
|
||||||
message(
|
|
||||||
FATAL_ERROR "LIB_NAME must be specified for sym_link_python_module()")
|
|
||||||
endif()
|
|
||||||
set(_lib_name ${_ARGS_LIB_NAME})
|
|
||||||
set(_module_name ${_ARGS_MODULE_NAME})
|
|
||||||
if(NOT DEFINED _ARGS_MODULE_NAME)
|
|
||||||
set(_module_name ${_lib_name}_module)
|
|
||||||
endif()
|
|
||||||
set(_plugin_name ${_ARGS_PLUGIN_NAME})
|
|
||||||
set(_deps ${_lib_name} ${_module_name})
|
|
||||||
if(DEFINED _ARGS_PLUGIN_NAME)
|
|
||||||
list(APPEND _deps ${_plugin_name} copy_${_plugin_name})
|
|
||||||
endif()
|
|
||||||
add_dependencies(${PROJECT_NAME} ${_deps})
|
|
||||||
sym_link_local_lib(${_lib_name})
|
|
||||||
set(_module_suffix cpython-310-x86_64-linux-gnu.so)
|
|
||||||
sym_link_file(
|
|
||||||
SOURCE
|
|
||||||
${CMAKE_BINARY_DIR}/src/lib_${_lib_name}/${_module_name}.${_module_suffix}
|
|
||||||
DEST
|
|
||||||
${CMAKE_CURRENT_BINARY_DIR}/${_module_name}.${_module_suffix}
|
|
||||||
INSTALL_DEST
|
|
||||||
${CMAKE_INSTALL_BINDIR}/)
|
|
||||||
if(DEFINED _ARGS_PLUGIN_NAME)
|
|
||||||
get_target_property(plugins ${_plugin_name} INTERFACE_PYTHON_SCRIPTS_DIR)
|
|
||||||
sym_link_dir(TARGET_NAME ${PROJECT_NAME}_link_${_plugin_name} SOURCE
|
|
||||||
${plugins} DEST ${CMAKE_CURRENT_BINARY_DIR}/${_plugin_name})
|
|
||||||
endif()
|
|
||||||
endfunction()
|
|
||||||
+21
-114
@@ -1,31 +1,26 @@
|
|||||||
services:
|
services:
|
||||||
sparkd:
|
sparkd:
|
||||||
volumes:
|
volumes:
|
||||||
- ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
|
- ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
|
||||||
build:
|
build:
|
||||||
dockerfile: src/app_sparkd/Dockerfile
|
dockerfile: src/app_sparkd/Dockerfile
|
||||||
args:
|
args:
|
||||||
- IMAGE=${SPARKD__IMAGE}
|
- IMAGE=${SPARKD__IMAGE}
|
||||||
- SOURCE_DIR=${SPARKD__SOURCE_DIR}
|
- SOURCE_DIR=${SPARKD__SOURCE_DIR}
|
||||||
- CONFIG_DIR=${SPARKD__CONFIG_DIR}
|
- CONFIG_DIR=${SPARKD__CONFIG_DIR}
|
||||||
- PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
|
|
||||||
shm_size: "${SPARKD__SHM_SIZE:-512mb}"
|
shm_size: "${SPARKD__SHM_SIZE:-512mb}"
|
||||||
command: >
|
command: >
|
||||||
sh -c "
|
sh -c "
|
||||||
echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd/data
|
echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
|
||||||
if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
|
if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
|
||||||
mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
|
mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
|
||||||
fi
|
fi
|
||||||
if [ -e "/home/bts/.local/share/bts/sparkd/conf/.env.local" ]; then
|
if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
|
||||||
echo 'Sourcing /home/bts/.local/share/bts/sparkd/conf/.env.local'
|
echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
|
||||||
set -a
|
set -a
|
||||||
. /home/bts/.local/share/bts/sparkd/conf/.env.local
|
. /home/bts/sparkdata/conf/.env.local
|
||||||
set +a
|
set +a
|
||||||
fi
|
fi
|
||||||
if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then
|
|
||||||
echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)'
|
|
||||||
cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/
|
|
||||||
fi
|
|
||||||
./sparkd run"
|
./sparkd run"
|
||||||
env_file:
|
env_file:
|
||||||
- .env
|
- .env
|
||||||
@@ -41,34 +36,29 @@ services:
|
|||||||
|
|
||||||
sparkd-symbols:
|
sparkd-symbols:
|
||||||
volumes:
|
volumes:
|
||||||
- ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/.local/share/bts/sparkd/data
|
- ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
|
||||||
- ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/.local/share/bts/sparkd/conf
|
- ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
|
||||||
build:
|
build:
|
||||||
dockerfile: src/app_sparkd/Dockerfile
|
dockerfile: src/server/sparkd/Dockerfile
|
||||||
args:
|
args:
|
||||||
- IMAGE=${SPARKD__IMAGE}
|
- IMAGE=${SPARKD__IMAGE}
|
||||||
- SOURCE_DIR=${SPARKD__SOURCE_DIR}
|
- SOURCE_DIR=${SPARKD__SOURCE_DIR}
|
||||||
- CONFIG_DIR=${SPARKD__CONFIG_DIR}
|
- CONFIG_DIR=${SPARKD__CONFIG_DIR}
|
||||||
- PYTHON_REQUIREMENTS=${SPARKD__PYTHON_REQUIREMENTS}
|
|
||||||
shm_size: "${SPARKD__SHM_SIZE:-512mb}"
|
shm_size: "${SPARKD__SHM_SIZE:-512mb}"
|
||||||
command: >
|
command: >
|
||||||
sh -c "
|
sh -c "
|
||||||
echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/sparkd
|
echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata
|
||||||
if [ -e "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" ]; then
|
if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
|
||||||
mv "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log" "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
|
mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
|
||||||
fi
|
fi
|
||||||
ls -alh /home/bts/.local/share/bts/sparkd/conf/
|
ls -alh /home/bts/sparkdata/conf/
|
||||||
cat /home/bts/.local/share/bts/sparkd/conf/.env.local
|
cat /home/bts/sparkdata/conf/.env.local
|
||||||
if [ -e "/home/bts/.local/share/bts/sparkd/conf/.env.local" ]; then
|
if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
|
||||||
echo 'Sourcing /home/bts/.local/share/bts/sparkd/conf/.env.local'
|
echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
|
||||||
set -a
|
set -a
|
||||||
. /home/bts/.local/share/bts/sparkd/conf/.env.local
|
. /home/bts/sparkdata/conf/.env.local
|
||||||
set +a
|
set +a
|
||||||
fi
|
fi
|
||||||
if [ -d /home/bts/.local/share/bts/sparkd/conf/plugins ]; then
|
|
||||||
echo 'Copying /home/bts/.local/share/bts/sparkd/conf/plugins to runtime dir (/home/bts/bin)'
|
|
||||||
cp -r /home/bts/.local/share/bts/sparkd/conf/plugins /home/bts/bin/
|
|
||||||
fi
|
|
||||||
./sparkd run"
|
./sparkd run"
|
||||||
env_file:
|
env_file:
|
||||||
- .env
|
- .env
|
||||||
@@ -91,10 +81,10 @@ services:
|
|||||||
- type: volume
|
- type: volume
|
||||||
read_only: true
|
read_only: true
|
||||||
source: sparkd_data
|
source: sparkd_data
|
||||||
target: /home/bts/.local/share/bts/sparkd/data/logging.sparkd
|
target: /home/bts/sparkdata/data/logging.sparkd
|
||||||
volume:
|
volume:
|
||||||
subpath: logging
|
subpath: logging.sparkd
|
||||||
command: ["tail", "-f", "/home/bts/.local/share/bts/sparkd/data/logging.sparkd/log"]
|
command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]
|
||||||
|
|
||||||
sparkd-ping-test:
|
sparkd-ping-test:
|
||||||
depends_on:
|
depends_on:
|
||||||
@@ -109,89 +99,6 @@ services:
|
|||||||
RUN apt-get update && apt-get -y install iputils-ping
|
RUN apt-get update && apt-get -y install iputils-ping
|
||||||
command: [ "ping", "sparkd" ]
|
command: [ "ping", "sparkd" ]
|
||||||
|
|
||||||
pubsub:
|
|
||||||
volumes:
|
|
||||||
- ${PUBSUB__DATA_DIR:-pubsub_data}:/home/bts/.local/share/bts/PubSubServer/data
|
|
||||||
build:
|
|
||||||
dockerfile: src/DataStreamSuite/PubSubServer/Dockerfile
|
|
||||||
args:
|
|
||||||
- SOURCE_DIR=${PUBSUB__SOURCE_DIR}
|
|
||||||
- CONFIG_DIR=${PUBSUB__CONFIG_DIR}
|
|
||||||
ports:
|
|
||||||
- 22019:22019
|
|
||||||
command: >
|
|
||||||
sh -c "
|
|
||||||
echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/PubSubServer
|
|
||||||
if [ -e "/home/bts/.local/share/bts/PubSubServer/data/logging.PubSubServer/log" ]; then
|
|
||||||
mv "/home/bts/.local/share/bts/PubSubServer/data/logging.PubSubServer/log" "/home/bts/.local/share/bts/PubSubServer/data/logging.PubSubServer/log_$(date '+%Y%m%d-%H%M')"
|
|
||||||
fi
|
|
||||||
if [ -e "/home/bts/.local/share/bts/PubSubServer/conf/.env" ]; then
|
|
||||||
echo 'Sourcing /home/bts/.local/share/bts/PubSubServer/conf/.env'
|
|
||||||
set -a
|
|
||||||
. /home/bts/.local/share/bts/PubSubServer/conf/.env
|
|
||||||
set +a
|
|
||||||
fi
|
|
||||||
if [ -e "/home/bts/.local/share/bts/PubSubServer/conf/.env.local" ]; then
|
|
||||||
echo 'Sourcing /home/bts/.local/share/bts/PubSubServer/conf/.env.local'
|
|
||||||
set -a
|
|
||||||
. /home/bts/.local/share/bts/PubSubServer/conf/.env.local
|
|
||||||
set +a
|
|
||||||
fi
|
|
||||||
./PubSubServer"
|
|
||||||
env_file:
|
|
||||||
- .env
|
|
||||||
- ${PUBSUB__CONFIG_DIR}/.env
|
|
||||||
|
|
||||||
md-publisher:
|
|
||||||
volumes:
|
|
||||||
- ${MD_PUBLISHER__DATA_DIR:-md_publisher_data}:/home/bts/.local/share/bts/DataStreamPublisher/data
|
|
||||||
- historic_md:/home/bts/.local/share/bts/databento
|
|
||||||
build:
|
|
||||||
dockerfile: src/DataStreamSuite/DataStreamPublisher/Dockerfile
|
|
||||||
args:
|
|
||||||
- SOURCE_DIR=${MD_PUBLISHER__SOURCE_DIR}
|
|
||||||
- CONFIG_DIR=${MD_PUBLISHER__CONFIG_DIR}
|
|
||||||
command: >
|
|
||||||
sh -c "
|
|
||||||
echo bts | sudo -S chown -R bts:bts /home/bts/.local/share/bts/DataStreamPublisher
|
|
||||||
if [ -e "/home/bts/.local/share/bts/DataStreamPublisher/data/logging.DataStreamPublisher/log" ]; then
|
|
||||||
mv "/home/bts/.local/share/bts/DataStreamPublisher/data/logging.DataStreamPublisher/log" "/home/bts/.local/share/bts/DataStreamPublisher/data/logging.DataStreamPublisher/log_$(date '+%Y%m%d-%H%M')"
|
|
||||||
fi
|
|
||||||
if [ -e "/home/bts/.local/share/bts/DataStreamPublisher/conf/.env" ]; then
|
|
||||||
echo 'Sourcing /home/bts/.local/share/bts/DataStreamPublisher/conf/.env'
|
|
||||||
set -a
|
|
||||||
. /home/bts/.local/share/bts/DataStreamPublisher/conf/.env
|
|
||||||
set +a
|
|
||||||
fi
|
|
||||||
if [ -e "/home/bts/.local/share/bts/DataStreamPublisher/conf/.env.local" ]; then
|
|
||||||
echo 'Sourcing /home/bts/.local/share/bts/DataStreamPublisher/conf/.env.local'
|
|
||||||
set -a
|
|
||||||
. /home/bts/.local/share/bts/DataStreamPublisher/conf/.env.local
|
|
||||||
set +a
|
|
||||||
fi
|
|
||||||
dockerize -wait tcp://pubsub:22019 -timeout 10s ./DataStreamPublisher"
|
|
||||||
env_file:
|
|
||||||
- .env
|
|
||||||
- ${MD_PUBLISHER__CONFIG_DIR}/.env
|
|
||||||
|
|
||||||
api-server:
|
|
||||||
depends_on:
|
|
||||||
- sparkd
|
|
||||||
ports:
|
|
||||||
- 8008:8008
|
|
||||||
- 9443:9443
|
|
||||||
build:
|
|
||||||
dockerfile: src/client/spark_api_server/Dockerfile
|
|
||||||
command: [ "dockerize", "-wait", "tcp://sparkd:8000", "-timeout", "10s", "./spark_api_server", "run" ]
|
|
||||||
environment:
|
|
||||||
- ASPNETCORE_ENVIRONMENT=Staging
|
|
||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
sparkd_data:
|
sparkd_data:
|
||||||
name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"
|
name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"
|
||||||
pubsub_data:
|
|
||||||
name: "pubsub_22019"
|
|
||||||
md_publisher_data:
|
|
||||||
name: "md_publisher_22019"
|
|
||||||
historic_md:
|
|
||||||
name: "historic_md"
|
|
||||||
Submodule
+1
Submodule docker/linux added at 6537812262.
Submodule
+1
Submodule docker/windows added at de7d40c09d.
@@ -0,0 +1,6 @@
|
|||||||
|
{
|
||||||
|
"sdk": {
|
||||||
|
"version": "6.0.200",
|
||||||
|
"rollForward": "latestMajor"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -16,16 +16,11 @@ jmespath==1.0.1
|
|||||||
lazy-object-proxy==1.6.0
|
lazy-object-proxy==1.6.0
|
||||||
MarkupSafe==3.0.2
|
MarkupSafe==3.0.2
|
||||||
mccabe==0.6.1
|
mccabe==0.6.1
|
||||||
numpy==2.2.6
|
|
||||||
platformdirs==2.3.0
|
platformdirs==2.3.0
|
||||||
protobuf==4.21.12
|
protobuf==4.21.12
|
||||||
pybars3==0.9.7
|
pybars3==0.9.7
|
||||||
pybind11==2.13.6
|
|
||||||
pybind11-stubgen==2.5.3
|
|
||||||
pybit==5.11.0
|
|
||||||
pycodestyle==2.7.0
|
pycodestyle==2.7.0
|
||||||
pycparser==2.20
|
pycparser==2.20
|
||||||
pycryptodome==3.23.0
|
|
||||||
pylint==2.10.2
|
pylint==2.10.2
|
||||||
PyMeta3==0.5.1
|
PyMeta3==0.5.1
|
||||||
python-dateutil==2.9.0.post0
|
python-dateutil==2.9.0.post0
|
||||||
@@ -36,7 +31,6 @@ six==1.17.0
|
|||||||
termcolor==1.1.0
|
termcolor==1.1.0
|
||||||
toml==0.10.2
|
toml==0.10.2
|
||||||
urllib3==2.0.3
|
urllib3==2.0.3
|
||||||
websocket-client==1.8.0
|
|
||||||
Werkzeug==3.1.3
|
Werkzeug==3.1.3
|
||||||
wrapt==1.12.1
|
wrapt==1.12.1
|
||||||
yapf==0.31.0
|
yapf==0.31.0
|
||||||
+103
-164
@@ -1,164 +1,103 @@
|
|||||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||||
# Visual Studio Version 17
|
# Visual Studio Version 17
|
||||||
VisualStudioVersion = 17.3.32922.545
|
VisualStudioVersion = 17.3.32922.545
|
||||||
MinimumVisualStudioVersion = 15.0.26124.0
|
MinimumVisualStudioVersion = 15.0.26124.0
|
||||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api", "src\client\spark_api\spark_api.csproj", "{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark", "src\client\spark\Spark.csproj", "{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{387566C6-C6F6-412C-A30C-9855BBCB4816}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_core", "src\client\spark_core\spark_core.csproj", "{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_sdk", "src\client\spark_sdk\spark_sdk.csproj", "{6963C314-83C4-4667-B0D0-C2F7AE208C29}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{ECD78CC5-138F-4809-A764-2358C2AC2E82}"
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_test", "src\client\spark_test\spark_test.csproj", "{ECD78CC5-138F-4809-A764-2358C2AC2E82}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_websocket", "src\client\spark_websocket\spark_websocket.csproj", "{3201359D-C071-4089-9EDE-B04D759F1508}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "spark_api_server", "src\client\spark_api_server\spark_api_server.csproj", "{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}"
|
Global
|
||||||
EndProject
|
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}"
|
Debug|x64 = Debug|x64
|
||||||
EndProject
|
MinSizeRel|x64 = MinSizeRel|x64
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}"
|
Release|x64 = Release|x64
|
||||||
EndProject
|
RelWithDebInfo|x64 = RelWithDebInfo|x64
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{51C40BA9-F226-4A35-939D-1ED6F5FF7299}"
|
EndGlobalSection
|
||||||
EndProject
|
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
||||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{4564E677-343A-46CB-AF02-AFC4E38BF94B}"
|
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
|
||||||
EndProject
|
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
|
||||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "client", "client", "{56A8A70A-DA03-4CED-A144-EC10F1379FD8}"
|
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
|
||||||
EndProject
|
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
|
||||||
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{915EF238-CA0D-4E73-AF50-88CB3F29C7AD}"
|
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
|
||||||
EndProject
|
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.Extensions.Hosting", "src\DataStreamSuite\Bts.Extensions.Hosting\Bts.Extensions.Hosting.csproj", "{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}"
|
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||||
EndProject
|
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||||
Global
|
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64
|
||||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64
|
||||||
Debug|x64 = Debug|x64
|
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.ActiveCfg = Release|x64
|
||||||
MinSizeRel|x64 = MinSizeRel|x64
|
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.Build.0 = Release|x64
|
||||||
Release|x64 = Release|x64
|
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.ActiveCfg = Release|x64
|
||||||
RelWithDebInfo|x64 = RelWithDebInfo|x64
|
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.Build.0 = Release|x64
|
||||||
EndGlobalSection
|
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||||
GlobalSection(ProjectConfigurationPlatforms) = postSolution
|
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.ActiveCfg = Debug|x64
|
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.ActiveCfg = Debug|x64
|
||||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Debug|x64.Build.0 = Debug|x64
|
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.Build.0 = Debug|x64
|
||||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.ActiveCfg = Release|x64
|
{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.ActiveCfg = Debug|x64
|
||||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.MinSizeRel|x64.Build.0 = Release|x64
|
{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.Build.0 = Debug|x64
|
||||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.ActiveCfg = Release|x64
|
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.ActiveCfg = Release|x64
|
||||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.Release|x64.Build.0 = Release|x64
|
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.Build.0 = Release|x64
|
||||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||||
{DC9A8975-8211-4FC9-A9BA-C8B9D26AD481}.RelWithDebInfo|x64.Build.0 = Release|x64
|
{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.ActiveCfg = Debug|x64
|
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
|
||||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Debug|x64.Build.0 = Debug|x64
|
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
|
||||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.ActiveCfg = Release|x64
|
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
|
||||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.MinSizeRel|x64.Build.0 = Release|x64
|
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
|
||||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.ActiveCfg = Release|x64
|
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
|
||||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.Release|x64.Build.0 = Release|x64
|
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
|
||||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||||
{9A4D4D07-1AB6-4B2B-B111-A4F34D6C267E}.RelWithDebInfo|x64.Build.0 = Release|x64
|
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.ActiveCfg = Debug|x64
|
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.ActiveCfg = Debug|x64
|
||||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Debug|x64.Build.0 = Debug|x64
|
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.Build.0 = Debug|x64
|
||||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.ActiveCfg = Debug|x64
|
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.ActiveCfg = Release|x64
|
||||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.MinSizeRel|x64.Build.0 = Debug|x64
|
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.Build.0 = Release|x64
|
||||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.ActiveCfg = Release|x64
|
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.ActiveCfg = Release|x64
|
||||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.Release|x64.Build.0 = Release|x64
|
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.Build.0 = Release|x64
|
||||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||||
{387566C6-C6F6-412C-A30C-9855BBCB4816}.RelWithDebInfo|x64.Build.0 = Release|x64
|
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.ActiveCfg = Debug|x64
|
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.ActiveCfg = Debug|x64
|
||||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Debug|x64.Build.0 = Debug|x64
|
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.Build.0 = Debug|x64
|
||||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.ActiveCfg = Release|x64
|
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.ActiveCfg = Debug|x64
|
||||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.MinSizeRel|x64.Build.0 = Release|x64
|
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.Build.0 = Debug|x64
|
||||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.ActiveCfg = Release|x64
|
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.ActiveCfg = Release|x64
|
||||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.Release|x64.Build.0 = Release|x64
|
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.Build.0 = Release|x64
|
||||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||||
{6E449EB6-20AC-4FA6-B8D9-10F4BB7E1FEE}.RelWithDebInfo|x64.Build.0 = Release|x64
|
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.ActiveCfg = Debug|x64
|
{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
|
||||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Debug|x64.Build.0 = Debug|x64
|
{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
|
||||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.ActiveCfg = Release|x64
|
{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
|
||||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.MinSizeRel|x64.Build.0 = Release|x64
|
{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
|
||||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.ActiveCfg = Release|x64
|
{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
|
||||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.Release|x64.Build.0 = Release|x64
|
{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
|
||||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||||
{6963C314-83C4-4667-B0D0-C2F7AE208C29}.RelWithDebInfo|x64.Build.0 = Release|x64
|
{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.ActiveCfg = Debug|x64
|
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
|
||||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Debug|x64.Build.0 = Debug|x64
|
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
|
||||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.ActiveCfg = Debug|x64
|
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
|
||||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.MinSizeRel|x64.Build.0 = Debug|x64
|
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
|
||||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.ActiveCfg = Release|x64
|
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
|
||||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.Release|x64.Build.0 = Release|x64
|
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
|
||||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
||||||
{ECD78CC5-138F-4809-A764-2358C2AC2E82}.RelWithDebInfo|x64.Build.0 = Release|x64
|
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
|
||||||
{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.ActiveCfg = Debug|x64
|
EndGlobalSection
|
||||||
{3201359D-C071-4089-9EDE-B04D759F1508}.Debug|x64.Build.0 = Debug|x64
|
GlobalSection(SolutionProperties) = preSolution
|
||||||
{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.ActiveCfg = Debug|x64
|
HideSolutionNode = FALSE
|
||||||
{3201359D-C071-4089-9EDE-B04D759F1508}.MinSizeRel|x64.Build.0 = Debug|x64
|
EndGlobalSection
|
||||||
{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.ActiveCfg = Release|x64
|
GlobalSection(ExtensibilityGlobals) = postSolution
|
||||||
{3201359D-C071-4089-9EDE-B04D759F1508}.Release|x64.Build.0 = Release|x64
|
SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
|
||||||
{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
EndGlobalSection
|
||||||
{3201359D-C071-4089-9EDE-B04D759F1508}.RelWithDebInfo|x64.Build.0 = Release|x64
|
GlobalSection(Performance) = preSolution
|
||||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.ActiveCfg = Debug|x64
|
HasPerformanceSessions = true
|
||||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Debug|x64.Build.0 = Debug|x64
|
EndGlobalSection
|
||||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.ActiveCfg = Debug|x64
|
EndGlobal
|
||||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.MinSizeRel|x64.Build.0 = Debug|x64
|
|
||||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.ActiveCfg = Release|x64
|
|
||||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.Release|x64.Build.0 = Release|x64
|
|
||||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.ActiveCfg = Release|x64
|
|
||||||
{2B8C5D01-FF78-4BFA-B332-5E430586DAF9}.RelWithDebInfo|x64.Build.0 = Release|x64
|
|
||||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
|
||||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{4E4F491F-C688-43C4-88DA-AC92BC5FA1B8}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
|
||||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
|
||||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{2AC54ADC-3246-427E-9CA9-FEA2DBCB9432}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
|
||||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
|
||||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{4E96E5CC-7791-40B3-A8A1-C34FBB736B3B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
|
||||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
|
||||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{51C40BA9-F226-4A35-939D-1ED6F5FF7299}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
|
||||||
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Debug|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Debug|x64.Build.0 = Debug|Any CPU
|
|
||||||
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.MinSizeRel|x64.Build.0 = Debug|Any CPU
|
|
||||||
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Release|x64.ActiveCfg = Release|Any CPU
|
|
||||||
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.Release|x64.Build.0 = Release|Any CPU
|
|
||||||
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
|
|
||||||
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
|
|
||||||
EndGlobalSection
|
|
||||||
GlobalSection(SolutionProperties) = preSolution
|
|
||||||
HideSolutionNode = FALSE
|
|
||||||
EndGlobalSection
|
|
||||||
GlobalSection(ExtensibilityGlobals) = postSolution
|
|
||||||
SolutionGuid = {422D1620-0166-4C26-9A81-CA8A2EB1AB58}
|
|
||||||
EndGlobalSection
|
|
||||||
GlobalSection(Performance) = preSolution
|
|
||||||
HasPerformanceSessions = true
|
|
||||||
EndGlobalSection
|
|
||||||
GlobalSection(NestedProjects) = preSolution
|
|
||||||
{56A8A70A-DA03-4CED-A144-EC10F1379FD8} = {4564E677-343A-46CB-AF02-AFC4E38BF94B}
|
|
||||||
{915EF238-CA0D-4E73-AF50-88CB3F29C7AD} = {4564E677-343A-46CB-AF02-AFC4E38BF94B}
|
|
||||||
{4E0E51AA-7B59-4808-B7CC-9EAEDF397460} = {915EF238-CA0D-4E73-AF50-88CB3F29C7AD}
|
|
||||||
EndGlobalSection
|
|
||||||
EndGlobal
|
|
||||||
@@ -8,6 +8,7 @@
|
|||||||
public int Integer;
|
public int Integer;
|
||||||
public double Real;
|
public double Real;
|
||||||
public string Identifier;
|
public string Identifier;
|
||||||
|
public DateTime DateTime;
|
||||||
public ImmutableList<IArithmeticExpression> Arguments;
|
public ImmutableList<IArithmeticExpression> Arguments;
|
||||||
public ImmutableList<Assignment> Assignments;
|
public ImmutableList<Assignment> Assignments;
|
||||||
public IArithmeticExpression Argument;
|
public IArithmeticExpression Argument;
|
||||||
|
|||||||
@@ -1,6 +1,4 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Runtime.Versioning;
|
|
||||||
|
|
||||||
using Microsoft.Win32;
|
using Microsoft.Win32;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG
|
||||||
@@ -12,7 +10,6 @@ namespace BCG
|
|||||||
rk.SetValue (name, value.ToString ());
|
rk.SetValue (name, value.ToString ());
|
||||||
}
|
}
|
||||||
|
|
||||||
[SupportedOSPlatform("windows")]
|
|
||||||
public static double getValue (RegistryKey rk, string name)
|
public static double getValue (RegistryKey rk, string name)
|
||||||
{
|
{
|
||||||
object v = rk.GetValue (name);
|
object v = rk.GetValue (name);
|
||||||
|
|||||||
@@ -1,11 +1,19 @@
|
|||||||
namespace Bts.BooleanExpressions;
|
namespace Bts.BooleanExpressions;
|
||||||
|
|
||||||
|
using bts;
|
||||||
|
|
||||||
class BooleanExpression : IBooleanExpression {
|
class BooleanExpression : IBooleanExpression {
|
||||||
readonly Func<IDictionary<string, object?>, bool> evaluate;
|
readonly Func<IDictionary<string, object?>, bool> evaluate;
|
||||||
|
|
||||||
public BooleanExpression(Func<IDictionary<string, object?>, bool> evaluate) =>
|
public BooleanExpression(Func<IDictionary<string, object?>, bool> evaluate) =>
|
||||||
this.evaluate = evaluate;
|
this.evaluate = evaluate;
|
||||||
|
|
||||||
public bool Evaluate(IDictionary<string, object?> env) =>
|
public bool Evaluate(IDictionary<string, object?> env) {
|
||||||
evaluate(env);
|
try {
|
||||||
|
return evaluate(env);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
Log.assert(false, $"BooleanExpression.Evaluate failed: {ex}");
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -47,6 +47,11 @@ IsOfType '/'
|
|||||||
|
|
||||||
%%
|
%%
|
||||||
|
|
||||||
|
{StringConstant} {
|
||||||
|
yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'", "'");
|
||||||
|
return (int)Token.STRING_CONSTANT;
|
||||||
|
}
|
||||||
|
|
||||||
{BooleanConstant} {
|
{BooleanConstant} {
|
||||||
yylval.Boolean = yytext.ToLower() switch {
|
yylval.Boolean = yytext.ToLower() switch {
|
||||||
"true" => true,
|
"true" => true,
|
||||||
@@ -57,11 +62,6 @@ IsOfType '/'
|
|||||||
return (int)Token.BOOLEAN_CONSTANT;
|
return (int)Token.BOOLEAN_CONSTANT;
|
||||||
}
|
}
|
||||||
|
|
||||||
{StringConstant} {
|
|
||||||
yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'", "'");
|
|
||||||
return (int)Token.STRING_CONSTANT;
|
|
||||||
}
|
|
||||||
|
|
||||||
{NullConstant} {
|
{NullConstant} {
|
||||||
yylval.Null = DBNull.Value;
|
yylval.Null = DBNull.Value;
|
||||||
return (int)Token.NULL_CONSTANT;
|
return (int)Token.NULL_CONSTANT;
|
||||||
|
|||||||
@@ -1,13 +0,0 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
|
||||||
|
|
||||||
<PropertyGroup>
|
|
||||||
<TargetFramework>net8.0</TargetFramework>
|
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
|
||||||
<Nullable>enable</Nullable>
|
|
||||||
</PropertyGroup>
|
|
||||||
|
|
||||||
<ItemGroup>
|
|
||||||
<PackageReference Include="Microsoft.Extensions.Hosting" Version="10.0.0" />
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
</Project>
|
|
||||||
@@ -1,71 +0,0 @@
|
|||||||
namespace Bts.Extensions.Hosting;
|
|
||||||
|
|
||||||
using Microsoft.Extensions.Configuration;
|
|
||||||
using Microsoft.Extensions.Hosting;
|
|
||||||
using Microsoft.Extensions.Logging;
|
|
||||||
|
|
||||||
using System.Diagnostics;
|
|
||||||
|
|
||||||
public static class HostBuilderExtensions {
|
|
||||||
public static IHostBuilder CreateBtsBuilder(string[] args) {
|
|
||||||
// Obtain calling assembly name here, rather than in the lambda where it is actually needed,
|
|
||||||
// because want the name of the assembly that called this function, not the assembly that will eventually invoke
|
|
||||||
// the lambda...that will be the Microsoft.Extensions.Hosting assembly.
|
|
||||||
// Get the name of the assembly that called this function. That will act as the "appsettings"
|
|
||||||
// json file prefix
|
|
||||||
var exeName = GetCallingAssemblyName();
|
|
||||||
return Host.CreateDefaultBuilder(args)
|
|
||||||
.ConfigureAppConfiguration((context, configBuilder) =>
|
|
||||||
ConfigureApp(configBuilder, context.HostingEnvironment.EnvironmentName, exeName));
|
|
||||||
}
|
|
||||||
|
|
||||||
public static void ConfigureApp(IConfigurationBuilder configBuilder, string? environment = null, string? prefix = null) {
|
|
||||||
var baseFileName = prefix is null ? "appsettings" : $"{prefix}.appsettings";
|
|
||||||
AddJsonFile($"{baseFileName}.json");
|
|
||||||
if (environment is not null) {
|
|
||||||
AddJsonFile($"{baseFileName}.{environment}.json");
|
|
||||||
}
|
|
||||||
|
|
||||||
var cfg = configBuilder.Build().GetSection("config")?.Value;
|
|
||||||
if (!SetBasePath(cfg)) {
|
|
||||||
AddJsonFile(cfg);
|
|
||||||
}
|
|
||||||
|
|
||||||
bool SetBasePath(string? path) {
|
|
||||||
if (!Directory.Exists(path)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
configBuilder.SetBasePath(path);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool AddJsonFile(string? path) {
|
|
||||||
if (!File.Exists(path)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
configBuilder.AddJsonFile(path);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static void ConfigureLogging(ILoggingBuilder loggingBuilder, ILoggerProvider loggerProvider) =>
|
|
||||||
loggingBuilder.AddFilter("Microsoft", LogLevel.Warning)
|
|
||||||
.AddFilter("System", LogLevel.Warning)
|
|
||||||
.AddFilter("NToastNotify", LogLevel.Warning)
|
|
||||||
.AddProvider(loggerProvider)
|
|
||||||
.AddSimpleConsole(opts => {
|
|
||||||
opts.SingleLine = true;
|
|
||||||
opts.TimestampFormat = "yyyy-MM-dd HH:mm:ss.ffffff|";
|
|
||||||
})
|
|
||||||
.AddDebug();
|
|
||||||
|
|
||||||
public static string GetCallingAssemblyName() =>
|
|
||||||
new StackTrace()
|
|
||||||
.GetFrame(2)
|
|
||||||
?.GetMethod()
|
|
||||||
?.Module
|
|
||||||
?.Assembly
|
|
||||||
?.GetName()
|
|
||||||
?.Name
|
|
||||||
?? throw new InvalidOperationException("Failed to determine name of calling assembly (calling method is null)");
|
|
||||||
}
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
|
||||||
|
|
||||||
<PropertyGroup>
|
|
||||||
<TargetFramework>net8.0</TargetFramework>
|
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
|
||||||
<Nullable>enable</Nullable>
|
|
||||||
</PropertyGroup>
|
|
||||||
|
|
||||||
</Project>
|
|
||||||
@@ -1,28 +0,0 @@
|
|||||||
namespace Bts.Net;
|
|
||||||
|
|
||||||
using System.Linq;
|
|
||||||
using System.Net;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Provides facilities to compare hosts for equality.
|
|
||||||
/// <para>
|
|
||||||
/// This class doesn't implement IEqualityComparer because a consistent hash code cannot be
|
|
||||||
/// guarenteed for equivalent host strings. The equals impl relies on Dns.GetHostAddresses, which
|
|
||||||
/// is non-determinstic (implementations that depends on mutable properties face a similar problem).
|
|
||||||
/// </para>
|
|
||||||
/// </summary>
|
|
||||||
public class HostComparer {
|
|
||||||
public static bool Equals(string? x, string? y) {
|
|
||||||
if (x is null) {
|
|
||||||
return y is null;
|
|
||||||
}
|
|
||||||
if (y is null) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return Dns.GetHostAddresses(x).Intersect(
|
|
||||||
Dns.GetHostAddresses(y), IpAddressComparer.Instance).Any();
|
|
||||||
}
|
|
||||||
|
|
||||||
public static bool Equals(IPHostEntry x, IPHostEntry y) =>
|
|
||||||
x.AddressList.Intersect(y.AddressList, IpAddressComparer.Instance).Any();
|
|
||||||
}
|
|
||||||
@@ -1,30 +0,0 @@
|
|||||||
namespace Bts.Net;
|
|
||||||
|
|
||||||
using System.Diagnostics.CodeAnalysis;
|
|
||||||
using System.Net;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Compares ip addresses. Unlike the default comparer, v4 and equivalent v6 addresses are
|
|
||||||
/// considered equal.
|
|
||||||
/// </summary>
|
|
||||||
public class IpAddressComparer : IEqualityComparer<IPAddress> {
|
|
||||||
public static IEqualityComparer<IPAddress> Instance { get; } = new IpAddressComparer();
|
|
||||||
|
|
||||||
public bool Equals(IPAddress? x, IPAddress? y) {
|
|
||||||
if (x is null) {
|
|
||||||
return y is null;
|
|
||||||
}
|
|
||||||
if (y is null) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return x.Equals(y) ||
|
|
||||||
(x.IsIPv4MappedToIPv6 && x.MapToIPv4().Equals(y)) ||
|
|
||||||
(y.IsIPv4MappedToIPv6 && y.MapToIPv4().Equals(x));
|
|
||||||
}
|
|
||||||
|
|
||||||
public int GetHashCode([DisallowNull] IPAddress address) =>
|
|
||||||
address.IsIPv4MappedToIPv6
|
|
||||||
// Return consistent hash code for v4 and equivalent v6
|
|
||||||
? address.MapToIPv4().GetHashCode()
|
|
||||||
: address.GetHashCode();
|
|
||||||
}
|
|
||||||
@@ -1,33 +0,0 @@
|
|||||||
namespace Bts.Net;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Net;
|
|
||||||
using System.Text.Json;
|
|
||||||
using System.Text.Json.Serialization;
|
|
||||||
|
|
||||||
public class IpHostEntryJsonConverter : JsonConverter<IPHostEntry> {
|
|
||||||
public override IPHostEntry? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
|
|
||||||
throw new NotImplementedException();
|
|
||||||
}
|
|
||||||
|
|
||||||
public override void Write(Utf8JsonWriter writer, IPHostEntry value, JsonSerializerOptions options) {
|
|
||||||
writer.WriteStartObject();
|
|
||||||
// HostName
|
|
||||||
writer.WriteString(nameof(value.HostName), value.HostName);
|
|
||||||
// Aliases
|
|
||||||
writer.WritePropertyName("Aliases");
|
|
||||||
writer.WriteStartArray();
|
|
||||||
foreach (var alias in value.Aliases) {
|
|
||||||
writer.WriteStringValue(alias);
|
|
||||||
}
|
|
||||||
writer.WriteEndArray();
|
|
||||||
// Addresses
|
|
||||||
writer.WritePropertyName("Addresses");
|
|
||||||
writer.WriteStartArray();
|
|
||||||
foreach (var address in value.AddressList) {
|
|
||||||
writer.WriteStringValue(address.ToString());
|
|
||||||
}
|
|
||||||
writer.WriteEndArray();
|
|
||||||
writer.WriteEndObject();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,39 +0,0 @@
|
|||||||
namespace Bts.Net;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.IO;
|
|
||||||
using System.Net;
|
|
||||||
using System.Text;
|
|
||||||
using System.Text.Json;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// This class seriailzes IpHostEntry to json (string). A common buffer is used to reduce memory
|
|
||||||
/// allocations on repeated calls to <see cref="Serialize"/>. For that reason, this class is NOT
|
|
||||||
/// thread safe.
|
|
||||||
/// </summary>
|
|
||||||
public class IpHostEntryJsonSerializer {
|
|
||||||
readonly MemoryStream buffer = new();
|
|
||||||
readonly IpHostEntryJsonConverter converter;
|
|
||||||
|
|
||||||
public IpHostEntryJsonSerializer(IpHostEntryJsonConverter? converter = null) =>
|
|
||||||
this.converter = converter ?? new() { };
|
|
||||||
|
|
||||||
public string? Serialize(IPHostEntry dp, JsonSerializerOptions? opts = null) {
|
|
||||||
opts ??= JsonSerializerOptions.Default;
|
|
||||||
buffer.Position = 0;
|
|
||||||
buffer.SetLength(0);
|
|
||||||
using (var jsonWriter = new Utf8JsonWriter(buffer)) {
|
|
||||||
converter.Write(jsonWriter, dp, opts);
|
|
||||||
}
|
|
||||||
var bytes = buffer.GetBuffer();
|
|
||||||
var span = new ArraySegment<byte>(bytes, 0, (int)buffer.Length);
|
|
||||||
return Encoding.UTF8.GetString(span);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void Serialize(IPHostEntry dp, Utf8JsonWriter jsonWriter, JsonSerializerOptions? opts = null) {
|
|
||||||
opts ??= JsonSerializerOptions.Default;
|
|
||||||
buffer.Position = 0;
|
|
||||||
buffer.SetLength(0);
|
|
||||||
converter.Write(jsonWriter, dp, opts);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,94 +0,0 @@
|
|||||||
namespace Bts.Net;
|
|
||||||
|
|
||||||
using System.Diagnostics.CodeAnalysis;
|
|
||||||
using System.Net;
|
|
||||||
using System.Text.RegularExpressions;
|
|
||||||
|
|
||||||
public static partial class IpUtils {
|
|
||||||
static readonly Regex ipv4Regex = new(string.Join('.', @"(((0|1)\d)|(2\d)|\d)?\d"));
|
|
||||||
|
|
||||||
// public static void ParseUri(string str, out string? scheme, out string? host, out string? port, out string? path, out string? queryParams) {
|
|
||||||
// if (Uri.TryCreate(str, UriKind.Absolute, out var uri)) {
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
// public static bool AreEndpointsEqual(Uri? x, Uri? y) {
|
|
||||||
// if (x is null) {
|
|
||||||
// return y is null;
|
|
||||||
// }
|
|
||||||
// if (y is null) {
|
|
||||||
// return false;
|
|
||||||
// }
|
|
||||||
// if (x.IsAbsoluteUri != y.IsAbsoluteUri) {
|
|
||||||
// return false;
|
|
||||||
// }
|
|
||||||
// if (x.IsAbsoluteUri) {
|
|
||||||
// if (x.Scheme != y.Scheme) {
|
|
||||||
// return false;
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// string? xHost = null;
|
|
||||||
// }
|
|
||||||
|
|
||||||
// static string ResolveHostIpv4(Uri uri) {
|
|
||||||
// // switch (uri.HostNameType) {
|
|
||||||
// // case UriHostNameType.IPv4:
|
|
||||||
// // return uri.Host;
|
|
||||||
// // case UriHostNameType.Dns:
|
|
||||||
// // break;
|
|
||||||
// // case UriHostNameType.IPv6:
|
|
||||||
// // var addresses = Dns.GetHostAddresses(uri.Host);
|
|
||||||
// // foreach (var address in addresses) {
|
|
||||||
// // if (address.IsIPv4MappedToIPv6) {
|
|
||||||
// // return address.MapToIPv4().Address;
|
|
||||||
// // }
|
|
||||||
// // }
|
|
||||||
// // break;
|
|
||||||
// // }
|
|
||||||
// var hostAddresses = Dns.GetHostAddresses(uri.Host);
|
|
||||||
// foreach (IPAddress address in hostAddresses) {
|
|
||||||
// if (address.Equals(remoteEndPoint.Address) ||
|
|
||||||
// (address.IsIPv4MappedToIPv6 && address.MapToIPv4().Equals(remoteEndPoint.Address)) ||
|
|
||||||
// (remoteEndPoint.Address.IsIPv4MappedToIPv6 && remoteEndPoint.Address.MapToIPv4().Equals(address))) {
|
|
||||||
// isMatch = true;
|
|
||||||
// break;
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
// public static bool AreEndpointsEqual(string? x, string? y) {
|
|
||||||
// if (x == y) {
|
|
||||||
// return true;
|
|
||||||
// }
|
|
||||||
// if (x is null) {
|
|
||||||
// return y is null;
|
|
||||||
// }
|
|
||||||
// if (y is null) {
|
|
||||||
// return false;
|
|
||||||
// }
|
|
||||||
// var xUri = new Uri(x);
|
|
||||||
// var yUri = new Uri(y);
|
|
||||||
// // return xUri.Equals(yUri);
|
|
||||||
// }
|
|
||||||
|
|
||||||
public static bool IsIpv4(string host) {
|
|
||||||
host = removeSchemaFromUrl(host);
|
|
||||||
host = removePortFromUrl(host);
|
|
||||||
return ipv4Regex.IsMatch(host);
|
|
||||||
}
|
|
||||||
|
|
||||||
static string removeSchemaFromUrl(string url) {
|
|
||||||
var i = url.IndexOf("://");
|
|
||||||
return i < 0 ? url : url.Substring(i + 3);
|
|
||||||
}
|
|
||||||
|
|
||||||
static string removePortFromUrl(string url) {
|
|
||||||
var match = MyRegex().Match(url);
|
|
||||||
return match.Success
|
|
||||||
? string.Concat(url.AsSpan(0, match.Index), url.AsSpan(match.Index + match.Length))
|
|
||||||
: url;
|
|
||||||
}
|
|
||||||
|
|
||||||
[GeneratedRegex(":\\d{1,4}")]
|
|
||||||
private static partial Regex MyRegex();
|
|
||||||
}
|
|
||||||
@@ -7,11 +7,14 @@
|
|||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
<PackageReference Include="Google.Protobuf" Version="3.24.3" />
|
||||||
<PackageReference Include="YaccLexTools" Version="1.2.2" />
|
<PackageReference Include="YaccLexTools" Version="1.2.2" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
<ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
|
||||||
<ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
|
<ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
|
||||||
|
<ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup Label="ColumnMapExpression">
|
<ItemGroup Label="ColumnMapExpression">
|
||||||
|
|||||||
@@ -1,11 +0,0 @@
|
|||||||
namespace bts.BlueOrb {
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
[DataStreamUserRespresentation("AddColumnsStream")]
|
|
||||||
public class AddColumnsStream: ViewStream {
|
|
||||||
public AddColumnsStream(ViewStream parent_str, string add_col_expr) {
|
|
||||||
ViewStream rs = new BCG.AddColumnsStream(parent_str, add_col_expr);
|
|
||||||
mySetRealSource(rs);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,118 +0,0 @@
|
|||||||
namespace Bts.DataStream;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using bts.collections;
|
|
||||||
using bts.utils;
|
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using Spark.Core.Collections;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
|
|
||||||
public class BlueAddIndexColumnTable : IBlueRowCollection {
|
|
||||||
readonly IBlueTableSchema schema;
|
|
||||||
readonly IList<object?>[] columnValues;
|
|
||||||
readonly int count;
|
|
||||||
|
|
||||||
BlueAddIndexColumnTable(IBlueTableSchema schema, IList<object?>[] columnValues, int? count = null) {
|
|
||||||
this.schema = schema;
|
|
||||||
this.columnValues = columnValues;
|
|
||||||
this.count = count ?? columnValues[0].Count;
|
|
||||||
}
|
|
||||||
|
|
||||||
IBlueTableSchema IBlueRowCollection.schema => schema;
|
|
||||||
|
|
||||||
public int Count => count;
|
|
||||||
|
|
||||||
public IBlueRow this[int id] =>
|
|
||||||
id > 0 && id <= count
|
|
||||||
? GetRow(id - 1)
|
|
||||||
: throw new KeyNotFoundException();
|
|
||||||
|
|
||||||
public static BlueAddIndexColumnTable Create(string tableName, ColumnInfo[] columnInfos, IList<object?>[] columnValues) {
|
|
||||||
if (columnInfos.Any(c => c.colName == "id")) {
|
|
||||||
throw new ArgumentException("'id' column already exists", nameof(columnInfos));
|
|
||||||
}
|
|
||||||
if (columnInfos.Length != columnValues.Length) {
|
|
||||||
throw new ArgumentException("columnValues length differs from columnInfos length", nameof(columnValues));
|
|
||||||
}
|
|
||||||
int count = columnValues.FirstOrDefault()?.Count ?? 0;
|
|
||||||
var origCols = columnInfos;
|
|
||||||
var origColValues = columnValues;
|
|
||||||
columnInfos = new ColumnInfo[origCols.Length + 1];
|
|
||||||
columnInfos[0] = new ColumnInfo("id", 0, typeof(int));
|
|
||||||
columnValues = new IList<object?>[origCols.Length + 1];
|
|
||||||
columnValues[0] = new StepList<int>(1, 1, count).WrapSelector(id => (object?)id, null);
|
|
||||||
for (int i = 1; i < columnInfos.Length; i++) {
|
|
||||||
columnInfos[i] = new ColumnInfo(origCols[i - 1].colName, i, origCols[i - 1].colType);
|
|
||||||
columnValues[i] = origColValues[i - 1];
|
|
||||||
}
|
|
||||||
var schema = new BlueTableSchema(columnInfos, tableName);
|
|
||||||
return new(schema, columnValues, count);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static BlueAddIndexColumnTable Create(IBlueTableSchema schema, IList<object?>[] columnValues) {
|
|
||||||
int count = columnValues.FirstOrDefault()?.Count ?? 0;
|
|
||||||
int? idColIdx = schema.TryResolveIndices("id");
|
|
||||||
var idColValues = new StepList<int>(1, 1, count).WrapSelector(id => (object?)id, null);
|
|
||||||
var columnInfos = schema.columnInfo;
|
|
||||||
if (idColIdx.HasValue) {
|
|
||||||
// Replace existing id column. columnValues may or may not contain the old ids (if its
|
|
||||||
// length is one less column than that of columnInfos, it doesn't contain them).
|
|
||||||
switch (columnValues.Length - schema.columnInfo.Length) {
|
|
||||||
case 0:
|
|
||||||
columnValues[idColIdx.Value] = idColValues;
|
|
||||||
break;
|
|
||||||
case -1:
|
|
||||||
columnValues = ArrayHelper.InsertExact(columnValues, idColIdx.Value, true, idColValues);
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
throw new ArgumentException("ColumnInfos length disagrees with columnValues length");
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if (columnValues.Length != columnInfos.Length) {
|
|
||||||
throw new ArgumentException("ColumnInfos length disagrees with columnValues length");
|
|
||||||
}
|
|
||||||
var origColInfos = columnInfos;
|
|
||||||
columnInfos = new ColumnInfo[columnInfos.Length + 1];
|
|
||||||
columnInfos[0] = new("id", 0, typeof(int));
|
|
||||||
for (int i = 1; i < columnInfos.Length; i++) {
|
|
||||||
var c = origColInfos[i - 1];
|
|
||||||
columnInfos[i] = new(c.colName, c.colNo + 1, c.colType);
|
|
||||||
}
|
|
||||||
columnValues = ArrayHelper.InsertExact(columnValues, 0, true, idColValues);
|
|
||||||
}
|
|
||||||
return new(new BlueTableSchema(schema.tableName, columnInfos), columnValues);
|
|
||||||
}
|
|
||||||
|
|
||||||
public bool containsID(int id) =>
|
|
||||||
id > 0 && id <= count;
|
|
||||||
|
|
||||||
public IList<object?> GetColumn(int index) =>
|
|
||||||
columnValues[index];
|
|
||||||
|
|
||||||
public IBlueRow GetRow(int index) =>
|
|
||||||
new BlueRowColumnWrapper(schema, index, columnValues);
|
|
||||||
|
|
||||||
public int IndexOf(IBlueRow item) {
|
|
||||||
int id = item.Id();
|
|
||||||
return id > 0 && id <= count && BlueRowEqualityComparer.Equals(GetRow(id - 1), item)
|
|
||||||
? id - 1
|
|
||||||
: -1;
|
|
||||||
}
|
|
||||||
|
|
||||||
public IEnumerator<IBlueRow> GetEnumerator() {
|
|
||||||
int count = Count;
|
|
||||||
for (int i = 0; i < count; i++) {
|
|
||||||
yield return GetRow(i);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
IEnumerator IEnumerable.GetEnumerator() =>
|
|
||||||
GetEnumerator();
|
|
||||||
}
|
|
||||||
@@ -1,56 +0,0 @@
|
|||||||
namespace Bts.DataStream;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using mbts.utils;
|
|
||||||
|
|
||||||
using System.Collections;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
|
|
||||||
public class BlueIndexedColumnTable : IBlueRowCollection {
|
|
||||||
readonly IBlueTableSchema schema;
|
|
||||||
readonly IList<object?>[] columnValues;
|
|
||||||
readonly Dictionary<int, int> rowIdToIndex;
|
|
||||||
|
|
||||||
public BlueIndexedColumnTable(IBlueTableSchema schema, IList<object?>[] columnValues) {
|
|
||||||
this.schema = schema;
|
|
||||||
this.columnValues = columnValues;
|
|
||||||
int idColumnIndex = schema.ResolveIndices("id");
|
|
||||||
rowIdToIndex = columnValues[idColumnIndex]
|
|
||||||
.Select((idObj, idx) => new KeyValuePair<int, int>(ExceptionAssert.Argument.Is<int>(idObj), idx))
|
|
||||||
.ToDictionary();
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow this[int id] =>
|
|
||||||
rowIdToIndex.TryGetValue(id, out var idx)
|
|
||||||
? GetRow(idx)
|
|
||||||
: throw new KeyNotFoundException();
|
|
||||||
|
|
||||||
IBlueTableSchema IBlueRowCollection.schema => schema;
|
|
||||||
|
|
||||||
public int Count => rowIdToIndex.Count;
|
|
||||||
|
|
||||||
public bool containsID(int id) =>
|
|
||||||
rowIdToIndex.ContainsKey(id);
|
|
||||||
|
|
||||||
public IEnumerator<IBlueRow> GetEnumerator() {
|
|
||||||
int count = Count;
|
|
||||||
for (int i = 0; i < count; i++) {
|
|
||||||
yield return GetRow(i);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow GetRow(int index) =>
|
|
||||||
new BlueRowColumnWrapper(schema, index, columnValues);
|
|
||||||
|
|
||||||
public int IndexOf(IBlueRow item) =>
|
|
||||||
rowIdToIndex.TryGetValue(item.Id(), out var idx) && BlueRowEqualityComparer.Equals(GetRow(idx), item)
|
|
||||||
? idx
|
|
||||||
: -1;
|
|
||||||
|
|
||||||
IEnumerator IEnumerable.GetEnumerator() =>
|
|
||||||
GetEnumerator();
|
|
||||||
}
|
|
||||||
@@ -1,21 +0,0 @@
|
|||||||
namespace Bts.DataStream;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using System.Collections.Generic;
|
|
||||||
|
|
||||||
public class BlueRowColumnWrapper : IBlueRow {
|
|
||||||
readonly int index;
|
|
||||||
readonly IList<object?>[] columns;
|
|
||||||
|
|
||||||
public BlueRowColumnWrapper(IBlueTableSchema schema, int index, IList<object?>[] columns) {
|
|
||||||
this.schema = schema;
|
|
||||||
this.index = index;
|
|
||||||
this.columns = columns;
|
|
||||||
}
|
|
||||||
|
|
||||||
public object? this[int index] =>
|
|
||||||
columns[index][this.index];
|
|
||||||
|
|
||||||
public IBlueTableSchema schema { get; }
|
|
||||||
}
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
namespace bts.BlueOrb {
|
|
||||||
using System;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
[DataStreamUserRespresentation("CSVReaderStream")]
|
|
||||||
public class CSVReaderStream : ViewStream {
|
|
||||||
public CSVReaderStream(string file_name)
|
|
||||||
: this(file_name, false) { }
|
|
||||||
public CSVReaderStream(string file_name, bool guess_schema) {
|
|
||||||
ViewStream rs = new BCG.CSVReaderStream(file_name);
|
|
||||||
if (guess_schema) {
|
|
||||||
rs = new SchemaGuesserStream(rs);
|
|
||||||
}
|
|
||||||
mySetRealSource(rs);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,13 +0,0 @@
|
|||||||
namespace bts.BlueOrb {
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
|
|
||||||
[DataStreamUserRespresentation("CSVSaverStream")]
|
|
||||||
public class CSVSaverStream: ViewStream {
|
|
||||||
public CSVSaverStream(ViewStream parent_str, string file_name) {
|
|
||||||
BCG.CSVSaverStream rs = new BCG.CSVSaverStream(parent_str, file_name, true, Int32.MaxValue, false);
|
|
||||||
mySetRealSource(rs);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,46 +0,0 @@
|
|||||||
using System;
|
|
||||||
|
|
||||||
namespace BCG {
|
|
||||||
[DataStreamUserRespresentation("CountStream")]
|
|
||||||
public class CountStream : ViewStream {
|
|
||||||
private DataPacket myLastFiredDataPacket;
|
|
||||||
private IBlueTableSchema mySchema = new BlueTableSchema("count",
|
|
||||||
new ColumnInfo("id", 0, typeof(int)),
|
|
||||||
new ColumnInfo("count", 1, typeof(int)));
|
|
||||||
private int myId = 0;
|
|
||||||
private ViewStream myVS;
|
|
||||||
|
|
||||||
public CountStream(ViewStream vs) {
|
|
||||||
myVS = vs;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
myVS.onChange += new ChangeDelegate(myHandleChanges);
|
|
||||||
}
|
|
||||||
protected override void myOnLastSubscriber() {
|
|
||||||
myVS.onChange -= new ChangeDelegate(myHandleChanges);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private void myHandleChanges(object newValue) {
|
|
||||||
myFireEvent(
|
|
||||||
myLastFiredDataPacket =
|
|
||||||
myLastFiredDataPacket == null ?
|
|
||||||
new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
|
|
||||||
new BlueTable(mySchema))
|
|
||||||
:
|
|
||||||
new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
|
|
||||||
myLastFiredDataPacket.newRows)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
private IBlueRowCollection myGetRows(object dp, int id) {
|
|
||||||
BlueTable retVal = new BlueTable(mySchema);
|
|
||||||
retVal.addRow
|
|
||||||
(new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
|
|
||||||
return retVal;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,156 +0,0 @@
|
|||||||
namespace bts.BlueOrb {
|
|
||||||
using BCG;
|
|
||||||
using System;
|
|
||||||
using System.Text.RegularExpressions;
|
|
||||||
|
|
||||||
[DataStreamUserRespresentation("FilterStream")]
|
|
||||||
public class FilterStream : ViewStream {
|
|
||||||
public class RegExpFilter : IStreamFilter {
|
|
||||||
private Regex[] myRegExps;
|
|
||||||
private string[] myCols;
|
|
||||||
|
|
||||||
public RegExpFilter(string col, string regExp)
|
|
||||||
: this(new string[] { col }, new string[] { regExp }) { }
|
|
||||||
|
|
||||||
public RegExpFilter(string[] cols, string[] regExps) {
|
|
||||||
myCols = cols;
|
|
||||||
myRegExps = new Regex[regExps.Length];
|
|
||||||
for (int i = 0; i < regExps.Length; i++)
|
|
||||||
myRegExps[i] = new Regex(regExps[i]);
|
|
||||||
}
|
|
||||||
|
|
||||||
#region IStreamFilter Members
|
|
||||||
public bool filter(IBlueRow dr) {
|
|
||||||
bool rv = true;
|
|
||||||
for (int i = 0; i < myCols.Length; i++) {
|
|
||||||
rv &= myRegExps[i].IsMatch(dr[myCols[i]] is DBNull ? "" : dr[myCols[i]].ToString());
|
|
||||||
if (!rv)
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
#endregion
|
|
||||||
}
|
|
||||||
private BlueTable myLastFiredRows;
|
|
||||||
private IBlueRowCollection myLastReceivedRows;
|
|
||||||
private IStreamFilter myFilter;
|
|
||||||
private ViewStream myParentStream;
|
|
||||||
private string myFilterString;
|
|
||||||
private bool myUseNewFilter = false;// Configuration.get<DataStreamConfig>().UseNewFilter;
|
|
||||||
|
|
||||||
#region constructors
|
|
||||||
public delegate bool OnFilter(IBlueRow r);
|
|
||||||
private class OnFilterWrapper : IStreamFilter {
|
|
||||||
public OnFilterWrapper(OnFilter f) { this.f = f; }
|
|
||||||
|
|
||||||
public bool filter(IBlueRow dr) { return f(dr); }
|
|
||||||
|
|
||||||
private OnFilter f;
|
|
||||||
}
|
|
||||||
public FilterStream(ViewStream vs, OnFilter f)
|
|
||||||
: this(vs, new OnFilterWrapper(f)) { }
|
|
||||||
|
|
||||||
|
|
||||||
public FilterStream(ViewStream vs, IStreamFilter filter) {
|
|
||||||
{
|
|
||||||
myParentStream = vs;
|
|
||||||
myFilter = filter;
|
|
||||||
myFilterString = filter.ToString();
|
|
||||||
if (filter is IDynamicStreamFilter)
|
|
||||||
(filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public FilterStream(ViewStream vs, string filterExpression)
|
|
||||||
: this(vs, new BooleanExpressionFilter(filterExpression)) {
|
|
||||||
myFilterString = filterExpression;
|
|
||||||
}
|
|
||||||
|
|
||||||
public FilterStream(ViewStream vs, string col, string regExp)
|
|
||||||
: this(vs, new RegExpFilter(col, regExp)) {
|
|
||||||
myFilterString = col + " , " + regExp;
|
|
||||||
}
|
|
||||||
|
|
||||||
public FilterStream(ViewStream vs)
|
|
||||||
: this(vs, new BinaryFilter(true)) {
|
|
||||||
myFilterString = true.ToString();
|
|
||||||
}
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
#region overridden methods
|
|
||||||
protected override void myOnFirstSubscriber() { myParentStream.onChange += new ChangeDelegate(myHandleCallbacks); }
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
|
||||||
myLastFiredRows = null;
|
|
||||||
myParentStream.onChange -= new ChangeDelegate(myHandleCallbacks);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myFireEvent(object o) {
|
|
||||||
DataPacket dp = o as DataPacket;
|
|
||||||
if (myLastValue == null || dp.addedRows.Count + dp.deletedRows.Count > 0)
|
|
||||||
base.myFireEvent(o);
|
|
||||||
}
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
private void myHandleCallbacks(object newValue) {
|
|
||||||
DataPacket retVal;
|
|
||||||
DataPacket arrivedPacket = newValue as DataPacket;
|
|
||||||
myLastReceivedRows = arrivedPacket.newRows;
|
|
||||||
if (myLastFiredRows != null) {
|
|
||||||
retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
|
|
||||||
new BlueTable(myLastFiredRows.schema));
|
|
||||||
foreach (IBlueRow r in arrivedPacket.deletedRows)
|
|
||||||
if (myLastFiredRows.containsID((int)r["id"])) {
|
|
||||||
myLastFiredRows.deleteRow(r);
|
|
||||||
(retVal.deletedRows as BlueTable).addRow(r);
|
|
||||||
}
|
|
||||||
foreach (IBlueRow r in arrivedPacket.addedRows)
|
|
||||||
if (myFilter.filter(r)) {
|
|
||||||
myLastFiredRows.addRow(r);
|
|
||||||
(retVal.addedRows as BlueTable).addRow(r);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
myLastFiredRows = new BlueTable(arrivedPacket.schema);
|
|
||||||
retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
|
|
||||||
foreach (IBlueRow r in arrivedPacket.newRows)
|
|
||||||
if (myFilter.filter(r))
|
|
||||||
myLastFiredRows.addRow(r);
|
|
||||||
}
|
|
||||||
myFireEvent(retVal);
|
|
||||||
}
|
|
||||||
|
|
||||||
public IStreamFilter filter {
|
|
||||||
set {
|
|
||||||
{
|
|
||||||
myFilter = value;
|
|
||||||
if (numberOfSubscribers > 0) {
|
|
||||||
if (myLastReceivedRows != null) {
|
|
||||||
BlueTable nRows = new BlueTable(myLastReceivedRows.schema);
|
|
||||||
foreach (IBlueRow r in myLastReceivedRows)
|
|
||||||
if (myFilter.filter(r))
|
|
||||||
nRows.addRow(r);
|
|
||||||
IBlueRowCollection dRows = new BlueTable(nRows.schema),
|
|
||||||
aRows = new BlueTable(nRows.schema);
|
|
||||||
BlueDataUtility.computeDifferences(nRows, myLastFiredRows, out dRows, out aRows);
|
|
||||||
myLastFiredRows = nRows;
|
|
||||||
myFireEvent(new DataPacket(nRows, aRows, dRows));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
get {
|
|
||||||
return myFilter;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
//public string filterString { set { filter = new BooleanExpressionFilter(value); } }
|
|
||||||
|
|
||||||
public override string ToString() {
|
|
||||||
return GetType().ToString() + " " + myFilterString;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void myHandleFilterChange() {
|
|
||||||
filter = myFilter;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,127 +0,0 @@
|
|||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Diagnostics;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Text.Json;
|
|
||||||
|
|
||||||
using bts;
|
|
||||||
//using bts.SilverPine;
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
namespace bts.BlueOrb {
|
|
||||||
[DataStreamUserRespresentation("GroupByStream")]
|
|
||||||
public class GroupByStream : ViewStream {
|
|
||||||
|
|
||||||
public interface INonIncGroupByEvaluator {
|
|
||||||
// grouped_rows
|
|
||||||
IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows);
|
|
||||||
string[] get_group_by_cols();
|
|
||||||
}
|
|
||||||
|
|
||||||
private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
|
|
||||||
Func<string[]> _groupby_cols_func;
|
|
||||||
Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
|
|
||||||
public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
|
|
||||||
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
|
|
||||||
_groupby_cols_func = groupby_cols_func;
|
|
||||||
_evaluate_func = evaluate_func;
|
|
||||||
}
|
|
||||||
public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
|
|
||||||
BlueTable rv = null;
|
|
||||||
foreach (var kv in grouped_rows) {
|
|
||||||
List<IBlueRow> g_rows = kv.Value;
|
|
||||||
List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
|
|
||||||
if (out_rows != null && out_rows.Count > 0) {
|
|
||||||
rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
|
|
||||||
}
|
|
||||||
if (rv != null)
|
|
||||||
rv.addRows(out_rows);
|
|
||||||
}
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
public string[] get_group_by_cols() {
|
|
||||||
return _groupby_cols_func();
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
private ViewStream _main_stream;
|
|
||||||
private INonIncGroupByEvaluator _groupby_func;
|
|
||||||
private BlueRowHasher _row_hasher;
|
|
||||||
private string[] _groupby_cols;
|
|
||||||
private Stopwatch _timer = new();
|
|
||||||
private int _n_total_fires = 0, _n_expensive_fires = 0;
|
|
||||||
|
|
||||||
public GroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) {
|
|
||||||
_main_stream = main_stream;
|
|
||||||
_groupby_func = groupby_func;
|
|
||||||
_groupby_cols = _groupby_func.get_group_by_cols();
|
|
||||||
_row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length);
|
|
||||||
}
|
|
||||||
|
|
||||||
public GroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
|
|
||||||
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
|
|
||||||
: this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
|
|
||||||
|
|
||||||
public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_text)
|
|
||||||
: this(parent_str, groupby_cols, standard_group_by_text, String.Empty) { }
|
|
||||||
|
|
||||||
// this is the main constructor that does everything
|
|
||||||
public GroupByStream(ViewStream parent_str, string groupby_cols, string standard_group_by_ops, string optional_sort_col) {
|
|
||||||
MultiRowOpHelper std_group_by = MultiRowOpHelper.parse_ops(standard_group_by_ops);
|
|
||||||
Func<string[]> groupby_cols_func = () => {
|
|
||||||
if (String.IsNullOrEmpty(groupby_cols))
|
|
||||||
return null;
|
|
||||||
return groupby_cols.Split(',').Select(x => x.Trim()).ToArray();
|
|
||||||
};
|
|
||||||
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
|
|
||||||
if (!String.IsNullOrEmpty(optional_sort_col))
|
|
||||||
group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
|
|
||||||
// this si where the group by aggregation happens
|
|
||||||
IBlueRow rv = MultiRowOpHelper.group_by_compute(group_rows, std_group_by);
|
|
||||||
if (rv == null)
|
|
||||||
return new List<IBlueRow>();
|
|
||||||
return new IBlueRow[] { rv }.ToList();
|
|
||||||
};
|
|
||||||
ViewStream rs = new GroupByStream(parent_str, groupby_cols_func, evaluate_func);
|
|
||||||
mySetRealSource(rs);
|
|
||||||
}
|
|
||||||
public GroupByStream(ViewStream parent_str, string standard_group_by_text) :
|
|
||||||
this(parent_str, String.Empty, standard_group_by_text) {
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
|
|
||||||
|
|
||||||
private void _main_stream_onchange(object nv) {
|
|
||||||
DataPacket dp = nv as DataPacket;
|
|
||||||
///grouping together
|
|
||||||
_timer.Restart();
|
|
||||||
IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ?
|
|
||||||
new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
|
|
||||||
BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
|
|
||||||
_timer.Stop();
|
|
||||||
var groupElapsedMs = _timer.ElapsedMilliseconds;
|
|
||||||
if (groupElapsedMs >= 50) {
|
|
||||||
Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
|
|
||||||
}
|
|
||||||
// Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
|
|
||||||
/// reducing
|
|
||||||
_timer.Restart();
|
|
||||||
IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
|
|
||||||
if (total_state == null)
|
|
||||||
// mayeb returna n empty table ?
|
|
||||||
return;
|
|
||||||
_timer.Stop();
|
|
||||||
var applyElapsedMs = _timer.ElapsedMilliseconds;
|
|
||||||
_n_total_fires++;
|
|
||||||
if (applyElapsedMs >= 50) {
|
|
||||||
_n_expensive_fires++;
|
|
||||||
Log.DataFlow.info(
|
|
||||||
$"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
|
|
||||||
}
|
|
||||||
myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,196 +0,0 @@
|
|||||||
namespace bts.BlueOrb {
|
|
||||||
using BCG;
|
|
||||||
using RowHash = System.Int32;
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Text;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
using System.Collections;
|
|
||||||
using System.Data.Entity.Core.Mapping;
|
|
||||||
|
|
||||||
[DataStreamUserRespresentation("JoinStream")]
|
|
||||||
public class SimpleJoinStream : ViewStream {
|
|
||||||
public class JoinBookkeepingData {
|
|
||||||
// initialized in cons
|
|
||||||
private readonly List<string> _join_col_names;
|
|
||||||
private readonly bool _is_left;
|
|
||||||
private readonly BlueRowHasher _hasher;
|
|
||||||
// initialized on first udpate
|
|
||||||
private IBlueTableSchema _schema;
|
|
||||||
private int[] _join_cols;
|
|
||||||
private IDictionary<RowHash, List<IBlueRow>> _hash_row_map = new Dictionary<RowHash, List<IBlueRow>>();
|
|
||||||
|
|
||||||
public JoinBookkeepingData(ViewStream vs, List<string> join_cols, bool is_left, BlueRowHasher row_hasher) {
|
|
||||||
_join_col_names = join_cols.Where(c => !String.IsNullOrEmpty(c)).ToList();
|
|
||||||
_is_left = is_left;
|
|
||||||
_hasher = row_hasher;
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueTableSchema schema => _schema;
|
|
||||||
|
|
||||||
public IDictionary<RowHash, List<IBlueRow>> rows => _hash_row_map;
|
|
||||||
|
|
||||||
public List<IBlueRow> get_matching_rows(int hash) {
|
|
||||||
List<IBlueRow> rv = null;
|
|
||||||
_hash_row_map.TryGetValue(hash, out rv);
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
private RowHash _get_hash(IBlueRow r) {
|
|
||||||
if (_join_cols == null || _join_cols.Length == 0) {
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
return _hasher.get_hash(r, _join_cols);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void update_data(DataPacket dp) {
|
|
||||||
if (!ready) {
|
|
||||||
_schema = dp.newRows.schema;
|
|
||||||
_join_cols = _join_col_names.Count == 0 ? new int[] { } : _join_col_names.Select(n => dp.schema[n].colNo).ToArray();
|
|
||||||
}
|
|
||||||
_hash_row_map.Clear();
|
|
||||||
foreach (IBlueRow r in dp.newRows) {
|
|
||||||
int hash = _get_hash(r);
|
|
||||||
List<IBlueRow> hash_rows = null;
|
|
||||||
if (!_hash_row_map.TryGetValue(hash, out hash_rows)) {
|
|
||||||
hash_rows = new List<IBlueRow>();
|
|
||||||
_hash_row_map.Add(hash, hash_rows);
|
|
||||||
}
|
|
||||||
hash_rows.Add(r);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public bool ready {
|
|
||||||
get {
|
|
||||||
return _schema != null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
private ViewStream _ls, _rs;
|
|
||||||
private JoinBookkeepingData _ldata, _rdata;
|
|
||||||
private ViewStreamDatabase _vsd;
|
|
||||||
private IBlueTableSchema _schema = null;
|
|
||||||
|
|
||||||
public SimpleJoinStream(ViewStream ls, string ljc, ViewStream rs, string rjc) :
|
|
||||||
this(ViewStreamDatabase.instance, ls, ljc, rs, rjc) { }
|
|
||||||
|
|
||||||
public SimpleJoinStream(ViewStreamDatabase vsd, ViewStream ls, string ljc, ViewStream rs, string rjc) {
|
|
||||||
_ls = ls;
|
|
||||||
_rs = rs;
|
|
||||||
List<string> lj_col_list = ljc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
|
|
||||||
List<string> rj_col_list = rjc.Split(',').Where(x => !String.IsNullOrEmpty(x)).ToList();
|
|
||||||
Log.assert(lj_col_list.Count == rj_col_list.Count);
|
|
||||||
int n_cols = lj_col_list.Count;
|
|
||||||
BlueRowHasher hasher = new BlueRowHasher(n_cols);
|
|
||||||
_ldata = new JoinBookkeepingData(ls, lj_col_list, true, hasher);
|
|
||||||
_rdata = new JoinBookkeepingData(rs, rj_col_list, false, hasher);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
_ls.onChange += _ls_onchange;
|
|
||||||
_rs.onChange += _rs_onchange;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _ls_onchange(object dp_obj) {
|
|
||||||
DataPacket dp = dp_obj as DataPacket;
|
|
||||||
_ldata.update_data(dp);
|
|
||||||
_form_and_fire();
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _rs_onchange(object dp_obj) {
|
|
||||||
DataPacket dp = dp_obj as DataPacket;
|
|
||||||
_rdata.update_data(dp);
|
|
||||||
_form_and_fire();
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
|
||||||
_ls.onChange -= _ls_onchange;
|
|
||||||
_rs.onChange -= _rs_onchange;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _build_schema() {
|
|
||||||
if (_schema != null)
|
|
||||||
return;
|
|
||||||
List<ColumnInfo> cis = new List<ColumnInfo>();
|
|
||||||
cis.Add(new ColumnInfo("id", 0, TypeCode.Int32));
|
|
||||||
foreach (ColumnInfo ci in _ldata.schema.columnInfo) {
|
|
||||||
string col_name = _rdata.schema.contains_column(ci.colName) ? $"left_{ci.colName}" : ci.colName;
|
|
||||||
cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
|
|
||||||
}
|
|
||||||
foreach (ColumnInfo ci in _rdata.schema.columnInfo) {
|
|
||||||
string col_name = _ldata.schema.contains_column(ci.colName) ? $"right_{ci.colName}" : ci.colName;
|
|
||||||
cis.Add(new ColumnInfo(col_name, cis.Count, ci.colType));
|
|
||||||
}
|
|
||||||
_schema = new BlueTableSchema($"{_ldata.schema.tableName}_join_{_rdata.schema.tableName}", cis.ToArray());
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _form_and_fire() {
|
|
||||||
if (!_ldata.ready || !_rdata.ready)
|
|
||||||
return;
|
|
||||||
_build_schema();
|
|
||||||
List<object[]> res_rows = new List<object[]>();
|
|
||||||
|
|
||||||
Action<object[], IBlueRow, bool> copy_row = (rval, r, is_left) => {
|
|
||||||
int offset = 1;
|
|
||||||
JoinBookkeepingData sdata = _ldata;
|
|
||||||
if (is_left) {
|
|
||||||
offset = 1;
|
|
||||||
sdata = _ldata;
|
|
||||||
} else {
|
|
||||||
sdata = _rdata;
|
|
||||||
offset = 1 + _ldata.schema.columnInfo.Length;
|
|
||||||
}
|
|
||||||
for (int i = 0; i < sdata.schema.columnInfo.Length; i++) {
|
|
||||||
rval[offset + i] = r[i];
|
|
||||||
}
|
|
||||||
};
|
|
||||||
Func<int, IBlueRow, object[]> build_row_array = (id, l_row) => {
|
|
||||||
object[] rv = new object[_schema.columnNames.Length];
|
|
||||||
for (int i = 0; i < _schema.columnNames.Length; i++) {
|
|
||||||
rv[i] = DBNull.Value;
|
|
||||||
}
|
|
||||||
rv[0] = id;
|
|
||||||
copy_row(rv, l_row, true);
|
|
||||||
return rv;
|
|
||||||
};
|
|
||||||
// this is where the join happens
|
|
||||||
foreach (var kv in _ldata.rows) {
|
|
||||||
RowHash hash = kv.Key;
|
|
||||||
List<IBlueRow> l_rows = kv.Value;
|
|
||||||
List<IBlueRow> r_rows = _rdata.get_matching_rows(hash);
|
|
||||||
|
|
||||||
foreach (IBlueRow lr in l_rows) {
|
|
||||||
if (r_rows != null) {
|
|
||||||
foreach (IBlueRow r in r_rows) {
|
|
||||||
object[] rowvals = build_row_array(res_rows.Count, lr);
|
|
||||||
copy_row(rowvals, r, false);
|
|
||||||
res_rows.Add(rowvals);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
object[] rowvals = build_row_array(res_rows.Count, lr);
|
|
||||||
res_rows.Add(rowvals);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
BlueTable rv = new BlueTable(_schema);
|
|
||||||
foreach (object[] rval in res_rows) {
|
|
||||||
rv.addRow(new BlueRow(_schema, rval));
|
|
||||||
}
|
|
||||||
var dp = BlueDataUtility.build_dp(rv, myLastValue as DataPacket);
|
|
||||||
myFireEvent(dp);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
[DataStreamUserRespresentation("FullJoinStream")]
|
|
||||||
public class FullJoinStream : ViewStream {
|
|
||||||
public FullJoinStream(ViewStream l_str, string ljcs, ViewStream r_str, string rjcs) {
|
|
||||||
ViewStream js = new SimpleJoinStream(l_str, ljcs, r_str, rjcs);
|
|
||||||
ViewStream rs = new bts.BlueOrb.FilterStream(js, r => !DBNull.Value.Equals(r["right_id"]));
|
|
||||||
mySetRealSource(rs);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -1,28 +0,0 @@
|
|||||||
//namespace bts.Blueorb {
|
|
||||||
// using BCG;
|
|
||||||
|
|
||||||
// public class JsonSaverStream : ViewStream {
|
|
||||||
// private ViewStream _stream;
|
|
||||||
// private string _file_name;
|
|
||||||
// public JsonSaverStream(ViewStream vs, string file_name) {
|
|
||||||
// _stream = vs;
|
|
||||||
// _file_name = file_name;
|
|
||||||
// }
|
|
||||||
|
|
||||||
// protected override void myOnFirstSubscriber() {
|
|
||||||
// _stream.onChange += _on_data;
|
|
||||||
// }
|
|
||||||
|
|
||||||
// private void _on_data(object dp_obj) {
|
|
||||||
// if (dp_obj == null) {
|
|
||||||
// return;
|
|
||||||
// }
|
|
||||||
// DataPacket dp = dp_obj as DataPacket;
|
|
||||||
// dp.newRows.write_to_json_file(dp.newRows, _file_name);
|
|
||||||
// }
|
|
||||||
|
|
||||||
// protected override void myOnLastSubscriber() {
|
|
||||||
// _stream.onChange -= _on_data;
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
//}
|
|
||||||
@@ -1,91 +0,0 @@
|
|||||||
using System;
|
|
||||||
using System.Collections;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
|
|
||||||
using Amazon.Runtime.Internal.Util;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
using BCG.Utilities;
|
|
||||||
|
|
||||||
using bts;
|
|
||||||
|
|
||||||
namespace bts.BlueOrb{
|
|
||||||
public class LiteralStream : ViewStream {
|
|
||||||
|
|
||||||
public static List<Tuple<ColumnInfo, object>> parse_columns(string key_value_pairs) {
|
|
||||||
List<Tuple<ColumnInfo, object>> rv = new List<Tuple<ColumnInfo, object>>();
|
|
||||||
string[] pairs = key_value_pairs.Split(' ');
|
|
||||||
foreach (string p in pairs) {
|
|
||||||
string[] kvp = p.Split(':');
|
|
||||||
string col_name = kvp[0];
|
|
||||||
string val = kvp[1];
|
|
||||||
var tup = BlueDataUtility.guess_type(val);
|
|
||||||
var ci = new ColumnInfo(col_name, rv.Count, tup.Item1);
|
|
||||||
rv.Add(Tuple.Create(ci, tup.Item2));
|
|
||||||
}
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
//literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
|
|
||||||
// col_name: col_value ... ; col_name: col_value
|
|
||||||
public LiteralStream(string table_name, string rows) {
|
|
||||||
Func<string, int, IBlueRow> get_row = (json_row, id) => {
|
|
||||||
Dictionary<string, object> map = new Dictionary<string, object>();
|
|
||||||
string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
|
|
||||||
//Logger.assert(col_vals.Length == 2);
|
|
||||||
foreach (string t in toks) {
|
|
||||||
var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
|
|
||||||
string col = col_vals[0];
|
|
||||||
string val = col_vals[1];
|
|
||||||
map.Add(col, val);
|
|
||||||
}
|
|
||||||
map["id"] = id;
|
|
||||||
return new BlueRow(map);
|
|
||||||
};
|
|
||||||
string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
|
|
||||||
BlueTable table = null;
|
|
||||||
foreach (string json_string in row_arr) {
|
|
||||||
IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
|
|
||||||
get_row(json_string, table == null ? 0 : table.Count);
|
|
||||||
table = table ?? new BlueTable(r.schema);
|
|
||||||
table.addRow(r);
|
|
||||||
}
|
|
||||||
ViewStream str = new BlueTableStream(table);
|
|
||||||
str = new SchemaGuesserStream(str);
|
|
||||||
mySetRealSource(str);
|
|
||||||
}
|
|
||||||
|
|
||||||
public LiteralStream(string tableName, string schema, string value) {
|
|
||||||
string[] columns = schema.Split(' ');
|
|
||||||
|
|
||||||
|
|
||||||
ColumnInfo[] ci = new ColumnInfo[columns.Length + 1];
|
|
||||||
ci[0] = new ColumnInfo("id", 0, typeof(int));
|
|
||||||
int n = 1;
|
|
||||||
foreach (string c in columns) {
|
|
||||||
string[] columnAndType = c.Split('/');
|
|
||||||
if (columnAndType.Length == 1)
|
|
||||||
ci[n] = new ColumnInfo(c, n, typeof(double));
|
|
||||||
else
|
|
||||||
ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true));
|
|
||||||
n++;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
BlueTable t = new BlueTable(new BlueTableSchema(tableName, ci));
|
|
||||||
string[] rows = value.Split(';');
|
|
||||||
n = 0;
|
|
||||||
foreach (string r in rows)
|
|
||||||
try {
|
|
||||||
if (r == string.Empty)
|
|
||||||
continue;
|
|
||||||
t.addRow(
|
|
||||||
BlueDataUtility.createRowFromCSVFileRow(r, t.schema, ++n));
|
|
||||||
} catch (Exception e) {
|
|
||||||
throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message));
|
|
||||||
}
|
|
||||||
|
|
||||||
mySetRealSource(new BlueRowCollectionStaticStream(t));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,89 +0,0 @@
|
|||||||
using System;
|
|
||||||
|
|
||||||
using BCG.Utilities.General;
|
|
||||||
using BCG;
|
|
||||||
using bts;
|
|
||||||
|
|
||||||
namespace bts.BlueOrb {
|
|
||||||
[DataStreamUserRespresentation("MillisecondStream")]
|
|
||||||
public class MillisecondStream : ViewStream, IWorker {
|
|
||||||
private DateTime _last_fire_time = DateTime.MinValue;
|
|
||||||
private readonly TimeSpan _period;
|
|
||||||
private readonly int _size;
|
|
||||||
private readonly ViewStreamDatabase _vsd;
|
|
||||||
private IBlueTableSchema _schema;
|
|
||||||
private bts.CircularBufferTable _table;
|
|
||||||
|
|
||||||
public MillisecondStream(int interval_msecs, int size)
|
|
||||||
: this(ViewStreamDatabase.instance, interval_msecs, size) {
|
|
||||||
}
|
|
||||||
|
|
||||||
public MillisecondStream(ViewStreamDatabase vsd, int interval_msecs, int size) {
|
|
||||||
_vsd = vsd;
|
|
||||||
_period = TimeSpan.FromMilliseconds(interval_msecs);
|
|
||||||
_size = size;
|
|
||||||
_schema = BlueDataUtility.form_schema("Milliseconds",
|
|
||||||
Tuple.Create("id", TypeCode.Int32),
|
|
||||||
Tuple.Create("time", TypeCode.DateTime),
|
|
||||||
Tuple.Create("display_time", TypeCode.String));
|
|
||||||
_table = new CircularBufferTable(_schema, _size);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
_vsd.addWorker(this);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
|
||||||
_vsd.removeWorker(this);
|
|
||||||
}
|
|
||||||
|
|
||||||
private int _id = 0;
|
|
||||||
public void doWork() {
|
|
||||||
if (DateTime.Now - _last_fire_time >= _period) {
|
|
||||||
_last_fire_time = DateTime.Now;
|
|
||||||
IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
|
|
||||||
_last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
|
|
||||||
_table = _table.Add(nr);
|
|
||||||
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
|
|
||||||
myFireEvent(dp);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
internal class MSecondStream : TimeUnitStream {
|
|
||||||
public MSecondStream(int intervalInms, int size)
|
|
||||||
: base(DateTime.Now, new TimeSpan(0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) { }
|
|
||||||
|
|
||||||
protected override DateTime myRoundTime(DateTime dt) { return dt; }
|
|
||||||
}
|
|
||||||
|
|
||||||
internal class DisplayableMillisecondStream : ViewStream, IStreamMapper {
|
|
||||||
private IBlueTableSchema mySchema = new BlueTableSchema("ms",
|
|
||||||
new ColumnInfo("id", 0, typeof(int)),
|
|
||||||
new ColumnInfo("time", 1, typeof(string)));
|
|
||||||
public DisplayableMillisecondStream(int intervalInMs, int sz) {
|
|
||||||
mySetRealSource(new MapStream(new MSecondStream(intervalInMs, sz), this));
|
|
||||||
}
|
|
||||||
#region IStreamMapper Members
|
|
||||||
|
|
||||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
|
|
||||||
return mySchema;
|
|
||||||
}
|
|
||||||
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
#region IDynamicStreamMapper Members
|
|
||||||
|
|
||||||
public IBlueRow map(IBlueRow row) {
|
|
||||||
DateTime time = row.get_datetime("time");
|
|
||||||
var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
|
|
||||||
return new BlueRow(mySchema, row["id"], time_txt);
|
|
||||||
}
|
|
||||||
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
private string myToString(DateTime dt) {
|
|
||||||
string sep = ":";
|
|
||||||
return dt.Hour + sep + dt.Minute + sep + dt.Second + sep + dt.Millisecond;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,169 +0,0 @@
|
|||||||
namespace bts.BlueOrb {
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
//using MathNet.Numerics.Statistics;
|
|
||||||
|
|
||||||
|
|
||||||
public enum MultiRowOp {
|
|
||||||
Mean,
|
|
||||||
Median,
|
|
||||||
Mode,
|
|
||||||
Range,
|
|
||||||
StdDev,
|
|
||||||
Max,
|
|
||||||
Min,
|
|
||||||
First,
|
|
||||||
Last,
|
|
||||||
Sum,
|
|
||||||
EdgeDiff,
|
|
||||||
EdgeDiffPercentage,
|
|
||||||
}
|
|
||||||
public class MultiRowOpHelper {
|
|
||||||
// input + op
|
|
||||||
public readonly List<Tuple<string, MultiRowOp>> operations;
|
|
||||||
|
|
||||||
public static MultiRowOpHelper parse_ops(string text) {
|
|
||||||
// text is inp_col: op, inp_col: op, inp_col:op
|
|
||||||
// ex: "open:First, close:Last, high:Max, low:Min, volume:Sum")
|
|
||||||
List<Tuple<string, MultiRowOp>> rv = new List<Tuple<string, MultiRowOp>>();
|
|
||||||
string[] calcs = text.Split(',');
|
|
||||||
foreach (string c in calcs) {
|
|
||||||
string[] toks = c.Split(':');
|
|
||||||
string inp_col = toks[0].Trim();
|
|
||||||
string op_txt = toks[1].Trim();
|
|
||||||
MultiRowOp op = Enum.Parse<MultiRowOp>(op_txt, true);
|
|
||||||
rv.Add(Tuple.Create(inp_col, op));
|
|
||||||
}
|
|
||||||
return new MultiRowOpHelper(rv);
|
|
||||||
}
|
|
||||||
|
|
||||||
public MultiRowOpHelper(List<Tuple<string, MultiRowOp>> operations_) {
|
|
||||||
operations = operations_;
|
|
||||||
}
|
|
||||||
|
|
||||||
// aggregates multiple rows into one - ie does group_by compute
|
|
||||||
public static IBlueRow group_by_compute(List<IBlueRow> rows, MultiRowOpHelper parameters) {
|
|
||||||
if (rows.Count() == 0) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
IBlueRow ir = rows.ElementAt(0);
|
|
||||||
IBlueTableSchema in_schema = ir.schema;
|
|
||||||
IBlueTableSchema out_schema = in_schema;
|
|
||||||
IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
|
|
||||||
new Dictionary<string, Tuple<string, MultiRowOp>>();
|
|
||||||
foreach (var t in parameters.operations) {
|
|
||||||
string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
|
|
||||||
out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
|
|
||||||
out_col_map[out_col_name] = t;
|
|
||||||
}
|
|
||||||
MutableBlueRow or = new MutableBlueRow(out_schema, ir);
|
|
||||||
foreach (var kv in out_col_map) {
|
|
||||||
string out_col_name = kv.Key;
|
|
||||||
string in_col_name = kv.Value.Item1;
|
|
||||||
MultiRowOp op = kv.Value.Item2;
|
|
||||||
IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
|
|
||||||
double comp_res = compute_op(col_vals, op);
|
|
||||||
or.setValue(out_col_name, comp_res);
|
|
||||||
}
|
|
||||||
return or;
|
|
||||||
}
|
|
||||||
|
|
||||||
// aggregates multiple rows into one, used by MultiRowRollingMathStream
|
|
||||||
public static IBlueRow group_by_compute(List<IBlueRow> rows, int start_idx, int n_rows, MultiRowOpHelper parameters) {
|
|
||||||
if (rows.Count() == 0) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
IBlueRow ir = rows.ElementAt(start_idx);
|
|
||||||
IBlueTableSchema in_schema = ir.schema;
|
|
||||||
IBlueTableSchema out_schema = in_schema;
|
|
||||||
// out_col vs inp_cal + type
|
|
||||||
IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
|
|
||||||
new Dictionary<string, Tuple<string, MultiRowOp>>();
|
|
||||||
foreach (var t in parameters.operations) {
|
|
||||||
string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
|
|
||||||
out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
|
|
||||||
out_col_map[out_col_name] = t;
|
|
||||||
}
|
|
||||||
MutableBlueRow or = new MutableBlueRow(out_schema, ir);
|
|
||||||
foreach (var kv in out_col_map) {
|
|
||||||
string out_col_name = kv.Key;
|
|
||||||
string in_col_name = kv.Value.Item1;
|
|
||||||
MultiRowOp op = kv.Value.Item2;
|
|
||||||
IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, start_idx, n_rows, in_col_name);
|
|
||||||
double comp_res = compute_op(col_vals, op);
|
|
||||||
or.setValue(out_col_name, comp_res);
|
|
||||||
or.setValue("id", ir["id"]);
|
|
||||||
}
|
|
||||||
return or;
|
|
||||||
}
|
|
||||||
|
|
||||||
// rolling_rows are the rolling window of rows. the 0th row is the one thats gettin
|
|
||||||
// calculated. rows 1 - n-1 are its window rows. the schema is based on the operations.
|
|
||||||
|
|
||||||
public IBlueRow rolling_calc(List<IBlueRow> rolling_rows) {
|
|
||||||
if (rolling_rows.Count() == 0) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
IBlueRow ir = rolling_rows.ElementAt(0);
|
|
||||||
IBlueTableSchema in_schema = ir.schema;
|
|
||||||
IBlueTableSchema out_schema = in_schema;
|
|
||||||
IDictionary<string, Tuple<string, MultiRowOp>> out_col_map =
|
|
||||||
new Dictionary<string, Tuple<string, MultiRowOp>>();
|
|
||||||
foreach (var t in operations) {
|
|
||||||
string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
|
|
||||||
out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
|
|
||||||
out_col_map[out_col_name] = t;
|
|
||||||
}
|
|
||||||
MutableBlueRow or = new MutableBlueRow(out_schema, ir);
|
|
||||||
foreach (var kv in out_col_map) {
|
|
||||||
string out_col_name = kv.Key;
|
|
||||||
string in_col_name = kv.Value.Item1;
|
|
||||||
MultiRowOp op = kv.Value.Item2;
|
|
||||||
IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rolling_rows, in_col_name);
|
|
||||||
double comp_res = compute_op(col_vals, op);
|
|
||||||
or.setValue(out_col_name, comp_res);
|
|
||||||
}
|
|
||||||
return or;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static double compute_op(IEnumerable<double> col_vals, MultiRowOp op) {
|
|
||||||
switch (op) {
|
|
||||||
case MultiRowOp.Max:
|
|
||||||
return col_vals.Max<double>();
|
|
||||||
case MultiRowOp.Min:
|
|
||||||
return col_vals.Min<double>();
|
|
||||||
case MultiRowOp.Mean:
|
|
||||||
return MathNet.Numerics.Statistics.Statistics.Mean(col_vals);
|
|
||||||
case MultiRowOp.Sum:
|
|
||||||
return col_vals.Sum();
|
|
||||||
case MultiRowOp.Median:
|
|
||||||
return MathNet.Numerics.Statistics.Statistics.Median(col_vals);
|
|
||||||
//case MultiRowOp.Mode:
|
|
||||||
// return MathNet.Numerics.Statistics.Statistics.(col_vals);
|
|
||||||
case MultiRowOp.First:
|
|
||||||
return col_vals.First();
|
|
||||||
case MultiRowOp.Last:
|
|
||||||
return col_vals.Last();
|
|
||||||
case MultiRowOp.StdDev:
|
|
||||||
return MathNet.Numerics.Statistics.Statistics.StandardDeviation(col_vals);
|
|
||||||
case MultiRowOp.EdgeDiff:
|
|
||||||
return col_vals.First() - col_vals.Last();
|
|
||||||
case MultiRowOp.EdgeDiffPercentage: {
|
|
||||||
double first = col_vals.First();
|
|
||||||
double last = col_vals.Last();
|
|
||||||
return ((first - last) / last) * 100.0;
|
|
||||||
return col_vals.First() - col_vals.Last();
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
// TODO
|
|
||||||
Log.assert(false);
|
|
||||||
return Double.NaN;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,40 +0,0 @@
|
|||||||
namespace bts.BlueOrb {
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Net;
|
|
||||||
|
|
||||||
[DataStreamUserRespresentation("MultiRowRollingMathStream")]
|
|
||||||
public class MultiRowRollingMathStream : ViewStream {
|
|
||||||
private ViewStream _sorted_parent_str;
|
|
||||||
private MultiRowOpHelper _op;
|
|
||||||
private int _rolling_window_sz;
|
|
||||||
public MultiRowRollingMathStream(ViewStream parent_str, string op_txt, string sort_exp, int rolling_window_sz) {
|
|
||||||
_sorted_parent_str = new SortStream(parent_str, sort_exp);
|
|
||||||
_op = MultiRowOpHelper.parse_ops(op_txt);
|
|
||||||
_rolling_window_sz = rolling_window_sz;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
_sorted_parent_str.onChange += _parent_str_onchange;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
|
||||||
_sorted_parent_str.onChange -= _parent_str_onchange;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _parent_str_onchange(object dp_obj) {
|
|
||||||
DataPacket dp = dp_obj as DataPacket;
|
|
||||||
var rows = dp.newRows.to_list();
|
|
||||||
BlueTable out_table = null;
|
|
||||||
for (int i = 0; i < rows.Count; i++) {
|
|
||||||
var out_r = MultiRowOpHelper.group_by_compute(rows, i, _rolling_window_sz, _op);
|
|
||||||
if (out_table == null)
|
|
||||||
out_table = new BlueTable(out_r.schema);
|
|
||||||
out_table.addRow(out_r);
|
|
||||||
}
|
|
||||||
var dp_out = BlueDataUtility.build_dp_inc(out_table, myLastValue as DataPacket, true);
|
|
||||||
myFireEvent(dp_out);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
namespace Bts.DataStream.Parquet;
|
|
||||||
|
|
||||||
using bts.collections;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
|
|
||||||
public interface IParquetColumnReader : IDisposable {
|
|
||||||
IList<object?> Read();
|
|
||||||
IList<object?> Read(int? start, int? stop);
|
|
||||||
}
|
|
||||||
|
|
||||||
public interface IParquetColumnReader<TLogical, TPhysical> : IParquetColumnReader {
|
|
||||||
new IList<TLogical?> Read();
|
|
||||||
new IList<TLogical?> Read(int? start, int? stop);
|
|
||||||
|
|
||||||
IList<object?> IParquetColumnReader.Read() =>
|
|
||||||
Read().WrapSelector(v => (object?)v, null);
|
|
||||||
|
|
||||||
IList<object?> IParquetColumnReader.Read(int? start, int? stop) =>
|
|
||||||
Read(start, stop).WrapSelector(v => (object?)v, null);
|
|
||||||
}
|
|
||||||
|
|
||||||
public interface IParquetColumnReader<T> : IParquetColumnReader<T, T> {
|
|
||||||
new IList<T?> Read();
|
|
||||||
|
|
||||||
IList<object?> IParquetColumnReader.Read() =>
|
|
||||||
Read().WrapSelector(v => (object?)v, null);
|
|
||||||
}
|
|
||||||
@@ -1,507 +0,0 @@
|
|||||||
namespace Bts.DataStream.Parquet;
|
|
||||||
|
|
||||||
using bts;
|
|
||||||
using bts.collections;
|
|
||||||
|
|
||||||
using mbts.utils;
|
|
||||||
|
|
||||||
using ParquetSharp;
|
|
||||||
|
|
||||||
using Spark.Core.Collections;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
|
|
||||||
public static class ParquetAdapter {
|
|
||||||
public static IParquetColumnReader GetColumnReader(this RowGroupReader rowGroupReader, int index, Type? colType = null) {
|
|
||||||
int rowCount = checked((int)rowGroupReader.MetaData.NumRows);
|
|
||||||
var colReader = rowGroupReader.Column(index);
|
|
||||||
return getColumnReader(colReader, rowCount, colType: colType);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static IList<object?>[] ReadColumns(this ParquetFileReader fileReader) {
|
|
||||||
var schema = fileReader.FileMetaData.Schema.ToBlueSchema("parquetReader");
|
|
||||||
var colInfos = schema.columnInfo;
|
|
||||||
var colTypes = Array.ConvertAll(colInfos, c => c.colType);
|
|
||||||
var colValues = new IList<object?>[fileReader.FileMetaData.NumColumns];
|
|
||||||
int rowCount = checked((int)fileReader.FileMetaData.NumRows);
|
|
||||||
for (int i = 0; i < colValues.Length; i++) {
|
|
||||||
colValues[i] = new List<object?>(rowCount);
|
|
||||||
}
|
|
||||||
int rowGroupCount = fileReader.FileMetaData.NumRowGroups;
|
|
||||||
for (int rowGroup = 0; rowGroup < rowGroupCount; ++rowGroup) {
|
|
||||||
using var rowGroupReader = fileReader.RowGroup(rowGroup);
|
|
||||||
rowGroupReader.ReadColumnsTo(ref colValues, colTypes);
|
|
||||||
}
|
|
||||||
return colValues;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static IList<object?>[] ReadColumns(this ParquetFileReader fileReader, int? start, int? stop) {
|
|
||||||
if (!start.HasValue && !stop.HasValue) {
|
|
||||||
return ReadColumns(fileReader);
|
|
||||||
}
|
|
||||||
var schema = fileReader.FileMetaData.ToBlueSchema("parquetReader");
|
|
||||||
var colInfos = schema.columnInfo;
|
|
||||||
var colTypes = Array.ConvertAll(colInfos, c => c.colType);
|
|
||||||
var colValues = new IList<object?>[fileReader.FileMetaData.NumColumns];
|
|
||||||
int rowCount = checked((int)fileReader.FileMetaData.NumRows);
|
|
||||||
start ??= 0;
|
|
||||||
stop ??= rowCount;
|
|
||||||
var sliceLength = stop.Value - start.Value;
|
|
||||||
for (int i = 0; i < colValues.Length; i++) {
|
|
||||||
colValues[i] = new List<object?>(sliceLength);
|
|
||||||
}
|
|
||||||
int numRowGroups = fileReader.FileMetaData.NumRowGroups;
|
|
||||||
int? numRowsPerGroup = null;
|
|
||||||
int rowIdx = 0;
|
|
||||||
for (int rowGroup = 0; rowGroup < numRowGroups; ++rowGroup) {
|
|
||||||
if (stop.Value <= rowIdx) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
using var rowGroupReader = fileReader.RowGroup(rowGroup);
|
|
||||||
int numRowsInGroup = checked((int)rowGroupReader.MetaData.NumRows);
|
|
||||||
numRowsPerGroup ??= numRowsInGroup;
|
|
||||||
if (rowGroup < numRowGroups - 1) {
|
|
||||||
Log.assert(numRowsInGroup == numRowsPerGroup.Value);
|
|
||||||
} else {
|
|
||||||
Log.assert(numRowsInGroup <= numRowsPerGroup.Value);
|
|
||||||
}
|
|
||||||
int startIdx = start.Value - rowIdx;
|
|
||||||
int stopIdx = stop.Value - rowIdx;
|
|
||||||
rowIdx += numRowsInGroup;
|
|
||||||
if (startIdx >= numRowsInGroup) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
startIdx = Math.Max(0, startIdx);
|
|
||||||
stopIdx = Math.Min(stopIdx, numRowsInGroup);
|
|
||||||
int n = rowGroupReader.ReadColumnsTo(ref colValues, colTypes, startIdx, stopIdx);
|
|
||||||
Log.assert(n == (stopIdx - startIdx));
|
|
||||||
}
|
|
||||||
return colValues;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static int ReadColumnsTo(this RowGroupReader rowGroupReader, ref IList<object?>[] columnValues, Type[] colTypes) {
|
|
||||||
int rowCount = checked((int)rowGroupReader.MetaData.NumRows);
|
|
||||||
for (int colIdx = rowGroupReader.MetaData.NumColumns - 1; colIdx > -1; colIdx--) {
|
|
||||||
using var colReader = rowGroupReader.GetColumnReader(colIdx, colTypes[colIdx]);
|
|
||||||
columnValues[colIdx].AddRange(colReader.Read());
|
|
||||||
}
|
|
||||||
return rowCount;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static int ReadColumnsTo(this RowGroupReader rowGroupReader, ref IList<object?>[] columnValues, Type[] colTypes, int? start, int? stop) {
|
|
||||||
if (!start.HasValue && !stop.HasValue) {
|
|
||||||
return ReadColumnsTo(rowGroupReader, ref columnValues, colTypes);
|
|
||||||
}
|
|
||||||
int numRows = checked((int)rowGroupReader.MetaData.NumRows);
|
|
||||||
if (start.HasValue && start.Value >= numRows) {
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
int? readRowCount = null;
|
|
||||||
for (int colIdx = rowGroupReader.MetaData.NumColumns - 1; colIdx > -1; colIdx--) {
|
|
||||||
using var colReader = rowGroupReader.GetColumnReader(colIdx, colTypes[colIdx]);
|
|
||||||
var colVals = colReader.Read(start, stop);
|
|
||||||
readRowCount ??= colVals.Count;
|
|
||||||
Log.assert(readRowCount == colVals.Count);
|
|
||||||
columnValues[colIdx].AddRange(colVals);
|
|
||||||
}
|
|
||||||
return readRowCount ?? throw new ArgumentException("Row group has no columns", nameof(rowGroupReader));
|
|
||||||
}
|
|
||||||
|
|
||||||
static IParquetColumnReader getColumnReader(ColumnReader reader, int bufferSize = 4096, Type? colType = null) {
|
|
||||||
var desc = reader.ColumnDescriptor;
|
|
||||||
var logicalType = desc.LogicalType;
|
|
||||||
return logicalType.Type switch {
|
|
||||||
LogicalTypeEnum.Int => IntReader(),
|
|
||||||
LogicalTypeEnum.String => new ParquetColumnReader<string?>(reader, bufferSize),
|
|
||||||
LogicalTypeEnum.Date => new ParquetColumnReader<DateTime?>(reader, bufferSize),
|
|
||||||
LogicalTypeEnum.Decimal => DecimalReader(),
|
|
||||||
LogicalTypeEnum.Timestamp => TimestampReader(),
|
|
||||||
LogicalTypeEnum.None => DefaultReader(),
|
|
||||||
_ => throw new NotImplementedException(),
|
|
||||||
};
|
|
||||||
|
|
||||||
IParquetColumnReader DefaultReader() {
|
|
||||||
return desc.PhysicalType switch {
|
|
||||||
PhysicalType.Boolean => new ParquetColumnReader<bool?>(reader, bufferSize),
|
|
||||||
PhysicalType.Int32 => new ParquetColumnReader<int?>(reader, bufferSize),
|
|
||||||
PhysicalType.Int64 => new ParquetColumnReader<long?>(reader, bufferSize),
|
|
||||||
PhysicalType.Float => new ParquetColumnReader<float?>(reader, bufferSize),
|
|
||||||
PhysicalType.Double => new ParquetColumnReader<double?>(reader, bufferSize),
|
|
||||||
_ => throw new NotImplementedException(),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
IParquetColumnReader TimestampReader() {
|
|
||||||
var tlt = ExceptionAssert.of_type<TimestampLogicalType>(logicalType);
|
|
||||||
return new ParquetColumnReader<DateTimeNanos?, DateTime?>(reader, dtn => dtn?.DateTime, bufferSize);
|
|
||||||
}
|
|
||||||
|
|
||||||
IParquetColumnReader IntReader() {
|
|
||||||
var ilt = ExceptionAssert.of_type<IntLogicalType>(logicalType);
|
|
||||||
if (!ilt.IsSigned) {
|
|
||||||
if (colType is null) {
|
|
||||||
return ilt.BitWidth switch {
|
|
||||||
8 => new ParquetColumnReader<byte?>(reader, bufferSize),
|
|
||||||
16 => new ParquetColumnReader<ushort?>(reader, bufferSize),
|
|
||||||
32 => new ParquetColumnReader<uint?>(reader, bufferSize),
|
|
||||||
64 => new ParquetColumnReader<ulong?>(reader, bufferSize),
|
|
||||||
_ => throw new NotImplementedException(),
|
|
||||||
};
|
|
||||||
} else if (colType == typeof(byte)) {
|
|
||||||
return ilt.BitWidth switch {
|
|
||||||
8 => new ParquetColumnReader<byte?>(reader, bufferSize),
|
|
||||||
16 => new ParquetColumnReader<ushort?, byte?>(reader, v => v.HasValue ? checked((byte)v.Value) : null, bufferSize),
|
|
||||||
32 => new ParquetColumnReader<uint?, ushort?>(reader, v => v.HasValue ? checked((byte)v.Value) : null, bufferSize),
|
|
||||||
64 => new ParquetColumnReader<ulong?, ushort?>(reader, v => v.HasValue ? checked((byte)v.Value) : null, bufferSize),
|
|
||||||
_ => throw new NotImplementedException(),
|
|
||||||
};
|
|
||||||
} else if (colType == typeof(ushort)) {
|
|
||||||
return ilt.BitWidth switch {
|
|
||||||
8 => new ParquetColumnReader<byte?, ushort?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
|
|
||||||
16 => new ParquetColumnReader<ushort?>(reader, bufferSize),
|
|
||||||
32 => new ParquetColumnReader<uint?, ushort?>(reader, v => v.HasValue ? checked((ushort)v.Value) : null, bufferSize),
|
|
||||||
64 => new ParquetColumnReader<ulong?, ushort?>(reader, v => v.HasValue ? checked((ushort)v.Value) : null, bufferSize),
|
|
||||||
_ => throw new NotImplementedException(),
|
|
||||||
};
|
|
||||||
} else if (colType == typeof(uint)) {
|
|
||||||
return ilt.BitWidth switch {
|
|
||||||
8 => new ParquetColumnReader<byte?, uint?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
|
|
||||||
16 => new ParquetColumnReader<ushort?, uint?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
|
|
||||||
32 => new ParquetColumnReader<uint?>(reader, bufferSize),
|
|
||||||
64 => new ParquetColumnReader<ulong?, uint?>(reader, v => v.HasValue ? checked((uint)v.Value) : null, bufferSize),
|
|
||||||
_ => throw new NotImplementedException(),
|
|
||||||
};
|
|
||||||
} else if (colType == typeof(ulong)) {
|
|
||||||
return ilt.BitWidth switch {
|
|
||||||
8 => new ParquetColumnReader<byte?, ulong?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
|
|
||||||
16 => new ParquetColumnReader<ushort?, ulong?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
|
|
||||||
32 => new ParquetColumnReader<uint?, ulong?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
|
|
||||||
64 => new ParquetColumnReader<ulong?>(reader, bufferSize),
|
|
||||||
_ => throw new NotImplementedException(),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
throw new NotImplementedException($"Expected signed int column type, but got {colType}");
|
|
||||||
} else {
|
|
||||||
if (colType is null) {
|
|
||||||
return ilt.BitWidth switch {
|
|
||||||
8 => new ParquetColumnReader<sbyte?>(reader, bufferSize),
|
|
||||||
16 => new ParquetColumnReader<short?>(reader, bufferSize),
|
|
||||||
32 => new ParquetColumnReader<int?>(reader, bufferSize),
|
|
||||||
64 => new ParquetColumnReader<long?>(reader, bufferSize),
|
|
||||||
_ => throw new NotImplementedException(),
|
|
||||||
};
|
|
||||||
} else if (colType == typeof(sbyte)) {
|
|
||||||
return ilt.BitWidth switch {
|
|
||||||
8 => new ParquetColumnReader<sbyte?>(reader, bufferSize),
|
|
||||||
16 => new ParquetColumnReader<short?, sbyte?>(reader, v => v.HasValue ? checked((sbyte)v.Value) : null, bufferSize),
|
|
||||||
32 => new ParquetColumnReader<int?, sbyte?>(reader, v => v.HasValue ? checked((sbyte)v.Value) : null, bufferSize),
|
|
||||||
64 => new ParquetColumnReader<long?, sbyte?>(reader, v => v.HasValue ? checked((sbyte)v.Value) : null, bufferSize),
|
|
||||||
_ => throw new NotImplementedException(),
|
|
||||||
};
|
|
||||||
} else if (colType == typeof(short)) {
|
|
||||||
return ilt.BitWidth switch {
|
|
||||||
8 => new ParquetColumnReader<char?, short?>(reader, v => v.HasValue ? checked((short)v.Value) : null, bufferSize),
|
|
||||||
16 => new ParquetColumnReader<short?>(reader, bufferSize),
|
|
||||||
32 => new ParquetColumnReader<int?, short?>(reader, v => v.HasValue ? checked((short)v.Value) : null, bufferSize),
|
|
||||||
64 => new ParquetColumnReader<long?, short?>(reader, v => v.HasValue ? checked((short)v.Value) : null, bufferSize),
|
|
||||||
_ => throw new NotImplementedException(),
|
|
||||||
};
|
|
||||||
} else if (colType == typeof(int)) {
|
|
||||||
return ilt.BitWidth switch {
|
|
||||||
8 => new ParquetColumnReader<char?, int?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
|
|
||||||
16 => new ParquetColumnReader<short?, int?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
|
|
||||||
32 => new ParquetColumnReader<int?>(reader, bufferSize),
|
|
||||||
64 => new ParquetColumnReader<long?, int?>(reader, v => v.HasValue ? checked((int)v.Value) : null, bufferSize),
|
|
||||||
_ => throw new NotImplementedException(),
|
|
||||||
};
|
|
||||||
} else if (colType == typeof(long)) {
|
|
||||||
return ilt.BitWidth switch {
|
|
||||||
8 => new ParquetColumnReader<char?, long?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
|
|
||||||
16 => new ParquetColumnReader<short?, long?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
|
|
||||||
32 => new ParquetColumnReader<int?, long?>(reader, v => v.HasValue ? v.Value : null, bufferSize),
|
|
||||||
64 => new ParquetColumnReader<long?>(reader, bufferSize),
|
|
||||||
_ => throw new NotImplementedException(),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
throw new NotImplementedException($"Expected unsigned int column type, but got {colType}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
IParquetColumnReader DecimalReader() {
|
|
||||||
var dlt = ExceptionAssert.of_type<DecimalLogicalType>(logicalType);
|
|
||||||
return new ParquetColumnReader<double?>(reader, bufferSize);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static IColumnWriter GetColumnWriter(TypeCode typeCode, ColumnWriter writer, int bufferSize = 4096) {
|
|
||||||
return typeCode switch {
|
|
||||||
TypeCode.Boolean => new ColumnWriter<bool?>(writer, bufferSize),
|
|
||||||
TypeCode.SByte => new ColumnWriter<sbyte?>(writer, bufferSize, GetInt8Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
|
|
||||||
TypeCode.Int16 => new ColumnWriter<short?>(writer, bufferSize, GetInt16Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
|
|
||||||
TypeCode.Int32 => new ColumnWriter<int?>(writer, bufferSize, GetInt32Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
|
|
||||||
TypeCode.Int64 => new ColumnWriter<long?>(writer, bufferSize, GetInt64Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
|
|
||||||
TypeCode.Byte => new ColumnWriter<byte?>(writer, bufferSize, GetUInt8Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
|
|
||||||
TypeCode.UInt16 => new ColumnWriter<ushort?>(writer, bufferSize, GetUInt16Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
|
|
||||||
TypeCode.UInt32 => new ColumnWriter<uint?>(writer, bufferSize, GetUInt32Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
|
|
||||||
TypeCode.UInt64 => new ColumnWriter<ulong?>(writer, bufferSize, GetUInt64Converter((IntLogicalType)writer.ColumnDescriptor.LogicalType)),
|
|
||||||
TypeCode.Single => new ColumnWriter<float?>(writer, bufferSize),
|
|
||||||
TypeCode.Double => new ColumnWriter<double?>(writer, bufferSize),
|
|
||||||
TypeCode.DateTime => new ColumnWriter<DateTimeNanos?>(writer, bufferSize, ConvertDateTimeObj),
|
|
||||||
TypeCode.String => new ColumnWriter<string?>(writer, bufferSize),
|
|
||||||
_ => throw new NotSupportedException($"Unsupported column TypeCode {typeCode}"),
|
|
||||||
};
|
|
||||||
|
|
||||||
static DateTimeNanos? ConvertDateTimeObj(object? obj) {
|
|
||||||
if (obj is DateTime dt) {
|
|
||||||
return new(dt);
|
|
||||||
}
|
|
||||||
if (obj is DateTimeNanos dtn) {
|
|
||||||
return dtn;
|
|
||||||
}
|
|
||||||
throw new InvalidCastException($"Unable to convert object of type {obj?.GetType()} to DateTimeNanos");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Func<object?, sbyte?> GetInt8Converter(IntLogicalType logicalType) =>
|
|
||||||
GetInt8Converter(logicalType.IsSigned, logicalType.BitWidth);
|
|
||||||
|
|
||||||
public static Func<object?, sbyte?> GetInt8Converter(bool isSigned, int bitWidth) =>
|
|
||||||
isSigned
|
|
||||||
? bitWidth switch {
|
|
||||||
8 => obj => obj as sbyte?,
|
|
||||||
16 => obj => obj is short v ? checked((sbyte)v) : null,
|
|
||||||
32 => obj => obj is int v ? checked((sbyte)v) : null,
|
|
||||||
64 => obj => obj is long v ? checked((sbyte)v) : null,
|
|
||||||
_ => throw new NotImplementedException(),
|
|
||||||
}
|
|
||||||
: bitWidth switch {
|
|
||||||
8 => obj => obj is byte v ? checked((sbyte)v) : null,
|
|
||||||
16 => obj => obj is ushort v ? checked((sbyte)v) : null,
|
|
||||||
32 => obj => obj is uint v ? checked((sbyte)v) : null,
|
|
||||||
64 => obj => obj is ulong v ? checked((sbyte)v) : null,
|
|
||||||
_ => throw new NotImplementedException(),
|
|
||||||
};
|
|
||||||
|
|
||||||
public static Func<object?, short?> GetInt16Converter(IntLogicalType logicalType) =>
|
|
||||||
GetInt16Converter(logicalType.IsSigned, logicalType.BitWidth);
|
|
||||||
|
|
||||||
public static Func<object?, short?> GetInt16Converter(bool isSigned, int bitWidth) =>
|
|
||||||
isSigned
|
|
||||||
? bitWidth switch {
|
|
||||||
8 => obj => obj is sbyte v ? v : null,
|
|
||||||
16 => obj => obj as short?,
|
|
||||||
32 => obj => obj is int v ? checked((short)v) : null,
|
|
||||||
64 => obj => obj is long v ? checked((short)v) : null,
|
|
||||||
_ => throw new NotImplementedException(),
|
|
||||||
}
|
|
||||||
: bitWidth switch {
|
|
||||||
8 => obj => obj is byte v ? v : null,
|
|
||||||
16 => obj => obj is ushort v ? checked((short)v) : null,
|
|
||||||
32 => obj => obj is uint v ? checked((short)v) : null,
|
|
||||||
64 => obj => obj is ulong v ? checked((short)v) : null,
|
|
||||||
_ => throw new NotImplementedException(),
|
|
||||||
};
|
|
||||||
|
|
||||||
public static Func<object?, int?> GetInt32Converter(IntLogicalType logicalType) =>
|
|
||||||
GetInt32Converter(logicalType.IsSigned, logicalType.BitWidth);
|
|
||||||
|
|
||||||
public static Func<object?, int?> GetInt32Converter(bool isSigned, int bitWidth) =>
|
|
||||||
isSigned
|
|
||||||
? bitWidth switch {
|
|
||||||
8 => obj => obj is sbyte v ? v : null,
|
|
||||||
16 => obj => obj is short v ? v : null,
|
|
||||||
32 => obj => obj as int?,
|
|
||||||
64 => obj => obj is long v ? checked((int)v) : null,
|
|
||||||
_ => throw new NotImplementedException(),
|
|
||||||
}
|
|
||||||
: bitWidth switch {
|
|
||||||
8 => obj => obj is byte v ? v : null,
|
|
||||||
16 => obj => obj is ushort v ? v : null,
|
|
||||||
32 => obj => obj is uint v ? checked((int)v) : null,
|
|
||||||
64 => obj => obj is ulong v ? checked((int)v) : null,
|
|
||||||
_ => throw new NotImplementedException(),
|
|
||||||
};
|
|
||||||
|
|
||||||
public static Func<object?, long?> GetInt64Converter(IntLogicalType logicalType) =>
|
|
||||||
GetInt64Converter(logicalType.IsSigned, logicalType.BitWidth);
|
|
||||||
|
|
||||||
public static Func<object?, long?> GetInt64Converter(bool isSigned, int bitWidth) =>
|
|
||||||
isSigned
|
|
||||||
? bitWidth switch {
|
|
||||||
8 => obj => obj is sbyte v ? v : null,
|
|
||||||
16 => obj => obj is short v ? v : null,
|
|
||||||
32 => obj => obj is int v ? v : null,
|
|
||||||
64 => obj => obj as long?,
|
|
||||||
_ => throw new NotImplementedException(),
|
|
||||||
}
|
|
||||||
: bitWidth switch {
|
|
||||||
8 => obj => obj is byte v ? v : null,
|
|
||||||
16 => obj => obj is ushort v ? v : null,
|
|
||||||
32 => obj => obj is uint v ? v : null,
|
|
||||||
64 => obj => obj is ulong v ? checked((long)v) : null,
|
|
||||||
_ => throw new NotImplementedException(),
|
|
||||||
};
|
|
||||||
|
|
||||||
public static Func<object?, byte?> GetUInt8Converter(IntLogicalType logicalType) =>
|
|
||||||
GetUInt8Converter(logicalType.IsSigned, logicalType.BitWidth);
|
|
||||||
|
|
||||||
public static Func<object?, byte?> GetUInt8Converter(bool isSigned, int bitWidth) =>
|
|
||||||
isSigned
|
|
||||||
? bitWidth switch {
|
|
||||||
8 => obj => obj is sbyte v ? checked((byte)v) : null,
|
|
||||||
16 => obj => obj is short v ? checked((byte)v) : null,
|
|
||||||
32 => obj => obj is int v ? checked((byte)v) : null,
|
|
||||||
64 => obj => obj is long v ? checked((byte)v) : null,
|
|
||||||
_ => throw new NotImplementedException(),
|
|
||||||
}
|
|
||||||
: bitWidth switch {
|
|
||||||
8 => obj => obj as byte?,
|
|
||||||
16 => obj => obj is ushort v ? checked((byte)v) : null,
|
|
||||||
32 => obj => obj is uint v ? checked((byte)v) : null,
|
|
||||||
64 => obj => obj is ulong v ? checked((byte)v) : null,
|
|
||||||
_ => throw new NotImplementedException(),
|
|
||||||
};
|
|
||||||
|
|
||||||
public static Func<object?, ushort?> GetUInt16Converter(IntLogicalType logicalType) =>
|
|
||||||
GetUInt16Converter(logicalType.IsSigned, logicalType.BitWidth);
|
|
||||||
|
|
||||||
public static Func<object?, ushort?> GetUInt16Converter(bool isSigned, int bitWidth) =>
|
|
||||||
isSigned
|
|
||||||
? bitWidth switch {
|
|
||||||
8 => obj => obj is sbyte v ? checked((ushort)v) : null,
|
|
||||||
16 => obj => obj is short v ? checked((ushort)v) : null,
|
|
||||||
32 => obj => obj is int v ? checked((ushort)v) : null,
|
|
||||||
64 => obj => obj is long v ? checked((ushort)v) : null,
|
|
||||||
_ => throw new NotImplementedException(),
|
|
||||||
}
|
|
||||||
: bitWidth switch {
|
|
||||||
8 => obj => obj is byte v ? v : null,
|
|
||||||
16 => obj => obj is ushort v ? v : null,
|
|
||||||
32 => obj => obj is uint v ? checked((ushort)v) : null,
|
|
||||||
64 => obj => obj is ulong v ? checked((ushort)v) : null,
|
|
||||||
_ => throw new NotImplementedException(),
|
|
||||||
};
|
|
||||||
|
|
||||||
public static Func<object?, uint?> GetUInt32Converter(IntLogicalType logicalType) =>
|
|
||||||
GetUInt32Converter(logicalType.IsSigned, logicalType.BitWidth);
|
|
||||||
|
|
||||||
public static Func<object?, uint?> GetUInt32Converter(bool isSigned, int bitWidth) =>
|
|
||||||
isSigned
|
|
||||||
? bitWidth switch {
|
|
||||||
8 => obj => obj is sbyte v ? checked((uint)v) : null,
|
|
||||||
16 => obj => obj is short v ? checked((uint)v) : null,
|
|
||||||
32 => obj => obj is int v ? checked((uint)v) : null,
|
|
||||||
64 => obj => obj is long v ? checked((uint)v) : null,
|
|
||||||
_ => throw new NotImplementedException(),
|
|
||||||
}
|
|
||||||
: bitWidth switch {
|
|
||||||
8 => obj => obj is byte v ? v : null,
|
|
||||||
16 => obj => obj is ushort v ? v : null,
|
|
||||||
32 => obj => obj is uint v ? v : null,
|
|
||||||
64 => obj => obj is ulong v ? checked((uint)v) : null,
|
|
||||||
_ => throw new NotImplementedException(),
|
|
||||||
};
|
|
||||||
|
|
||||||
public static Func<object?, ulong?> GetUInt64Converter(IntLogicalType logicalType) =>
|
|
||||||
GetUInt64Converter(logicalType.IsSigned, logicalType.BitWidth);
|
|
||||||
|
|
||||||
public static Func<object?, ulong?> GetUInt64Converter(bool isSigned, int bitWidth) =>
|
|
||||||
isSigned
|
|
||||||
? bitWidth switch {
|
|
||||||
8 => obj => obj is sbyte v ? checked((ulong)v) : null,
|
|
||||||
16 => obj => obj is short v ? checked((ulong)v) : null,
|
|
||||||
32 => obj => obj is int v ? checked((ulong)v) : null,
|
|
||||||
64 => obj => obj is long v ? checked((ulong)v) : null,
|
|
||||||
_ => throw new NotImplementedException(),
|
|
||||||
}
|
|
||||||
: bitWidth switch {
|
|
||||||
8 => obj => obj is byte v ? v : null,
|
|
||||||
16 => obj => obj is ushort v ? v : null,
|
|
||||||
32 => obj => obj is uint v ? v : null,
|
|
||||||
64 => obj => obj is ulong v ? v : null,
|
|
||||||
_ => throw new NotImplementedException(),
|
|
||||||
};
|
|
||||||
|
|
||||||
public static string ColumnEnumTypeKey(string colName) =>
|
|
||||||
$"{colName}.EnumType";
|
|
||||||
|
|
||||||
public static KeyValuePair<string, string> ColumnEnumTypeEntry(string colName, Type enumType) {
|
|
||||||
if (enumType.IsNullable()) {
|
|
||||||
enumType = enumType.GetGenericArguments()[0];
|
|
||||||
}
|
|
||||||
if (!enumType.IsEnum) {
|
|
||||||
throw new ArgumentException($"Must be enum type (actually {enumType})", nameof(enumType));
|
|
||||||
}
|
|
||||||
return new(ColumnEnumTypeKey(colName), enumType.ToString());
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Type? GetColumnEnumType(IReadOnlyDictionary<string, string>? keyValueMetadata, string columnName) {
|
|
||||||
if (keyValueMetadata is not null) {
|
|
||||||
var key = ColumnEnumTypeKey(columnName);
|
|
||||||
if (keyValueMetadata.TryGetValue(key, out var typeName)) {
|
|
||||||
return Type.GetType(typeName);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
public interface IColumnWriter : IDisposable {
|
|
||||||
void Write(IReadOnlyCollection<object?> values);
|
|
||||||
}
|
|
||||||
|
|
||||||
public interface IColumnWriter<T> : IColumnWriter {
|
|
||||||
void Write(IReadOnlyCollection<T?> values);
|
|
||||||
}
|
|
||||||
|
|
||||||
class ColumnWriter<T> : IColumnWriter<T> {
|
|
||||||
readonly T?[] buffer;
|
|
||||||
readonly LogicalColumnWriter<T> writer;
|
|
||||||
readonly Func<object?, T?> convert;
|
|
||||||
|
|
||||||
public ColumnWriter(ColumnWriter writer, int bufferSize)
|
|
||||||
: this(writer, new T?[bufferSize]) { }
|
|
||||||
|
|
||||||
public ColumnWriter(ColumnWriter writer, int bufferSize, Func<object?, T?> convert)
|
|
||||||
: this(writer, new T?[bufferSize], convert) { }
|
|
||||||
|
|
||||||
public ColumnWriter(ColumnWriter writer, T?[] buffer)
|
|
||||||
: this(writer, buffer, obj => (T?)obj) { }
|
|
||||||
|
|
||||||
public ColumnWriter(ColumnWriter writer, T?[] buffer, Func<object?, T?> convert) {
|
|
||||||
this.buffer = buffer;
|
|
||||||
this.writer = writer.LogicalWriter<T>();
|
|
||||||
this.convert = convert;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void Write(IReadOnlyCollection<T?> values) {
|
|
||||||
if (buffer.Length == 0) {
|
|
||||||
throw new InvalidOperationException("Cannot write because buffer length is 0");
|
|
||||||
}
|
|
||||||
var count = values.Count;
|
|
||||||
var nPages = count / buffer.Length;
|
|
||||||
using var en = values.GetEnumerator();
|
|
||||||
for (int pg = 0; pg < nPages; pg++) {
|
|
||||||
for (int i = 0; i < buffer.Length; i++) {
|
|
||||||
Log.assert(en.MoveNext(), "Enumeration failed before reaching expected count");
|
|
||||||
buffer[i] = en.Current;
|
|
||||||
}
|
|
||||||
writer.WriteBatch(buffer!);
|
|
||||||
}
|
|
||||||
var remainder = count - nPages * buffer.Length;
|
|
||||||
if (remainder != 0) {
|
|
||||||
for (int i = 0; i < remainder; i++) {
|
|
||||||
Log.assert(en.MoveNext(), "Enumeration failed before reaching expected count");
|
|
||||||
buffer[i] = en.Current;
|
|
||||||
}
|
|
||||||
writer.WriteBatch(buffer!, 0, remainder);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public void Dispose() =>
|
|
||||||
writer.Dispose();
|
|
||||||
|
|
||||||
void IColumnWriter.Write(IReadOnlyCollection<object?> values) =>
|
|
||||||
Write(ReadOnlySelectorCollection.Create(values, convert));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,111 +0,0 @@
|
|||||||
namespace Bts.DataStream.Parquet;
|
|
||||||
|
|
||||||
using bts;
|
|
||||||
using bts.collections;
|
|
||||||
|
|
||||||
using ParquetSharp;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
|
|
||||||
class ParquetColumnReader<T> : IParquetColumnReader<T> {
|
|
||||||
protected readonly LogicalColumnReader<T> reader;
|
|
||||||
protected readonly T?[] buffer;
|
|
||||||
|
|
||||||
public ParquetColumnReader(ColumnReader reader, int bufferSize)
|
|
||||||
: this(reader, new T?[bufferSize]) { }
|
|
||||||
|
|
||||||
protected ParquetColumnReader(ColumnReader reader, T?[] buffer) {
|
|
||||||
this.buffer = buffer;
|
|
||||||
this.reader = reader.LogicalReader<T>();
|
|
||||||
}
|
|
||||||
|
|
||||||
~ParquetColumnReader() =>
|
|
||||||
Dispose();
|
|
||||||
|
|
||||||
public IList<T?> Read() {
|
|
||||||
if (buffer.Length == 0) {
|
|
||||||
throw new InvalidOperationException("Cannot write because buffer length is 0");
|
|
||||||
}
|
|
||||||
int n = reader.ReadBatch(buffer!);
|
|
||||||
return buffer.WrapSubRange(0, n);
|
|
||||||
}
|
|
||||||
|
|
||||||
public IList<T?> Read(int? start, int? stop) {
|
|
||||||
if (buffer.Length == 0) {
|
|
||||||
throw new InvalidOperationException("Cannot write because buffer length is 0");
|
|
||||||
}
|
|
||||||
start ??= 0;
|
|
||||||
stop ??= buffer.Length;
|
|
||||||
int length = stop.Value - start.Value;
|
|
||||||
if (length <= 0) {
|
|
||||||
return new List<T?>(0);
|
|
||||||
}
|
|
||||||
if (start.Value > 0) {
|
|
||||||
var nSkipped = checked((int)reader.Skip(start.Value));
|
|
||||||
Log.assert(nSkipped >= 0 && nSkipped <= start.Value);
|
|
||||||
if (nSkipped < start.Value) {
|
|
||||||
return new List<T?>(0);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
int n = reader.ReadBatch(buffer!, 0, length);
|
|
||||||
return buffer.WrapSubRange(0, n);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void Dispose() {
|
|
||||||
GC.SuppressFinalize(this);
|
|
||||||
reader.Dispose();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
class ParquetColumnReader<TIn, TOut> : IParquetColumnReader<TOut> {
|
|
||||||
readonly Func<TIn, TOut?> convert;
|
|
||||||
readonly LogicalColumnReader<TIn> reader;
|
|
||||||
readonly TIn[] buffer;
|
|
||||||
|
|
||||||
public ParquetColumnReader(ColumnReader reader, Func<TIn, TOut?> convert, int bufferSize)
|
|
||||||
: this(reader, convert, new TIn[bufferSize]) { }
|
|
||||||
|
|
||||||
public ParquetColumnReader(ColumnReader reader, Func<TIn, TOut?> convert, TIn[] buffer) {
|
|
||||||
this.convert = convert;
|
|
||||||
this.reader = reader.LogicalReader<TIn>();
|
|
||||||
this.buffer = buffer;
|
|
||||||
}
|
|
||||||
|
|
||||||
~ParquetColumnReader() =>
|
|
||||||
Dispose();
|
|
||||||
|
|
||||||
public IList<TOut?> Read() {
|
|
||||||
if (buffer.Length == 0) {
|
|
||||||
throw new InvalidOperationException("Cannot write because buffer length is 0");
|
|
||||||
}
|
|
||||||
int n = reader.ReadBatch(buffer!);
|
|
||||||
return buffer.WrapSelector(convert, null).WrapSubRange(0, n);
|
|
||||||
}
|
|
||||||
|
|
||||||
public IList<TOut?> Read(int? start, int? stop) {
|
|
||||||
if (buffer.Length == 0) {
|
|
||||||
throw new InvalidOperationException("Cannot write because buffer length is 0");
|
|
||||||
}
|
|
||||||
start ??= 0;
|
|
||||||
stop ??= buffer.Length;
|
|
||||||
int length = stop.Value - start.Value;
|
|
||||||
if (length <= 0) {
|
|
||||||
return new List<TOut?>(0);
|
|
||||||
}
|
|
||||||
if (start.Value > 0) {
|
|
||||||
var nSkipped = checked((int)reader.Skip(start.Value));
|
|
||||||
Log.assert(nSkipped >= 0 && nSkipped <= start.Value);
|
|
||||||
if (nSkipped < start.Value) {
|
|
||||||
return new List<TOut?>(0);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
int n = reader.ReadBatch(buffer!, 0, length);
|
|
||||||
return buffer.WrapSelector(convert, null).WrapSubRange(0, n);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void Dispose() {
|
|
||||||
GC.SuppressFinalize(this);
|
|
||||||
reader.Dispose();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,159 +0,0 @@
|
|||||||
namespace Bts.DataStream.Parquet;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using bts;
|
|
||||||
using bts.collections;
|
|
||||||
|
|
||||||
using Microsoft.Scripting.Utils;
|
|
||||||
|
|
||||||
using ParquetSharp;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
|
|
||||||
public static class ParquetExtensions {
|
|
||||||
public static int RowCount(this FileMetaData fileMetaData) =>
|
|
||||||
checked((int)fileMetaData.NumRows);
|
|
||||||
|
|
||||||
public static IBlueTableSchema ToBlueSchema(this SchemaDescriptor schemaDescriptor, string tableName = "parquet", IReadOnlyDictionary<string, string>? keyValueMetadata = null) {
|
|
||||||
var colInfos = new ColumnInfo[schemaDescriptor.NumColumns];
|
|
||||||
for (int i = 0; i < colInfos.Length; i++) {
|
|
||||||
colInfos[i] = schemaDescriptor.BlueColumn(i, keyValueMetadata);
|
|
||||||
}
|
|
||||||
return new BlueTableSchema(tableName, colInfos);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static IBlueTableSchema ToBlueSchema(this FileMetaData fileMetaData, string tableName = "parquet") =>
|
|
||||||
ToBlueSchema(fileMetaData.Schema, tableName, fileMetaData.KeyValueMetadata);
|
|
||||||
|
|
||||||
public static ColumnInfo BlueColumn(this SchemaDescriptor schemaDescriptor, int index, IReadOnlyDictionary<string, string>? keyValueMetadata = null) {
|
|
||||||
var c = schemaDescriptor.Column(index);
|
|
||||||
var lt = c.LogicalType;
|
|
||||||
if (lt.Type == LogicalTypeEnum.Enum) {
|
|
||||||
var enumType = ParquetAdapter.GetColumnEnumType(keyValueMetadata, c.Name);
|
|
||||||
if (enumType is not null) {
|
|
||||||
return new(c.Name, index, enumType);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return new(c.Name, index, getTypeCode(c.LogicalType, c.PhysicalType));
|
|
||||||
}
|
|
||||||
|
|
||||||
public static IBlueRowCollection ReadBlueTable(this ParquetFileReader fileReader, int? start = null, int? stop = null) {
|
|
||||||
var schema = fileReader.FileMetaData.ToBlueSchema("parquetReader");
|
|
||||||
var colInfos = schema.columnInfo;
|
|
||||||
IBlueRowCollection? rv;
|
|
||||||
if (!schema.columnInfo.Any(c => c.colName == "id")) {
|
|
||||||
rv = BlueAddIndexColumnTable.Create(schema, fileReader.ReadColumns(start, stop));
|
|
||||||
} else {
|
|
||||||
rv = new BlueIndexedColumnTable(schema, fileReader.ReadColumns(start, stop));
|
|
||||||
}
|
|
||||||
Log.assert(!schema.columnInfo.Any(c => c is null));
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static (Column[] Column, IReadOnlyDictionary<string, string> Meta) ToParquetSchema(this IBlueTableSchema schema) {
|
|
||||||
var blueColumns = schema.columnInfo;
|
|
||||||
var parquetColumns = new Column[blueColumns.Length];
|
|
||||||
var parquetMeta = new Dictionary<string, string>();
|
|
||||||
KeyValuePair<string, string>[] parquetColMeta;
|
|
||||||
for (int i = 0; i < blueColumns.Length; i++) {
|
|
||||||
(parquetColumns[i], parquetColMeta) = ToParquetColumn(blueColumns[i]);
|
|
||||||
parquetMeta.AddRange(parquetColMeta);
|
|
||||||
}
|
|
||||||
return (parquetColumns, parquetMeta);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static (Column Column, KeyValuePair<string, string>[] Meta) ToParquetColumn(this ColumnInfo columnInfo) {
|
|
||||||
var colType = columnInfo.colType;
|
|
||||||
bool isNullable = colType.IsNullable();
|
|
||||||
if (isNullable) {
|
|
||||||
colType = colType.GetGenericArguments()[0];
|
|
||||||
}
|
|
||||||
KeyValuePair<string, string>[] meta;
|
|
||||||
TypeCode typeCode;
|
|
||||||
var colName = columnInfo.colName;
|
|
||||||
if (colType.IsEnum) {
|
|
||||||
typeCode = colType.GetEnumUnderlyingType().GetTypeCode();
|
|
||||||
meta = new[] { ParquetAdapter.ColumnEnumTypeEntry(columnInfo.colName, colType) };
|
|
||||||
} else {
|
|
||||||
typeCode = colType.GetTypeCode();
|
|
||||||
meta = Array.Empty<KeyValuePair<string, string>>();
|
|
||||||
}
|
|
||||||
return (ToParquetColumn(colName, typeCode), meta);
|
|
||||||
|
|
||||||
static Column ToParquetColumn(string colName, TypeCode typeCode) =>
|
|
||||||
typeCode switch {
|
|
||||||
TypeCode.Boolean => new Column<bool?>(colName),
|
|
||||||
TypeCode.Byte => new Column<byte?>(colName),
|
|
||||||
TypeCode.Int16 => new Column<short?>(colName),
|
|
||||||
TypeCode.Int32 => new Column<int?>(colName),
|
|
||||||
TypeCode.Int64 => new Column<long?>(colName),
|
|
||||||
TypeCode.UInt16 => new Column<ushort?>(colName),
|
|
||||||
TypeCode.UInt32 => new Column<uint?>(colName),
|
|
||||||
TypeCode.UInt64 => new Column<ulong?>(colName),
|
|
||||||
TypeCode.Single => new Column<float?>(colName),
|
|
||||||
TypeCode.Double => new Column<double?>(colName),
|
|
||||||
TypeCode.DateTime => new Column<DateTimeNanos?>(colName),
|
|
||||||
TypeCode.String => new Column<string?>(colName),
|
|
||||||
_ => throw new NotSupportedException($"Unsupported column TypeCode {typeCode}"),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
static TypeCode getTypeCode(LogicalType logicalType, PhysicalType physicalType) {
|
|
||||||
return logicalType.Type switch {
|
|
||||||
// LogicalTypeEnum.Int => TypeCode.Int32,
|
|
||||||
// LogicalTypeEnum.Decimal => TypeCode.Double,
|
|
||||||
LogicalTypeEnum.Date or LogicalTypeEnum.Time or LogicalTypeEnum.Timestamp => TypeCode.DateTime,
|
|
||||||
LogicalTypeEnum.String => TypeCode.String,
|
|
||||||
LogicalTypeEnum.Int => FromInt(),
|
|
||||||
LogicalTypeEnum.Decimal => FromDecimal(),
|
|
||||||
_ => FromPhysicalType(),
|
|
||||||
};
|
|
||||||
|
|
||||||
TypeCode FromPhysicalType() =>
|
|
||||||
physicalType switch {
|
|
||||||
PhysicalType.Boolean => TypeCode.Boolean,
|
|
||||||
PhysicalType.Int32 => TypeCode.Int32,
|
|
||||||
PhysicalType.Int64 => TypeCode.Int64,
|
|
||||||
PhysicalType.Float => TypeCode.Single,
|
|
||||||
PhysicalType.Double => TypeCode.Double,
|
|
||||||
_ => throw new NotImplementedException($"TypeCode not implemented for {logicalType} (logical) {physicalType} (physical)"),
|
|
||||||
};
|
|
||||||
|
|
||||||
TypeCode FromInt() {
|
|
||||||
var ilt = (IntLogicalType)logicalType;
|
|
||||||
var bitWidth = ilt.BitWidth;
|
|
||||||
return ilt.IsSigned ? FromSignedInt() : FromUnsignedInt();
|
|
||||||
|
|
||||||
TypeCode FromSignedInt() =>
|
|
||||||
bitWidth switch {
|
|
||||||
8 => TypeCode.SByte,
|
|
||||||
16 => TypeCode.Int16,
|
|
||||||
32 => TypeCode.Int32,
|
|
||||||
64 => TypeCode.Int64,
|
|
||||||
_ => throw new NotSupportedException($"Bit width not supported for signed int ({bitWidth})"),
|
|
||||||
};
|
|
||||||
|
|
||||||
TypeCode FromUnsignedInt() =>
|
|
||||||
bitWidth switch {
|
|
||||||
8 => TypeCode.Byte,
|
|
||||||
16 => TypeCode.UInt16,
|
|
||||||
32 => TypeCode.UInt32,
|
|
||||||
64 => TypeCode.UInt64,
|
|
||||||
_ => throw new NotSupportedException($"Bit width not supported for unsigned int ({bitWidth})"),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
TypeCode FromDecimal() {
|
|
||||||
var dlt = (DecimalLogicalType)logicalType;
|
|
||||||
var scale = dlt.Scale;
|
|
||||||
var precision = dlt.Precision;
|
|
||||||
return scale switch {
|
|
||||||
0 => throw new NotSupportedException($"Scale not supported for decimal ({scale})"),
|
|
||||||
_ => TypeCode.Double,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,234 +0,0 @@
|
|||||||
namespace Bts.DataStream.Parquet;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using bts;
|
|
||||||
using bts.collections;
|
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using ParquetSharp;
|
|
||||||
|
|
||||||
using Spark.Core.Collections;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Diagnostics.CodeAnalysis;
|
|
||||||
|
|
||||||
public class ParquetGroupTable : IBlueRowCollection, IDisposable {
|
|
||||||
readonly RowGroupReader reader;
|
|
||||||
readonly IBlueTableSchema schema;
|
|
||||||
readonly bool leaveOpen;
|
|
||||||
readonly int fullCount;
|
|
||||||
readonly object syncRoot = new();
|
|
||||||
readonly int? start;
|
|
||||||
readonly int? stop;
|
|
||||||
readonly int startIndex;
|
|
||||||
readonly int stopIndex;
|
|
||||||
readonly int idColumnIndex;
|
|
||||||
readonly int idColumnIndexInFile;
|
|
||||||
readonly Func<int, int>? rowIdFactory;
|
|
||||||
DataCache? dataCacheBacking;
|
|
||||||
|
|
||||||
public ParquetGroupTable(
|
|
||||||
RowGroupReader reader,
|
|
||||||
IBlueTableSchema schema,
|
|
||||||
int? start = null,
|
|
||||||
int? stop = null,
|
|
||||||
int idColumnIndex = -1,
|
|
||||||
int idColumnIndexInFile = -1,
|
|
||||||
Func<int, int>? rowIdFactory = null,
|
|
||||||
bool leaveOpen = false
|
|
||||||
) {
|
|
||||||
this.reader = reader;
|
|
||||||
this.schema = schema;
|
|
||||||
this.start = start;
|
|
||||||
this.stop = stop;
|
|
||||||
this.idColumnIndex = idColumnIndex;
|
|
||||||
this.idColumnIndexInFile = idColumnIndexInFile;
|
|
||||||
this.rowIdFactory = rowIdFactory;
|
|
||||||
this.leaveOpen = leaveOpen;
|
|
||||||
fullCount = checked((int)reader.MetaData.NumRows);
|
|
||||||
Log.assert(fullCount >= 0);
|
|
||||||
startIndex = SliceList.GetStartIndex(start, fullCount, true);
|
|
||||||
stopIndex = SliceList.GetStopIndex(stop, fullCount, true);
|
|
||||||
stopIndex = Math.Max(stopIndex, startIndex);
|
|
||||||
Log.assert(
|
|
||||||
startIndex >= 0
|
|
||||||
&& startIndex <= fullCount
|
|
||||||
&& stopIndex >= 0
|
|
||||||
&& stopIndex <= fullCount
|
|
||||||
&& stopIndex >= startIndex);
|
|
||||||
}
|
|
||||||
|
|
||||||
~ParquetGroupTable() =>
|
|
||||||
Dispose();
|
|
||||||
|
|
||||||
public int Count => stopIndex - startIndex;
|
|
||||||
|
|
||||||
IBlueTableSchema IBlueRowCollection.schema => schema;
|
|
||||||
|
|
||||||
DataCache dataCache =>
|
|
||||||
EnsureInit();
|
|
||||||
|
|
||||||
public IBlueRow this[int id] => dataCache[id];
|
|
||||||
|
|
||||||
public bool containsID(int id) =>
|
|
||||||
dataCache.RowIdToIndex.ContainsKey(id);
|
|
||||||
|
|
||||||
public IBlueRow GetRow(int index) =>
|
|
||||||
dataCache.Rows[index];
|
|
||||||
|
|
||||||
public int IndexOf(IBlueRow item) {
|
|
||||||
int id = item.Id();
|
|
||||||
return dataCache.RowIdToIndex.TryGetValue(id, out var idx) && BlueRowEqualityComparer.Equals(dataCache.Rows[idx], item)
|
|
||||||
? idx
|
|
||||||
: -1;
|
|
||||||
}
|
|
||||||
|
|
||||||
public bool TryGetValue(int id, [NotNullWhen(true)] out IBlueRow? row) {
|
|
||||||
if (dataCache.RowIdToIndex.TryGetValue(id, out var idx)) {
|
|
||||||
row = dataCache.Rows[idx];
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
row = null;
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
public IEnumerator<IBlueRow> GetEnumerator() {
|
|
||||||
for (int i = 0; i < dataCache.Rows.Length; i++) {
|
|
||||||
yield return dataCache.Rows[i];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public void Dispose() {
|
|
||||||
GC.SuppressFinalize(this);
|
|
||||||
if (!leaveOpen) {
|
|
||||||
reader.Dispose();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
[MemberNotNull(nameof(dataCacheBacking))]
|
|
||||||
internal DataCache EnsureInit() {
|
|
||||||
if (dataCacheBacking.HasValue) {
|
|
||||||
return dataCacheBacking.Value;
|
|
||||||
}
|
|
||||||
lock (syncRoot) {
|
|
||||||
if (dataCacheBacking.HasValue) {
|
|
||||||
return dataCacheBacking.Value;
|
|
||||||
}
|
|
||||||
var rows = new IBlueRow[Count];
|
|
||||||
int numColumns = reader.MetaData.NumColumns;
|
|
||||||
var blueColumns = schema.columnInfo;
|
|
||||||
Log.assert((blueColumns.Length - numColumns) switch {
|
|
||||||
0 or 1 => true,
|
|
||||||
_ => false,
|
|
||||||
});
|
|
||||||
var columnValues = allocateColumnValues(
|
|
||||||
reader, blueColumns.Length, Count, idColumnIndex, idColumnIndexInFile, startIndex, stopIndex, rowIdFactory);
|
|
||||||
var rowIdToIndex = new Dictionary<int, int>();
|
|
||||||
for (int i = 0; i < rows.Length; i++) {
|
|
||||||
rows[i] = new BlueRowColumnWrapper(schema, i, columnValues);
|
|
||||||
}
|
|
||||||
dataCacheBacking = new(rows, rowIdToIndex, columnValues);
|
|
||||||
}
|
|
||||||
return dataCacheBacking.Value;
|
|
||||||
}
|
|
||||||
|
|
||||||
static IList<object?>[] allocateColumnValues(
|
|
||||||
RowGroupReader reader,
|
|
||||||
int numColumns,
|
|
||||||
int numRows,
|
|
||||||
int idColIdx,
|
|
||||||
int idColIdxInFile,
|
|
||||||
int startIndex,
|
|
||||||
int stopIndex,
|
|
||||||
Func<int, int>? idFactory
|
|
||||||
) {
|
|
||||||
if (idColIdx < 0 || idColIdx >= numColumns) {
|
|
||||||
throw new ArgumentException("Id column index out of bounds", nameof(idColIdx));
|
|
||||||
}
|
|
||||||
var columnValues = new IList<object?>[numColumns];
|
|
||||||
if (idColIdxInFile == -1) {
|
|
||||||
// Generate ids (there is no id column in the file)
|
|
||||||
ArgumentNullException.ThrowIfNull(idFactory);
|
|
||||||
for (int i = 0; i < numColumns; i++) {
|
|
||||||
if (i < idColIdx) {
|
|
||||||
ReadColumn(i, i);
|
|
||||||
} else if (i == idColIdx) {
|
|
||||||
columnValues[idColIdx] = new StepList<int>(0, 1, numRows).WrapSelector(idx => (object?)idFactory(idx), null);
|
|
||||||
} else {
|
|
||||||
ReadColumn(i - 1, i);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else if (idFactory is not null) {
|
|
||||||
// Generate ids (skip id column in the file)
|
|
||||||
for (int i = 0; i < numColumns; i++) {
|
|
||||||
if (i == idColIdx) {
|
|
||||||
columnValues[idColIdx] = new StepList<int>(0, 1, numRows).WrapSelector(idx => (object?)idFactory(idx), null);
|
|
||||||
} else {
|
|
||||||
if (i > idColIdx && i < idColIdxInFile) {
|
|
||||||
ReadColumn(i - 1, i);
|
|
||||||
} else if (i > idColIdxInFile && i < idColIdx) {
|
|
||||||
ReadColumn(i + 1, i);
|
|
||||||
} else {
|
|
||||||
ReadColumn(i, i);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// Get ids from the file (output table id column index may differ from file)
|
|
||||||
// Case 1) File id col index > output
|
|
||||||
// src cols: a, b, c, d, (id), e, f, ...
|
|
||||||
// dst cols: a, b, (id), c, d, e, f, ...
|
|
||||||
//
|
|
||||||
// Case 2) File id col index == output
|
|
||||||
// src cols: a, b, (id), c, d, e, f, ...
|
|
||||||
// dst cols: a, b, (id), c, d, e, f, ...
|
|
||||||
//
|
|
||||||
// Case 3) File id col index < output
|
|
||||||
// src cols: a, b, (id), c, d, e, f, ...
|
|
||||||
// dst cols: a, b, c, d, (id), e, f, ...
|
|
||||||
|
|
||||||
for (int i = 0; i < numColumns; i++) {
|
|
||||||
if (i == idColIdx) {
|
|
||||||
ReadColumn(idColIdxInFile, idColIdx);
|
|
||||||
} else if (i > idColIdx && i < idColIdxInFile) {
|
|
||||||
ReadColumn(i - 1, i);
|
|
||||||
} else if (i > idColIdxInFile && i < idColIdx) {
|
|
||||||
ReadColumn(i, i - 1);
|
|
||||||
} else {
|
|
||||||
ReadColumn(i, i);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return columnValues;
|
|
||||||
|
|
||||||
void ReadColumn(int srcIdx, int dstIdx) {
|
|
||||||
using var colReader = reader.GetColumnReader(srcIdx);
|
|
||||||
columnValues[dstIdx] = colReader.Read(startIndex, stopIndex);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
IEnumerator IEnumerable.GetEnumerator() =>
|
|
||||||
GetEnumerator();
|
|
||||||
|
|
||||||
internal readonly struct DataCache {
|
|
||||||
public readonly IBlueRow[] Rows;
|
|
||||||
public readonly Dictionary<int, int> RowIdToIndex;
|
|
||||||
public readonly IList<object?>[] ColumnValues;
|
|
||||||
|
|
||||||
public DataCache(IBlueRow[] rows, Dictionary<int, int> rowIdToIndex, IList<object?>[] columnValues) {
|
|
||||||
Rows = rows;
|
|
||||||
RowIdToIndex = rowIdToIndex;
|
|
||||||
ColumnValues = columnValues;
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow this[int id] =>
|
|
||||||
Rows[RowIdToIndex[id]];
|
|
||||||
|
|
||||||
public IBlueRow GetRow(int index) =>
|
|
||||||
Rows[index];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,65 +0,0 @@
|
|||||||
namespace Bts.DataStream.Parquet;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using bts;
|
|
||||||
using bts.collections;
|
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
using Bts.PubSub;
|
|
||||||
|
|
||||||
using Spark.Core.Disposables;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Threading;
|
|
||||||
|
|
||||||
[DataStreamUserRespresentation(false)]
|
|
||||||
public class ParquetReaderStream : ViewStream {
|
|
||||||
readonly string file;
|
|
||||||
readonly int? start;
|
|
||||||
readonly int? stop;
|
|
||||||
readonly IWorkScheduler workScheduler;
|
|
||||||
IBlueRowCollection? table;
|
|
||||||
IDisposable? workerSub;
|
|
||||||
|
|
||||||
public ParquetReaderStream(string file, IWorkScheduler workScheduler) {
|
|
||||||
this.file = file;
|
|
||||||
this.workScheduler = workScheduler;
|
|
||||||
}
|
|
||||||
|
|
||||||
public ParquetReaderStream(string file, int? start, int? stop, IWorkScheduler workScheduler) {
|
|
||||||
this.file = file;
|
|
||||||
this.start = start;
|
|
||||||
this.stop = stop;
|
|
||||||
this.workScheduler = workScheduler;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
Start();
|
|
||||||
var worker = new AnonymousWorker(DoWork);
|
|
||||||
workScheduler.Add(worker);
|
|
||||||
workerSub = Disposable.Create(() => workScheduler.Remove(worker));
|
|
||||||
|
|
||||||
async void Start() {
|
|
||||||
try {
|
|
||||||
Volatile.Write(ref table, await ParquetTableReader.ReadAsync(file, start, stop));
|
|
||||||
} catch (Exception ex) {
|
|
||||||
Log.DataFlow.error($"[parquet_reader_stream_err.json]{{ \"err\": \"{ex}\" }}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void DoWork() {
|
|
||||||
var table = Volatile.Read(ref this.table);
|
|
||||||
if (table is null) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
Disposable.dispose_thread_safe(ref workerSub);
|
|
||||||
if (table.schema.columnInfo.Length > 1) {
|
|
||||||
myFireEvent(new DataPacket(table, table, new BlueTableImmutable(table.schema)));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() =>
|
|
||||||
Disposable.dispose_thread_safe(ref workerSub);
|
|
||||||
}
|
|
||||||
@@ -1,26 +0,0 @@
|
|||||||
// namespace Bts.DataStream.Parquet;
|
|
||||||
|
|
||||||
// using BCG;
|
|
||||||
|
|
||||||
// using global::Parquet.Data;
|
|
||||||
|
|
||||||
// public class ParquetRow : IBlueRow {
|
|
||||||
// readonly int id;
|
|
||||||
// readonly int rowIndex;
|
|
||||||
// readonly DataColumn[] data;
|
|
||||||
|
|
||||||
// public ParquetRow(int id, int rowIndex, DataColumn[] data, IBlueTableSchema schema) {
|
|
||||||
// this.id = id;
|
|
||||||
// this.rowIndex = rowIndex;
|
|
||||||
// this.data = data;
|
|
||||||
// Schema = schema;
|
|
||||||
// }
|
|
||||||
|
|
||||||
// public object? this[int index] =>
|
|
||||||
// index == 0
|
|
||||||
// ? id
|
|
||||||
// : data[index - 1].Data.GetValue(rowIndex);
|
|
||||||
|
|
||||||
// public IBlueTableSchema Schema { get; }
|
|
||||||
// IBlueTableSchema IBlueRow.schema => Schema;
|
|
||||||
// }
|
|
||||||
@@ -1,145 +0,0 @@
|
|||||||
namespace Bts.DataStream.Parquet;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using bts;
|
|
||||||
using bts.collections;
|
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using ParquetSharp;
|
|
||||||
|
|
||||||
using Spark.Core.Collections;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
|
|
||||||
public class ParquetTable : IBlueRowCollection, IDisposable {
|
|
||||||
readonly SortedList<int, ParquetGroupTable> groups = new();
|
|
||||||
readonly int fullCount;
|
|
||||||
readonly IBlueTableSchema schema;
|
|
||||||
readonly ParquetFileReader? reader;
|
|
||||||
readonly int startIndex;
|
|
||||||
readonly int stopIndex;
|
|
||||||
|
|
||||||
public ParquetTable(ParquetFileReader reader, string tableName = "parquet", int? start = null, int? stop = null, bool leaveOpen = false) {
|
|
||||||
schema = reader.FileMetaData.ToBlueSchema(tableName);
|
|
||||||
int idColIdx;
|
|
||||||
if (schema.TryResolveIndices("id", out int idColIdxInFile)) {
|
|
||||||
idColIdx = idColIdxInFile;
|
|
||||||
} else {
|
|
||||||
var origCols = schema.columnInfo;
|
|
||||||
var cols = new ColumnInfo[origCols.Length + 1];
|
|
||||||
idColIdx = 0;
|
|
||||||
cols[0] = new("id", 0, typeof(int));
|
|
||||||
for (int i = 0; i < origCols.Length; i++) {
|
|
||||||
var c = origCols[i];
|
|
||||||
cols[i + 1] = new(c.colName, i + 1, c.colType);
|
|
||||||
}
|
|
||||||
schema = new BlueTableSchema(cols, schema.tableName);
|
|
||||||
}
|
|
||||||
fullCount = checked((int)reader.FileMetaData.NumRows);
|
|
||||||
Log.assert(fullCount >= 0);
|
|
||||||
startIndex = SliceList.GetStartIndex(start, fullCount, true);
|
|
||||||
stopIndex = SliceList.GetStopIndex(stop, fullCount, true);
|
|
||||||
stopIndex = Math.Max(stopIndex, startIndex);
|
|
||||||
Log.assert(startIndex >= 0 && startIndex <= fullCount && stopIndex >= 0 && stopIndex <= fullCount && stopIndex >= startIndex);
|
|
||||||
int numRowGroups = reader.FileMetaData.NumRowGroups;
|
|
||||||
int rowGroupLastIdx = 0;
|
|
||||||
object syncRoot = new();
|
|
||||||
for (int i = 0; i < numRowGroups && rowGroupLastIdx < Count; i++) {
|
|
||||||
int? grpStart = start.HasValue ? startIndex - rowGroupLastIdx : null;
|
|
||||||
int? grpStop = stop.HasValue ? stopIndex - rowGroupLastIdx : null;
|
|
||||||
int offset = rowGroupLastIdx;
|
|
||||||
var grp = new ParquetGroupTable(reader.RowGroup(i), schema, grpStart, grpStop, idColIdx, idColIdxInFile, GetAutoId);
|
|
||||||
if (grp.Count == 0) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
rowGroupLastIdx += grp.Count;
|
|
||||||
groups.Add(rowGroupLastIdx, grp);
|
|
||||||
|
|
||||||
int GetAutoId(int index) =>
|
|
||||||
index + offset;
|
|
||||||
}
|
|
||||||
Parallel.For(0, groups.Count, i => groups.GetValueAtIndex(i).EnsureInit());
|
|
||||||
if (!leaveOpen) {
|
|
||||||
this.reader = reader;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
~ParquetTable() =>
|
|
||||||
Dispose();
|
|
||||||
|
|
||||||
public IBlueTableSchema Schema => schema;
|
|
||||||
|
|
||||||
public int Count => stopIndex - startIndex;
|
|
||||||
|
|
||||||
IBlueTableSchema IBlueRowCollection.schema => schema;
|
|
||||||
|
|
||||||
public IBlueRow this[int index] {
|
|
||||||
get {
|
|
||||||
foreach (var grp in groups.Values) {
|
|
||||||
if (grp.TryGetValue(index, out var r)) {
|
|
||||||
return r;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
throw new KeyNotFoundException();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static ParquetTable FromFile(string path, int? start = null, int? stop = null, string tableName = "parquet") =>
|
|
||||||
new(new(path), tableName, start, stop, false);
|
|
||||||
|
|
||||||
public IBlueRow GetRow(int index) {
|
|
||||||
if (index < 0 || index >= Count) {
|
|
||||||
throw new IndexOutOfRangeException();
|
|
||||||
}
|
|
||||||
int rowGroupIdx = groups.Keys.BinarySearch(index);
|
|
||||||
if (rowGroupIdx < 0) {
|
|
||||||
rowGroupIdx = ~rowGroupIdx;
|
|
||||||
}
|
|
||||||
Log.assert(rowGroupIdx <= groups.Count);
|
|
||||||
var grp = groups[rowGroupIdx];
|
|
||||||
int idxWithinGrp = index;
|
|
||||||
if (rowGroupIdx > 0) {
|
|
||||||
idxWithinGrp -= groups.Keys[rowGroupIdx - 1];
|
|
||||||
}
|
|
||||||
return grp[idxWithinGrp];
|
|
||||||
}
|
|
||||||
|
|
||||||
public bool containsID(int id) =>
|
|
||||||
groups.Values.Any(rgl => rgl.containsID(id));
|
|
||||||
|
|
||||||
public int IndexOf(IBlueRow item) {
|
|
||||||
int idx;
|
|
||||||
int grpFirstIdx = 0;
|
|
||||||
foreach (var (grpLastIdx, grp) in groups) {
|
|
||||||
idx = grp.IndexOf(item);
|
|
||||||
if (idx >= 0) {
|
|
||||||
return idx + grpFirstIdx;
|
|
||||||
}
|
|
||||||
grpFirstIdx = grpLastIdx;
|
|
||||||
}
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
|
|
||||||
public IEnumerator<IBlueRow> GetEnumerator() {
|
|
||||||
foreach (var rgl in groups.Values) {
|
|
||||||
foreach (var r in rgl) {
|
|
||||||
yield return r;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public void Dispose() {
|
|
||||||
GC.SuppressFinalize(this);
|
|
||||||
groups.Values.DisposeAll();
|
|
||||||
reader?.Dispose();
|
|
||||||
}
|
|
||||||
|
|
||||||
IEnumerator IEnumerable.GetEnumerator() =>
|
|
||||||
GetEnumerator();
|
|
||||||
}
|
|
||||||
@@ -1,32 +0,0 @@
|
|||||||
#nullable enable
|
|
||||||
namespace Bts.DataStream.Parquet;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using ParquetSharp;
|
|
||||||
|
|
||||||
using System.IO;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
|
|
||||||
public class ParquetTableReader {
|
|
||||||
public static Task<IBlueRowCollection> ReadAsync(Stream stream, bool leaveOpen = false, int? start = null, int? stop = null) =>
|
|
||||||
Task.Run(() => Read(stream, leaveOpen, start, stop));
|
|
||||||
|
|
||||||
public static Task<IBlueRowCollection> ReadAsync(string path, int? start = null, int? stop = null) =>
|
|
||||||
Task.Run(() => Read(path, start, stop));
|
|
||||||
|
|
||||||
public static IBlueRowCollection Read(Stream stream, bool leaveOpen = false, int? start = null, int? stop = null) {
|
|
||||||
using var fileReader = new ParquetFileReader(stream, leaveOpen);
|
|
||||||
var rv = fileReader.ReadBlueTable(start, stop);
|
|
||||||
rv.schema.Validate();
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static IBlueRowCollection Read(string path, int? start = null, int? stop = null) {
|
|
||||||
IBlueRowCollection rv = ParquetTable.FromFile(path, start, stop);
|
|
||||||
rv.schema.Validate();
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,38 +0,0 @@
|
|||||||
namespace Bts.DataStream.Parquet;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using bts;
|
|
||||||
|
|
||||||
using ParquetSharp;
|
|
||||||
|
|
||||||
using System.IO;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
|
|
||||||
public static class ParquetTableWriter {
|
|
||||||
public static Task WriteAsync(IBlueRowCollection blueRows, Stream outStream, bool leaveOpen = false) =>
|
|
||||||
Task.Run(() => Write(blueRows, outStream, leaveOpen));
|
|
||||||
|
|
||||||
public static void Write(IBlueRowCollection blueRows, Stream outStream, bool leaveOpen = false) {
|
|
||||||
var (parquetColumns, parquetMeta) = blueRows.schema.ToParquetSchema();
|
|
||||||
using var schema = Column.CreateSchemaNode(parquetColumns);
|
|
||||||
using var writerProps = createWriterProperties();
|
|
||||||
using var writer = new ParquetFileWriter(outStream, schema, writerProps, keyValueMetadata: parquetMeta, leaveOpen: leaveOpen);
|
|
||||||
using var rowGroup = writer.AppendRowGroup();
|
|
||||||
for (int i = 0; i < blueRows.schema.columnInfo.Length; i++) {
|
|
||||||
var col = blueRows.schema.columnInfo[i];
|
|
||||||
Log.assert(col.colNo == i);
|
|
||||||
using var colWriter = ParquetAdapter.GetColumnWriter(col.type_code(), rowGroup.NextColumn());
|
|
||||||
colWriter.Write(blueRows.GetColumn(i));
|
|
||||||
}
|
|
||||||
writer.Close();
|
|
||||||
}
|
|
||||||
|
|
||||||
static WriterProperties createWriterProperties(Compression compression = Compression.Snappy) {
|
|
||||||
using var builder = new WriterPropertiesBuilder();
|
|
||||||
builder.Compression(compression);
|
|
||||||
return builder.Build();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,87 +0,0 @@
|
|||||||
namespace Bts.DataStream.Parquet;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using bts.collections;
|
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
using Bts.PubSub;
|
|
||||||
|
|
||||||
using mbts.utils;
|
|
||||||
|
|
||||||
using Spark.Core.Disposables;
|
|
||||||
using Spark.Core.Threading;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.IO;
|
|
||||||
|
|
||||||
[DataStreamUserRespresentation(false)]
|
|
||||||
public class ParquetWriterStream : ViewStream {
|
|
||||||
readonly string file;
|
|
||||||
readonly ViewStream inputStream;
|
|
||||||
readonly IWorkScheduler workScheduler;
|
|
||||||
readonly AtomicGate gate = new();
|
|
||||||
IDataPacket? pendingOutput;
|
|
||||||
IDisposable? dispose;
|
|
||||||
|
|
||||||
public ParquetWriterStream(ViewStream inputStream, string file, IWorkScheduler workScheduler) {
|
|
||||||
this.inputStream = inputStream;
|
|
||||||
this.file = file;
|
|
||||||
this.workScheduler = workScheduler;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
BlueTableImmutable.DataPacketBuilder? outputBuilder = null;
|
|
||||||
bool isFirstUpdate = false;
|
|
||||||
var worker = new AnonymousWorker(DoWork);
|
|
||||||
inputStream.onChange += OnInputStreamChange;
|
|
||||||
workScheduler.Add(worker);
|
|
||||||
dispose = Disposable.Create(Dispose);
|
|
||||||
|
|
||||||
void Dispose() {
|
|
||||||
workScheduler.Remove(worker);
|
|
||||||
inputStream.onChange -= OnInputStreamChange;
|
|
||||||
}
|
|
||||||
|
|
||||||
void OnInputStreamChange(object? dpObj) {
|
|
||||||
var dp = ExceptionAssert.Argument.Is<IDataPacket>(dpObj);
|
|
||||||
using (gate.BeginTransaction()) {
|
|
||||||
outputBuilder ??= new BlueTableImmutable(dp.Schema).ToDataPacketBuilder();
|
|
||||||
foreach (IBlueRow r in dp.addedRows) {
|
|
||||||
outputBuilder.AddOrUpdate(r);
|
|
||||||
}
|
|
||||||
foreach (IBlueRow r in dp.deletedRows) {
|
|
||||||
outputBuilder.Remove(r.Id());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async void DoWork() {
|
|
||||||
using (gate.BeginTransaction()) {
|
|
||||||
if (outputBuilder is null) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (pendingOutput is not null) {
|
|
||||||
myFireEvent(pendingOutput);
|
|
||||||
pendingOutput = null;
|
|
||||||
}
|
|
||||||
var dp = outputBuilder.ToDataPacket();
|
|
||||||
if (isFirstUpdate) {
|
|
||||||
isFirstUpdate = false;
|
|
||||||
} else if (!dp.HasChanges()) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
{
|
|
||||||
using var outStream = File.OpenWrite(file);
|
|
||||||
await ParquetTableWriter.WriteAsync(dp.newRows, outStream);
|
|
||||||
}
|
|
||||||
// Fire the update at the beginning of the next doWork call so it happens on the
|
|
||||||
// 'work' thread.
|
|
||||||
pendingOutput = dp;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() =>
|
|
||||||
Disposable.dispose_thread_safe(ref dispose);
|
|
||||||
}
|
|
||||||
@@ -1,188 +0,0 @@
|
|||||||
namespace bts.BlueOrb {
|
|
||||||
using BCG;
|
|
||||||
using System;
|
|
||||||
using System.Collections;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
|
|
||||||
#region mappings
|
|
||||||
internal class ProjectData {
|
|
||||||
// schema from the base stream
|
|
||||||
public readonly IBlueTableSchema in_schema;
|
|
||||||
// this is the projected schema
|
|
||||||
public readonly IBlueTableSchema out_schema;
|
|
||||||
// projected cols to the input cols, for cols who's names are changed
|
|
||||||
public readonly IDictionary<string, string> mapped_to_orig_col_map = new Dictionary<string, string>();
|
|
||||||
public ProjectData(IBlueTableSchema in_schema_, IBlueTableSchema out_schema_,
|
|
||||||
IDictionary<string, string> mapped_to_orig_col_map_) {
|
|
||||||
in_schema = in_schema_;
|
|
||||||
out_schema = out_schema_;
|
|
||||||
mapped_to_orig_col_map = mapped_to_orig_col_map_;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
|
|
||||||
#region Row with mappings build in
|
|
||||||
internal class NewProjectRow : IBlueRow {
|
|
||||||
#region IBlueRow members
|
|
||||||
private ProjectData _project_data;
|
|
||||||
private IBlueRow _row;
|
|
||||||
public NewProjectRow(ProjectData project_data, IBlueRow row) {
|
|
||||||
_project_data = project_data;
|
|
||||||
_row = row;
|
|
||||||
}
|
|
||||||
public IBlueTableSchema schema { get { return _project_data.out_schema; } }
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
#region ICollection members
|
|
||||||
public int Count { get { return schema.columnNames.Length; } }
|
|
||||||
|
|
||||||
public bool IsSynchronized { get { return false; } }
|
|
||||||
|
|
||||||
public object SyncRoot { get { return this; } }
|
|
||||||
|
|
||||||
public void CopyTo(Array array, int index) {
|
|
||||||
int i = index;
|
|
||||||
foreach (object o in this as IEnumerable)
|
|
||||||
array.SetValue(o, i++);
|
|
||||||
}
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
#region IDictionary members
|
|
||||||
public bool IsFixedSize { get { return true; } }
|
|
||||||
|
|
||||||
public bool IsReadOnly { get { return true; } }
|
|
||||||
|
|
||||||
public object this[object key] {
|
|
||||||
get {
|
|
||||||
if (key is string)
|
|
||||||
return this[(string)key];
|
|
||||||
return this[(int)key];
|
|
||||||
}
|
|
||||||
set { throw new NotSupportedException("The IBlueRow is readonly"); }
|
|
||||||
}
|
|
||||||
|
|
||||||
public object this[int index] {
|
|
||||||
get {
|
|
||||||
return this[schema.columnInfo[index].colName];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public object this[string col_name] {
|
|
||||||
get {
|
|
||||||
string mapped_to_col = null;
|
|
||||||
if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out mapped_to_col)) {
|
|
||||||
return _row[mapped_to_col];
|
|
||||||
}
|
|
||||||
return _row[col_name];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public ICollection Keys { get { return schema.columnNames; } }
|
|
||||||
|
|
||||||
public ICollection Values {
|
|
||||||
get {
|
|
||||||
object[] retVal = new object[schema.columnNames.Length];
|
|
||||||
CopyTo(retVal, 0);
|
|
||||||
return retVal;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
|
|
||||||
|
|
||||||
public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
|
|
||||||
|
|
||||||
public bool Contains(object key) { return schema[(string)key] != null; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
|
|
||||||
/// iterate through the values, typecast this into an IEnumerable
|
|
||||||
/// </summary>
|
|
||||||
/// <returns></returns>
|
|
||||||
public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
|
|
||||||
|
|
||||||
IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
|
|
||||||
|
|
||||||
public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
|
|
||||||
#endregion
|
|
||||||
}
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
[DataStreamUserRespresentation("ProjectStream")]
|
|
||||||
public class NewProjectStream : ViewStream, IStreamMapper {
|
|
||||||
|
|
||||||
private ViewStream _stream;
|
|
||||||
private string _project_expression;
|
|
||||||
private ProjectData _project_data = null;
|
|
||||||
//public ProjectData
|
|
||||||
public NewProjectStream(ViewStream vs, string project_expression) {
|
|
||||||
_stream = vs;
|
|
||||||
_project_expression = project_expression;
|
|
||||||
_stream = new MapStream(vs, this);
|
|
||||||
mySetRealSource(_stream);
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) {
|
|
||||||
_project_data = _create_project_data(input_schema, _project_expression);
|
|
||||||
return _project_data.out_schema;
|
|
||||||
}
|
|
||||||
|
|
||||||
public IBlueRow map(IBlueRow row) {
|
|
||||||
return new NewProjectRow(_project_data, row);
|
|
||||||
}
|
|
||||||
|
|
||||||
// this is where most of the logic is, in terms of translating cols
|
|
||||||
private static ProjectData _create_project_data(IBlueTableSchema input_schema,
|
|
||||||
string project_expression) {
|
|
||||||
project_expression = project_expression.Trim();
|
|
||||||
string[] project_toks = project_expression.Split(' ', ',');
|
|
||||||
HashSet<string> removed_cols = new HashSet<string>();
|
|
||||||
Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
|
|
||||||
// by default every col will show up, so if we have the projection as (a:a1)
|
|
||||||
// it will convert a to a1,
|
|
||||||
// modifications (6/30/2025):
|
|
||||||
// only the columns represented here, either mapped
|
|
||||||
// or not, will show
|
|
||||||
// examples: if a schema has cols a b c
|
|
||||||
// project(parent_str, "a:aa b") -> aa b
|
|
||||||
foreach (string o_p_tok in project_toks) {
|
|
||||||
string p_tok = o_p_tok.Trim();
|
|
||||||
if (p_tok.Contains(":")) {
|
|
||||||
// explicit mapping
|
|
||||||
string[] map_toks = p_tok.Split(":");
|
|
||||||
string orig_col = map_toks[0];
|
|
||||||
string mapped_col = map_toks[1];
|
|
||||||
orig_to_mapped.Add(orig_col, mapped_col);
|
|
||||||
} else {
|
|
||||||
// this is a col_name explicitely stated
|
|
||||||
if (input_schema.columnNames.Contains(p_tok)) {
|
|
||||||
string orig_col = p_tok;
|
|
||||||
string mapped_col = orig_col;
|
|
||||||
orig_to_mapped.Add(orig_col, mapped_col);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// make the new schema
|
|
||||||
List<ColumnInfo> new_cols = new List<ColumnInfo>();
|
|
||||||
foreach (ColumnInfo ci in input_schema.columnInfo) {
|
|
||||||
if (ci.colName == "id") {
|
|
||||||
ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
|
|
||||||
new_cols.Add(nc);
|
|
||||||
continue;
|
|
||||||
} else if (orig_to_mapped.ContainsKey(ci.colName)) {
|
|
||||||
string mapped_name = orig_to_mapped[ci.colName];
|
|
||||||
ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
|
|
||||||
new_cols.Add(nc);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
|
|
||||||
var mapped_to_orig_map = orig_to_mapped.ToDictionary(kv => kv.Value, kv => kv.Key);
|
|
||||||
return new ProjectData(input_schema, new_schema, mapped_to_orig_map);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,128 +0,0 @@
|
|||||||
namespace bts.BlueOrb {
|
|
||||||
using BCG;
|
|
||||||
using System.IO;
|
|
||||||
using System.Diagnostics;
|
|
||||||
using System;
|
|
||||||
|
|
||||||
[DataStreamUserRespresentation("PythonDataframeStream")]
|
|
||||||
public class PythonDataframeStream : ViewStream {
|
|
||||||
private ViewStream _parent_str;
|
|
||||||
private readonly string _in_csv_file_name;
|
|
||||||
private readonly string _out_csv_filename;
|
|
||||||
private readonly string _transform_function;
|
|
||||||
public PythonDataframeStream(ViewStream parent_str, string transform_function) {
|
|
||||||
var guid = System.Guid.NewGuid();
|
|
||||||
_in_csv_file_name = $"pdfs_in_{guid}.csv";
|
|
||||||
_out_csv_filename = $"pdfs_out_{guid}.csv";
|
|
||||||
_parent_str = new CSVSaverStream(parent_str, _in_csv_file_name);
|
|
||||||
_transform_function = transform_function;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
_parent_str.onChange += _parent_str_onchange;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
|
||||||
_parent_str.onChange -= _parent_str_onchange;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _parent_str_onchange(object dp_obj) {
|
|
||||||
Log.assert(dp_obj != null);
|
|
||||||
// the file _in_csv_file_name is now populated.
|
|
||||||
// call py_func_wrapper_main and pass this file name as well
|
|
||||||
// as the output file name
|
|
||||||
// py_func_wrapper_main reads _in_csv_file_name into the df and calls
|
|
||||||
// transform function, which returns the tranformed df. py_func_wrapper_main then
|
|
||||||
// writes it into the _out_csv_filename file.
|
|
||||||
// now this function should read _out_csv_filename into a dp
|
|
||||||
// and fire it
|
|
||||||
_compute_and_write();
|
|
||||||
//ViewStream rs = new SchemaGuesserStream(new CSVReaderStream(_out_csv_filename));
|
|
||||||
ViewStream rs = new CSVReaderStream(_out_csv_filename);
|
|
||||||
IBlueRowCollection nrows = null;
|
|
||||||
rs.onChange += (dp_obj) => {
|
|
||||||
nrows = (dp_obj as DataPacket).newRows;
|
|
||||||
};
|
|
||||||
var dp = BlueDataUtility.build_dp(nrows, myLastValue as DataPacket);
|
|
||||||
myFireEvent(dp);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _compute_and_write() {
|
|
||||||
File.WriteAllText("user_code.py", _transform_function);
|
|
||||||
// wrapper function
|
|
||||||
string py_func_wrapper_main = @"
|
|
||||||
import sys
|
|
||||||
from user_code import transform
|
|
||||||
import pandas as pd
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
input_file_name = sys.argv[1]
|
|
||||||
output_file_name = sys.argv[2]
|
|
||||||
# 1. load input_file_name into df
|
|
||||||
# 2. do some test calc
|
|
||||||
# 3. write the output of 2 into output_file_name
|
|
||||||
df = pd.read_csv(input_file_name)
|
|
||||||
df_out = transform(df)
|
|
||||||
df_out.to_csv(output_file_name, index=False)";
|
|
||||||
|
|
||||||
|
|
||||||
File.WriteAllText("runner.py", py_func_wrapper_main);
|
|
||||||
|
|
||||||
ProcessStartInfo psi = new ProcessStartInfo {
|
|
||||||
FileName = "python",
|
|
||||||
Arguments = $"runner.py {_in_csv_file_name} {_out_csv_filename}",
|
|
||||||
RedirectStandardOutput = false,
|
|
||||||
UseShellExecute = false
|
|
||||||
};
|
|
||||||
var proc = Process.Start(psi);
|
|
||||||
proc.WaitForExit();
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
#region Unused test code
|
|
||||||
private void _simple_test() {
|
|
||||||
|
|
||||||
// actual function that needs to be called ////////////////////////
|
|
||||||
string py_real_function = @"
|
|
||||||
def test(x, out_path):
|
|
||||||
rv = x * 10
|
|
||||||
with open(out_path, 'w') as f:
|
|
||||||
f.write(str(rv))
|
|
||||||
return rv";
|
|
||||||
// the function is in a string object.
|
|
||||||
File.WriteAllText("user_code.py", py_real_function);
|
|
||||||
|
|
||||||
|
|
||||||
// runner app (top level main function) //////////////////////////////
|
|
||||||
string py_func_wrapper_main = @"
|
|
||||||
import sys
|
|
||||||
from user_code import test
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
arg = int(sys.argv[1])
|
|
||||||
out_path = sys.argv[2]
|
|
||||||
rv = test(arg, out_path)
|
|
||||||
rv = rv + 12
|
|
||||||
with open(out_path, 'w') as f:to \
|
|
||||||
f.write(str(rv))";
|
|
||||||
|
|
||||||
File.WriteAllText("runner.py", py_func_wrapper_main);
|
|
||||||
|
|
||||||
string outputPath = "py_out.txt";
|
|
||||||
ProcessStartInfo psi = new ProcessStartInfo {
|
|
||||||
FileName = "python",
|
|
||||||
Arguments = $"runner.py 4 {outputPath}",
|
|
||||||
RedirectStandardOutput = false,
|
|
||||||
UseShellExecute = false
|
|
||||||
};
|
|
||||||
var proc = Process.Start(psi);
|
|
||||||
proc.WaitForExit();
|
|
||||||
|
|
||||||
// Now read the output written by the function
|
|
||||||
string result = File.ReadAllText(outputPath);
|
|
||||||
Console.WriteLine(result); // Should print "6"
|
|
||||||
|
|
||||||
}
|
|
||||||
#endregion
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,30 +0,0 @@
|
|||||||
using BCG;
|
|
||||||
|
|
||||||
namespace bts.BlueOrb {
|
|
||||||
public class SchemaGuesserStream : ViewStream {
|
|
||||||
|
|
||||||
private IBlueRowCollection _rows = null;
|
|
||||||
private ViewStream _vs;
|
|
||||||
public SchemaGuesserStream(ViewStream vs) {
|
|
||||||
_vs = vs;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
_vs.onChange += _on_data;
|
|
||||||
}
|
|
||||||
protected override void myOnLastSubscriber() {
|
|
||||||
_vs.onChange -= _on_data;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _on_data(object obj) {
|
|
||||||
DataPacket dp = obj as DataPacket;
|
|
||||||
if (_rows == null) {
|
|
||||||
_rows = BlueDataUtility.get_typed_table(dp.newRows);
|
|
||||||
}
|
|
||||||
var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
|
|
||||||
myFireEvent(dp_out);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,67 +0,0 @@
|
|||||||
using Bts.ArithmeticExpressions;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Text;
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
namespace bts.BlueOrb {
|
|
||||||
|
|
||||||
[DataStreamUserRespresentation("SingleRowMathStream")]
|
|
||||||
public class MathStream : ViewStream, IStreamMapper {
|
|
||||||
private IArithmeticProgram myExp;
|
|
||||||
private BooleanExpressionFilter _bool;
|
|
||||||
List<string> myExtraColumns = new List<string>();
|
|
||||||
BlueTableSchema mySchema;
|
|
||||||
|
|
||||||
|
|
||||||
public MathStream(ViewStream vs, string mathExpression)
|
|
||||||
: this(vs, mathExpression, string.Empty) {
|
|
||||||
}
|
|
||||||
|
|
||||||
public MathStream(ViewStream vs, string mathExpression, string boolean_expression) {
|
|
||||||
myExp = ArithmeticProgramParser.Parse(mathExpression) ??
|
|
||||||
throw new Exception($"Failed to parse ArithmeticExpression '{mathExpression}'");
|
|
||||||
myExtraColumns = myExp.OutputIdentifiers.ToList();
|
|
||||||
if (!string.IsNullOrEmpty(boolean_expression))
|
|
||||||
_bool = new BooleanExpressionFilter(boolean_expression);
|
|
||||||
mySetRealSource(new MapStream(vs, this));
|
|
||||||
}
|
|
||||||
|
|
||||||
#region IStreamMapper Members
|
|
||||||
|
|
||||||
public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) {
|
|
||||||
if (mySchema == null) {
|
|
||||||
Dictionary<string, ColumnInfo> colIdx = new Dictionary<string, ColumnInfo>();
|
|
||||||
foreach (ColumnInfo ci in inputSchema.columnInfo)
|
|
||||||
colIdx[ci.colName] = ci;
|
|
||||||
|
|
||||||
foreach (string ncn in myExtraColumns) {
|
|
||||||
ColumnInfo newC;
|
|
||||||
if (colIdx.ContainsKey(ncn))
|
|
||||||
newC = new ColumnInfo(colIdx[ncn].colName, colIdx[ncn].colNo, typeof(double));
|
|
||||||
else
|
|
||||||
newC = new ColumnInfo(ncn, colIdx.Count, typeof(double));
|
|
||||||
colIdx[ncn] = newC;
|
|
||||||
}
|
|
||||||
mySchema = new BlueTableSchema(colIdx.Values.ToArray(), "math(" + inputSchema.tableName + ")");
|
|
||||||
}
|
|
||||||
return mySchema;
|
|
||||||
}
|
|
||||||
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
#region IDynamicStreamMapper Members
|
|
||||||
|
|
||||||
public IBlueRow map(IBlueRow row) {
|
|
||||||
MutableBlueRow result = new MutableBlueRow(mySchema, row);
|
|
||||||
if (_bool == null || _bool.filter(row)) {
|
|
||||||
myExp.Evaluate(result.AsDictionary());
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
#endregion
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,17 +0,0 @@
|
|||||||
namespace bts.BlueOrb {
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Text;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
|
|
||||||
[DataStreamUserRespresentation("SortStream")]
|
|
||||||
public class SortStream: ViewStream {
|
|
||||||
public SortStream (ViewStream parent_stream, string sort_expr) {
|
|
||||||
ViewStream rs = new BCG.SortStream(parent_stream, sort_expr);
|
|
||||||
mySetRealSource(rs);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,16 +0,0 @@
|
|||||||
namespace bts.BlueOrb {
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Text;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
[DataStreamUserRespresentation("TimeBasedRollingFilterStream")]
|
|
||||||
public class TimeBasedRollingFilterStream : ViewStream {
|
|
||||||
public TimeBasedRollingFilterStream(ViewStream parent_str, string time_col, double n_seconds) {
|
|
||||||
ViewStream rs = new BCG.TimeBasedFilterStream(ViewStreamDatabase.instance, parent_str, time_col, TimeSpan.FromSeconds(n_seconds));
|
|
||||||
mySetRealSource(rs);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,81 +0,0 @@
|
|||||||
using BCG;
|
|
||||||
|
|
||||||
using bts.utils;
|
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using mbts.utils;
|
|
||||||
|
|
||||||
using Spark.Core.Collections;
|
|
||||||
using Spark.Core.Threading;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Collections.Immutable;
|
|
||||||
using System.Linq;
|
|
||||||
|
|
||||||
namespace bts.BlueOrb {
|
|
||||||
// Keeps adding changing rows to the stream, hence preserving the history
|
|
||||||
// and creating a time series, for example this can be used to make bbo updates
|
|
||||||
// into a time series of prices/qty
|
|
||||||
[DataStreamUserRespresentation("TimeSeriesStream")]
|
|
||||||
public class TimeSeriesStream : ViewStream {
|
|
||||||
readonly ViewStream _stream;
|
|
||||||
readonly string timeColumn;
|
|
||||||
readonly AtomicGate gate = new();
|
|
||||||
CircularBufferTable? _table;
|
|
||||||
private int _id = 0;
|
|
||||||
private int _size = 0;
|
|
||||||
|
|
||||||
public TimeSeriesStream(ViewStream stream)
|
|
||||||
: this(stream, Int32.MaxValue) { }
|
|
||||||
|
|
||||||
public TimeSeriesStream(ViewStream stream, int size, string timeColumn = "ts_event") {
|
|
||||||
_stream = stream;
|
|
||||||
_size = size;
|
|
||||||
this.timeColumn = timeColumn;
|
|
||||||
}
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
|
||||||
_stream.onChange += new ChangeDelegate(_onstream_change);
|
|
||||||
}
|
|
||||||
protected override void myOnLastSubscriber() {
|
|
||||||
_stream.onChange -= new ChangeDelegate(_onstream_change);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private void _onstream_change(object o) {
|
|
||||||
var dp = ExceptionAssert.Argument.Is<IDataPacket>(o);
|
|
||||||
DateTime now = DateTime.Now;
|
|
||||||
using var _ = gate.BeginTransaction();
|
|
||||||
if (_table == null) {
|
|
||||||
var colInfos = dp.newRows.schema.columnInfo;
|
|
||||||
if (colInfos.TryGetFirst(c => c.colName == timeColumn, out var currTimeCol, out var currTimeColIdx)) {
|
|
||||||
colInfos = colInfos.ToArray();
|
|
||||||
colInfos[currTimeColIdx] = new(timeColumn, currTimeCol.colNo, typeof(DateTime));
|
|
||||||
} else {
|
|
||||||
colInfos = ArrayHelper.add(colInfos, new(timeColumn, colInfos.Length, typeof(DateTime)));
|
|
||||||
}
|
|
||||||
_table = new CircularBufferTable(new BlueTableSchema(colInfos, dp.newRows.schema.tableName), _size);
|
|
||||||
}
|
|
||||||
var addedRowsBd = new BlueTableImmutable(_table.schema).ToDataPacketBuilder();
|
|
||||||
var deletedRowsBd = new BlueTableImmutable(_table.schema).ToDataPacketBuilder();
|
|
||||||
foreach (IBlueRow r in dp.addedRows) {
|
|
||||||
MutableBlueRow ar = new MutableBlueRow(_table.schema, r);
|
|
||||||
ar.setValue(timeColumn, now);
|
|
||||||
int id = _id++;
|
|
||||||
ar.setValue("id", id);
|
|
||||||
_table = _table.Add(ar, out var oldRow);
|
|
||||||
addedRowsBd.AddOrUpdate(ar);
|
|
||||||
if (oldRow is not null) {
|
|
||||||
deletedRowsBd.AddOrUpdate(oldRow);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var addedRows = addedRowsBd.ToImmutable();
|
|
||||||
var deletedRows = deletedRowsBd.ToImmutable();
|
|
||||||
var dp_out = new DataPacket(_table, addedRows, deletedRows);
|
|
||||||
myFireEvent(dp_out);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@@ -1,197 +0,0 @@
|
|||||||
namespace Bts.DataStream;
|
|
||||||
|
|
||||||
using bts;
|
|
||||||
|
|
||||||
using Spark.Core.Collections;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Collections.Immutable;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Maps a timeseries of keys to unique int32 ids.
|
|
||||||
/// </summary>
|
|
||||||
public class TimeseriesIndex<T> where T : notnull {
|
|
||||||
static readonly IComparer<KeyValuePair<DateTime, int>> idTimeseriesComp =
|
|
||||||
KeyValuePairComparer.KeyComparer<DateTime, int>();
|
|
||||||
|
|
||||||
readonly ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>> nameToId;
|
|
||||||
readonly ImmutableDictionary<int, TimeseriesKey<T>> idToName;
|
|
||||||
readonly int idCounter;
|
|
||||||
|
|
||||||
public TimeseriesIndex()
|
|
||||||
: this(ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>>.Empty, ImmutableDictionary<int, TimeseriesKey<T>>.Empty, 0) { }
|
|
||||||
|
|
||||||
TimeseriesIndex(ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>> nameToId, ImmutableDictionary<int, TimeseriesKey<T>> idToName, int idCounter) {
|
|
||||||
this.nameToId = nameToId;
|
|
||||||
this.idToName = idToName;
|
|
||||||
this.idCounter = idCounter;
|
|
||||||
}
|
|
||||||
|
|
||||||
public int Count => idToName.Count;
|
|
||||||
|
|
||||||
public ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>> NameToId => nameToId;
|
|
||||||
public ImmutableDictionary<int, TimeseriesKey<T>> IdToName => idToName;
|
|
||||||
|
|
||||||
public bool ContainsId(int id) =>
|
|
||||||
idToName.ContainsKey(id);
|
|
||||||
|
|
||||||
public int GetNearest(TimeseriesKey<T> key) {
|
|
||||||
if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) {
|
|
||||||
throw new KeyNotFoundException();
|
|
||||||
}
|
|
||||||
int idx = ids.BinarySearch(new(key.Timestamp, -1), idTimeseriesComp);
|
|
||||||
if (idx < 0) {
|
|
||||||
idx = ~idx;
|
|
||||||
if (idx == 0) {
|
|
||||||
throw new KeyNotFoundException();
|
|
||||||
}
|
|
||||||
idx--;
|
|
||||||
}
|
|
||||||
return ids[idx].Value;
|
|
||||||
}
|
|
||||||
|
|
||||||
public bool TryGetNearest(TimeseriesKey<T> key, out int result) {
|
|
||||||
if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) {
|
|
||||||
result = -1;
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
int idx = ids.BinarySearch(new(key.Timestamp, -1), idTimeseriesComp);
|
|
||||||
if (idx < 0) {
|
|
||||||
idx = ~idx;
|
|
||||||
if (idx == 0) {
|
|
||||||
result = -1;
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
idx--;
|
|
||||||
}
|
|
||||||
result = ids[idx].Value;
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
public TimeseriesIndex<T> GetNearestOrAdd(TimeseriesKey<T> key, out int result) {
|
|
||||||
int idx;
|
|
||||||
if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) {
|
|
||||||
ids = ImmutableList<KeyValuePair<DateTime, int>>.Empty;
|
|
||||||
idx = 0;
|
|
||||||
} else {
|
|
||||||
idx = ids.BinarySearch(new(key.Timestamp, -1), idTimeseriesComp);
|
|
||||||
if (idx < 0) {
|
|
||||||
idx = ~idx;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (idx == 0) {
|
|
||||||
result = idCounter + 1;
|
|
||||||
ids = ids.Insert(0, new(key.Timestamp, result));
|
|
||||||
return new(
|
|
||||||
nameToId.SetItem(key.Name, ids),
|
|
||||||
idToName.SetItem(result, key),
|
|
||||||
result);
|
|
||||||
}
|
|
||||||
result = ids[idx].Value;
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
public TimeseriesIndex<T> GetOrAdd(TimeseriesKey<T> key, out int result) {
|
|
||||||
int idx;
|
|
||||||
if (!nameToId.TryGetValue(key.Name, out var ids) || ids.Count == 0) {
|
|
||||||
ids = ImmutableList<KeyValuePair<DateTime, int>>.Empty;
|
|
||||||
idx = 0;
|
|
||||||
} else {
|
|
||||||
idx = ids.BinarySearch(new(key.Timestamp, -1), idTimeseriesComp);
|
|
||||||
if (idx < 0) {
|
|
||||||
idx = ~idx;
|
|
||||||
} else {
|
|
||||||
result = ids[idx].Value;
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
result = idCounter + 1;
|
|
||||||
ids = ids.Insert(idx, new(key.Timestamp, result));
|
|
||||||
return new(
|
|
||||||
nameToId.SetItem(key.Name, ids),
|
|
||||||
idToName.SetItem(result, key),
|
|
||||||
result);
|
|
||||||
}
|
|
||||||
|
|
||||||
public IList<KeyValuePair<DateTime, int>> GetIds(T name) =>
|
|
||||||
nameToId[name];
|
|
||||||
|
|
||||||
public Builder ToBuilder() =>
|
|
||||||
new(this);
|
|
||||||
|
|
||||||
public class Builder {
|
|
||||||
readonly ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>>.Builder nameToId;
|
|
||||||
readonly ImmutableDictionary<int, TimeseriesKey<T>>.Builder idToName;
|
|
||||||
int idCounter;
|
|
||||||
|
|
||||||
public Builder(TimeseriesIndex<T> source)
|
|
||||||
: this(source.nameToId, source.idToName, source.idCounter) { }
|
|
||||||
|
|
||||||
public Builder(ImmutableDictionary<T, ImmutableList<KeyValuePair<DateTime, int>>> nameToId, ImmutableDictionary<int, TimeseriesKey<T>> idToName, int idCounter) {
|
|
||||||
this.nameToId = nameToId.ToBuilder();
|
|
||||||
this.idToName = idToName.ToBuilder();
|
|
||||||
this.idCounter = idCounter;
|
|
||||||
}
|
|
||||||
|
|
||||||
public int GetNearest(TimeseriesKey<T> key) =>
|
|
||||||
getOrAdd(key, false, false);
|
|
||||||
|
|
||||||
public int GetNearestOrAdd(TimeseriesKey<T> key) =>
|
|
||||||
getOrAdd(key, false, true);
|
|
||||||
|
|
||||||
public int GetOrAdd(TimeseriesKey<T> key) =>
|
|
||||||
getOrAdd(key, true, true);
|
|
||||||
|
|
||||||
public int GetLatest(T key) =>
|
|
||||||
nameToId.TryGetValue(key, out var ids)
|
|
||||||
? ids[^1].Value
|
|
||||||
: throw new KeyNotFoundException();
|
|
||||||
|
|
||||||
public bool TryGetLatest(T key, out int result) {
|
|
||||||
if (nameToId.TryGetValue(key, out var ids)) {
|
|
||||||
result = ids[^1].Value;
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
result = 0;
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
public ImmutableList<KeyValuePair<DateTime, int>> GetIds(T name) =>
|
|
||||||
nameToId.TryGetValue(name, out var ids)
|
|
||||||
? ids
|
|
||||||
: ImmutableList<KeyValuePair<DateTime, int>>.Empty;
|
|
||||||
|
|
||||||
public TimeseriesIndex<T> ToImmutable() =>
|
|
||||||
new(nameToId.ToImmutable(), idToName.ToImmutable(), idCounter);
|
|
||||||
|
|
||||||
int getOrAdd(TimeseriesKey<T> key, bool requireExactTimestamp, bool canAdd) {
|
|
||||||
var ids = nameToId.GetOrAdd(key.Name, _ => ImmutableList<KeyValuePair<DateTime, int>>.Empty);
|
|
||||||
int idx = ids.BinarySearch(new(key.Timestamp, 0), idTimeseriesComp);
|
|
||||||
int id;
|
|
||||||
if (idx < 0) {
|
|
||||||
Log.Gen.info($"Adding key to timeseries index ({key.Timestamp}, {key.Name})");
|
|
||||||
// We didn't get an exact match. If requireExactTimestamp is false and a record with an
|
|
||||||
// earlier Timestamp exists, use that. Otherwise, add a new record with this Timestamp.
|
|
||||||
idx = ~idx;
|
|
||||||
if (requireExactTimestamp || idx == 0) {
|
|
||||||
if (!canAdd) {
|
|
||||||
throw new KeyNotFoundException();
|
|
||||||
}
|
|
||||||
id = ++idCounter;
|
|
||||||
ids = ids.Insert(idx, new(key.Timestamp, id));
|
|
||||||
nameToId[key.Name] = ids;
|
|
||||||
idToName.Add(id, key);
|
|
||||||
} else {
|
|
||||||
if (idx == ids.Count) {
|
|
||||||
--idx;
|
|
||||||
}
|
|
||||||
id = ids[idx].Value;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
id = ids[idx].Value;
|
|
||||||
}
|
|
||||||
return id;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,22 +0,0 @@
|
|||||||
namespace Bts.DataStream;
|
|
||||||
|
|
||||||
using Spark.Core.Collections;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
|
|
||||||
public readonly record struct TimeseriesKey<T>(T Name, DateTime Timestamp) : IDualComparable<TimeseriesKey<T>> {
|
|
||||||
public int CompareTo(TimeseriesKey<T> other) {
|
|
||||||
var rv = Timestamp.CompareTo(other.Timestamp);
|
|
||||||
if (rv == 0) {
|
|
||||||
rv = Comparer<T>.Default.Compare(Name, other.Name);
|
|
||||||
}
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static implicit operator (T Name, DateTime Timestamp)(TimeseriesKey<T> key) =>
|
|
||||||
(key.Name, key.Timestamp);
|
|
||||||
|
|
||||||
public static implicit operator TimeseriesKey<T>((T Name, DateTime Timestamp) tup) =>
|
|
||||||
new(tup.Name, tup.Timestamp);
|
|
||||||
}
|
|
||||||
@@ -1,11 +0,0 @@
|
|||||||
namespace bts.BlueOrb {
|
|
||||||
using BCG;
|
|
||||||
[DataStreamUserRespresentation("TopStream")]
|
|
||||||
public class TopStream : ViewStream {
|
|
||||||
public TopStream(ViewStream vs, int n_rows, string sort_col) {
|
|
||||||
ViewStream rs = new BCG.TopStream (vs, n_rows, sort_col);
|
|
||||||
mySetRealSource(rs);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -1,16 +0,0 @@
|
|||||||
namespace bts.BlueOrb {
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
|
|
||||||
using mbts;
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
[DataStreamUserRespresentation("UnionStream")]
|
|
||||||
public class UnionStream : ViewStream {
|
|
||||||
public UnionStream(ViewStream left_stream, ViewStream right_stream) {
|
|
||||||
ViewStream rs = new BCG.MutableAppendStream(left_stream, right_stream);
|
|
||||||
mySetRealSource(rs);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -17,11 +17,11 @@
|
|||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="IronPython" Version="3.4.2" />
|
<PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
|
||||||
<PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
|
<PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
|
||||||
<PackageReference Include="MongoDB.Bson" Version="2.23.1" />
|
<PackageReference Include="MongoDB.Bson" Version="2.23.1" />
|
||||||
<PackageReference Include="MongoDB.Driver" Version="2.23.1" />
|
<PackageReference Include="MongoDB.Driver" Version="2.23.1" />
|
||||||
<PackageReference Include="ParquetSharp" Version="20.0.0" />
|
<PackageReference Include="Parquet.Net" Version="5.0.1" />
|
||||||
<PackageReference Include="System.Data.SQLite" Version="1.0.118" />
|
<PackageReference Include="System.Data.SQLite" Version="1.0.118" />
|
||||||
<PackageReference Include="MathNet.Numerics" Version="5.0.0" />
|
<PackageReference Include="MathNet.Numerics" Version="5.0.0" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|||||||
@@ -2,277 +2,25 @@ namespace Bts.DataStream;
|
|||||||
|
|
||||||
using BCG;
|
using BCG;
|
||||||
|
|
||||||
using bts;
|
|
||||||
using bts.api.core;
|
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
using Bts.PubSub;
|
|
||||||
|
|
||||||
using Microsoft.Extensions.Logging;
|
|
||||||
|
|
||||||
using Spark.Core.Disposables;
|
using Spark.Core.Disposables;
|
||||||
using Spark.Core.Threading;
|
|
||||||
|
|
||||||
using System;
|
using System;
|
||||||
using System.Linq;
|
|
||||||
using System.Reactive.Linq;
|
using System.Reactive.Linq;
|
||||||
using System.Threading;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
|
|
||||||
public static class DataStreamExtensions {
|
public static class DataStreamExtensions {
|
||||||
public static ViewStream AsViewStream<T>(this IObservable<IDataPacket<T>> source, string? name = null, ILogger? logger = null) where T : IBlueRow =>
|
public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
|
||||||
source as ViewStream ?? new ObservableViewStream(source.Select(dp => new DataPacket(dp.newRows, dp.addedRows, dp.deletedRows)), name, logger);
|
source as ViewStream ?? new ObservableViewStream(source);
|
||||||
|
|
||||||
// public static IObservable<IDataPacket<T>> SelectDataPacket<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> selector) where T : IBlueRow =>
|
public static IObservable<DataPacket> AsObservable(this ViewStream source) {
|
||||||
// source.Select(dp => {
|
return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
|
||||||
// dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
|
|
||||||
// return new DataPacket<T>(CastRows(newRows), CastRows(addedRows), CastRows(deletedRows));
|
|
||||||
|
|
||||||
// IBlueRowCollection<T> CastRows(ICollection rows) =>
|
IDisposable Subscribe(IObserver<DataPacket> observer) {
|
||||||
// new BlueTableSelectorWrapper<T>(
|
source.onChange += OnChange;
|
||||||
// rows as IBlueRowCollection ?? new BlueRowCollection(rows, dp.Schema), selector);
|
return Disposable.Create(() => source.onChange -= OnChange);
|
||||||
// });
|
|
||||||
|
|
||||||
public static IObservable<TableUpdate<IBlueRow, int>> AsTables(this IObservable<IDataPacket> source) =>
|
void OnChange(object? newValue) =>
|
||||||
source.AsTables<IBlueRow, int>(r => r, r => r.Id());
|
observer.OnNext(newValue as DataPacket ?? throw new ArgumentException(
|
||||||
|
$"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
|
||||||
public static IObservable<TableUpdate<T, int>> AsTables<T>(this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector) where T : IBlueRow =>
|
|
||||||
source.AsTables<T, int>(valueSelector, r => r.Id());
|
|
||||||
|
|
||||||
public static IObservable<TableUpdate<T, int>> AsTables<T>(
|
|
||||||
this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, int> keySelector) =>
|
|
||||||
source.AsTables<T, int>(valueSelector, keySelector);
|
|
||||||
|
|
||||||
public static IObservable<TableUpdate<T, TKey>> AsTables<T, TKey>(
|
|
||||||
this IObservable<IDataPacket> source, Func<IBlueRow, T> valueSelector, Func<T, TKey> keySelector, Func<IBlueRow, TKey>? rowKeySelector = null) where TKey : IEquatable<TKey> {
|
|
||||||
rowKeySelector ??= r => keySelector(valueSelector(r));
|
|
||||||
return source.Scan(TableUpdate<T, TKey>.mk_empty(keySelector), (res, dp) => {
|
|
||||||
var bd = res.to_builder();
|
|
||||||
dp.get_new_added_deleted(out var newRows, out var addedRows, out var deletedRows);
|
|
||||||
foreach (IBlueRow r in addedRows) {
|
|
||||||
bd.add_or_update(valueSelector(r));
|
|
||||||
}
|
|
||||||
foreach (IBlueRow r in deletedRows) {
|
|
||||||
bd.remove(rowKeySelector(r));
|
|
||||||
}
|
|
||||||
return bd.to_table();
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
public static IObservable<IDataPacket> AsDataPackets(this IObservable<TableUpdate<IBlueRow, int>> source) =>
|
|
||||||
source.Scan(
|
|
||||||
(Table: (BlueTableImmutable?)null, Dp: (IDataPacket?)null),
|
|
||||||
(res, table) => {
|
|
||||||
var resTable = res.Table;
|
|
||||||
if (resTable is null) {
|
|
||||||
if (table.state.Count == 0) {
|
|
||||||
return (null, null);
|
|
||||||
}
|
|
||||||
resTable = new(table.state.Values.First().schema);
|
|
||||||
}
|
|
||||||
var bd = resTable.ToDataPacketBuilder();
|
|
||||||
foreach (var r in table.removed) {
|
|
||||||
bd.Remove(r.Id());
|
|
||||||
}
|
|
||||||
foreach (var r in table.added) {
|
|
||||||
bd.AddOrUpdate(r);
|
|
||||||
}
|
|
||||||
var resDp = bd.ToDataPacket(out var newRows);
|
|
||||||
return (newRows, resDp);
|
|
||||||
})
|
|
||||||
.Where(t => t.Dp is not null && (t.Dp.addedRows.Count > 0 || t.Dp.deletedRows.Count > 0))
|
|
||||||
.Select(t => t.Dp!);
|
|
||||||
|
|
||||||
public static IObservable<IDataPacket> AsDataPackets<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
|
|
||||||
source.Scan(
|
|
||||||
(Table: new BlueTableImmutable(schema), Dp: (IDataPacket)new DataPacket(schema)),
|
|
||||||
(res, table) => {
|
|
||||||
var bd = res.Table.ToDataPacketBuilder();
|
|
||||||
foreach (var r in table.removed) {
|
|
||||||
bd.Remove(r.Id());
|
|
||||||
}
|
|
||||||
foreach (var r in table.added) {
|
|
||||||
bd.AddOrUpdate(r);
|
|
||||||
}
|
|
||||||
var resDp = bd.ToDataPacket(out var newRows);
|
|
||||||
return (newRows, resDp);
|
|
||||||
})
|
|
||||||
.Where(t => t.Dp.addedRows.Count > 0 || t.Dp.deletedRows.Count > 0)
|
|
||||||
.Select(t => t.Dp);
|
|
||||||
|
|
||||||
public static IObservable<IDataPacket> AsDataPackets<T, TKey>(this IObservable<TableUpdate<T, TKey>> source, BlueTableSchemaAdapter<T> schema) where TKey : IEquatable<TKey> {
|
|
||||||
return Observable.Create<IDataPacket>(Subscribe);
|
|
||||||
|
|
||||||
IDisposable Subscribe(IObserver<IDataPacket> o) {
|
|
||||||
var adapter = new BlueTableImmutableAdapter<T>(schema);
|
|
||||||
IDataPacket initDataPacket = new DataPacket(schema);
|
|
||||||
return source.Scan((Adapter: adapter, DataPacket: initDataPacket), OnUpdate).Select(res => res.DataPacket).Subscribe(o);
|
|
||||||
|
|
||||||
(BlueTableImmutableAdapter<T> Adapter, IDataPacket DataPacket) OnUpdate(
|
|
||||||
(BlueTableImmutableAdapter<T> Adapter, IDataPacket DataPacket) state, TableUpdate<T, TKey> tableUpdate) {
|
|
||||||
var bd = state.Adapter.ToDataPacketBuilder();
|
|
||||||
bd.Update(tableUpdate);
|
|
||||||
var newRows = bd.ToImmutable(out var dp);
|
|
||||||
return (newRows, dp);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static ViewStream AsViewStream<T>(this IObservable<TableUpdate<T, int>> source, IBlueTableSchema schema) where T : IBlueRow =>
|
|
||||||
source.AsDataPackets(schema).AsViewStream();
|
|
||||||
|
|
||||||
public static IObservable<IDataPacket> OnWorkScheduler(this IObservable<IDataPacket> source, IWorkScheduler scheduler) =>
|
|
||||||
Observable.Create<IDataPacket>(o => {
|
|
||||||
var inputGate = new AtomicGate();
|
|
||||||
var outputGate = new AtomicGate();
|
|
||||||
BlueTableImmutable? lastTable = null;
|
|
||||||
BlueTableImmutable.DataPacketBuilder? bd = null;
|
|
||||||
IDataPacket? pendingDpOut = null;
|
|
||||||
return Disposable.Create(source.Subscribe(OnNext), scheduler.Subscribe(DoWork));
|
|
||||||
|
|
||||||
void OnNext(IDataPacket value) {
|
|
||||||
using (inputGate.BeginTransaction()) {
|
|
||||||
lastTable ??= new(value.newRows.schema);
|
|
||||||
bd ??= lastTable.ToDataPacketBuilder();
|
|
||||||
if (pendingDpOut is null) {
|
|
||||||
// The last outgoing datapacket was just fired. Don't merge this update with the
|
|
||||||
// previous one (otherwise its adds/removes would be fired twice).
|
|
||||||
bd.Reset();
|
|
||||||
}
|
|
||||||
foreach (IBlueRow r in value.deletedRows) {
|
|
||||||
bd.Remove(r.Id());
|
|
||||||
}
|
|
||||||
foreach (IBlueRow r in value.addedRows) {
|
|
||||||
bd.AddOrUpdate(r);
|
|
||||||
}
|
|
||||||
pendingDpOut = bd.ToDataPacket(out lastTable, false);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void DoWork() {
|
|
||||||
IDataPacket? dpOut = null;
|
|
||||||
if (outputGate.TryEnter()) {
|
|
||||||
using var exit = Disposable.Create(outputGate.Exit);
|
|
||||||
(dpOut, pendingDpOut) = (pendingDpOut, dpOut);
|
|
||||||
if (dpOut is not null) {
|
|
||||||
o.OnNext(dpOut);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
public static IObservable<IDataPacket> NonIncrementalAsyncTransform(this IObservable<IDataPacket> source, Func<IBlueRowCollection, Task<IBlueRowCollection?>> transformAsync, IWorkScheduler workScheduler) {
|
|
||||||
const int STATE_IDLE = 0;
|
|
||||||
const int STATE_BUSY = 1;
|
|
||||||
const int STATE_DISPOSED = 2;
|
|
||||||
return Observable.Create<IDataPacket>(Subscribe);
|
|
||||||
IDisposable Subscribe(IObserver<IDataPacket> o) {
|
|
||||||
var syncRoot = new object();
|
|
||||||
IBlueRowCollection? lastTableIn = null;
|
|
||||||
BlueTableImmutable? lastTableOut = null;
|
|
||||||
int nConcurrentDoWorks = 0;
|
|
||||||
int state = STATE_IDLE;
|
|
||||||
IDisposable? dispose = null;
|
|
||||||
dispose = Disposable.Create(source.Subscribe(OnNext), workScheduler.Subscribe(DoWork));
|
|
||||||
return Disposable.Create(Dispose);
|
|
||||||
|
|
||||||
void Dispose() {
|
|
||||||
lock (syncRoot) {
|
|
||||||
switch (state) {
|
|
||||||
case STATE_DISPOSED:
|
|
||||||
return;
|
|
||||||
case STATE_BUSY:
|
|
||||||
state = STATE_DISPOSED;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
state = STATE_DISPOSED;
|
|
||||||
}
|
|
||||||
Disposable.dispose(ref dispose);
|
|
||||||
}
|
|
||||||
|
|
||||||
void OnNext(IDataPacket value) =>
|
|
||||||
Volatile.Write(ref lastTableIn, value.newRows);
|
|
||||||
|
|
||||||
async void DoWork() {
|
|
||||||
lock (syncRoot) {
|
|
||||||
nConcurrentDoWorks++;
|
|
||||||
switch (state) {
|
|
||||||
case STATE_DISPOSED:
|
|
||||||
case STATE_BUSY:
|
|
||||||
return;
|
|
||||||
case STATE_IDLE:
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
Log.assert(false, $"Unexpected state ({StateName(state)}) when DoWork called");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
state = STATE_BUSY;
|
|
||||||
}
|
|
||||||
var dp = await PublishTable();
|
|
||||||
bool isDisposed = false;
|
|
||||||
lock (syncRoot) {
|
|
||||||
switch (state) {
|
|
||||||
case STATE_DISPOSED:
|
|
||||||
isDisposed = true;
|
|
||||||
break;
|
|
||||||
case STATE_BUSY:
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
Log.assert(false, $"Unexpected state ({StateName(state)}) after building output table (async)");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (isDisposed) {
|
|
||||||
Disposable.dispose(ref dispose);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (dp is not null) {
|
|
||||||
o.OnNext(dp);
|
|
||||||
}
|
|
||||||
lock (syncRoot) {
|
|
||||||
nConcurrentDoWorks = 0;
|
|
||||||
switch (state) {
|
|
||||||
case STATE_DISPOSED:
|
|
||||||
isDisposed = true;
|
|
||||||
break;
|
|
||||||
case STATE_BUSY:
|
|
||||||
state = STATE_IDLE;
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
Log.assert(false, $"Unexpected state ({StateName(state)}) after firing output table");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (isDisposed) {
|
|
||||||
Disposable.dispose(ref dispose);
|
|
||||||
}
|
|
||||||
|
|
||||||
async Task<DataPacket?> PublishTable() {
|
|
||||||
var tableIn = Interlocked.Exchange(ref lastTableIn, null);
|
|
||||||
if (tableIn is null) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
var tableOut = await transformAsync(tableIn);
|
|
||||||
if (tableOut is null) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
lastTableOut ??= new(tableOut.schema);
|
|
||||||
var bd = lastTableOut.ToDataPacketBuilder();
|
|
||||||
bd.Clear();
|
|
||||||
foreach (IBlueRow r in tableOut) {
|
|
||||||
bd.AddOrUpdate(r, BlueRowEqualityComparer.Instance);
|
|
||||||
}
|
|
||||||
return bd.ToDataPacket(out lastTableOut);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static string StateName(int state) =>
|
|
||||||
state switch {
|
|
||||||
STATE_IDLE => "IDLE",
|
|
||||||
STATE_BUSY => "BUSY",
|
|
||||||
STATE_DISPOSED => "DISPOSED",
|
|
||||||
_ => $"STATE_{state}",
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,26 +1,26 @@
|
|||||||
namespace bts.BlueOrb {
|
namespace bts.SilverPine {
|
||||||
using System;
|
using System;
|
||||||
using BCG;
|
using BCG;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using Bts.DataStreamExpressions;
|
using Bts.DataStreamExpressions;
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
// All this does is take in a dsl expression, parse it and fire the data that come out of running it.
|
// All this does is take in a dsl expression, parse it and fire the data that come out of running it.
|
||||||
// So for example, if an expresison is passed
|
// So for example, if an expresison is passed
|
||||||
[DataStreamUserRespresentation("dsl")]
|
[DataStreamUserRespresentation("dsl")]
|
||||||
public class DSLStream : ViewStream {
|
public class DSLStream : ViewStream {
|
||||||
private readonly string _dsl;
|
private readonly string _dsl;
|
||||||
public DSLStream(string dsl, IDataStreamFactory streamFactory) {
|
public DSLStream(string dsl) {
|
||||||
Log.Gen.info($"DSLStream.cons dsl: {dsl}");
|
Log.Gen.info($"DSLStream.cons dsl: {dsl}");
|
||||||
_dsl = dsl;
|
_dsl = dsl;
|
||||||
var rs = build_stream(streamFactory, dsl) ?? new LiteralStream("error", $"msg:invalid dsl");
|
ViewStream rs = build_stream(dsl);
|
||||||
|
rs = rs ?? new LiteralStream("error", $"msg:invalid dsl \"{dsl}\"");
|
||||||
mySetRealSource(rs);
|
mySetRealSource(rs);
|
||||||
}
|
}
|
||||||
|
|
||||||
static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression) {
|
static ViewStream? build_stream(string stream_expression) {
|
||||||
try {
|
try {
|
||||||
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
|
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
|
||||||
var rv = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
|
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
|
||||||
return rv;
|
return rv;
|
||||||
|
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
@@ -29,4 +29,4 @@ namespace bts.BlueOrb {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -40,7 +40,7 @@ namespace BCG {
|
|||||||
protected override void myOnFirstSubscriber() {
|
protected override void myOnFirstSubscriber() {
|
||||||
MasterWorker.getInstance(false).addWorker(this, _send_period);
|
MasterWorker.getInstance(false).addWorker(this, _send_period);
|
||||||
var server_rci = new RemoteConnectionInfo(_ip, _port);
|
var server_rci = new RemoteConnectionInfo(_ip, _port);
|
||||||
bts.AsyncTCPConnector_AsyncWrapper.connect_to(server_rci, this);
|
bts.AsyncTCPConnector_APM.connect_to(server_rci, this);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
protected override void myOnLastSubscriber() {
|
||||||
@@ -76,4 +76,4 @@ namespace BCG {
|
|||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
namespace Bts.DataStream;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
using bts;
|
||||||
|
|
||||||
|
using System;
|
||||||
|
|
||||||
|
public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
|
||||||
|
readonly IObservable<DataPacket> source;
|
||||||
|
IDisposable? subscription;
|
||||||
|
|
||||||
|
public ObservableViewStream(IObservable<DataPacket> source) =>
|
||||||
|
this.source = source;
|
||||||
|
|
||||||
|
public void Dispose() {
|
||||||
|
GC.SuppressFinalize(this);
|
||||||
|
if (subscription is not null) {
|
||||||
|
subscription.Dispose();
|
||||||
|
subscription = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
Log.isnull(subscription);
|
||||||
|
subscription = source.Subscribe(myFireEvent);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnLastSubscriber() {
|
||||||
|
Log.notnull(subscription);
|
||||||
|
subscription.Dispose();
|
||||||
|
subscription = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
IDisposable IObservable<DataPacket>.Subscribe(IObserver<DataPacket> observer) =>
|
||||||
|
source.Subscribe(observer);
|
||||||
|
}
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
|
||||||
|
using Bts.DataStreamExpressions;
|
||||||
|
using BCG.Utilities.General;
|
||||||
|
using bts;
|
||||||
|
|
||||||
|
namespace BCG {
|
||||||
|
// listens to the requests posted on "cloud_streams" request table, and
|
||||||
|
// if unfulfilled starts a ViewPubsliher
|
||||||
|
public class RemoteQueryManager : IWorker {
|
||||||
|
public class StreamPubUnit {
|
||||||
|
public string name;
|
||||||
|
public string stream_expression;
|
||||||
|
public ViewStream stream;
|
||||||
|
public bool persist;
|
||||||
|
public ViewPublisher view_publisher;
|
||||||
|
|
||||||
|
public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_) {
|
||||||
|
name = name_;
|
||||||
|
stream_expression = stream_expression_;
|
||||||
|
stream = stream_;
|
||||||
|
persist = persist_;
|
||||||
|
view_publisher = view_publisher_;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
static ViewStream? build_stream(string stream_expression)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
|
||||||
|
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
|
||||||
|
return rv;
|
||||||
|
|
||||||
|
} catch (Exception ex) {
|
||||||
|
bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
private IDictionary<string, StreamPubUnit>
|
||||||
|
_stream_pub_map = new Dictionary<string, StreamPubUnit>();
|
||||||
|
private ViewStream _stream = new TableStream("cloud_streams");
|
||||||
|
private PubTableBroker _broker = new PubTableBroker();
|
||||||
|
private ManagedSubscriber _sub = ManagedSubscriber.instance;
|
||||||
|
|
||||||
|
|
||||||
|
public RemoteQueryManager() {
|
||||||
|
_stream.onChange += _on_str;
|
||||||
|
_sub.subscribe("RemoteRequest", (string tag, byte[] data) => {
|
||||||
|
RemoteRequest req = RemoteRequest.construct_from_bytes(data);
|
||||||
|
_spawn_remote_stream(req.published_table_name, req.stream_expression, false);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
private void _on_str(object obj)
|
||||||
|
{
|
||||||
|
if (obj is not DataPacket dp) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
foreach (IBlueRow r in dp.newRows)
|
||||||
|
{
|
||||||
|
string name = r.get<string>("name");
|
||||||
|
bool persist = r.get<bool>("persist");
|
||||||
|
string stream_exp = r.get<string>("stream_exp");
|
||||||
|
if (!_stream_pub_map.ContainsKey(name)) {
|
||||||
|
_spawn_remote_stream(name, stream_exp, persist);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _spawn_remote_stream(string name, string stream_expression, bool persist) {
|
||||||
|
bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
|
||||||
|
ViewStream stream = build_stream(stream_expression);
|
||||||
|
_expose_to_ws(name, stream);
|
||||||
|
if (persist) {
|
||||||
|
stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000); // 10MBs
|
||||||
|
}
|
||||||
|
ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
|
||||||
|
StreamPubUnit u = new StreamPubUnit(name, stream_expression, stream, persist, view_pub);
|
||||||
|
_stream_pub_map.Add(name, u);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _expose_to_ws(string name, ViewStream stream) {
|
||||||
|
// TODO: john
|
||||||
|
}
|
||||||
|
|
||||||
|
public void doWork() {
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,168 +0,0 @@
|
|||||||
using bts;
|
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
using Bts.DataStreamExpressions;
|
|
||||||
using Bts.PubSub;
|
|
||||||
|
|
||||||
using Microsoft.Extensions.Hosting;
|
|
||||||
|
|
||||||
using Spark.Core.Disposables;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.IO;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Reactive.Linq;
|
|
||||||
using System.Threading;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
|
|
||||||
namespace BCG {
|
|
||||||
// listens to the requests posted on "cloud_streams" request table, and
|
|
||||||
// if unfulfilled starts a ViewPubsliher
|
|
||||||
public class RemoteQueryManager : IHostedService {
|
|
||||||
public class StreamPubUnit {
|
|
||||||
public string name;
|
|
||||||
public string stream_expression;
|
|
||||||
public ViewStream stream;
|
|
||||||
public bool persist;
|
|
||||||
public ViewPublisher view_publisher;
|
|
||||||
|
|
||||||
public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_) {
|
|
||||||
name = name_;
|
|
||||||
stream_expression = stream_expression_;
|
|
||||||
stream = stream_;
|
|
||||||
persist = persist_;
|
|
||||||
view_publisher = view_publisher_;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
static ViewStream? build_stream(IDataStreamFactory streamFactory, string stream_expression)
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
|
|
||||||
var rv = exp.getView(new Dictionary<string, StatementExpression>(0), streamFactory);
|
|
||||||
return rv;
|
|
||||||
|
|
||||||
} catch (Exception ex) {
|
|
||||||
bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
private IDictionary<string, StreamPubUnit>
|
|
||||||
_stream_pub_map = new Dictionary<string, StreamPubUnit>();
|
|
||||||
readonly IPubTableBroker _broker;
|
|
||||||
readonly IManagedSubscriber _sub;
|
|
||||||
readonly IDataStreamFactory streamFactory;
|
|
||||||
readonly ITableStreamFactory tableStreamFactory;
|
|
||||||
readonly object syncRoot = new();
|
|
||||||
IDisposable? cloudStreamsSub;
|
|
||||||
IDisposable? remoteRequestsSub;
|
|
||||||
State state;
|
|
||||||
|
|
||||||
public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker, ITableStreamFactory tableStreamFactory, IEnumerable<IDataStreamFactory> streamFactories) {
|
|
||||||
_sub = managedSubscriber;
|
|
||||||
_broker = pubTableBroker;
|
|
||||||
this.tableStreamFactory = tableStreamFactory;
|
|
||||||
var streamFactoryList = streamFactories.ToList();
|
|
||||||
streamFactory = streamFactoryList.Count == 1
|
|
||||||
? streamFactoryList[0]
|
|
||||||
: new CompositeDataStreamFactory(streamFactoryList);
|
|
||||||
}
|
|
||||||
|
|
||||||
enum State {
|
|
||||||
Idle = 0,
|
|
||||||
Running = 1,
|
|
||||||
Disposed = 2,
|
|
||||||
WaitingToDispose = 3,
|
|
||||||
}
|
|
||||||
|
|
||||||
public Task StartAsync(CancellationToken cancellationToken) {
|
|
||||||
lock (syncRoot) {
|
|
||||||
switch (state) {
|
|
||||||
case State.Idle:
|
|
||||||
break;
|
|
||||||
case State.WaitingToDispose:
|
|
||||||
case State.Disposed:
|
|
||||||
throw new ObjectDisposedException(nameof(RemoteQueryManager));
|
|
||||||
default:
|
|
||||||
return Task.CompletedTask;
|
|
||||||
}
|
|
||||||
state = State.Running;
|
|
||||||
}
|
|
||||||
if (cloudStreamsSub is not null) {
|
|
||||||
throw new InvalidOperationException("cloudStreamsSub already created");
|
|
||||||
}
|
|
||||||
if (remoteRequestsSub is not null) {
|
|
||||||
throw new InvalidOperationException("remoteRequestsSub already created");
|
|
||||||
}
|
|
||||||
var cloudStream = tableStreamFactory.Create("cloud_streams")
|
|
||||||
?? throw new Exception("Failed to create TableStream 'cloud_stream'");
|
|
||||||
cloudStreamsSub = cloudStream.AsObservable().Subscribe(_on_str);
|
|
||||||
remoteRequestsSub = _sub.Observe("RemoteRequest").Subscribe(data => {
|
|
||||||
RemoteRequest req = RemoteRequest.construct_from_bytes(data);
|
|
||||||
_spawn_remote_stream(req.published_table_name, req.stream_expression, false);
|
|
||||||
});
|
|
||||||
lock (syncRoot) {
|
|
||||||
if (state == State.WaitingToDispose) {
|
|
||||||
state = State.Disposed;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (state == State.Disposed) {
|
|
||||||
Disposable.dispose(ref cloudStreamsSub);
|
|
||||||
Disposable.dispose(ref remoteRequestsSub);
|
|
||||||
}
|
|
||||||
return Task.CompletedTask;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Task StopAsync(CancellationToken cancellationToken) {
|
|
||||||
lock (syncRoot) {
|
|
||||||
switch (state) {
|
|
||||||
case State.Running:
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
return Task.CompletedTask;
|
|
||||||
}
|
|
||||||
state = State.Disposed;
|
|
||||||
}
|
|
||||||
Disposable.dispose(ref cloudStreamsSub);
|
|
||||||
Disposable.dispose(ref remoteRequestsSub);
|
|
||||||
return Task.CompletedTask;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _on_str(IDataPacket dp)
|
|
||||||
{
|
|
||||||
foreach (IBlueRow r in dp.newRows)
|
|
||||||
{
|
|
||||||
string name = r.get<string>("name");
|
|
||||||
bool persist = r.get<bool>("persist");
|
|
||||||
string stream_exp = r.get<string>("stream_exp");
|
|
||||||
if (!_stream_pub_map.ContainsKey(name)) {
|
|
||||||
_spawn_remote_stream(name, stream_exp, persist);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _spawn_remote_stream(string name, string stream_expression, bool persist) {
|
|
||||||
bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
|
|
||||||
var stream = build_stream(streamFactory, stream_expression);
|
|
||||||
if (stream is null) {
|
|
||||||
Log.DataFlow.error($"Failed to spawn remote stream. name: {name}; expression: {stream_expression}");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
_expose_to_ws(name, stream);
|
|
||||||
if (persist) {
|
|
||||||
string tempDir = Path.GetTempPath();
|
|
||||||
Directory.CreateDirectory(tempDir); // make sure it exists
|
|
||||||
|
|
||||||
stream = new PersistorStream(stream, name, tempDir, 10_000_000); // 10 MB
|
|
||||||
}
|
|
||||||
ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
|
|
||||||
StreamPubUnit u = new StreamPubUnit(name, stream_expression, stream, persist, view_pub);
|
|
||||||
_stream_pub_map.Add(name, u);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _expose_to_ws(string name, ViewStream stream) {
|
|
||||||
// TODO: john
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -2,30 +2,17 @@ using BCG;
|
|||||||
|
|
||||||
using bts;
|
using bts;
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using Spark.Core.Disposables;
|
|
||||||
|
|
||||||
using System;
|
using System;
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG {
|
||||||
/// <summary>
|
// this is a stream so we can run it from the DSQA (since dsqa can only load streams)
|
||||||
/// This is a stream so we can run it from the DSQA (since dsqa can only load streams).
|
[DataStreamUserRespresentation("remoteQuery")]
|
||||||
/// <para>
|
|
||||||
/// Note: It's hidden because this stream is now created by an IDataStreamFactory
|
|
||||||
/// (see RemoteQueryStreamFactory.cs for details)
|
|
||||||
/// </para>
|
|
||||||
/// </summary>
|
|
||||||
[DataStreamUserRespresentation(false)]
|
|
||||||
public class RemoteQueryStream : ViewStream {
|
public class RemoteQueryStream : ViewStream {
|
||||||
private readonly string _name, _stream_exp;
|
private readonly string _name, _stream_exp;
|
||||||
readonly ITableStreamFactory tableStreamFactory;
|
private PublisherCore _publisher_core = new PublisherCore();
|
||||||
readonly IPublisherCore _publisher_core;
|
private ViewStream _source_str;
|
||||||
IDisposable? subscription;
|
|
||||||
|
|
||||||
public RemoteQueryStream(ITableStreamFactory tableStreamFactory, IPublisherCore publisherCore, params string[] lines) {
|
public RemoteQueryStream(params string[] lines) {
|
||||||
this.tableStreamFactory = tableStreamFactory;
|
|
||||||
_publisher_core = publisherCore;
|
|
||||||
_stream_exp = get_expression(lines);
|
_stream_exp = get_expression(lines);
|
||||||
_name = Guid.NewGuid().ToString();
|
_name = Guid.NewGuid().ToString();
|
||||||
_name = $"remotereq_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds().ToString()}";
|
_name = $"remotereq_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds().ToString()}";
|
||||||
@@ -34,17 +21,17 @@ namespace BCG {
|
|||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
protected override void myOnFirstSubscriber() {
|
||||||
Log.isnull(subscription);
|
|
||||||
RemoteRequest rem_req = new RemoteRequest(_stream_exp, _name);
|
RemoteRequest rem_req = new RemoteRequest(_stream_exp, _name);
|
||||||
byte[] rem_req_bytes = rem_req.serialize();
|
byte[] rem_req_bytes = rem_req.serialize();
|
||||||
_publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
|
_publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
|
||||||
var sourceStream = tableStreamFactory.Create(_name);
|
_source_str = new TableStream(_name);
|
||||||
sourceStream.onChange += _handle_data;
|
_source_str.onChange += new ChangeDelegate(_handle_data);
|
||||||
subscription = Disposable.Create(() => sourceStream.onChange -= _handle_data);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() =>
|
protected override void myOnLastSubscriber() {
|
||||||
Disposable.dispose(ref subscription);
|
//maybe there should be a way to turn remote requests off ?
|
||||||
|
_source_str.onChange -= new ChangeDelegate(_handle_data);
|
||||||
|
}
|
||||||
|
|
||||||
static string get_expression(params string[] lines) {
|
static string get_expression(params string[] lines) {
|
||||||
for (int i = 0; i < lines.Length; i++) {
|
for (int i = 0; i < lines.Length; i++) {
|
||||||
|
|||||||
@@ -1,51 +1,33 @@
|
|||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG {
|
||||||
public class AddColumnsStream : ViewStream {
|
public class AddColumnsStream : ViewStream {
|
||||||
public static Tuple<string, Type, object> parse_single_tok(string exp) {
|
public static Tuple<string, Type, object> parse_exp(string exp) {
|
||||||
string[] toks = exp.Split(':');
|
string[] toks = exp.Split(':');
|
||||||
string col_name = toks[0].Trim();
|
string col_name = toks[0];
|
||||||
string col_value = toks[1].Trim();
|
string col_value = toks[1];
|
||||||
Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
|
Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
|
||||||
return Tuple.Create(col_name, tup.Item1, tup.Item2);
|
return Tuple.Create(col_name, tup.Item1, tup.Item2);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static List<Tuple<string, Type, object>> parse_exp(string exp) {
|
|
||||||
string[] toks = exp.Split(",");
|
|
||||||
List<Tuple<string, Type, object>> rv = new List<Tuple<string, Type, object>>();
|
|
||||||
foreach (string s in toks) {
|
|
||||||
var tup = parse_single_tok(s.Trim());
|
|
||||||
rv.Add(tup);
|
|
||||||
}
|
|
||||||
return rv;
|
|
||||||
}
|
|
||||||
|
|
||||||
private Tuple<string, Type, object> _new_col;
|
private Tuple<string, Type, object> _new_col;
|
||||||
private List<Tuple<string, Type, object>> _new_col_map;
|
|
||||||
private IBlueTableSchema _out_schema;
|
private IBlueTableSchema _out_schema;
|
||||||
|
|
||||||
public AddColumnsStream(ViewStream vs, string col_def) {
|
public AddColumnsStream(ViewStream vs, string col_def) {
|
||||||
_new_col_map = parse_exp(col_def);
|
_new_col = parse_exp(col_def);
|
||||||
ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
|
ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
|
||||||
mySetRealSource(rs);
|
mySetRealSource(rs);
|
||||||
}
|
}
|
||||||
|
|
||||||
private IBlueTableSchema _get_schema(IBlueTableSchema input_schema) {
|
private IBlueTableSchema _get_schema(IBlueTableSchema input_schema) {
|
||||||
if (_out_schema == null) {
|
if (_out_schema == null)
|
||||||
_out_schema = input_schema;
|
_out_schema = BlueDataUtility.add_column(input_schema, _new_col.Item1, _new_col.Item2);
|
||||||
foreach (var m in _new_col_map) {
|
|
||||||
_out_schema = BlueDataUtility.add_column(_out_schema, m.Item1, m.Item2);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return _out_schema;
|
return _out_schema;
|
||||||
}
|
}
|
||||||
|
|
||||||
private IBlueRow _map(IBlueRow r) {
|
private IBlueRow _map(IBlueRow r) {
|
||||||
MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
|
MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
|
||||||
foreach (var m in _new_col_map) {
|
rv.setValue(_new_col.Item1, _new_col.Item3);
|
||||||
rv.setValue(m.Item1, m.Item3);
|
|
||||||
}
|
|
||||||
return rv;
|
return rv;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,44 +1,52 @@
|
|||||||
using System;
|
using System;
|
||||||
|
|
||||||
namespace bts.BlueOrb {
|
namespace BCG
|
||||||
using BCG;
|
{
|
||||||
[DataStreamUserRespresentation("oldCount")]
|
[DataStreamUserRespresentation ("count")]
|
||||||
public class CountStream : ViewStream {
|
public class CountStream : ViewStream
|
||||||
|
{
|
||||||
private DataPacket myLastFiredDataPacket;
|
private DataPacket myLastFiredDataPacket;
|
||||||
private IBlueTableSchema mySchema = new BlueTableSchema("count",
|
private IBlueTableSchema mySchema = new BlueTableSchema ("count",
|
||||||
new ColumnInfo("id", 0, typeof(int)),
|
new ColumnInfo ("id", 0, typeof(int)),
|
||||||
new ColumnInfo("count", 1, typeof(int)));
|
new ColumnInfo ("count", 1, typeof(int)));
|
||||||
private int myId = 0;
|
private int myId = 0;
|
||||||
private ViewStream myVS;
|
private ViewStream myVS;
|
||||||
|
|
||||||
public CountStream(ViewStream vs) {
|
public CountStream (ViewStream vs)
|
||||||
|
{
|
||||||
myVS = vs;
|
myVS = vs;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
public CountStream (string tableName)
|
||||||
|
: this (new TableStream (tableName))
|
||||||
|
{}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber () {
|
||||||
myVS.onChange += new ChangeDelegate(myHandleChanges);
|
myVS.onChange += new ChangeDelegate(myHandleChanges);
|
||||||
}
|
}
|
||||||
protected override void myOnLastSubscriber() {
|
protected override void myOnLastSubscriber () {
|
||||||
myVS.onChange -= new ChangeDelegate(myHandleChanges);
|
myVS.onChange -= new ChangeDelegate(myHandleChanges);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private void myHandleChanges(object newValue) {
|
private void myHandleChanges(object newValue)
|
||||||
myFireEvent(
|
{
|
||||||
|
myFireEvent (
|
||||||
myLastFiredDataPacket =
|
myLastFiredDataPacket =
|
||||||
myLastFiredDataPacket == null ?
|
myLastFiredDataPacket == null ?
|
||||||
new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
|
new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId),
|
||||||
new BlueTable(mySchema))
|
new BlueTable (mySchema))
|
||||||
:
|
:
|
||||||
new DataPacket(myGetRows(newValue, ++myId), myGetRows(newValue, myId),
|
new DataPacket (myGetRows (newValue, ++myId), myGetRows (newValue, myId),
|
||||||
myLastFiredDataPacket.newRows)
|
myLastFiredDataPacket.newRows)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
private IBlueRowCollection myGetRows(object dp, int id) {
|
private IBlueRowCollection myGetRows (object dp, int id)
|
||||||
BlueTable retVal = new BlueTable(mySchema);
|
{
|
||||||
|
BlueTable retVal = new BlueTable (mySchema);
|
||||||
retVal.addRow
|
retVal.addRow
|
||||||
(new BlueRow(mySchema, id, (dp as DataPacket).newRows.Count));
|
(new BlueRow (mySchema, id, (dp as DataPacket).newRows.Count));
|
||||||
return retVal;
|
return retVal;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,106 +1,93 @@
|
|||||||
using bts;
|
|
||||||
|
|
||||||
using Bts.DataStream;
|
using Bts.DataStream;
|
||||||
using Bts.DataStream.Primitives.Filter;
|
|
||||||
|
|
||||||
using mbts.config;
|
using mbts.config;
|
||||||
|
|
||||||
using System;
|
using System;
|
||||||
using System.Text.RegularExpressions;
|
using System.Text.RegularExpressions;
|
||||||
|
|
||||||
namespace BCG.StreamWorld.Addition {
|
namespace BCG.StreamWorld.Addition
|
||||||
[DataStreamUserRespresentation(false)]
|
{
|
||||||
public class FilterStream : ViewStream {
|
[DataStreamUserRespresentation (false)]
|
||||||
|
public class FilterStream : ViewStream
|
||||||
|
{
|
||||||
private ViewStream myBaseStream;
|
private ViewStream myBaseStream;
|
||||||
private BlueTable myBaseTable, myStateTable;
|
private BlueTable myBaseTable, myStateTable;
|
||||||
private IStreamFilter myFilter;
|
private IStreamFilter myFilter;
|
||||||
|
|
||||||
public FilterStream(ViewStream vs)
|
public FilterStream(ViewStream vs)
|
||||||
: this(vs, "1=1") { }
|
: this(vs, "1=1")
|
||||||
|
{ }
|
||||||
|
|
||||||
public FilterStream(ViewStream vs, string filterExp)
|
public FilterStream(ViewStream vs, string filterExp)
|
||||||
: this(vs, new BooleanExpressionFilter(filterExp)) { }
|
: this(vs, new BooleanExpressionFilter(filterExp))
|
||||||
|
{ }
|
||||||
|
|
||||||
public FilterStream(ViewStream vs, IStreamFilter filter) {
|
public FilterStream(ViewStream vs, IStreamFilter filter)
|
||||||
|
{
|
||||||
myBaseStream = vs;
|
myBaseStream = vs;
|
||||||
myFilter = filter;
|
myFilter = filter;
|
||||||
vs.ErrorChanged += OnError;
|
|
||||||
if (filter is IDynamicStreamFilter)
|
if (filter is IDynamicStreamFilter)
|
||||||
(filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
|
(filter as IDynamicStreamFilter).onFilterChange += new FilterChangeDelegate(myHandleFilterChange);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber() {
|
protected override void myOnFirstSubscriber()
|
||||||
|
{
|
||||||
myBaseStream.onChange += new ChangeDelegate(myHandleChanges);
|
myBaseStream.onChange += new ChangeDelegate(myHandleChanges);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myOnLastSubscriber() {
|
protected override void myOnLastSubscriber()
|
||||||
|
{
|
||||||
myBaseStream.onChange -= new ChangeDelegate(myHandleChanges);
|
myBaseStream.onChange -= new ChangeDelegate(myHandleChanges);
|
||||||
myBaseTable.clear();
|
myBaseTable.clear();
|
||||||
myStateTable.clear();
|
myStateTable.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
bool filterRow(IBlueRow r) {
|
private void myHandleChanges(object newValue)
|
||||||
try {
|
{
|
||||||
return myFilter.filter(r);
|
DataPacket dp = newValue as DataPacket;
|
||||||
} catch (Exception ex) {
|
if (myStateTable == null)
|
||||||
Log.Gen.error($"FilterStream delegate threw exception: {ex}");
|
{
|
||||||
OnError(new StreamFilterException(StreamId, StreamName, null, ex, r));
|
bts.Log.assert(myBaseTable == null);
|
||||||
|
myStateTable = new BlueTable(dp.schema);
|
||||||
|
myBaseTable = new BlueTable(dp.schema);
|
||||||
}
|
}
|
||||||
return false;
|
BlueTable dels = new BlueTable(dp.schema),
|
||||||
|
adds = new BlueTable(dp.schema);
|
||||||
|
foreach (IBlueRow r in dp.deletedRows)
|
||||||
|
{
|
||||||
|
myBaseTable.deleteRow(r);
|
||||||
|
if (myStateTable.containsID((int)r["id"]))
|
||||||
|
{
|
||||||
|
dels.addRow(r);
|
||||||
|
myStateTable.deleteRow(r);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
foreach (IBlueRow r in dp.addedRows)
|
||||||
|
{
|
||||||
|
myBaseTable.addRow(r);
|
||||||
|
if (myFilter.filter(r))
|
||||||
|
{
|
||||||
|
adds.addRow(r);
|
||||||
|
myStateTable.addRow(r);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ((dels.Count > 0 || adds.Count > 0) || (myLastValue == null))
|
||||||
|
myFireEvent(new DataPacket(myStateTable, adds, dels));
|
||||||
}
|
}
|
||||||
|
|
||||||
private void myHandleChanges(object newValue) {
|
public IStreamFilter filter
|
||||||
try {
|
{
|
||||||
DataPacket dp = newValue as DataPacket;
|
set
|
||||||
if (myStateTable == null) {
|
{
|
||||||
bts.Log.assert(myBaseTable == null);
|
|
||||||
myStateTable = new BlueTable(dp.schema);
|
|
||||||
myBaseTable = new BlueTable(dp.schema);
|
|
||||||
}
|
|
||||||
BlueTable dels = new BlueTable(dp.schema),
|
|
||||||
adds = new BlueTable(dp.schema);
|
|
||||||
foreach (IBlueRow r in dp.deletedRows) {
|
|
||||||
myBaseTable.deleteRow(r);
|
|
||||||
if (myStateTable.containsID((int)r["id"])) {
|
|
||||||
dels.addRow(r);
|
|
||||||
myStateTable.deleteRow(r);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
foreach (IBlueRow r in dp.addedRows) {
|
|
||||||
myBaseTable.addRow(r);
|
|
||||||
if (filterRow(r)) {
|
|
||||||
adds.addRow(r);
|
|
||||||
myStateTable.addRow(r);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if ((dels.Count > 0 || adds.Count > 0) || (myLastValue == null))
|
|
||||||
myFireEvent(new DataPacket(myStateTable, adds, dels));
|
|
||||||
} catch (Exception ex) {
|
|
||||||
OnError(ex);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public IStreamFilter filter {
|
|
||||||
set {
|
|
||||||
myFilter = value;
|
myFilter = value;
|
||||||
if (myBaseTable != null) {
|
if (myBaseTable != null)
|
||||||
|
{
|
||||||
IBlueRowCollection oldState = new BlueTable(myStateTable);
|
IBlueRowCollection oldState = new BlueTable(myStateTable);
|
||||||
myStateTable.clear();
|
myStateTable.clear();
|
||||||
// If the filter throws an exception, we can rethrow it as a StreamFilterException
|
foreach (IBlueRow r in myBaseTable)
|
||||||
// and include the 'currentRow' (for which the filter failed) to provide context.
|
if (myFilter.filter(r))
|
||||||
IBlueRow? currentRow = null;
|
myStateTable.addRow(r);
|
||||||
foreach (IBlueRow r in myBaseTable) {
|
|
||||||
try {
|
|
||||||
currentRow = r;
|
|
||||||
if (filterRow(r)) {
|
|
||||||
myStateTable.addRow(r);
|
|
||||||
}
|
|
||||||
} catch (Exception ex) {
|
|
||||||
Log.Gen.error($"FilterStream delegate threw exception: {ex}");
|
|
||||||
OnError(new StreamFilterException(StreamId, StreamName, null, ex, currentRow));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
IBlueRowCollection dels, adds;
|
IBlueRowCollection dels, adds;
|
||||||
BlueDataUtility.computeDifferences(myStateTable, oldState, out dels, out adds);
|
BlueDataUtility.computeDifferences(myStateTable, oldState, out dels, out adds);
|
||||||
myFireEvent(new DataPacket(myStateTable, adds, dels));
|
myFireEvent(new DataPacket(myStateTable, adds, dels));
|
||||||
@@ -235,36 +222,32 @@ namespace BCG
|
|||||||
DataPacket retVal;
|
DataPacket retVal;
|
||||||
DataPacket arrivedPacket = newValue as DataPacket;
|
DataPacket arrivedPacket = newValue as DataPacket;
|
||||||
myLastReceivedRows = arrivedPacket.newRows;
|
myLastReceivedRows = arrivedPacket.newRows;
|
||||||
try {
|
if (myLastFiredRows != null)
|
||||||
if (myLastFiredRows != null)
|
{
|
||||||
{
|
retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
|
||||||
retVal = new DataPacket(myLastFiredRows, new BlueTable(myLastFiredRows.schema),
|
new BlueTable(myLastFiredRows.schema));
|
||||||
new BlueTable(myLastFiredRows.schema));
|
foreach (IBlueRow r in arrivedPacket.deletedRows)
|
||||||
foreach (IBlueRow r in arrivedPacket.deletedRows)
|
if (myLastFiredRows.containsID((int)r["id"]))
|
||||||
if (myLastFiredRows.containsID((int)r["id"]))
|
{
|
||||||
{
|
myLastFiredRows.deleteRow(r);
|
||||||
myLastFiredRows.deleteRow(r);
|
(retVal.deletedRows as BlueTable).addRow(r);
|
||||||
(retVal.deletedRows as BlueTable).addRow(r);
|
}
|
||||||
}
|
foreach (IBlueRow r in arrivedPacket.addedRows)
|
||||||
foreach (IBlueRow r in arrivedPacket.addedRows)
|
if (myFilter.filter(r))
|
||||||
if (filterRow(r))
|
{
|
||||||
{
|
myLastFiredRows.addRow(r);
|
||||||
myLastFiredRows.addRow(r);
|
(retVal.addedRows as BlueTable).addRow(r);
|
||||||
(retVal.addedRows as BlueTable).addRow(r);
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
myLastFiredRows = new BlueTable(arrivedPacket.schema);
|
|
||||||
retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
|
|
||||||
foreach (IBlueRow r in arrivedPacket.newRows)
|
|
||||||
if (filterRow(r))
|
|
||||||
myLastFiredRows.addRow(r);
|
|
||||||
}
|
|
||||||
myFireEvent(retVal);
|
|
||||||
} catch (Exception ex) {
|
|
||||||
OnError(ex);
|
|
||||||
}
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
myLastFiredRows = new BlueTable(arrivedPacket.schema);
|
||||||
|
retVal = new DataPacket(myLastFiredRows, myLastFiredRows, new BlueTable(myLastFiredRows.schema));
|
||||||
|
foreach (IBlueRow r in arrivedPacket.newRows)
|
||||||
|
if (myFilter.filter(r))
|
||||||
|
myLastFiredRows.addRow(r);
|
||||||
|
}
|
||||||
|
myFireEvent(retVal);
|
||||||
}
|
}
|
||||||
|
|
||||||
public IStreamFilter filter
|
public IStreamFilter filter
|
||||||
@@ -282,20 +265,16 @@ namespace BCG
|
|||||||
{
|
{
|
||||||
if (myLastReceivedRows != null)
|
if (myLastReceivedRows != null)
|
||||||
{
|
{
|
||||||
try {
|
|
||||||
BlueTable nRows = new BlueTable(myLastReceivedRows.schema);
|
BlueTable nRows = new BlueTable(myLastReceivedRows.schema);
|
||||||
foreach (IBlueRow r in myLastReceivedRows)
|
foreach (IBlueRow r in myLastReceivedRows)
|
||||||
if (filterRow(r))
|
if (myFilter.filter(r))
|
||||||
nRows.addRow(r);
|
nRows.addRow(r);
|
||||||
IBlueRowCollection dRows = new BlueTable(nRows.schema),
|
IBlueRowCollection dRows = new BlueTable(nRows.schema),
|
||||||
aRows = new BlueTable(nRows.schema);
|
aRows = new BlueTable(nRows.schema);
|
||||||
BlueDataUtility.computeDifferences(nRows, myLastFiredRows, out dRows, out aRows);
|
BlueDataUtility.computeDifferences(nRows, myLastFiredRows, out dRows, out aRows);
|
||||||
myLastFiredRows = nRows;
|
myLastFiredRows = nRows;
|
||||||
myFireEvent(new DataPacket(nRows, aRows, dRows));
|
myFireEvent(new DataPacket(nRows, aRows, dRows));
|
||||||
} catch (Exception ex) {
|
}
|
||||||
OnError(ex);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -319,15 +298,5 @@ namespace BCG
|
|||||||
{
|
{
|
||||||
filter = myFilter;
|
filter = myFilter;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool filterRow(IBlueRow r) {
|
|
||||||
try {
|
|
||||||
return myFilter.filter(r);
|
|
||||||
} catch (Exception ex) {
|
|
||||||
Log.Gen.error($"FilterStream delegate threw exception: {ex}");
|
|
||||||
OnError(new StreamFilterException(StreamId, StreamName, null, ex, r));
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,19 +0,0 @@
|
|||||||
#nullable enable
|
|
||||||
namespace Bts.DataStream.Primitives.Filter;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
|
|
||||||
public class StreamFilterException : Exception {
|
|
||||||
public StreamFilterException(Guid streamId, string streamName, string? message, Exception? innerException, IBlueRow? badRow = null)
|
|
||||||
: base(message, innerException) {
|
|
||||||
StreamId = streamId;
|
|
||||||
StreamName = streamName;
|
|
||||||
BadRow = badRow;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Guid StreamId { get; }
|
|
||||||
public string StreamName { get; }
|
|
||||||
public IBlueRow? BadRow { get; }
|
|
||||||
}
|
|
||||||
@@ -52,14 +52,14 @@ namespace BCG {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive)
|
public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive)
|
||||||
: this(vsd, rowTimeColumnName, rowTimeToLive, rowTimeToLive) {
|
: this(vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) {
|
||||||
}
|
}
|
||||||
|
|
||||||
public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filter_period) {
|
public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) {
|
||||||
myVsd = vsd;
|
myVsd = vsd;
|
||||||
_time_col = rowTimeColumnName;
|
_time_col = rowTimeColumnName;
|
||||||
_ttl = rowTimeToLive;
|
_ttl = rowTimeToLive;
|
||||||
_filter_period = filter_period;
|
myFilterFrequency = filterFrequency;
|
||||||
myNextFilter = DateTime.MinValue;
|
myNextFilter = DateTime.MinValue;
|
||||||
|
|
||||||
// add worker
|
// add worker
|
||||||
@@ -71,10 +71,11 @@ namespace BCG {
|
|||||||
|
|
||||||
#region IStreamFilter
|
#region IStreamFilter
|
||||||
public bool filter(IBlueRow row) {
|
public bool filter(IBlueRow row) {
|
||||||
DateTime now_utc = DateTime.UtcNow;
|
DateTime now = DateTime.Now;
|
||||||
DateTime row_time_utc = row.get_datetime(_time_col, DateTime.Now).ToUniversalTime();
|
|
||||||
if (now_utc > row_time_utc) {
|
DateTime row_time = row.get_datetime(_time_col, DateTime.Now);
|
||||||
TimeSpan span = now_utc - row_time_utc;
|
if (now > row_time) {
|
||||||
|
TimeSpan span = now - row_time;
|
||||||
if (span >= _ttl) {
|
if (span >= _ttl) {
|
||||||
return (false);
|
return (false);
|
||||||
}
|
}
|
||||||
@@ -89,9 +90,9 @@ namespace BCG {
|
|||||||
|
|
||||||
if (now > myNextFilter) {
|
if (now > myNextFilter) {
|
||||||
// fire event
|
// fire event
|
||||||
onFilterChange?.Invoke();
|
onFilterChange();
|
||||||
|
|
||||||
myNextFilter = now.Add(_filter_period);
|
myNextFilter = now.Add(myFilterFrequency);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#endregion
|
#endregion
|
||||||
@@ -107,7 +108,7 @@ namespace BCG {
|
|||||||
private ViewStreamDatabase myVsd;
|
private ViewStreamDatabase myVsd;
|
||||||
private string _time_col;
|
private string _time_col;
|
||||||
private TimeSpan _ttl;
|
private TimeSpan _ttl;
|
||||||
private TimeSpan _filter_period;
|
private TimeSpan myFilterFrequency;
|
||||||
private DateTime myNextFilter;
|
private DateTime myNextFilter;
|
||||||
#endregion
|
#endregion
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ using System.Collections.Generic;
|
|||||||
using System.Linq;
|
using System.Linq;
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG {
|
||||||
internal class GroupByStream : MethodDrivenStream {
|
public class GroupByStream : MethodDrivenStream {
|
||||||
private class MyFuncGroupByEvaluator : IGroupByEvaluator {
|
private class MyFuncGroupByEvaluator : IGroupByEvaluator {
|
||||||
private string[] _groupby_cols;
|
private string[] _groupby_cols;
|
||||||
private Func<DataPacket, IBlueRow> _groupby_func;
|
private Func<DataPacket, IBlueRow> _groupby_func;
|
||||||
|
|||||||
+1
-5
@@ -8,7 +8,7 @@ using bts;
|
|||||||
using bts.SilverPine;
|
using bts.SilverPine;
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG {
|
||||||
//[DataStreamUserRespresentation("AggregateStream")]
|
[DataStreamUserRespresentation("AggregateStream")]
|
||||||
public class NonIncGroupByStream : ViewStream {
|
public class NonIncGroupByStream : ViewStream {
|
||||||
private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
|
private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
|
||||||
Func<string[]> _groupby_cols_func;
|
Func<string[]> _groupby_cols_func;
|
||||||
@@ -96,10 +96,6 @@ namespace BCG {
|
|||||||
_timer.Restart();
|
_timer.Restart();
|
||||||
IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
|
IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
|
||||||
_timer.Stop();
|
_timer.Stop();
|
||||||
if (total_state is null) {
|
|
||||||
Log.Gen.warn("NonIncGroupByStream _groupby_func.evaluate returned null (ignoring last update).");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
var applyElapsedMs = _timer.ElapsedMilliseconds;
|
var applyElapsedMs = _timer.ElapsedMilliseconds;
|
||||||
_n_total_fires++;
|
_n_total_fires++;
|
||||||
if (applyElapsedMs >= 50) {
|
if (applyElapsedMs >= 50) {
|
||||||
|
|||||||
@@ -1,31 +1,31 @@
|
|||||||
//using System;
|
using System;
|
||||||
//using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
//using System.Linq;
|
using System.Linq;
|
||||||
//using System.Text;
|
using System.Text;
|
||||||
|
|
||||||
//namespace BCG
|
namespace BCG
|
||||||
//{
|
{
|
||||||
// public class FullJoinStream : ViewStream
|
public class FullJoinStream : ViewStream
|
||||||
// {
|
{
|
||||||
// private ViewStream myMainStream;
|
private ViewStream myMainStream;
|
||||||
// public FullJoinStream(ViewStream v1, string joinCol1, string cmExp1,
|
public FullJoinStream(ViewStream v1, string joinCol1, string cmExp1,
|
||||||
// ViewStream v2, string joinCol2, string cmExp2, string empty_col)
|
ViewStream v2, string joinCol2, string cmExp2, string empty_col)
|
||||||
// {
|
{
|
||||||
// myMainStream = new LeftJoinStream(v1, joinCol1, cmExp1, v2, joinCol2, cmExp2);
|
myMainStream = new LeftJoinStream(v1, joinCol1, cmExp1, v2, joinCol2, cmExp2);
|
||||||
// ViewStream filtered = new LeftJoinStream(v2, joinCol2, cmExp2, v1, joinCol1, cmExp1);
|
ViewStream filtered = new LeftJoinStream(v2, joinCol2, cmExp2, v1, joinCol1, cmExp1);
|
||||||
// filtered = new FilterStream(filtered, String.Format("{0}=null", empty_col));
|
filtered = new FilterStream(filtered, String.Format("{0}=null", empty_col));
|
||||||
// myMainStream = new AppendStream(myMainStream, filtered);
|
myMainStream = new AppendStream(myMainStream, filtered);
|
||||||
// }
|
}
|
||||||
|
|
||||||
// protected override void myOnFirstSubscriber()
|
protected override void myOnFirstSubscriber()
|
||||||
// {
|
{
|
||||||
// myMainStream.onChange += myHandleMainStreamChange;
|
myMainStream.onChange += myHandleMainStreamChange;
|
||||||
// }
|
}
|
||||||
|
|
||||||
// private void myHandleMainStreamChange(object newData)
|
private void myHandleMainStreamChange(object newData)
|
||||||
// {
|
{
|
||||||
// IBlueRowCollection result = (newData as DataPacket).newRows;
|
IBlueRowCollection result = (newData as DataPacket).newRows;
|
||||||
// myFireEvent(new DataPacket(result, result, myLastValue == null ? new BlueTable(result.schema) : (myLastValue as DataPacket).newRows));
|
myFireEvent(new DataPacket(result, result, myLastValue == null ? new BlueTable(result.schema) : (myLastValue as DataPacket).newRows));
|
||||||
// }
|
}
|
||||||
// }
|
}
|
||||||
//}
|
}
|
||||||
@@ -1,7 +1,9 @@
|
|||||||
#nullable disable
|
using System;
|
||||||
using System;
|
|
||||||
using System.Collections;
|
using System.Collections;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
|
using System.Text;
|
||||||
|
using BCG.Utilities.General;
|
||||||
|
using BCG.Utilities;
|
||||||
|
|
||||||
namespace BCG
|
namespace BCG
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -10,8 +10,6 @@ using BCG.Utilities;
|
|||||||
using bts;
|
using bts;
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG {
|
||||||
// Prefixing with "old" to avoid name conflict with the BlueOrb LiteralStream
|
|
||||||
[DataStreamUserRespresentation("oldLiteral")]
|
|
||||||
public class LiteralStream : ViewStream {
|
public class LiteralStream : ViewStream {
|
||||||
|
|
||||||
public static List<Tuple<ColumnInfo, object>> parse_columns(string key_value_pairs) {
|
public static List<Tuple<ColumnInfo, object>> parse_columns(string key_value_pairs) {
|
||||||
|
|||||||
@@ -1,10 +1,6 @@
|
|||||||
using bts;
|
|
||||||
|
|
||||||
using Bts.DataStream;
|
using Bts.DataStream;
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using mbts.config;
|
using mbts.config;
|
||||||
using mbts.utils;
|
|
||||||
|
|
||||||
using System;
|
using System;
|
||||||
|
|
||||||
@@ -19,8 +15,8 @@ namespace BCG {
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
[DataStreamUserRespresentation(false)]
|
[DataStreamUserRespresentation(false)]
|
||||||
public class MapStream : MethodDrivenStream {
|
public class MapStream : MethodDrivenStream {
|
||||||
protected IStreamMapper? myMapper;
|
protected IStreamMapper myMapper;
|
||||||
protected IChangeableStreamMapper? myChangeableMapper;
|
protected IChangeableStreamMapper myChangeableMapper = null;
|
||||||
private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;
|
private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;
|
||||||
|
|
||||||
public class FuncWrapper : IStreamMapper {
|
public class FuncWrapper : IStreamMapper {
|
||||||
@@ -75,7 +71,7 @@ namespace BCG {
|
|||||||
|
|
||||||
|
|
||||||
protected override void myHandleFirstCall() {
|
protected override void myHandleFirstCall() {
|
||||||
ArgumentNullException.ThrowIfNull(myMapper);
|
|
||||||
IBlueTableSchema outputSchema = myMapper.getOutputSchema(myLastArrivedPacket.newRows.schema);
|
IBlueTableSchema outputSchema = myMapper.getOutputSchema(myLastArrivedPacket.newRows.schema);
|
||||||
myLastFiredPacket = new DataPacket();
|
myLastFiredPacket = new DataPacket();
|
||||||
myLastFiredPacket.newRows = new BlueTable(outputSchema);
|
myLastFiredPacket.newRows = new BlueTable(outputSchema);
|
||||||
@@ -86,34 +82,28 @@ namespace BCG {
|
|||||||
}
|
}
|
||||||
|
|
||||||
protected override void myHandleIncrementalAdd() {
|
protected override void myHandleIncrementalAdd() {
|
||||||
if (myLastArrivedPacket.addedRows.Count == 0) {
|
//t.start ();
|
||||||
Log.Gen.warn("MapStream.myHandleIncrementalAdd called with 0 addedRows");
|
myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows.schema);
|
||||||
return;
|
|
||||||
}
|
|
||||||
ArgumentNullException.ThrowIfNull(myMapper);
|
|
||||||
var addedRows = new BlueTable(myLastFiredPacket.newRows.schema);
|
|
||||||
var newRows = ExceptionAssert.Cast(myLastFiredPacket.newRows).To<BlueTable>();
|
|
||||||
foreach (IBlueRow row in myLastArrivedPacket.addedRows) {
|
foreach (IBlueRow row in myLastArrivedPacket.addedRows) {
|
||||||
var newRow = myMapper.map(row);
|
IBlueRow newRow = myMapper.map(row);
|
||||||
newRows.addRow(newRow);
|
(myLastFiredPacket.newRows as BlueTable).addRow(newRow);
|
||||||
addedRows.addRow(newRow);
|
(myLastFiredPacket.addedRows as BlueTable).addRow(newRow);
|
||||||
}
|
}
|
||||||
myLastFiredPacket.addedRows = addedRows;
|
//t.stop ();
|
||||||
|
//System.Diagnostics.Trace.WriteLine ("adding " + myLastFiredPacket.addedRows.Count.ToString () + " : " + t.lastIntervalTime);
|
||||||
|
//System.Diagnostics.Trace.WriteLine (" ");
|
||||||
}
|
}
|
||||||
|
|
||||||
protected override void myHandleIncrementalDelete() {
|
protected override void myHandleIncrementalDelete() {
|
||||||
if (myLastArrivedPacket.deletedRows.Count == 0) {
|
//t.start ();
|
||||||
Log.Gen.warn("MapStream.myHandleIncrementalDelete called with 0 deletedRows");
|
myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows.schema);
|
||||||
return;
|
|
||||||
}
|
|
||||||
var deletedRows = new BlueTable(myLastFiredPacket.newRows.schema);
|
|
||||||
var newRows = ExceptionAssert.Cast(myLastFiredPacket.newRows).To<BlueTable>();
|
|
||||||
foreach (IBlueRow row in myLastArrivedPacket.deletedRows) {
|
foreach (IBlueRow row in myLastArrivedPacket.deletedRows) {
|
||||||
var delRow = newRows[row.Id()];
|
IBlueRow delRow = (myLastFiredPacket.newRows as BlueTable)[(int)row["id"]];
|
||||||
deletedRows.addRow(delRow);
|
(myLastFiredPacket.deletedRows as BlueTable).addRow(delRow);
|
||||||
newRows.deleteRow(row);
|
(myLastFiredPacket.newRows as BlueTable).deleteRow(row);
|
||||||
}
|
}
|
||||||
myLastFiredPacket.deletedRows = deletedRows;
|
//t.stop ();
|
||||||
|
//System.Diagnostics.Trace.WriteLine ("deleting " + myLastFiredPacket.deletedRows.Count.ToString () + " : " + t.lastIntervalTime);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,9 +1,11 @@
|
|||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using mbts.utils;
|
|
||||||
|
|
||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
|
using System.Text;
|
||||||
|
|
||||||
|
using BCG.Utilities;
|
||||||
|
using BCG.Utilities.General;
|
||||||
|
|
||||||
|
using mbts;
|
||||||
|
|
||||||
namespace BCG {
|
namespace BCG {
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@@ -54,15 +56,15 @@ namespace BCG {
|
|||||||
return rows == null ? new IBlueRow[] { } : new List<IBlueRow>(rows).ToArray();
|
return rows == null ? new IBlueRow[] { } : new List<IBlueRow>(rows).ToArray();
|
||||||
}
|
}
|
||||||
|
|
||||||
public event MapperChangeDelegate? onMapperChange;
|
public event MapperChangeDelegate onMapperChange;
|
||||||
|
private void _fake() { onMapperChange(); }
|
||||||
}
|
}
|
||||||
|
|
||||||
//maps the id of the underlying stream to the
|
//maps the id of the underlying stream to the
|
||||||
//rows of the resulting stream
|
//rows of the resulting stream
|
||||||
private Dictionary<int, List<IBlueRow>> myMap = new Dictionary<int, List<IBlueRow>>();
|
private Dictionary<int, List<IBlueRow>> myMap = new Dictionary<int, List<IBlueRow>>();
|
||||||
private int myID = 0;
|
private int myID = 0;
|
||||||
private BlueTable? myRows;
|
private BlueTable myRows, myBaseRows;
|
||||||
private BlueTable? myBaseRows;
|
|
||||||
private ViewStream myStream;
|
private ViewStream myStream;
|
||||||
private ISuperStreamMapper myMapper;
|
private ISuperStreamMapper myMapper;
|
||||||
public bool generate_ids = true;
|
public bool generate_ids = true;
|
||||||
@@ -90,40 +92,49 @@ namespace BCG {
|
|||||||
myRows = null;
|
myRows = null;
|
||||||
myBaseRows = null;
|
myBaseRows = null;
|
||||||
myMap.Clear();
|
myMap.Clear();
|
||||||
|
//myMapper.onMapperChange -= new MapperChangeDelegate(myHandleMapChange);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void myHandleData(object newValue) {
|
private void myHandleData(object newValue) {
|
||||||
var dp = ExceptionAssert.Cast(newValue).To<DataPacket>();
|
DataPacket idp = newValue as DataPacket;
|
||||||
myRows ??= new BlueTable(myMapper.getOutputSchema(dp.schema));
|
//HighPerformanceTimer timer = new HighPerformanceTimer();
|
||||||
myBaseRows ??= new BlueTable(dp.schema);
|
//timer.start();
|
||||||
|
DataPacket dp = newValue as DataPacket;
|
||||||
|
myRows = myRows ?? new BlueTable(myMapper.getOutputSchema(dp.schema));
|
||||||
|
myBaseRows = myBaseRows ?? new BlueTable(dp.schema);
|
||||||
BlueDataUtility.absorbChanges(myBaseRows, dp.deletedRows, dp.addedRows);
|
BlueDataUtility.absorbChanges(myBaseRows, dp.deletedRows, dp.addedRows);
|
||||||
var dels = new BlueTable(myRows.schema);
|
BlueTable dels = new BlueTable(myRows.schema),
|
||||||
var adds = new BlueTable(myRows.schema);
|
adds = new BlueTable(myRows.schema);
|
||||||
|
|
||||||
foreach (IBlueRow r in dp.deletedRows) {
|
foreach (IBlueRow r in dp.deletedRows) {
|
||||||
var mappedRows = myMap[r.Id()];
|
List<IBlueRow> mappedRows = myMap[(int)r["id"]];
|
||||||
foreach (IBlueRow mr in mappedRows) {
|
foreach (IBlueRow mr in mappedRows) {
|
||||||
dels.addRow(mr);
|
dels.addRow(mr);
|
||||||
myRows.deleteRow(mr);
|
myRows.deleteRow(mr);
|
||||||
}
|
}
|
||||||
myMap.Remove(r.Id());
|
myMap.Remove((int)r["id"]);
|
||||||
}
|
}
|
||||||
foreach (IBlueRow r in dp.addedRows) {
|
foreach (IBlueRow r in dp.addedRows) {
|
||||||
IBlueRow[] mappedRows = myMapper.map(r);
|
IBlueRow[] mappedRows = myMapper.map(r);
|
||||||
foreach (IBlueRow mr in mappedRows) {
|
foreach (IBlueRow mr in mappedRows) {
|
||||||
IBlueRow idChangedMapRow = generate_ids ? BlueDataUtility.getNewReplacedRow(mr, myID++, "id") : mr;
|
IBlueRow idChangedMapRow = generate_ids ? BlueDataUtility.getNewReplacedRow(mr, myID++, "id") : mr;
|
||||||
myMap.GetOrAdd(r.Id(), _ => new()).Add(idChangedMapRow);
|
if (!myMap.ContainsKey((int)r["id"])) {
|
||||||
|
myMap.Add((int)r["id"], new List<IBlueRow>());
|
||||||
|
}
|
||||||
|
myMap[(int)r["id"]].Add(idChangedMapRow);
|
||||||
myRows.addRow(idChangedMapRow);
|
myRows.addRow(idChangedMapRow);
|
||||||
adds.addRow(idChangedMapRow);
|
adds.addRow(idChangedMapRow);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
//timer.stop();
|
||||||
|
// if (timer.lastIntervalTime > 20)
|
||||||
|
// {}//mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("SuperMapper:myHandlerData for mapper {0} took {1} msecs", myMapper.GetType().ToString(), timer.lastIntervalTime));
|
||||||
myFireEvent(new DataPacket(myRows, adds, dels));
|
myFireEvent(new DataPacket(myRows, adds, dels));
|
||||||
}
|
}
|
||||||
|
|
||||||
private void myHandleMapChange() {
|
private void myHandleMapChange() {
|
||||||
if (myBaseRows is not null) {
|
if (myBaseRows != null)
|
||||||
myHandleData(new DataPacket(myBaseRows, new BlueTable(myBaseRows), new BlueTable(myBaseRows)));
|
myHandleData(new DataPacket(myBaseRows, new BlueTable(myBaseRows), new BlueTable(myBaseRows)));
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -70,8 +70,7 @@ namespace bts
|
|||||||
private readonly int _msg_size;
|
private readonly int _msg_size;
|
||||||
private readonly TimeSpan _publish_period;
|
private readonly TimeSpan _publish_period;
|
||||||
private DateTime _last_publish_time = DateTime.MinValue;
|
private DateTime _last_publish_time = DateTime.MinValue;
|
||||||
private IPublisherCore _pub_core;
|
private PublisherCore _pub_core = new PublisherCore();
|
||||||
readonly ISubscriberCore subscriberCore;
|
|
||||||
private BlueTable _table;
|
private BlueTable _table;
|
||||||
private int _n_pubs = 0;
|
private int _n_pubs = 0;
|
||||||
private readonly int _max_pubs;
|
private readonly int _max_pubs;
|
||||||
@@ -83,10 +82,8 @@ namespace bts
|
|||||||
private readonly int _payload_sz;
|
private readonly int _payload_sz;
|
||||||
private const byte _payload_byte = (byte)'A';
|
private const byte _payload_byte = (byte)'A';
|
||||||
private readonly string _tag_name;
|
private readonly string _tag_name;
|
||||||
public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, IPublisherCore publisherCore, ISubscriberCore subscriberCore, ViewStreamDatabase vsd)
|
public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd)
|
||||||
{
|
{
|
||||||
_pub_core = publisherCore;
|
|
||||||
this.subscriberCore = subscriberCore;
|
|
||||||
_vsd = vsd;
|
_vsd = vsd;
|
||||||
_payload_sz = payload_size;
|
_payload_sz = payload_size;
|
||||||
var schema = BlueDataUtility.form_schema("PubSubTest",
|
var schema = BlueDataUtility.form_schema("PubSubTest",
|
||||||
@@ -106,6 +103,9 @@ namespace bts
|
|||||||
_tag_name = tag_name;
|
_tag_name = tag_name;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size, string tag_name)
|
||||||
|
: this(publish_period_seconds, max_pubs, msg_size, tag_name, ViewStreamDatabase.instance) { }
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber()
|
protected override void myOnFirstSubscriber()
|
||||||
{
|
{
|
||||||
_vsd.addWorker(this);
|
_vsd.addWorker(this);
|
||||||
@@ -132,31 +132,33 @@ namespace bts
|
|||||||
|
|
||||||
private void _subscribe_thread_func()
|
private void _subscribe_thread_func()
|
||||||
{
|
{
|
||||||
var sub = subscriberCore.Updates().Subscribe(onUpdate);
|
|
||||||
// subscription is done in another thread
|
// subscription is done in another thread
|
||||||
subscriberCore.subscribe(_tag_name);
|
SubscriberCore sub_core = new SubscriberCore();
|
||||||
|
sub_core.subscribe(_tag_name);
|
||||||
while (_continue_sub_thread)
|
while (_continue_sub_thread)
|
||||||
{
|
{
|
||||||
// TODO recv_id
|
// TODO recv_id
|
||||||
subscriberCore.get_updates(TimeSpan.FromMilliseconds(250));
|
var updates = sub_core.get_updates(TimeSpan.FromMilliseconds(250));
|
||||||
|
if (updates == null || updates.Count() == 0)
|
||||||
|
continue;
|
||||||
|
_log_updates(updates);
|
||||||
|
_n_updates += updates.Count();
|
||||||
|
if (updates.Count() > 0)
|
||||||
|
{
|
||||||
|
var server_msg = updates.Last();
|
||||||
|
if (server_msg.data != null && server_msg.data.Length > 0)
|
||||||
|
{
|
||||||
|
_last_recvd_msg = PayloadMessage.deserialize(server_msg.data);
|
||||||
|
_last_recvd_msg.travel_epoch_ms = epoch_now();
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// why are we getting empty or null data ? (server is sending data regardless?)
|
||||||
|
}
|
||||||
|
_fire_dirty = true;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
sub.Dispose();
|
}
|
||||||
|
|
||||||
void onUpdate(IEnumerable<PubSubServerMessage> updates) {
|
|
||||||
_log_updates(updates);
|
|
||||||
_n_updates += updates.Count();
|
|
||||||
if (updates.Count() > 0) {
|
|
||||||
var server_msg = updates.Last();
|
|
||||||
if (server_msg.data != null && server_msg.data.Length > 0) {
|
|
||||||
_last_recvd_msg = PayloadMessage.deserialize(server_msg.data);
|
|
||||||
_last_recvd_msg.travel_epoch_ms = epoch_now();
|
|
||||||
} else {
|
|
||||||
// why are we getting empty or null data ? (server is sending data regardless?)
|
|
||||||
}
|
|
||||||
_fire_dirty = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void _publish_next_packet()
|
private void _publish_next_packet()
|
||||||
{
|
{
|
||||||
@@ -222,4 +224,4 @@ namespace bts
|
|||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -8,11 +8,10 @@ using System.IO;
|
|||||||
using System.Linq;
|
using System.Linq;
|
||||||
using System.Runtime.CompilerServices;
|
using System.Runtime.CompilerServices;
|
||||||
using bts.api;
|
using bts.api;
|
||||||
using Bts.PubSub.Routing;
|
|
||||||
|
|
||||||
namespace bts
|
namespace bts
|
||||||
{
|
{
|
||||||
[DataStreamUserRespresentation(false)]
|
[DataStreamUserRespresentation("pss_echo")]
|
||||||
public class PubsubEchoTestStream : ViewStream, IWorker
|
public class PubsubEchoTestStream : ViewStream, IWorker
|
||||||
{
|
{
|
||||||
public class PayloadMessage
|
public class PayloadMessage
|
||||||
@@ -77,7 +76,7 @@ namespace bts
|
|||||||
private readonly int _msg_size;
|
private readonly int _msg_size;
|
||||||
private readonly TimeSpan _publish_period;
|
private readonly TimeSpan _publish_period;
|
||||||
private DateTime _last_publish_time = DateTime.MinValue;
|
private DateTime _last_publish_time = DateTime.MinValue;
|
||||||
private IPublisherCore _pub_core;
|
private PublisherCore _pub_core;
|
||||||
private ISubscriberCore _sub_core;
|
private ISubscriberCore _sub_core;
|
||||||
private BlueTable _table;
|
private BlueTable _table;
|
||||||
private int _n_pubs = 0;
|
private int _n_pubs = 0;
|
||||||
@@ -96,21 +95,10 @@ namespace bts
|
|||||||
};
|
};
|
||||||
IOState _state = IOState.SEND;
|
IOState _state = IOState.SEND;
|
||||||
|
|
||||||
public PubsubEchoTestStream(
|
public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd)
|
||||||
string server_url,
|
{
|
||||||
TimeSpan publish_period,
|
var route = new PubSubRoute(server_url, ".*");
|
||||||
int payload_size,
|
_router = new PubSubRouter(route);
|
||||||
string tag_name,
|
|
||||||
IPublisherCore publisherCore,
|
|
||||||
ISubscriberCore subscriberCore,
|
|
||||||
ViewStreamDatabase vsd
|
|
||||||
) {
|
|
||||||
_router = new PubSubRouter(new PubSubRouterConfig() {
|
|
||||||
PublisherRoutes = new() { new(server_url, ".*") },
|
|
||||||
SubscriberRoutes = new() { new(server_url, ".*") },
|
|
||||||
});
|
|
||||||
_pub_core = publisherCore;
|
|
||||||
_sub_core = subscriberCore;
|
|
||||||
_vsd = vsd;
|
_vsd = vsd;
|
||||||
_payload_sz = payload_size;
|
_payload_sz = payload_size;
|
||||||
var schema = BlueDataUtility.form_schema("PubSubTest",
|
var schema = BlueDataUtility.form_schema("PubSubTest",
|
||||||
@@ -126,8 +114,13 @@ namespace bts
|
|||||||
_tag_name = tag_name;
|
_tag_name = tag_name;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public PubsubEchoTestStream(string server_url, int publish_period_msecs, int payload_size, string tag_name)
|
||||||
|
: this(server_url, TimeSpan.FromMilliseconds(publish_period_msecs), payload_size, tag_name, ViewStreamDatabase.instance) { }
|
||||||
|
|
||||||
protected override void myOnFirstSubscriber()
|
protected override void myOnFirstSubscriber()
|
||||||
{
|
{
|
||||||
|
_pub_core = new PublisherCore(_router);
|
||||||
|
_sub_core = new SubscriberCore(_router);
|
||||||
_sub_core.subscribe(_tag_name);
|
_sub_core.subscribe(_tag_name);
|
||||||
_vsd.addWorker(this);
|
_vsd.addWorker(this);
|
||||||
|
|
||||||
@@ -227,4 +220,4 @@ namespace bts
|
|||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -66,7 +66,7 @@ namespace bts {
|
|||||||
if (_in_rows.Count > 0) {
|
if (_in_rows.Count > 0) {
|
||||||
// fire row
|
// fire row
|
||||||
IBlueRow curr_row = _in_rows.ElementAt(_in_row_ptr++);
|
IBlueRow curr_row = _in_rows.ElementAt(_in_row_ptr++);
|
||||||
_out_table = _out_table.Add(curr_row);
|
_out_table.add_row(curr_row);
|
||||||
DataPacket dp = BlueDataUtility.build_dp_inc(_out_table,
|
DataPacket dp = BlueDataUtility.build_dp_inc(_out_table,
|
||||||
myLastValue == null ? null : myLastValue as DataPacket, false);
|
myLastValue == null ? null : myLastValue as DataPacket, false);
|
||||||
myFireEvent(dp);
|
myFireEvent(dp);
|
||||||
@@ -74,4 +74,4 @@ namespace bts {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
@@ -95,7 +95,7 @@ namespace BCG {
|
|||||||
sub_msg.serialize_into_str(str);
|
sub_msg.serialize_into_str(str);
|
||||||
str.Position = 0;
|
str.Position = 0;
|
||||||
bts.SubscribeMsg sub_msg_ = new bts.SubscribeMsg();
|
bts.SubscribeMsg sub_msg_ = new bts.SubscribeMsg();
|
||||||
sub_msg_.construct_deserialize(str);
|
sub_msg_.consruct_deserialize(str);
|
||||||
bool res = sub_msg.Equals(sub_msg_);
|
bool res = sub_msg.Equals(sub_msg_);
|
||||||
return new TestResult {
|
return new TestResult {
|
||||||
test_name = "SubscribeMsg",
|
test_name = "SubscribeMsg",
|
||||||
@@ -117,7 +117,7 @@ namespace BCG {
|
|||||||
pub_msg.serialize_into_str(str);
|
pub_msg.serialize_into_str(str);
|
||||||
str.Position = 0;
|
str.Position = 0;
|
||||||
var pub_msg_ = new bts.PublishMsg();
|
var pub_msg_ = new bts.PublishMsg();
|
||||||
pub_msg_.construct_deserialize(str);
|
pub_msg_.consruct_deserialize(str);
|
||||||
bool res = pub_msg.Equals(pub_msg_);
|
bool res = pub_msg.Equals(pub_msg_);
|
||||||
return new TestResult {
|
return new TestResult {
|
||||||
test_name = "PublishMsg",
|
test_name = "PublishMsg",
|
||||||
@@ -135,7 +135,7 @@ namespace BCG {
|
|||||||
msg.serialize_into_str(str);
|
msg.serialize_into_str(str);
|
||||||
str.Position = 0;
|
str.Position = 0;
|
||||||
var msg_ = new bts.PubSubServerMessage();
|
var msg_ = new bts.PubSubServerMessage();
|
||||||
msg_.construct_deserialize(str);
|
msg_.consruct_deserialize(str);
|
||||||
bool res = msg.Equals(msg_);
|
bool res = msg.Equals(msg_);
|
||||||
return new TestResult {
|
return new TestResult {
|
||||||
test_name = "PubSubServerMessage",
|
test_name = "PubSubServerMessage",
|
||||||
@@ -156,7 +156,7 @@ namespace BCG {
|
|||||||
msg.serialize_into_str(str);
|
msg.serialize_into_str(str);
|
||||||
str.Position = 0;
|
str.Position = 0;
|
||||||
var msg_ = new PubSubClientMessage();
|
var msg_ = new PubSubClientMessage();
|
||||||
msg_.construct_deserialize(str);
|
msg_.consruct_deserialize(str);
|
||||||
var res = msg.Equals(msg_);
|
var res = msg.Equals(msg_);
|
||||||
return new TestResult {
|
return new TestResult {
|
||||||
test_name = "PubSubClientMessage",
|
test_name = "PubSubClientMessage",
|
||||||
@@ -617,15 +617,14 @@ namespace BCG {
|
|||||||
|
|
||||||
|
|
||||||
private static TestResult _run_single_tag_data_test_clean() {
|
private static TestResult _run_single_tag_data_test_clean() {
|
||||||
// var tagHandlerFactory
|
|
||||||
// create message
|
// create message
|
||||||
var test_name = MethodBase.GetCurrentMethod()?.Name;
|
var test_name = MethodBase.GetCurrentMethod().Name;
|
||||||
var schema = create_schema_with_all_types();
|
var schema = create_schema_with_all_types();
|
||||||
var tag = "DS_" + schema.tableName;
|
var tag = "DS_" + schema.tableName;
|
||||||
BlueTable table = new BlueTable(schema);
|
BlueTable table = new BlueTable(schema);
|
||||||
IStreamIOMsgSerializer serializer = new StreamIOSerializationController();
|
IStreamIOMsgSerializer serializer = new StreamIOSerializationController();
|
||||||
IStreamMsgDeserilizer deser = new StreamIODeserializationController();
|
IStreamMsgDeserilizer deser = new StreamIODeserializationController();
|
||||||
bts.SingleTagData st_data = new bts.SingleTagData(tag, DSTagManager.DataStreamTagType, new PubsubServerStreamTagHandler());
|
bts.SingleTagData st_data = new bts.SingleTagData(tag, DSTagManager.DataStreamTagType);
|
||||||
// create with 100 rows
|
// create with 100 rows
|
||||||
int n_rows = 100;
|
int n_rows = 100;
|
||||||
bool is_succ = true;
|
bool is_succ = true;
|
||||||
|
|||||||
@@ -0,0 +1,51 @@
|
|||||||
|
using System;
|
||||||
|
|
||||||
|
using BCG;
|
||||||
|
|
||||||
|
namespace bts.SilverPine {
|
||||||
|
// Keeps adding changing rows to the stream, hence preserving the history
|
||||||
|
// and creating a time series, for example this can be used to make bbo updates
|
||||||
|
// into a time series of prices/qty
|
||||||
|
public class TimeSeriesStream : ViewStream {
|
||||||
|
private BlueTable _table;
|
||||||
|
private ViewStream _stream;
|
||||||
|
private int _id = 0;
|
||||||
|
|
||||||
|
public TimeSeriesStream(ViewStream stream) {
|
||||||
|
_stream = stream;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void myOnFirstSubscriber() {
|
||||||
|
_stream.onChange += new ChangeDelegate(_onstream_change);
|
||||||
|
}
|
||||||
|
protected override void myOnLastSubscriber() {
|
||||||
|
_stream.onChange -= new ChangeDelegate(_onstream_change);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private void _onstream_change(object o) {
|
||||||
|
DataPacket dp = o as DataPacket;
|
||||||
|
DateTime now = DateTime.Now;
|
||||||
|
if (_table == null) {
|
||||||
|
IBlueTableSchema schema = new BlueTableSchema(dp.newRows.schema.columnInfo, dp.newRows.schema.tableName);
|
||||||
|
schema = schema.add_columns(Tuple.Create(typeof(DateTime), "timestamp"));
|
||||||
|
_table = new BlueTable(schema);
|
||||||
|
}
|
||||||
|
BlueTable added_rows = new BlueTable(_table.schema);
|
||||||
|
foreach (IBlueRow r in dp.addedRows) {
|
||||||
|
MutableBlueRow ar = new MutableBlueRow(_table.schema, r);
|
||||||
|
ar.setValue("timestamp", now);
|
||||||
|
ar.setValue("id", _id++);
|
||||||
|
added_rows.addRow(ar);
|
||||||
|
}
|
||||||
|
_table.addRows(added_rows);
|
||||||
|
myFireEvent(new DataPacket(_table, added_rows, new BlueTable(_table.schema)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@@ -1,121 +1,121 @@
|
|||||||
// using System;
|
using System;
|
||||||
// using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
// using System.IO;
|
using System.IO;
|
||||||
// using System.Security.Cryptography.X509Certificates;
|
using System.Security.Cryptography.X509Certificates;
|
||||||
|
|
||||||
// using BCG;
|
using BCG;
|
||||||
// using BCG.Utilities.General;
|
using BCG.Utilities.General;
|
||||||
|
|
||||||
// using HtmlAgilityPack;
|
using HtmlAgilityPack;
|
||||||
|
|
||||||
// namespace BCG {
|
namespace BCG {
|
||||||
// class Program {
|
class Program {
|
||||||
// // static void Main() {
|
static void Main() {
|
||||||
// // string url = "http://quotes.toscrape.com";
|
string url = "http://quotes.toscrape.com";
|
||||||
// // List<string> quotes = ScrapeQuotes(url);
|
List<string> quotes = ScrapeQuotes(url);
|
||||||
|
|
||||||
// // Console.WriteLine("Quotes from http://quotes.toscrape.com:");
|
Console.WriteLine("Quotes from http://quotes.toscrape.com:");
|
||||||
// // foreach (var quote in quotes) {
|
foreach (var quote in quotes) {
|
||||||
// // Console.WriteLine(quote);
|
Console.WriteLine(quote);
|
||||||
// // }
|
}
|
||||||
// // }
|
}
|
||||||
|
|
||||||
// static List<string> ScrapeQuotes(string url) {
|
static List<string> ScrapeQuotes(string url) {
|
||||||
// List<string> quotes = new List<string>();
|
List<string> quotes = new List<string>();
|
||||||
|
|
||||||
// // Create a new HtmlWeb instance
|
// Create a new HtmlWeb instance
|
||||||
// HtmlWeb web = new HtmlWeb();
|
HtmlWeb web = new HtmlWeb();
|
||||||
|
|
||||||
// try {
|
try {
|
||||||
// // Load the HTML document from the specified URL
|
// Load the HTML document from the specified URL
|
||||||
// HtmlDocument document = web.Load(url);
|
HtmlDocument document = web.Load(url);
|
||||||
|
|
||||||
// // Extract quotes from the HTML document
|
// Extract quotes from the HTML document
|
||||||
// var quoteNodes = document.DocumentNode.SelectNodes("//span[@class='text']");
|
var quoteNodes = document.DocumentNode.SelectNodes("//span[@class='text']");
|
||||||
// if (quoteNodes != null) {
|
if (quoteNodes != null) {
|
||||||
// foreach (var quoteNode in quoteNodes) {
|
foreach (var quoteNode in quoteNodes) {
|
||||||
// string quote = quoteNode.InnerText.Trim();
|
string quote = quoteNode.InnerText.Trim();
|
||||||
// quotes.Add(quote);
|
quotes.Add(quote);
|
||||||
// }
|
}
|
||||||
// }
|
}
|
||||||
// } catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
// Console.WriteLine($"Error: {ex.Message}");
|
Console.WriteLine($"Error: {ex.Message}");
|
||||||
// }
|
}
|
||||||
|
|
||||||
// return quotes;
|
return quotes;
|
||||||
// }
|
}
|
||||||
// }
|
}
|
||||||
|
|
||||||
// public class WebScapperStream : ViewStream, IWorker {
|
public class WebScapperStream : ViewStream, IWorker {
|
||||||
// private readonly string _url;
|
private readonly string _url;
|
||||||
// private readonly string _xpath;
|
private readonly string _xpath;
|
||||||
// private readonly ViewStreamDatabase _vsd;
|
private readonly ViewStreamDatabase _vsd;
|
||||||
// private IDictionary<string, int> _line_id_map = new Dictionary<string, int>();
|
private IDictionary<string, int> _line_id_map = new Dictionary<string, int>();
|
||||||
// private readonly IBlueTableSchema _schema;
|
private readonly IBlueTableSchema _schema;
|
||||||
|
|
||||||
// public WebScapperStream()
|
public WebScapperStream()
|
||||||
// :
|
:
|
||||||
// this("http://quotes.toscrape.com", "//span[@class='text']", ViewStreamDatabase.instance) { }
|
this("http://quotes.toscrape.com", "//span[@class='text']", ViewStreamDatabase.instance) { }
|
||||||
// public WebScapperStream(string url, string xpath)
|
public WebScapperStream(string url, string xpath)
|
||||||
// : this(url, xpath, ViewStreamDatabase.instance) { }
|
: this(url, xpath, ViewStreamDatabase.instance) { }
|
||||||
// public WebScapperStream(string url, string xpath, ViewStreamDatabase vsd) {
|
public WebScapperStream(string url, string xpath, ViewStreamDatabase vsd) {
|
||||||
// bts.Log.Gen.info($"WebScapperStream cons url: {url} xpath: {xpath}");
|
bts.Log.Gen.info($"WebScapperStream cons url: {url} xpath: {xpath}");
|
||||||
// _schema = BlueDataUtility.form_schema($"{url}_{xpath}",
|
_schema = BlueDataUtility.form_schema($"{url}_{xpath}",
|
||||||
// Tuple.Create("id", TypeCode.Int32),
|
Tuple.Create("id", TypeCode.Int32),
|
||||||
// Tuple.Create("url", TypeCode.UInt32),
|
Tuple.Create("url", TypeCode.UInt32),
|
||||||
// Tuple.Create("line", TypeCode.String));
|
Tuple.Create("line", TypeCode.String));
|
||||||
// _url = url;
|
_url = url;
|
||||||
// _xpath = xpath;
|
_xpath = xpath;
|
||||||
// _vsd = vsd;
|
_vsd = vsd;
|
||||||
// }
|
}
|
||||||
|
|
||||||
// protected override void myOnFirstSubscriber() {
|
protected override void myOnFirstSubscriber() {
|
||||||
// MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
|
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
|
||||||
// }
|
}
|
||||||
|
|
||||||
// protected override void myOnLastSubscriber() {
|
protected override void myOnLastSubscriber() {
|
||||||
// MasterWorker.getInstance(false).removeWorker(this);
|
MasterWorker.getInstance(false).removeWorker(this);
|
||||||
// }
|
}
|
||||||
|
|
||||||
// public void doWork() {
|
public void doWork() {
|
||||||
// var lines = scrape_lines(_url, _xpath);
|
var lines = scrape_lines(_url, _xpath);
|
||||||
// Func<string, int> get_id = (line) => {
|
Func<string, int> get_id = (line) => {
|
||||||
// int rv = 0;
|
int rv = 0;
|
||||||
// if (!_line_id_map.TryGetValue(line, out rv))
|
if (!_line_id_map.TryGetValue(line, out rv))
|
||||||
// _line_id_map.Add(line, _line_id_map.Count);
|
_line_id_map.Add(line, _line_id_map.Count);
|
||||||
// return rv;
|
return rv;
|
||||||
// };
|
};
|
||||||
// BlueTable nrows = new BlueTable(_schema);
|
BlueTable nrows = new BlueTable(_schema);
|
||||||
// foreach (var l in lines) {
|
foreach (var l in lines) {
|
||||||
// bts.Log.Gen.info($"line: {l}");
|
bts.Log.Gen.info($"line: {l}");
|
||||||
// var id = get_id(l);
|
var id = get_id(l);
|
||||||
// BlueRow nr = new BlueRow(_schema, get_id(l), _url, l);
|
BlueRow nr = new BlueRow(_schema, get_id(l), _url, l);
|
||||||
// nrows.addRow(nr);
|
nrows.addRow(nr);
|
||||||
// }
|
}
|
||||||
// var dp = BlueDataUtility.build_dp_inc(nrows, myLastValue as DataPacket, true);
|
var dp = BlueDataUtility.build_dp_inc(nrows, myLastValue as DataPacket, true);
|
||||||
// myFireEvent(dp);
|
myFireEvent(dp);
|
||||||
// }
|
}
|
||||||
|
|
||||||
// static List<string> scrape_lines(string url, string xpath) {
|
static List<string> scrape_lines(string url, string xpath) {
|
||||||
// List<string> quotes = new List<string>();
|
List<string> quotes = new List<string>();
|
||||||
// // Create a new HtmlWeb instance
|
// Create a new HtmlWeb instance
|
||||||
// HtmlWeb web = new HtmlWeb();
|
HtmlWeb web = new HtmlWeb();
|
||||||
// try {
|
try {
|
||||||
// // Load the HTML document from the specified URL
|
// Load the HTML document from the specified URL
|
||||||
// HtmlDocument document = web.Load(url);
|
HtmlDocument document = web.Load(url);
|
||||||
// // Extract quotes from the HTML document
|
// Extract quotes from the HTML document
|
||||||
// var quote_nodes = document.DocumentNode.SelectNodes(xpath);
|
var quote_nodes = document.DocumentNode.SelectNodes(xpath);
|
||||||
// if (quote_nodes != null) {
|
if (quote_nodes != null) {
|
||||||
// foreach (var qn in quote_nodes) {
|
foreach (var qn in quote_nodes) {
|
||||||
// string quote = qn.InnerText.Trim();
|
string quote = qn.InnerText.Trim();
|
||||||
// quotes.Add(quote);
|
quotes.Add(quote);
|
||||||
// }
|
}
|
||||||
// }
|
}
|
||||||
// } catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
// Console.WriteLine($"Error: {ex.Message}");
|
Console.WriteLine($"Error: {ex.Message}");
|
||||||
// }
|
}
|
||||||
|
|
||||||
// return quotes;
|
return quotes;
|
||||||
// }
|
}
|
||||||
// }
|
}
|
||||||
// }
|
}
|
||||||
@@ -2,16 +2,14 @@ namespace Bts.DataStream;
|
|||||||
|
|
||||||
using bts;
|
using bts;
|
||||||
|
|
||||||
using Bts.DataStreamCore;
|
|
||||||
|
|
||||||
using MongoDB.Bson;
|
using MongoDB.Bson;
|
||||||
|
|
||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
|
|
||||||
public static class MongoSchemaAdaptor {
|
public static class MongoSchemaAdaptor {
|
||||||
public static BlueTableSchemaAdapter<BsonDocument> InferSchema(IEnumerable<BsonDocument> docs) {
|
public static BlueTableSchemaAdaptor<BsonDocument> InferSchema(IEnumerable<BsonDocument> docs) {
|
||||||
var schemaAdaptor = new BlueTableSchemaAdapter<BsonDocument>("MongoCollectionStream");
|
var schemaAdaptor = new BlueTableSchemaAdaptor<BsonDocument>("MongoCollectionStream");
|
||||||
foreach (var doc in docs) {
|
foreach (var doc in docs) {
|
||||||
foreach (var el in doc) {
|
foreach (var el in doc) {
|
||||||
if (schemaAdaptor.Contains(el.Name)) {
|
if (schemaAdaptor.Contains(el.Name)) {
|
||||||
|
|||||||
@@ -1,54 +0,0 @@
|
|||||||
namespace Bts.DataStream;
|
|
||||||
|
|
||||||
using BCG;
|
|
||||||
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Data;
|
|
||||||
using System.Data.SQLite;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Threading;
|
|
||||||
|
|
||||||
public class SQLiteEnumerable : IAsyncEnumerable<IBlueRowCollection> {
|
|
||||||
readonly string location;
|
|
||||||
readonly string statement;
|
|
||||||
readonly int pageSize;
|
|
||||||
|
|
||||||
public SQLiteEnumerable(string location, string statement, int pageSize = 65536) {
|
|
||||||
this.location = location;
|
|
||||||
this.statement = statement;
|
|
||||||
this.pageSize = pageSize;
|
|
||||||
}
|
|
||||||
|
|
||||||
public async IAsyncEnumerator<IBlueRowCollection> GetAsyncEnumerator(CancellationToken cancellationToken = default) {
|
|
||||||
var dataTable = new DataTable();
|
|
||||||
await using var conn = new SQLiteConnection($"Data Source={location};Version=3");
|
|
||||||
await conn.OpenAsync(cancellationToken);
|
|
||||||
await using var cmd = new SQLiteCommand(statement, conn);
|
|
||||||
await using var reader = cmd.ExecuteReader();
|
|
||||||
var schema = await reader.GetColumnSchemaAsync(cancellationToken);
|
|
||||||
var columns = schema.OrderBy(c => c.ColumnOrdinal ?? int.MaxValue)
|
|
||||||
.Select((c, i) => new ColumnInfo(c.ColumnName, c.ColumnOrdinal ?? i, c.DataType))
|
|
||||||
.Append(new ColumnInfo("id", schema.Count, typeof(int)))
|
|
||||||
.ToArray();
|
|
||||||
var blueSchema = new BlueTableSchema("anon", columns);
|
|
||||||
BlueTable? table = null;
|
|
||||||
int rowId = 1;
|
|
||||||
while (await reader.ReadAsync(cancellationToken)) {
|
|
||||||
table ??= new BlueTable(blueSchema);
|
|
||||||
var rowVals = new object[blueSchema.columnInfo.Length];
|
|
||||||
if (reader.GetValues(rowVals) != rowVals.Length - 1) {
|
|
||||||
throw new InvalidOperationException("Sqlite reader value count for single row not equal to column count");
|
|
||||||
}
|
|
||||||
rowVals[^1] = rowId++;
|
|
||||||
table.addRow(new BlueRow(blueSchema, rowVals));
|
|
||||||
if (table.Count == pageSize) {
|
|
||||||
yield return table;
|
|
||||||
table = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (table is not null) {
|
|
||||||
yield return table;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -3,8 +3,6 @@ using System.Data;
|
|||||||
using System.Data.SQLite;
|
using System.Data.SQLite;
|
||||||
using System.Diagnostics;
|
using System.Diagnostics;
|
||||||
using System.Collections;
|
using System.Collections;
|
||||||
using System.IO;
|
|
||||||
using System.Text;
|
|
||||||
|
|
||||||
|
|
||||||
using BCG;
|
using BCG;
|
||||||
@@ -12,14 +10,12 @@ using System.Linq;
|
|||||||
using System.Data.Entity;
|
using System.Data.Entity;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Data.SQLite;
|
using System.Data.SQLite;
|
||||||
using System.Runtime.InteropServices;
|
|
||||||
|
|
||||||
|
|
||||||
|
namespace bts.SilverPine {
|
||||||
namespace bts.BlueOrb{
|
|
||||||
|
|
||||||
public static class SQLiteHelper {
|
public static class SQLiteHelper {
|
||||||
public static string location = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "C:\\BlueOrb_Data\\" : Path.Combine(Environment.GetEnvironmentVariable("HOME"), "BlueOrb_Data");
|
public static string location = "C:\\Users\\m_tah\\development\\data\\";
|
||||||
public static string db_name = "stream_data.db";
|
public static string db_name = "stream_data.db";
|
||||||
|
|
||||||
public static DataTable execute_select_query(string connection_string, string query) {
|
public static DataTable execute_select_query(string connection_string, string query) {
|
||||||
|
|||||||
Loaded 100 of 1193 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user