Compare commits

..
Author SHA1 Message Date
Mohammad Arham a986dacc60 file restructuring for sdk 2024-04-17 16:17:26 +00:00
Mohammad Arham 5b1bf06feb active cme working small fixex and conf left 2024-04-17 01:13:23 -04:00
John Colvin bc6dc44555 Add Activ md libs to sparkd
WARNING: This breaks the sparkd clang build (only works with gcc)
2024-04-12 12:30:36 -04:00
John Colvin 01e62b9aae build: Upgrade gcc to v12 (explicitly) 2024-04-12 12:28:48 -04:00
1151 changed files with 53222 additions and 69721 deletions

No files matched your search

+2 -2
View File
@@ -9,7 +9,7 @@
] ]
}, },
"fable": { "fable": {
"version": "4.24.0", "version": "4.0.6",
"commands": [ "commands": [
"fable" "fable"
] ]
@@ -27,4 +27,4 @@
] ]
} }
} }
} }
-3
View File
@@ -87,9 +87,6 @@ RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
# Pcap # Pcap
RUN apt-get update && apt-get install -y libpcap0.8-dev RUN apt-get update && apt-get install -y libpcap0.8-dev
# Databento (deps)
RUN apt-get update && apt-get -y install libzstd-dev
# Set up default executables (update-alternatives) # Set up default executables (update-alternatives)
RUN \ RUN \
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \ update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
+18 -8
View File
@@ -1,10 +1,15 @@
{ {
"dockerFile": "Dockerfile", "dockerFile": "Dockerfile",
// Creates local dir to mount to HOME in docker
"initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
"hostRequirements": { "hostRequirements": {
"memory": "8gb", "memory": "4gb",
"storage": "10gb" "storage": "6gb"
}, },
"postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
"containerUser": "bts", "containerUser": "bts",
"workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
"workspaceFolder": "/home/bts/spark",
"updateRemoteUserUID": true, "updateRemoteUserUID": true,
"shutdownAction": "none", "shutdownAction": "none",
// Note: If running multiple containers on the same host, each must use a // Note: If running multiple containers on the same host, each must use a
@@ -12,7 +17,7 @@
// example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc. // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
"runArgs": [ "runArgs": [
"--shm-size", "--shm-size",
"8gb", "2gb",
"-p", "-p",
"8000:8000", "8000:8000",
"-p", "-p",
@@ -20,18 +25,23 @@
"-p", "-p",
"8002:8002", "8002:8002",
"-p", "-p",
"8003:8003", "8003:8003"
"-p", ],
"12345:12345" "mounts": [
"source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
], ],
"customizations": { "customizations": {
"vscode": { "vscode": {
"settings": {
"clangd.arguments": [
"--compile-commands-dir=${workspaceFolder}/build/clang/debug"
]
},
"extensions": [ "extensions": [
"eamodio.gitlens", "eamodio.gitlens",
"llvm-vs-code-extensions.vscode-clangd", "llvm-vs-code-extensions.vscode-clangd",
"ms-vscode.cmake-tools", "ms-vscode.cmake-tools",
"ms-vscode.cpptools", "ms-vscode.cpptools"
"xaver.clang-format"
] ]
} }
} }
-3
View File
@@ -1,3 +0,0 @@
*.cmd text=auto eol=crlf
*.cs text=auto eol=crlf
*.sh text=auto eol=lf
+1 -3
View File
@@ -8,9 +8,7 @@ cmake-build-debug/
.idea/ .idea/
.dir-locals.el .dir-locals.el
*.pyc *.pyc
**.vscode/* **.vscode
!.vscode/launch.json
!.vscode/tasks.json
.vs .vs
obj/ obj/
*.csproj.user *.csproj.user
-329
View File
@@ -1,329 +0,0 @@
{
"version": "0.2.0",
"configurations": [
{
"name": ".NET Core Attach",
"type": "coreclr",
"request": "attach"
},
{
"name": "DataStreamPublisher",
"type": "coreclr",
"request": "launch",
"preLaunchTask": "Build DataStream (Debug)",
"program": "",
"linux": {
"program": "DataStreamPublisher",
},
"windows": {
"program": "DataStreamPublisher.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
],
"cwd": "build/Debug/bin/net7.0",
"stopAtEntry": false,
"console": "integratedTerminal"
},
{
"name": "(gdb) Launch",
"type": "cppdbg",
"request": "launch",
"program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
"args": [
"run"
],
"stopAtEntry": false,
"cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
"environment": [],
"externalConsole": false,
"MIMode": "gdb",
},
{
"name": "spreader tests2",
"type": "cppdbg",
"request": "launch",
"program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
"args": [
"test",
"[spreader2]"
],
"stopAtEntry": false,
"cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
"environment": [],
"externalConsole": false,
"MIMode": "gdb",
},
{
"name": "spreader tests",
"type": "cppdbg",
"request": "launch",
"program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
"args": [
"test",
"[spreader]"
],
"stopAtEntry": false,
"cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
"environment": [],
"externalConsole": false,
"MIMode": "gdb",
},
{
"name": "DataStreamQueryAnalyzer",
"type": "coreclr",
"request": "launch",
//"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
"program": "",
"windows": {
"program": "DataStreamQueryAnalyzer.exe",
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
],
"cwd": "build/Debug/bin/net7.0-windows",
"env": {
"PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
},
},
"stopAtEntry": false,
"console": "integratedTerminal",
},
{
"name": "PubSubServer",
"type": "coreclr",
"request": "launch",
"preLaunchTask": "Build DataStream (Debug)",
"program": "",
"linux": {
"program": "PubSubServer",
},
"windows": {
"program": "PubSubServer.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
],
"cwd": "build/Debug/bin/net7.0",
"stopAtEntry": false,
"console": "integratedTerminal"
},
{
"name": "Spark",
"type": "coreclr",
"request": "launch",
"program": "",
"windows": {
"preLaunchTask": "Build Spark (Debug)",
"program": "Spark.exe",
"cwd": "build/Debug/bin/net7.0-windows",
},
"stopAtEntry": false,
"console": "integratedTerminal"
},
{
"name": "Spark (Skip Build)",
"type": "coreclr",
"request": "launch",
"program": "",
"windows": {
"program": "Spark.exe",
"cwd": "build/Debug/bin/net7.0-windows",
},
"stopAtEntry": false,
"console": "integratedTerminal"
},
{
"name": "Spark (Parameterized)",
"type": "coreclr",
"request": "launch",
"preLaunchTask": "Build Spark (Debug)",
"program": "",
"windows": {
"program": "Spark.exe",
"args": [
"--sparkd",
"${input:sparkConfigServer}"
],
"cwd": "build/Debug/bin/net7.0-windows",
},
"stopAtEntry": false,
"console": "integratedTerminal"
},
{
"name": "Spark Test",
"type": "coreclr",
"request": "launch",
"program": "",
"windows": {
"preLaunchTask": "Build Spark (Debug)",
"program": "Spark.Console.exe",
"cwd": "build/Debug/bin/net7.0",
"args": [
"test",
"${input:sparkTest}"
]
},
"stopAtEntry": false,
"console": "integratedTerminal"
},
{
"name": "Spark Test (Skip Build)",
"type": "coreclr",
"request": "launch",
"program": "",
"windows": {
"program": "Spark.Console.exe",
"cwd": "build/Debug/bin/net7.0",
"args": [
"test",
"${input:sparkTest}"
]
},
"stopAtEntry": false,
"console": "integratedTerminal"
},
{
"name": "Sparkd",
"type": "cppdbg",
"request": "launch",
"program": "",
"linux": {
"preLaunchTask": "Build Sparkd (Debug)",
"program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
"args": [
"run"
],
"stopAtEntry": false,
"cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
"environment": [],
"externalConsole": false,
"MIMode": "gdb",
"setupCommands": [
{
"description": "Enable pretty-printing for gdb",
"text": "-enable-pretty-printing",
"ignoreFailures": true
}
]
}
},
{
"name": "Sparkd (Skip Build)",
"type": "cppdbg",
"request": "launch",
"program": "",
"linux": {
"program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
"args": [
"run"
],
"stopAtEntry": false,
"cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
"environment": [],
"externalConsole": false,
"MIMode": "gdb",
"setupCommands": [
{
"description": "Enable pretty-printing for gdb",
"text": "-enable-pretty-printing",
"ignoreFailures": true
}
]
}
},
{
"name": "Sparkd Test",
"type": "cppdbg",
"request": "launch",
"program": "",
"linux": {
"preLaunchTask": "Build Sparkd (Debug)",
"program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
"args": [
"test",
"${input:sparkdTest}",
],
"stopAtEntry": false,
"cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
"environment": [],
"externalConsole": false,
"MIMode": "gdb",
"setupCommands": [
{
"description": "Enable pretty-printing for gdb",
"text": "-enable-pretty-printing",
"ignoreFailures": true
}
]
}
},
{
"name": "Sparkd Test (Skip Build)",
"type": "cppdbg",
"request": "launch",
"program": "",
"linux": {
"program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
"args": [
"test",
"${input:sparkdTest}",
],
"stopAtEntry": false,
"cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
"environment": [],
"externalConsole": false,
"MIMode": "gdb",
"setupCommands": [
{
"description": "Enable pretty-printing for gdb",
"text": "-enable-pretty-printing",
"ignoreFailures": true
}
]
}
},
],
"inputs": [
{
"id": "dataStreamPublisherConfig",
"type": "pickString",
"description": "Publisher Configuration",
"options": [
"MarketDataPublisher",
"SimpleEchoClient",
"SimpleTestDataPublisher",
"TestDataPublisher",
],
"default": "MarketDataPublisher",
},
{
"id": "pubSubServerConfig",
"type": "pickString",
"description": "PubSub Configuration",
"options": [
"PubSubServer",
"SimpleEchoClient",
],
"default": "PubSubServer",
},
{
"id": "sparkConfigServer",
"type": "promptString",
"description": "Config Server URL",
"default": "http://localhost:8000/config",
},
{
"id": "sparkTest",
"type": "promptString",
"description": "Test to run",
},
{
"id": "sparkdTest",
"type": "promptString",
"description": "Test to run",
},
]
}
-930
View File
@@ -1,930 +0,0 @@
{
"version": "2.0.0",
"tasks": [
{
"label": "Build Api Server",
"group": "build",
"type": "process",
"command": "",
"linux": {
"command": "tools/build_api_server.sh",
},
"windows": {
"command": "tools/build_api_server.cmd",
},
"args": [
"--configuration",
"${input:buildConfiguration}",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Api Server (Debug)",
"group": "build",
"type": "process",
"command": "",
"linux": {
"command": "tools/build_api_server.sh",
},
"windows": {
"command": "tools/build_api_server.cmd",
},
"args": [
"--configuration",
"Debug",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Api Server (Release)",
"group": "build",
"type": "process",
"command": "",
"windows": {
"command": "tools/build_api_server.cmd",
},
"linux": {
"command": "tools/build_api_server.sh",
},
"args": [
"--configuration",
"Release",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build DataStream",
"group": "build",
"type": "process",
"command": "",
"linux": {
"command": "tools/build_data_stream.sh",
},
"windows": {
"command": "tools/build_data_stream.cmd",
},
"args": [
"--configuration",
"${input:buildConfiguration}",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build DataStream (Debug)",
"group": "build",
"type": "process",
"command": "",
"linux": {
"command": "tools/build_data_stream.sh",
},
"windows": {
"command": "tools/build_data_stream.cmd",
},
"args": [
"--configuration",
"Debug",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build DataStream (Release)",
"group": "build",
"type": "process",
"command": "",
"linux": {
"command": "tools/build_data_stream.sh",
},
"windows": {
"command": "tools/build_data_stream.cmd",
},
"args": [
"--configuration",
"Release",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Spark",
"group": "build",
"type": "process",
"command": "",
"windows": {
"command": "tools/build_spark.cmd",
},
"args": [
"--configuration",
"${input:buildConfiguration}",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Spark (Debug)",
"group": "build",
"type": "process",
"command": "",
"windows": {
"command": "tools/build_spark.cmd",
},
"args": [
"--configuration",
"Debug",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Spark (Release)",
"group": "build",
"type": "process",
"command": "",
"windows": {
"command": "tools/build_spark.cmd",
},
"args": [
"--configuration",
"Release",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Sparkd",
"group": "build",
"type": "shell",
"command": "",
"linux": {
"command": "tools/build_sparkd.sh",
"args": [
"--preset",
"${input:sparkdCmakePreset}",
],
},
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Sparkd (Debug)",
"group": "build",
"type": "shell",
"command": "",
"linux": {
"command": "tools/build_sparkd.sh",
"args": [
"--build-type",
"Debug",
"--compiler",
"${input:sparkdCompiler}"
],
},
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Sparkd (Debug | Gcc)",
"group": "build",
"type": "shell",
"command": "",
"linux": {
"command": "tools/build_sparkd.sh",
"args": [
"--build-type",
"Debug",
"--compiler",
"gcc"
],
},
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Sparkd (Debug | Clang)",
"group": "build",
"type": "shell",
"command": "",
"linux": {
"command": "tools/build_sparkd.sh",
"args": [
"--build-type",
"Debug",
"--compiler",
"clang"
],
},
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Sparkd (Release)",
"group": "build",
"type": "shell",
"command": "",
"linux": {
"command": "tools/build_sparkd.sh",
"args": [
"--build-type",
"Release",
"--compiler",
"${input:sparkdCompiler}"
],
},
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Sparkd (Release | Gcc)",
"group": "build",
"type": "shell",
"command": "",
"linux": {
"command": "tools/build_sparkd.sh",
"args": [
"--build-type",
"Release",
"--compiler",
"gcc"
],
},
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Sparkd (Release | Clang)",
"group": "build",
"type": "shell",
"command": "",
"linux": {
"command": "tools/build_sparkd.sh",
"args": [
"--build-type",
"Release",
"--compiler",
"clang"
],
},
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Edit DataStream Config",
"type": "process",
"command": "",
"linux": {
"command": "editor",
"args": [
"${env:HOME}/.local/share/DataStream/conf/spark.conf"
],
},
"windows": {
"command": "vim",
"args": [
"${env:APPDATA}/bts/DataStream/conf/spark.conf"
],
},
"presentation": {
"reveal": "always",
"focus": true,
"panel": "shared",
"showReuseMessage": false,
"close": true,
},
"icon": {
"id": "wrench"
},
},
{
"label": "Edit Spark Config",
"type": "process",
"command": "",
"windows": {
"command": "vim",
"args": [
"${env:APPDATA}/bts/conf/spark.conf"
],
},
"presentation": {
"reveal": "always",
"focus": true,
"panel": "dedicated",
"showReuseMessage": false,
"close": true,
},
"icon": {
"id": "wrench"
},
},
{
"label": "Run DataStreamQueryAnalyzer",
"dependsOn": "Build DataStream",
"type": "process",
"command": "",
"windows": {
"command": "DataStreamQueryAnalyzer.exe",
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
],
"options": {
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
},
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run DataStreamQueryAnalyzer (Debug)",
"dependsOn": "Build DataStream (Debug)",
"type": "process",
"command": "",
"windows": {
"command": "DataStreamQueryAnalyzer.exe",
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
],
"options": {
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
},
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run DataStreamQueryAnalyzer (Release)",
"dependsOn": "Build DataStream (Release)",
"type": "process",
"command": "",
"windows": {
"command": "DataStreamQueryAnalyzer.exe",
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
],
"options": {
"cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
},
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run DataStreamPublisher",
"dependsOn": "Build DataStream",
"type": "process",
"command": "",
"linux": {
"command": "DataStreamPublisher",
},
"windows": {
"command": "DataStreamPublisher.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
],
"options": {
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run DataStreamPublisher (Debug)",
"dependsOn": "Build DataStream (Debug)",
"type": "process",
"command": "",
"linux": {
"command": "DataStreamPublisher",
},
"windows": {
"command": "DataStreamPublisher.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
],
"options": {
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run DataStreamPublisher (Release)",
"dependsOn": "Build DataStream (Release)",
"type": "process",
"command": "",
"linux": {
"command": "DataStreamPublisher",
},
"windows": {
"command": "DataStreamPublisher.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
],
"options": {
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run PubSubServer",
"dependsOn": "Build DataStream",
"type": "process",
"command": "",
"linux": {
"command": "PubSubServer",
},
"windows": {
"command": "PubSubServer.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
],
"options": {
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run PubSubServer (Debug)",
"dependsOn": "Build DataStream (Debug)",
"type": "process",
"command": "",
"linux": {
"command": "PubSubServer",
},
"windows": {
"command": "PubSubServer.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
],
"options": {
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run PubSubServer (Debug | Skip Build)",
"type": "process",
"command": "",
"linux": {
"command": "PubSubServer",
},
"windows": {
"command": "PubSubServer.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
],
"options": {
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run PubSubServer (Release)",
"dependsOn": "Build DataStream (Release)",
"type": "process",
"command": "",
"linux": {
"command": "PubSubServer",
},
"windows": {
"command": "PubSubServer.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
],
"options": {
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run PubSubServer (Release | Skip Build)",
"type": "process",
"command": "",
"linux": {
"command": "PubSubServer",
},
"windows": {
"command": "PubSubServer.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
],
"options": {
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run Spark",
"dependsOn": "Build Spark",
"type": "process",
"command": "",
"windows": {
"command": "Spark.exe",
"options": {
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
},
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
}
},
{
"label": "Run Spark (Debug)",
"dependsOn": "Build Spark (Debug)",
"type": "process",
"command": "",
"windows": {
"command": "Spark.exe",
"options": {
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
},
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
}
},
{
"label": "Run Spark (Release)",
"dependsOn": "Build Spark (Release)",
"type": "process",
"command": "",
"windows": {
"command": "Spark.exe",
"options": {
"cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
},
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
}
},
{
"label": "Run Spark (Skip Build)",
"type": "process",
"command": "",
"windows": {
"command": "Spark.exe",
"options": {
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
},
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
}
},
{
"label": "Run Spark Config Query",
"dependsOn": "Build Spark",
"type": "process",
"command": "",
"linux": {
"command": "Spark.Console",
},
"windows": {
"command": "Spark.Console.exe",
},
"args": [
"config",
"--query",
"${input:sparkConfigQuery}",
],
"options": {
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run Sparkd",
"dependsOn": "Build Sparkd",
"type": "process",
"command": "",
"linux": {
"command": "sparkd",
},
"args": [
"run"
],
"options": {
"cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run Sparkd (Skip Build)",
"type": "process",
"command": "",
"linux": {
"command": "sparkd",
},
"args": [
"run"
],
"options": {
"cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run Sparkd Config Query",
"dependsOn": "Build Sparkd",
"type": "process",
"command": "",
"linux": {
"command": "sparkd",
},
"args": [
"config",
"--query",
"${input:sparkdConfigQuery}",
],
"options": {
"cwd": "${workspaceFolder}/build/${input:sparkdCompiler}/${input:sparkdBuildType}/server"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Test Spark (Debug)",
"dependsOn": "Build Spark (Debug)",
"type": "process",
"icon": {
"id": "coverage",
"color": "terminal.ansiYellow",
},
"command": "",
"windows": {
"command": "Spark.Console.exe",
"args": [
"test",
"${input:sparkTest}"
],
"options": {
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
}
},
},
{
"label": "Test Sparkd",
"dependsOn": "Build Sparkd",
"type": "process",
"command": "",
"linux": {
"command": "sparkd",
},
"args": [
"test",
"${input:sparkdTest}",
],
"options": {
"cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
},
"icon": {
"id": "coverage",
"color": "terminal.ansiYellow",
},
},
{
"label": "Test Sparkd (Skip Build)",
"type": "process",
"command": "",
"linux": {
"command": "sparkd",
},
"args": [
"test",
"${input:sparkdTest}",
],
"options": {
"cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
},
"icon": {
"id": "coverage",
"color": "terminal.ansiYellow",
},
},
{
"label": "View DataStreamQueryAnalyzer Logs (Last Run)",
"type": "process",
"command": "",
"windows": {
"command": "tools/logging/show_last_run.cmd",
"args": [
"DataStream",
"DataStreamQueryAnalyzer.exe",
"'Starting DataStreamQueryAnalyzer'"
]
},
"presentation": {
"reveal": "always",
"focus": true,
"panel": "shared",
"showReuseMessage": false,
"close": true,
},
"icon": {
"id": "wrench"
},
},
{
"label": "View PubSubServer Logs (Last Run)",
"type": "process",
"command": "",
"linux": {
"command": "tools/logging/show_last_run.sh",
"args": [
"${env:HOME}/.local/share/DataStream/data/logging.PubSubServer",
"Starting PubSubServer"
]
},
"windows": {
"command": "tools/logging/show_last_run.cmd",
"args": [
"DataStream",
"PubSubServer.exe",
"Starting PubSubServer"
]
},
"presentation": {
"reveal": "always",
"focus": true,
"panel": "shared",
"showReuseMessage": false,
"close": true,
},
"icon": {
"id": "wrench"
},
},
],
"inputs": [
{
"id": "buildConfiguration",
"type": "pickString",
"description": "Configuration",
"options": [
"Debug",
"Release",
],
"default": "Debug",
},
{
"id": "sparkdCmakePreset",
"type": "pickString",
"description": "Configuration",
"options": [
"sparkd-debug-clang",
"sparkd-debug-gcc",
"sparkd-release-clang",
"sparkd-release-gcc",
],
"default": "sparkd-debug-clang",
},
{
"id": "sparkdCompiler",
"type": "pickString",
"description": "Configuration",
"options": [
"clang",
"gcc",
],
"default": "sparkd-debug-clang",
},
{
"id": "dataStreamPublisherConfig",
"type": "pickString",
"description": "Publisher Configuration",
"options": [
"MarketDataPublisher",
"SimpleEchoClient",
"SimpleTestDataPublisher",
"TestDataPublisher",
],
"default": "MarketDataPublisher",
},
{
"id": "pubSubServerConfig",
"type": "pickString",
"description": "PubSub Configuration",
"options": [
"PubSubServer",
"SimpleEchoClient",
],
"default": "PubSubServer",
},
{
"id": "sparkConfigQuery",
"type": "promptString",
"description": "Config Type(s)"
},
{
"id": "sparkdConfigQuery",
"type": "promptString",
"description": "Config Type(s)"
},
{
"id": "sparkTest",
"type": "promptString",
"description": "Test to run",
},
{
"id": "sparkdTest",
"type": "promptString",
"description": "Test to run",
},
]
}
+20 -30
View File
@@ -1,6 +1,5 @@
cmake_minimum_required(VERSION 3.12) cmake_minimum_required(VERSION 3.12)
project(spark) project(spark)
include(GNUInstallDirs)
set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD 20)
if(WIN32) if(WIN32)
@@ -15,17 +14,12 @@ set(
CACHE CACHE
STRING STRING
"External dependencies root folder (should contain folders: lib, bin, include") "External dependencies root folder (should contain folders: lib, bin, include")
# SHARED_MEMORY_PREFIX serves as a kind of namespace so multiple applications
# can run simultaneously without fighting over the same shm files (e.g., sparkd,
# api server, etc.).
set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
set(SPARK_BIN "${EXTERNALS_ROOT}/bin") set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include") set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>") set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
set(BUILD_API_SERVER OFF CACHE BOOL "Build the api server")
set(BUILD_DOCS OFF CACHE BOOL "Build documentation") set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
set(BUILD_SDK OFF CACHE BOOL "Build the sdk") set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
if (WIN32) if (WIN32)
set(BUILD_SPARKD OFF CACHE BOOL "Build sparkd") set(BUILD_SPARKD OFF CACHE BOOL "Build sparkd")
set(BUILD_ADMIN_WEB_APP OFF CACHE BOOL "Build the Admin WebApp") set(BUILD_ADMIN_WEB_APP OFF CACHE BOOL "Build the Admin WebApp")
@@ -37,21 +31,29 @@ endif()
include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib) include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
link_directories(${SPARK_LIB}) link_directories(${SPARK_LIB})
add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}") if(BUILD_API_SERVER)
add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}") add_compile_definitions(SPARK_ENABLE_API_SERVER)
if(DEFINED APP_DATA_DIR_OVERRIDE)
add_compile_definitions(APP_DATA_DIR_OVERRIDE="${APP_DATA_DIR_OVERRIDE}")
endif() endif()
add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
if(WIN32) if(WIN32)
add_compile_definitions(SHARED_MEMORY_PREFIX="spark")
add_compile_definitions(APP_DATA_DIR="\$\{APPDATA\}/bts/")
# subdirs # subdirs
add_subdirectory(src/lib) add_subdirectory(src/lib)
else() else()
# make sparkd look for libspark.so in its directory # make sparkd look for libspark.so in its directory
set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
set(CMAKE_INSTALL_RPATH ".") set(CMAKE_INSTALL_RPATH ".")
set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")
if(BUILD_API_SERVER)
add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/spark_api/")
add_compile_definitions(SHARED_MEMORY_PREFIX="spark_api")
elseif(BUILD_SPARKD)
add_compile_definitions(SHARED_MEMORY_PREFIX="sparkd")
add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/sparkdata/")
if(BUILD_SPARKD)
# Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
# read by their code). We override some below. # read by their code). We override some below.
add_compile_definitions(SPARK_ENABLE_CCAPI) add_compile_definitions(SPARK_ENABLE_CCAPI)
@@ -88,15 +90,9 @@ else()
add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr") add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
# ** Binance USDS Futures ** # ** Binance USDS Futures **
add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://fapi.binance.com") add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/fapi/v1/exchangeInfo") add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/fapi/v1/ticker/24hr") add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
add_compile_definitions(BINANCE_USDS_FUTURES_USER_DATA_PATH="/api/v3/userDataStream")
# ** Binance Coin Futures **
add_compile_definitions(CCAPI_BINANCE_COIN_FUTURES_URL_REST_BASE="https://dapi.binance.com")
add_compile_definitions(BINANCE_COIN_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
add_compile_definitions(BINANCE_COIN_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
# ** Deribit ** # ** Deribit **
add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument") add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
@@ -116,7 +112,6 @@ else()
# subdirs # subdirs
if (BUILD_SPARKD) if (BUILD_SPARKD)
add_subdirectory(src/lib_binance_execution)
add_subdirectory(src/server/sparkd) add_subdirectory(src/server/sparkd)
add_subdirectory(src/data_files) add_subdirectory(src/data_files)
endif() endif()
@@ -138,8 +133,6 @@ else()
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
COMMAND COMMAND
ln ARGS -sf ln ARGS -sf
${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0 ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
@@ -153,14 +146,11 @@ else()
install( install(
DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins" DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
DESTINATION "${CMAKE_INSTALL_BINDIR}") DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
install( install(
FILES FILES
"${SPARK_LIB}/libACE-5.7.5.so" "${SPARK_LIB}/libACE-5.7.5.so"
"${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0" "${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
"${SPARK_LIB}/libonixs-fix-engine.so.4.10.1" DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
"${SPARK_LIB}/libonixs-fix-engine.so.4"
"${SPARK_LIB}/libonixs-fix-engine.so"
DESTINATION "${CMAKE_INSTALL_BINDIR}")
endif() endif()
endif() endif()
+75 -118
View File
@@ -15,30 +15,22 @@
"rhs": "Windows" "rhs": "Windows"
}, },
"generator": "Visual Studio 17 2022", "generator": "Visual Studio 17 2022",
"binaryDir": "${sourceDir}/build/${presetName}", "binaryDir": "${sourceDir}/build",
"environment": { "environment": {
"SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600" "SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
}, },
"cacheVariables": { "cacheVariables": {
"APP_DATA_DIR": "${dollar}{APPDATA}/bts",
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_DATA_DIR}",
"CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}", "CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
"CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd", "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
"CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT", "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
"CMAKE_CXX_STANDARD": "20", "CMAKE_CXX_STANDARD": "20",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
"CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG", "CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
"CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG", "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
"CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake", "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
"EXTERNALS_ROOT": "${sourceDir}/externals/windows", "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf", "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0", "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
"DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows", "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
"EXPORT_MD_CONSUMER": {
"type": "BOOL",
"value": "ON"
},
"SHARED_MEMORY_PREFIX": "spark"
} }
}, },
@@ -54,15 +46,13 @@
"binaryDir": "${sourceDir}/build/${presetName}", "binaryDir": "${sourceDir}/build/${presetName}",
"cacheVariables": { "cacheVariables": {
"platform": "", "platform": "",
"APP_DATA_DIR": "${dollar}{HOME}/sparkdata",
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
"CMAKE_BUILD_WITH_INSTALL_RPATH": { "CMAKE_BUILD_WITH_INSTALL_RPATH": {
"type": "BOOL", "type": "BOOL",
"value": "ON" "value": "ON"
}, },
"CMAKE_CXX_STANDARD": "20", "CMAKE_CXX_STANDARD": "20",
"CMAKE_INSTALL_RPATH": ".", "CMAKE_INSTALL_RPATH": ".",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}", "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}/sparkd",
"CMAKE_EXE_LINKER_FLAGS": "-Wno-undef", "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
"EXTERNALS_ROOT": "${sourceDir}/externals/linux", "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
"BUILD_SDK": { "BUILD_SDK": {
@@ -72,8 +62,7 @@
"BUILD_ADMIN_WEB_APP": { "BUILD_ADMIN_WEB_APP": {
"type": "BOOL", "type": "BOOL",
"value": "ON" "value": "ON"
}, }
"SHARED_MEMORY_PREFIX": "sparkd"
} }
}, },
@@ -106,72 +95,98 @@
{ {
"name": "sparkd-debug-clang", "name": "sparkd-debug-clang",
"inherits": "sparkd-debug", "inherits": "sparkd-debug",
"displayName": "Sparkd (Debug | Clang)", "displayName": "Debug (clang)",
"description": "Debug build compiled w/ clang", "description": "Debug build compiled w/ clang",
"binaryDir": "${sourceDir}/build/clang/debug",
"cacheVariables": { "cacheVariables": {
"CMAKE_C_COMPILER": "clang", "CMAKE_C_COMPILER": "clang",
"CMAKE_CXX_COMPILER": "clang++", "CMAKE_CXX_COMPILER": "clang++",
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG" "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/debug/sparkd"
} }
}, },
{ {
"name": "sparkd-debug-gcc", "name": "sparkd-debug-gcc",
"inherits": "sparkd-debug", "inherits": "sparkd-debug",
"displayName": "Sparkd (Debug | Gcc)", "displayName": "Debug (gcc)",
"description": "Debug build compiled w/ gcc", "description": "Debug build compiled w/ gcc",
"binaryDir": "${sourceDir}/build/gcc/debug",
"cacheVariables": { "cacheVariables": {
"CMAKE_C_COMPILER": "gcc", "CMAKE_C_COMPILER": "gcc-12",
"CMAKE_CXX_COMPILER": "g++", "CMAKE_CXX_COMPILER": "g++-12",
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG" "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/debug/sparkd"
} }
}, },
{ {
"name": "sparkd-release-clang", "name": "sparkd-release-clang",
"inherits": "sparkd-release", "inherits": "sparkd-release",
"displayName": "Sparkd (Release | Clang)", "displayName": "Release (clang)",
"description": "Release build compiled w/ clang", "description": "Release build compiled w/ clang",
"binaryDir": "${sourceDir}/build/clang/release",
"cacheVariables": { "cacheVariables": {
"CMAKE_C_COMPILER": "clang", "CMAKE_C_COMPILER": "clang",
"CMAKE_CXX_COMPILER": "clang++", "CMAKE_CXX_COMPILER": "clang++",
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3" "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/release/sparkd"
} }
}, },
{ {
"name": "sparkd-release-gcc", "name": "sparkd-release-gcc",
"inherits": "sparkd-release", "inherits": "sparkd-release",
"displayName": "Sparkd (Release | Gcc)", "displayName": "Release (gcc)",
"description": "Release build compiled w/ gcc", "description": "Release build compiled w/ gcc",
"binaryDir": "${sourceDir}/build/gcc/release",
"cacheVariables": { "cacheVariables": {
"CMAKE_C_COMPILER": "gcc", "CMAKE_C_COMPILER": "gcc",
"CMAKE_CXX_COMPILER": "g++", "CMAKE_CXX_COMPILER": "g++",
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers -Wno-stringop-overflow --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3" "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers -Wno-stringop-overflow --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/release/sparkd"
}
},
{
"name": "spark-api-server-base",
"hidden": true,
"binaryDir": "${sourceDir}/build/${presetName}",
"cacheVariables": {
"BUILD_API_SERVER": {
"type": "bool",
"value": "ON"
},
"CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
} }
}, },
{ {
"name": "spark-api-server-windows", "name": "spark-api-server-windows",
"inherits": "spark", "inherits": "spark",
"displayName": "Spark Api Server (Windows)", "displayName": "Spark Api Server",
"cacheVariables": { "cacheVariables": {
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}", "BUILD_API_SERVER": {
"SHARED_MEMORY_PREFIX": "spark_api_server" "type": "bool",
"value": "ON"
}
} }
}, },
{ {
"name": "spark-api-server-linux", "name": "spark-api-server-linux",
"inherits": "sparkd", "inherits": "sparkd",
"displayName": "Spark Api Server (Linux)", "displayName": "Spark Api Server",
"generator": "Ninja Multi-Config", "generator": "Ninja Multi-Config",
"cacheVariables": { "cacheVariables": {
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
"BUILD_ADMIN_WEB_APP": { "BUILD_ADMIN_WEB_APP": {
"type": "BOOL", "type": "BOOL",
"value": "OFF" "value": "OFF"
}, },
"BUILD_API_SERVER": {
"type": "bool",
"value": "ON"
},
"BUILD_SPARKD": { "BUILD_SPARKD": {
"type": "BOOL", "type": "BOOL",
"value": "OFF" "value": "OFF"
@@ -182,11 +197,7 @@
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3", "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}", "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf", "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0", "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0"
"EXPORT_MD_CONSUMER": {
"type": "BOOL",
"value": "ON"
}
} }
}, },
@@ -209,62 +220,13 @@
}, },
{ {
"name": "data-stream-suite-windows", "name": "data-stream-suite",
"condition": {
"type": "equals",
"lhs": "${hostSystemName}",
"rhs": "Windows"
},
"inherits": "spark", "inherits": "spark",
"displayName": "Data Stream Suite (Windows)", "displayName": "Data Stream Suite",
"description": "Data Stream utils ported over from Edge", "description": "Data Stream utils ported over from Edge",
"binaryDir": "${sourceDir}/build/${presetName}", "binaryDir": "${sourceDir}/build/${presetName}",
"cacheVariables": { "cacheVariables": {
"APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream", "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf"
"APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
"SHARED_MEMORY_PREFIX": "data_stream"
}
},
{
"name": "data-stream-suite-linux",
"displayName": "Data Stream Suite (Linux)",
"description": "Data Stream utils ported over from Edge",
"condition": {
"type": "equals",
"lhs": "${hostSystemName}",
"rhs": "Linux"
},
"inherits": "sparkd",
"generator": "Ninja Multi-Config",
"cacheVariables": {
"APP_DATA_DIR": "${dollar}{HOME}/.local/share/DataStream",
"APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
"BUILD_ADMIN_WEB_APP": {
"type": "BOOL",
"value": "OFF"
},
"BUILD_SDK": {
"type": "BOOL",
"value": "OFF"
},
"BUILD_SPARKD": {
"type": "BOOL",
"value": "OFF"
},
"CMAKE_C_COMPILER": "clang",
"CMAKE_CXX_COMPILER": "clang++",
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
"EXPORT_MD_CONSUMER": {
"type": "BOOL",
"value": "ON"
},
"SHARED_MEMORY_PREFIX": "data_stream"
} }
} }
], ],
@@ -282,7 +244,7 @@
{ {
"name": "spark-debug", "name": "spark-debug",
"displayName": "Spark (Debug)", "displayName": "Debug",
"inherits": "spark", "inherits": "spark",
"configurePreset": "spark", "configurePreset": "spark",
"configuration": "Debug" "configuration": "Debug"
@@ -290,7 +252,7 @@
{ {
"name": "spark-release", "name": "spark-release",
"displayName": "Spark (Release)", "displayName": "Release",
"inherits": "spark", "inherits": "spark",
"configurePreset": "spark", "configurePreset": "spark",
"configuration": "Release" "configuration": "Release"
@@ -303,34 +265,43 @@
"lhs": "${hostSystemName}", "lhs": "${hostSystemName}",
"rhs": "Linux" "rhs": "Linux"
}, },
"hidden": true "hidden": true,
"targets": [
"data_files_dir",
"admin_web_app_bundle",
"install",
"libspark",
"rapidjson_project",
"server_layout",
"sparkd"
]
}, },
{ {
"name": "sparkd-debug-clang", "name": "sparkd-debug-clang",
"configurePreset": "sparkd-debug-clang", "configurePreset": "sparkd-debug-clang",
"displayName": "Sparkd (Debug | Clang)", "displayName": "Debug (clang)",
"inherits": "sparkd" "inherits": "sparkd"
}, },
{ {
"name": "sparkd-debug-gcc", "name": "sparkd-debug-gcc",
"configurePreset": "sparkd-debug-gcc", "configurePreset": "sparkd-debug-gcc",
"displayName": "Sparkd (Debug | Gcc)", "displayName": "Debug (gcc)",
"inherits": "sparkd" "inherits": "sparkd"
}, },
{ {
"name": "sparkd-release-clang", "name": "sparkd-release-clang",
"configurePreset": "sparkd-release-clang", "configurePreset": "sparkd-release-clang",
"displayName": "Sparkd (Release | Clang)", "displayName": "Release (clang)",
"inherits": "sparkd" "inherits": "sparkd"
}, },
{ {
"name": "sparkd-release-gcc", "name": "sparkd-release-gcc",
"configurePreset": "sparkd-release-gcc", "configurePreset": "sparkd-release-gcc",
"displayName": "Sparkd (Release | Gcc)", "displayName": "Release (gcc)",
"inherits": "sparkd" "inherits": "sparkd"
}, },
@@ -347,14 +318,14 @@
{ {
"name": "spark-api-server-windows-debug", "name": "spark-api-server-windows-debug",
"displayName": "Spark Api Server (Windows | Debug)", "displayName": "Spark Api Server (Debug)",
"inherits": "spark-api-server-windows", "inherits": "spark-api-server-windows",
"configuration": "Debug" "configuration": "Debug"
}, },
{ {
"name": "spark-api-server-windows-release", "name": "spark-api-server-windows-release",
"displayName": "Spark Api Server (Windows | Release)", "displayName": "Spark Api Server (Debug)",
"inherits": "spark-api-server-windows", "inherits": "spark-api-server-windows",
"configuration": "Release" "configuration": "Release"
}, },
@@ -373,14 +344,14 @@
{ {
"name": "spark-api-server-linux-debug", "name": "spark-api-server-linux-debug",
"inherits": "spark-api-server-linux", "inherits": "spark-api-server-linux",
"displayName": "Spark Api Server (Linux | Debug)", "displayName": "Spark Api Server (Debug)",
"configuration": "Debug" "configuration": "Debug"
}, },
{ {
"name": "spark-api-server-linux-release", "name": "spark-api-server-linux-release",
"inherits": "spark-api-server-linux", "inherits": "spark-api-server-linux",
"displayName": "Spark Api Server (Linux | Release)", "displayName": "Spark Api Server (Release)",
"configuration": "Release" "configuration": "Release"
}, },
@@ -400,30 +371,16 @@
}, },
{ {
"name": "data-stream-suite-windows-debug", "name": "data-stream-suite-debug",
"displayName": "Data Stream Suite (Windows | Debug)", "displayName": "DataStreamSuite (Debug)",
"configurePreset": "data-stream-suite-windows", "configurePreset": "data-stream-suite",
"configuration": "Debug" "configuration": "Debug"
}, },
{ {
"name": "data-stream-suite-windows-release", "name": "data-stream-suite-release",
"displayName": "Data Stream Suite (Windows | Release)", "displayName": "DataStreamSuite (Release)",
"configurePreset": "data-stream-suite-windows", "configurePreset": "data-stream-suite",
"configuration": "Release"
},
{
"name": "data-stream-suite-linux-debug",
"displayName": "Data Stream Suite (Linux | Debug)",
"configurePreset": "data-stream-suite-linux",
"configuration": "Debug"
},
{
"name": "data-stream-suite-linux-release",
"displayName": "Data Stream Suite (Linux | Release)",
"configurePreset": "data-stream-suite-linux",
"configuration": "Release" "configuration": "Release"
} }
] ]
-139
View File
@@ -14,34 +14,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}" Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
EndProject EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{EF26A475-6101-4C9C-A717-D60BB92A2302}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{911B08A3-515A-441D-84C8-8A43740B5FC1}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{14140310-2233-430A-821B-F3058A25234B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExpressionCore", "src\DataStreamSuite\ExpressionCore\ExpressionCore.csproj", "{6F22DB96-6EAB-4034-99D9-287021C36F01}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{A5C33956-A41F-49B3-A41D-1082CBA59D02}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ColumnMapExpressions", "src\DataStreamSuite\ColumnMapExpressions\ColumnMapExpressions.csproj", "{B945A183-2174-4ACB-B289-8A5C546656FE}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{9F6D7DE8-185B-4C1D-B809-3314D092DF28}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
EndProject
Global Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|x64 = Debug|x64 Debug|x64 = Debug|x64
@@ -98,102 +70,6 @@ Global
{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64 {89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64 {89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64 {89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.ActiveCfg = Debug|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.Build.0 = Debug|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.ActiveCfg = Release|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.Build.0 = Release|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.ActiveCfg = Debug|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.Build.0 = Debug|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.ActiveCfg = Release|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.Build.0 = Release|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.ActiveCfg = Debug|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.Build.0 = Debug|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.Release|x64.ActiveCfg = Release|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.Release|x64.Build.0 = Release|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.ActiveCfg = Debug|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.Build.0 = Debug|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.ActiveCfg = Release|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.Build.0 = Release|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.ActiveCfg = Debug|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.Build.0 = Debug|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.ActiveCfg = Release|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.Build.0 = Release|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.ActiveCfg = Debug|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.Build.0 = Debug|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.ActiveCfg = Release|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.Build.0 = Release|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.ActiveCfg = Debug|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.Build.0 = Debug|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.ActiveCfg = Release|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.Build.0 = Release|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.ActiveCfg = Debug|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.Build.0 = Debug|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.ActiveCfg = Release|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.Build.0 = Release|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.ActiveCfg = Debug|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.Build.0 = Debug|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.ActiveCfg = Release|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.Build.0 = Release|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
EndGlobalSection EndGlobalSection
GlobalSection(SolutionProperties) = preSolution GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE HideSolutionNode = FALSE
@@ -204,19 +80,4 @@ Global
GlobalSection(Performance) = preSolution GlobalSection(Performance) = preSolution
HasPerformanceSessions = true HasPerformanceSessions = true
EndGlobalSection EndGlobalSection
GlobalSection(NestedProjects) = preSolution
{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
{EF26A475-6101-4C9C-A717-D60BB92A2302} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
{911B08A3-515A-441D-84C8-8A43740B5FC1} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
{14140310-2233-430A-821B-F3058A25234B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
{6F22DB96-6EAB-4034-99D9-287021C36F01} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
{A5C33956-A41F-49B3-A41D-1082CBA59D02} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
{B945A183-2174-4ACB-B289-8A5C546656FE} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
EndGlobalSection
EndGlobal EndGlobal
-49
View File
@@ -1,49 +0,0 @@
include_guard(DIRECTORY)
function(protobuf_generate_fs)
set(_options)
set(_one_value_args TARGET_NAME OUT_DIR PROTOC)
set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
if (NOT DEFINED _ARGS_TARGET_NAME)
set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_fs_protos")
endif()
if (NOT DEFINED _ARGS_PROTOC)
set(_ARGS_PROTOC "${SPARK_BIN}/protoc")
endif()
set(_proto_include_cli_args)
foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
list(APPEND _proto_include_cli_args -I ${_dir})
endforeach()
set(_outputs)
foreach(_proto_source ${_ARGS_PROTO_SOURCES})
# get_filename_component(_abs_proto_source ${_proto_source} ABSOLUTE)
get_filename_component(_proto_source_we ${_proto_source} NAME_WE)
# get_filename_component(_proto_source_dir ${_proto_source} DIRECTORY)
list(APPEND _outputs ${_ARGS_OUT_DIR}/${_proto_source_we}.fable.fs)
endforeach()
set(_venv_exec ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh)
set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py)
set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/templates/fs/fable.fs)
add_custom_command(
OUTPUT ${_outputs}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
COMMAND ${_venv_exec}
${_ARGS_PROTOC}
--plugin=protoc-gen-service=${_plugin_py}
--service_out=lang=fs,strip_out_path=1:${_ARGS_OUT_DIR}
${_proto_include_cli_args}
${_ARGS_PROTO_SOURCES}
DEPENDS
${_venv_exec}
${_plugin_py}
${_template_fs}
${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
${_ARGS_PROTO_SOURCES}
)
add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS "${_outputs}")
endfunction()
-33
View File
@@ -1,33 +0,0 @@
include_guard(DIRECTORY)
function(protobuf_generate_js)
set(_options)
set(_one_value_args TARGET_NAME OUTPUT)
set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
if (NOT DEFINED _ARGS_TARGET_NAME)
set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_js_protos")
endif()
set(_proto_include_cli_args)
foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
list(APPEND _proto_include_cli_args -p ${_dir})
endforeach()
add_custom_command(
OUTPUT ${_ARGS_OUTPUT}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
COMMAND npx
--no-install pbjs
-t json-module
-w commonjs
-o ${_ARGS_OUTPUT}
${_proto_include_cli_args}
${_ARGS_PROTO_SOURCES}
DEPENDS
${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
${_ARGS_PROTO_SOURCES}
)
add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS ${_ARGS_OUTPUT})
endfunction()
-49
View File
@@ -1,49 +0,0 @@
include_guard(DIRECTORY)
include(${CMAKE_SOURCE_DIR}/cmake/TernaryOperator.cmake)
function(sym_link_dir)
set(_options)
set(_one_value_args TARGET_NAME SOURCE DEST COMMENT)
set(_multi_value_args)
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
add_custom_command(
OUTPUT ${_ARGS_DEST}
COMMAND ln ARGS -sf ${_ARGS_SOURCE} ${_ARGS_DEST}
)
set_ternary(_comment _ARGS_COMMENT "${_ARGS_COMMENT}" "linking dir ${_ARGS_SOURCE} -> ${_ARGS_DEST}")
add_custom_target(
${_ARGS_TARGET_NAME} ALL
DEPENDS ${_ARGS_DEST}
COMMENT "${_comment}"
)
endfunction()
function(sym_link_file)
set(_options)
set(_one_value_args SOURCE DEST INSTALL_DEST)
set(_multi_value_args)
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
file(
CREATE_LINK
${_ARGS_SOURCE}
${_ARGS_DEST}
SYMBOLIC
)
if (DEFINED _ARGS_INSTALL_DEST)
install(
FILES
${_ARGS_SOURCE}
DESTINATION ${_ARGS_INSTALL_DEST}
)
endif()
endfunction()
function(sym_link_local_lib LIB_NAME)
set(_lib_file_name "${CMAKE_SHARED_LIBRARY_PREFIX}${LIB_NAME}.so")
file(
CREATE_LINK
"${CMAKE_BINARY_DIR}/src/lib_${LIB_NAME}/${_lib_file_name}"
"${CMAKE_CURRENT_BINARY_DIR}/${_lib_file_name}"
SYMBOLIC
)
endfunction()
-9
View File
@@ -1,9 +0,0 @@
include_guard(GLOBAL)
macro(set_ternary RESULT CONDITION TRUE_VALUE FALSE_VALUE)
if(${CONDITION})
set(${RESULT} ${TRUE_VALUE})
else()
set(${RESULT} ${FALSE_VALUE})
endif()
endmacro()
File diff suppressed because it is too large. Load diff
+34 -15
View File
@@ -1,22 +1,41 @@
using System.Diagnostics; using mbts.spark;
namespace BCG.Utilities.General; using System;
using System.Runtime;
/// <summary> namespace BCG.Utilities.General
/// Summary description for Crono. {
/// </summary> /// <summary>
public class Crono { /// Summary description for Crono.
readonly Stopwatch myTimer = new(); /// </summary>
public class Crono
{
public Crono()
{
myTimer = new HighPerformanceTimer();
}
public void start() => public void start()
myTimer.Start(); {myTimer.start();}
public void stop() => public void stop()
myTimer.Stop(); {myTimer.stop();}
public TimeSpan time => public TimeSpan time
myTimer.Elapsed; {
get
{
return new TimeSpan((long) (myTimer.totalMilliseconds * TimeSpan.TicksPerMillisecond));
}
}
public double miliseconds
{
get
{return myTimer.totalMilliseconds;}
}
private HighPerformanceTimer myTimer;
}
public double miliseconds =>
myTimer.Elapsed.TotalMilliseconds;
} }
+192 -192
View File
@@ -1,192 +1,192 @@
using System; using System;
using System.Collections; using System.Collections;
using System.Collections.Generic; using System.Collections.Generic;
using System.Collections.Concurrent; using System.Collections.Concurrent;
using System.Reflection; using System.Reflection;
using Microsoft.Win32; using Microsoft.Win32;
namespace BCG namespace BCG
{ {
public enum RegistryRoot { HKCU, HKLM } public enum RegistryRoot { HKCU, HKLM }
[Serializable] [Serializable]
public class BlueRegistry public class BlueRegistry
{ {
public const string BlueSubKey = @"Software\Blue Capital Group\"; public const string BlueSubKey = @"Software\Blue Capital Group\";
public static readonly string ApplicationPrefix; public static readonly string ApplicationPrefix;
protected string myFullPath; protected string myFullPath;
public BlueRegistry(string subPath) public BlueRegistry(string subPath)
{ {
myFullPath = string.Concat(BlueRegistry.ApplicationPrefix, @"\", subPath); myFullPath = string.Concat(BlueRegistry.ApplicationPrefix, @"\", subPath);
} }
public BlueRegistry(BlueRegistry parent, string subPath) public BlueRegistry(BlueRegistry parent, string subPath)
{ {
myFullPath = string.Concat(parent.myFullPath, @"\", subPath); myFullPath = string.Concat(parent.myFullPath, @"\", subPath);
} }
static BlueRegistry() static BlueRegistry()
{ {
string app = "Applications"; string app = "Applications";
Assembly ass = Assembly.GetEntryAssembly(); Assembly ass = Assembly.GetEntryAssembly();
if (ass != null) if (ass != null)
{ {
app = ass.GetName().Name; app = ass.GetName().Name;
object[] attrs = ass.GetCustomAttributes( object[] attrs = ass.GetCustomAttributes(
Type.GetType("System.Reflection.AssemblyTitleAttribute"), true); Type.GetType("System.Reflection.AssemblyTitleAttribute"), true);
if (attrs.Length > 0 && ((AssemblyTitleAttribute)attrs[0]).Title != string.Empty) if (attrs.Length > 0 && ((AssemblyTitleAttribute)attrs[0]).Title != string.Empty)
{ {
app = ((AssemblyTitleAttribute)attrs[0]).Title; app = ((AssemblyTitleAttribute)attrs[0]).Title;
} }
} }
ApplicationPrefix = string.Concat(BlueRegistry.BlueSubKey, app); ApplicationPrefix = string.Concat(BlueRegistry.BlueSubKey, app);
} }
public object getValue(string name) public object getValue(string name)
{ {
return getValue(RegistryRoot.HKCU, name); return getValue(RegistryRoot.HKCU, name);
} }
public object getValue(string name, object defaultValue) public object getValue(string name, object defaultValue)
{ {
return getValue(RegistryRoot.HKCU, name, defaultValue); return getValue(RegistryRoot.HKCU, name, defaultValue);
} }
public void setValue(string name, object value) public void setValue(string name, object value)
{ {
setValue(RegistryRoot.HKCU, name, value); setValue(RegistryRoot.HKCU, name, value);
} }
public void deleteValue(string name) public void deleteValue(string name)
{ {
deleteValue(RegistryRoot.HKCU, name); deleteValue(RegistryRoot.HKCU, name);
} }
private RegistryKey myGetRegistryKey(RegistryRoot root) private RegistryKey myGetRegistryKey(RegistryRoot root)
{ {
if (root == RegistryRoot.HKCU) if (root == RegistryRoot.HKCU)
return Registry.CurrentUser.CreateSubKey(myFullPath); return Registry.CurrentUser.CreateSubKey(myFullPath);
else else
return Registry.LocalMachine.CreateSubKey(myFullPath); return Registry.LocalMachine.CreateSubKey(myFullPath);
} }
#region Generic Methods #region Generic Methods
public virtual void setValue(RegistryRoot root, string name, object value) internal virtual void setValue(RegistryRoot root, string name, object value)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
rk.SetValue(name, value); rk.SetValue(name, value);
} }
} }
internal virtual object getValue(RegistryRoot root, string name) internal virtual object getValue(RegistryRoot root, string name)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
return rk.GetValue(name); return rk.GetValue(name);
} }
} }
internal virtual void deleteValue(RegistryRoot root, string name) internal virtual void deleteValue(RegistryRoot root, string name)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
rk.DeleteValue(name); rk.DeleteValue(name);
} }
} }
public virtual object getValue(RegistryRoot root, string name, object defaultValue) internal virtual object getValue(RegistryRoot root, string name, object defaultValue)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
return rk.GetValue(name, defaultValue); return rk.GetValue(name, defaultValue);
} }
} }
#endregion #endregion
#region Double #region Double
internal virtual void setDouble(RegistryRoot root, string name, double value) internal virtual void setDouble(RegistryRoot root, string name, double value)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
DoubleRegistryHandler.setValue(rk, name, value); DoubleRegistryHandler.setValue(rk, name, value);
} }
} }
internal virtual double getDouble(RegistryRoot root, string name) internal virtual double getDouble(RegistryRoot root, string name)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
return DoubleRegistryHandler.getValue(rk, name); return DoubleRegistryHandler.getValue(rk, name);
} }
} }
#endregion #endregion
#region Enum #region Enum
internal virtual void setEnum(RegistryRoot root, string name, Enum value) internal virtual void setEnum(RegistryRoot root, string name, Enum value)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
EnumRegistryHandler.setValue(rk, name, value); EnumRegistryHandler.setValue(rk, name, value);
} }
} }
internal virtual Enum getEnum(RegistryRoot root, string name, Type enumType) internal virtual Enum getEnum(RegistryRoot root, string name, Type enumType)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
return EnumRegistryHandler.getValue(rk, name, enumType); return EnumRegistryHandler.getValue(rk, name, enumType);
} }
} }
#endregion #endregion
#region Array #region Array
internal virtual void setArray(RegistryRoot root, string name, Array array) internal virtual void setArray(RegistryRoot root, string name, Array array)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
ArrayRegistryHandler.setValue(rk, name, array); ArrayRegistryHandler.setValue(rk, name, array);
} }
} }
internal virtual Array getArray(RegistryRoot root, string name, Type elementType) internal virtual Array getArray(RegistryRoot root, string name, Type elementType)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
return ArrayRegistryHandler.getValue(rk, name, elementType); return ArrayRegistryHandler.getValue(rk, name, elementType);
} }
} }
#endregion #endregion
#region SortedList #region SortedList
internal virtual void setSortedList(RegistryRoot root, string name, SortedList slist) internal virtual void setSortedList(RegistryRoot root, string name, SortedList slist)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
SortedListRegistryHandler.setValue(rk, name, slist); SortedListRegistryHandler.setValue(rk, name, slist);
} }
} }
internal virtual SortedList getSortedList(RegistryRoot root, string name) internal virtual SortedList getSortedList(RegistryRoot root, string name)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
return SortedListRegistryHandler.getValue(rk, name); return SortedListRegistryHandler.getValue(rk, name);
} }
} }
internal virtual void deleteSortedList(RegistryRoot root, string name) internal virtual void deleteSortedList(RegistryRoot root, string name)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
SortedListRegistryHandler.deleteValue(rk, name); SortedListRegistryHandler.deleteValue(rk, name);
} }
} }
#endregion #endregion
} }
} }
@@ -1,9 +1,10 @@
<Project Sdk="Microsoft.NET.Sdk"> <Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup> <PropertyGroup>
<TargetFramework>net7.0</TargetFramework> <TargetFramework>net7.0-windows</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings> <ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable> <Nullable>enable</Nullable>
<UseWindowsForms>true</UseWindowsForms>
</PropertyGroup> </PropertyGroup>
</Project> </Project>
@@ -1,166 +1,166 @@
using System; using System;
using System.Reflection; using System.Reflection;
using System.Windows.Forms; using System.Windows.Forms;
using System.Drawing; using System.Drawing;
using System.Diagnostics; using System.Diagnostics;
using Microsoft.Win32; using Microsoft.Win32;
namespace BCG namespace BCG
{ {
public class PersistWindowState public class PersistWindowState
{ {
// event info that allows form to persist extra window state data // event info that allows form to persist extra window state data
public delegate void WindowStateHandler (object sender, BlueRegistry bluereg); public delegate void WindowStateHandler (object sender, BlueRegistry bluereg);
public event WindowStateHandler LoadState; public event WindowStateHandler LoadState;
public event WindowStateHandler SaveState; public event WindowStateHandler SaveState;
private Form myParent; private Form myParent;
private BlueRegistry myRegistry; private BlueRegistry myRegistry;
private int myNormalLeft; private int myNormalLeft;
private int myNormalTop; private int myNormalTop;
private int myNormalWidth; private int myNormalWidth;
private int myNormalHeight; private int myNormalHeight;
private bool myResizable = true; private bool myResizable = true;
private FormWindowState myWindowState; private FormWindowState myWindowState;
private bool myAllowSaveMinimized = false; private bool myAllowSaveMinimized = false;
private bool mySavePosition = true; private bool mySavePosition = true;
private bool mySaveState = true; private bool mySaveState = true;
public PersistWindowState (bool savePosition) public PersistWindowState (bool savePosition)
{ {
mySavePosition = savePosition; mySavePosition = savePosition;
} }
public PersistWindowState () public PersistWindowState ()
{ {
} }
public bool saveState public bool saveState
{ {
[DebuggerStepThrough] [DebuggerStepThrough]
set { mySaveState = value; } set { mySaveState = value; }
} }
public Form parent public Form parent
{ {
[DebuggerStepThrough] [DebuggerStepThrough]
get { return myParent; } get { return myParent; }
set set
{ {
myParent = value; myParent = value;
myRegistry = new BlueRegistry (myParent.Text); myRegistry = new BlueRegistry (myParent.Text);
// subscribe to parent form's events // subscribe to parent form's events
myParent.Closing += new System.ComponentModel.CancelEventHandler(myOnClosing); myParent.Closing += new System.ComponentModel.CancelEventHandler(myOnClosing);
myParent.Resize += new System.EventHandler(myOnResize); myParent.Resize += new System.EventHandler(myOnResize);
myParent.Move += new System.EventHandler(myOnMove); myParent.Move += new System.EventHandler(myOnMove);
myParent.Load += new System.EventHandler(myOnLoad); myParent.Load += new System.EventHandler(myOnLoad);
// get initial width and height in case form is never resized // get initial width and height in case form is never resized
myNormalWidth = myParent.Width; myNormalWidth = myParent.Width;
myNormalHeight = myParent.Height; myNormalHeight = myParent.Height;
} }
} }
public BlueRegistry blueRegistry public BlueRegistry blueRegistry
{ {
[DebuggerStepThrough] [DebuggerStepThrough]
get { return myRegistry; } get { return myRegistry; }
[DebuggerStepThrough] [DebuggerStepThrough]
set { myRegistry = value; } set { myRegistry = value; }
} }
public bool resizable public bool resizable
{ {
[DebuggerStepThrough] [DebuggerStepThrough]
set { myResizable = value; } set { myResizable = value; }
} }
public bool allowSaveMinimized public bool allowSaveMinimized
{ {
[DebuggerStepThrough] [DebuggerStepThrough]
set { myAllowSaveMinimized = value; } set { myAllowSaveMinimized = value; }
} }
private void myOnResize(object sender, System.EventArgs e) private void myOnResize(object sender, System.EventArgs e)
{ {
// save width and height // save width and height
if (myParent.WindowState == FormWindowState.Normal && myResizable) if (myParent.WindowState == FormWindowState.Normal && myResizable)
{ {
myNormalWidth = myParent.Width; myNormalWidth = myParent.Width;
myNormalHeight = myParent.Height; myNormalHeight = myParent.Height;
} }
} }
private void myOnMove(object sender, System.EventArgs e) private void myOnMove(object sender, System.EventArgs e)
{ {
// save position // save position
if(myParent.WindowState == FormWindowState.Normal) if(myParent.WindowState == FormWindowState.Normal)
{ {
myNormalLeft = myParent.Left; myNormalLeft = myParent.Left;
myNormalTop = myParent.Top; myNormalTop = myParent.Top;
} }
// save state // save state
myWindowState = myParent.WindowState; myWindowState = myParent.WindowState;
} }
private void myOnClosing(object sender, System.ComponentModel.CancelEventArgs e) private void myOnClosing(object sender, System.ComponentModel.CancelEventArgs e)
{ {
// save position, size and state // save position, size and state
if (mySavePosition) if (mySavePosition)
{ {
myRegistry.setValue (RegistryRoot.HKCU, "Left", myNormalLeft); myRegistry.setValue (RegistryRoot.HKCU, "Left", myNormalLeft);
myRegistry.setValue (RegistryRoot.HKCU, "Top", myNormalTop); myRegistry.setValue (RegistryRoot.HKCU, "Top", myNormalTop);
} }
if (myResizable) if (myResizable)
{ {
myRegistry.setValue (RegistryRoot.HKCU, "Width", myNormalWidth); myRegistry.setValue (RegistryRoot.HKCU, "Width", myNormalWidth);
myRegistry.setValue (RegistryRoot.HKCU, "Height", myNormalHeight); myRegistry.setValue (RegistryRoot.HKCU, "Height", myNormalHeight);
} }
// check if we are allowed to save the state as minimized (not normally) // check if we are allowed to save the state as minimized (not normally)
if (!myAllowSaveMinimized) if (!myAllowSaveMinimized)
{ {
if(myWindowState == FormWindowState.Minimized) if(myWindowState == FormWindowState.Minimized)
myWindowState = FormWindowState.Normal; myWindowState = FormWindowState.Normal;
} }
if (mySaveState) if (mySaveState)
myRegistry.setValue (RegistryRoot.HKCU, "WindowState", (int) myWindowState); myRegistry.setValue (RegistryRoot.HKCU, "WindowState", (int) myWindowState);
// fire SaveState event // fire SaveState event
if (SaveState != null) if (SaveState != null)
SaveState (this, myRegistry); SaveState (this, myRegistry);
} }
private void myOnLoad (object sender, System.EventArgs e) private void myOnLoad (object sender, System.EventArgs e)
{ {
if (mySavePosition) if (mySavePosition)
{ {
int left = (int) myRegistry.getValue (RegistryRoot.HKCU, "Left", myParent.Left); int left = (int) myRegistry.getValue (RegistryRoot.HKCU, "Left", myParent.Left);
int top = (int) myRegistry.getValue (RegistryRoot.HKCU, "Top", myParent.Top); int top = (int) myRegistry.getValue (RegistryRoot.HKCU, "Top", myParent.Top);
myParent.Location = new Point (left, top); myParent.Location = new Point (left, top);
} }
if (myResizable) if (myResizable)
{ {
int width = (int) myRegistry.getValue (RegistryRoot.HKCU, "Width", myParent.Width); int width = (int) myRegistry.getValue (RegistryRoot.HKCU, "Width", myParent.Width);
int height = (int) myRegistry.getValue (RegistryRoot.HKCU, "Height", myParent.Height); int height = (int) myRegistry.getValue (RegistryRoot.HKCU, "Height", myParent.Height);
myParent.Size = new Size (width, height); myParent.Size = new Size (width, height);
} }
if (mySaveState) if (mySaveState)
{ {
FormWindowState windowState = (FormWindowState) myRegistry.getValue (RegistryRoot.HKCU, "WindowState", FormWindowState windowState = (FormWindowState) myRegistry.getValue (RegistryRoot.HKCU, "WindowState",
(int) myParent.WindowState); (int) myParent.WindowState);
myParent.WindowState = windowState; myParent.WindowState = windowState;
} }
// fire LoadState event // fire LoadState event
if(LoadState != null) if(LoadState != null)
LoadState (this, myRegistry); LoadState (this, myRegistry);
} }
} }
} }
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
@@ -18,11 +18,8 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="HtmlAgilityPack" Version="1.11.57" /> <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
<PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
<PackageReference Include="MongoDB.Bson" Version="2.23.1" /> <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
<PackageReference Include="MongoDB.Driver" Version="2.23.1" /> <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
<PackageReference Include="Parquet.Net" Version="5.0.1" />
<PackageReference Include="System.Data.SQLite" Version="1.0.118" />
</ItemGroup> </ItemGroup>
</Project> </Project>
@@ -1,26 +0,0 @@
namespace Bts.DataStream;
using BCG;
using Spark.Core.Disposables;
using System;
using System.Reactive.Linq;
public static class DataStreamExtensions {
public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
source as ViewStream ?? new ObservableViewStream(source);
public static IObservable<DataPacket> AsObservable(this ViewStream source) {
return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
IDisposable Subscribe(IObserver<DataPacket> observer) {
source.onChange += OnChange;
return Disposable.Create(() => source.onChange -= OnChange);
void OnChange(object? newValue) =>
observer.OnNext(newValue as DataPacket ?? throw new ArgumentException(
$"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
}
}
}
@@ -1,32 +0,0 @@
namespace bts.SilverPine {
using System;
using BCG;
using System.Collections.Generic;
using Bts.DataStreamExpressions;
// All this does is take in a dsl expression, parse it and fire the data that come out of running it.
// So for example, if an expresison is passed
[DataStreamUserRespresentation("dsl")]
public class DSLStream : ViewStream {
private readonly string _dsl;
public DSLStream(string dsl) {
Log.Gen.info($"DSLStream.cons dsl: {dsl}");
_dsl = dsl;
ViewStream rs = build_stream(dsl);
rs = rs ?? new LiteralStream("error", $"msg:invalid dsl \"{dsl}\"");
mySetRealSource(rs);
}
static ViewStream? build_stream(string stream_expression) {
try {
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
return rv;
} catch (Exception ex) {
Log.Gen.error($"Could not build stream: {stream_expression} ex: {ex}");
return null;
}
}
}
}
@@ -1,29 +0,0 @@
namespace Bts.DataStream;
using BCG;
using bts;
using System;
using System.IO;
public class PandasScriptStream : ViewStream {
public PandasScriptStream(ViewStream vs, string file)
: this(vs, file, null) { }
public PandasScriptStream(ViewStream vs, string file, string? pythonExe)
: this(vs, file, pythonExe, ViewStreamDatabase.instance) { }
public PandasScriptStream(ViewStream vs, string file, string? pythonExe, ViewStreamDatabase vsd) {
mySetRealSource(new PandasStream(vs, readFile(), pythonExe, vsd));
string readFile() {
try {
return File.ReadAllText(file);
} catch (Exception ex) {
Log.Gen.error($"PandasScriptStream error reading file: {ex}");
throw;
}
}
}
}
@@ -1,130 +1,62 @@
namespace Bts.DataStream; // using System;
// using System.Diagnostics;
using BCG;
using BCG.Extensions; // using BCG;
using BCG.Utilities.General; // using BCG.Utilities.General;
using bts;
using bts.collections; // namespace bts {
using Bts.DataStreamCore; // public class PandasStream : ViewStream, IWorker {
// //C:\Users\Owner\AppData\Local\Microsoft\WindowsApps\python.exe
using Spark.Core.Disposables;
// public PandasStream(ViewStream vs, ViewStreamDatabase vsd, string pandas_code) {
using System;
using System.Collections.Generic; // }
using System.Diagnostics;
using System.IO; // public PandasStream(ViewStream vs, string pandas_code) :
using System.Linq; // this(vs, ViewStreamDatabase.instance, pandas_code) {
using System.Runtime.CompilerServices; // }
using System.Text;
public class PandasStream : ViewStream, IWorker { // protected override void myOnFirstSubscriber() {
readonly string inputFile = $"PandasStream_{Guid.NewGuid()}.csv";
readonly string outputFile = $"PandasStream_{Guid.NewGuid()}.csv"; // // string python_code = @"
readonly string pythonExe; // // import winsound
readonly ViewStreamDatabase vsd; // // frequency = 2500 # Set frequency (in Hz)
readonly CSVSaverStream inputCsvStream; // // duration = 1000 # Set duration (in milliseconds)
readonly Lazy<string> pythonCode; // // winsound.Beep(frequency, duration)
IDictionary<int, IBlueRow>? lastRowMap = null; // // ";
int inputVersion = 0; // string python_code = @"
int outputVersion = 0; // file_path = \"C:\\Temp\\test2.txt\\" # r prefix for raw string to avoid escape characters
DateTime lastFireTime = DateTime.MinValue; // with open(file_path, \"w\") as f:
TimeSpan fireThrottle = TimeSpan.FromSeconds(3); // f.write(\"Hello, world!\n\")
IDisposable? dispose; // ";
public PandasStream(ViewStream vs, string dataFrameExpression) // execute_python_code(python_code);
: this(vs, dataFrameExpression, null, ViewStreamDatabase.instance) { } // }
public PandasStream(ViewStream vs, string dataFrameExpression, string? pythonExe, ViewStreamDatabase vsd) { // static void execute_python_code(string python_code) {
pythonExe ??= Environment.GetEnvironmentVariable("PYTHON_EXE") ?? @"C:\Users\johnc\GoogleDrive\Repos\bts\spark\.venv\Scripts\python.exe"; // ProcessStartInfo start = new ProcessStartInfo();
this.pythonExe = pythonExe; // start.FileName = "python"; // Python executable
this.vsd = vsd; // start.Arguments = "-c \"" + python_code.Replace("\"", "\\\"") + "\""; // Pass code as argument
// This should fire an update when it writes to the output file // start.UseShellExecute = false;
inputCsvStream = new CSVSaverStream(vs, inputFile, true, int.MaxValue, true); // start.RedirectStandardOutput = true;
pythonCode = new(getPythonCode);
// using (Process process = Process.Start(start)) {
string getPythonCode() { // using (System.IO.StreamReader reader = process.StandardOutput) {
var imports = // string result = reader.ReadToEnd();
@"import numpy as np // Console.Write(result);
import pandas as pd // }
import sys // }
"; // }
var bd = new StringBuilder(imports);
bd.AppendLine($"df=pd.read_csv('{inputFile}')"); // protected override void myOnLastSubscriber() {
bd.AppendLine(dataFrameExpression); // }
bd.AppendLine($"df.to_csv('{outputFile}', index=False)");
return bd.ToString(); // public void doWork() {
}
} // }
protected override void myOnFirstSubscriber() { // }
vsd.addWorker(this); // }
inputCsvStream.onChange += onInputChanged;
this.dispose = Disposable.Create(dispose);
void onInputChanged(object newValue) =>
inputVersion++;
void dispose() {
inputCsvStream.onChange -= onInputChanged;
vsd.removeWorker(this);
deleteFile(inputFile);
deleteFile(outputFile);
}
void deleteFile(string file, [CallerArgumentExpression(nameof(file))] string name = "") {
if (!File.Exists(file)) {
return;
}
try {
File.Delete(file);
} catch (Exception ex) {
Log.Gen.error($"PandasStream. Error deleting {name} file '{file}': {ex}");
}
}
}
protected override void myOnLastSubscriber() =>
Disposable.dispose(ref dispose);
static void executePythonCode(string pythonCode, string? pythonExe = null) {
pythonExe ??= "python";
var start = new ProcessStartInfo {
FileName = pythonExe,
Arguments = "-c \"" + pythonCode.Replace("\"", "\\\"") + "\"",
UseShellExecute = false,
RedirectStandardOutput = true
};
using var process = Process.Start(start)
?? throw new InvalidOperationException("Failed to launch python");
process.WaitForExit();
}
void IWorker.doWork() {
if (outputVersion == inputVersion) {
return;
}
var now = DateTime.Now;
var elapsedSinceLastFire = now - lastFireTime;
if (elapsedSinceLastFire < fireThrottle) {
return;
}
outputVersion = inputVersion;
executePythonCode(pythonCode.Value, pythonExe);
var rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(outputFile, out var schema);
var rowMap = rows as IDictionary<int, IBlueRow> ?? rows.Cast<IBlueRow>().ToDictionary(r => r.Id());
var comp = BlueRowEqualityComparer.Instance;
var deletedRows = new BlueTable(schema);
var addedRows = new BlueTable(schema);
if (lastRowMap is not null) {
deletedRows.SetItems(lastRowMap.Where(oldRow =>
!(rowMap.TryGetValue(oldRow.Key, out var newRow) && comp.Equals(newRow, oldRow.Value))));
addedRows.SetItems(rowMap.Where(newRow =>
!(lastRowMap.TryGetValue(newRow.Key, out var oldRow) && comp.Equals(oldRow, newRow.Value))));
}
lastRowMap = rowMap;
lastFireTime = DateTime.Now;
myFireEvent(new DataPacket(rows, addedRows, deletedRows));
}
}
@@ -39,8 +39,8 @@ namespace BCG {
protected override void myOnFirstSubscriber() { protected override void myOnFirstSubscriber() {
MasterWorker.getInstance(false).addWorker(this, _send_period); MasterWorker.getInstance(false).addWorker(this, _send_period);
var server_rci = new RemoteConnectionInfo(_ip, _port); var server_address = new BlutoSubAddress(_ip, _port);
bts.AsyncTCPConnector_APM.connect_to(server_rci, this); bts.AsyncTCPConnector_APM.connect_to(server_address, this);
} }
protected override void myOnLastSubscriber() { protected override void myOnLastSubscriber() {
@@ -1,37 +0,0 @@
namespace Bts.DataStream;
using BCG;
using bts;
using System;
public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
readonly IObservable<DataPacket> source;
IDisposable? subscription;
public ObservableViewStream(IObservable<DataPacket> source) =>
this.source = source;
public void Dispose() {
GC.SuppressFinalize(this);
if (subscription is not null) {
subscription.Dispose();
subscription = null;
}
}
protected override void myOnFirstSubscriber() {
Log.isnull(subscription);
subscription = source.Subscribe(myFireEvent);
}
protected override void myOnLastSubscriber() {
Log.notnull(subscription);
subscription.Dispose();
subscription = null;
}
IDisposable IObservable<DataPacket>.Subscribe(IObserver<DataPacket> observer) =>
source.Subscribe(observer);
}
@@ -0,0 +1,32 @@
using BCG;
namespace BCG
{
// this is a stream so we can run it from the DSQA (since dsqa can only load streams)
[DataStreamUserRespresentation("add_to_cloud")]
public class AddToCloudStream : ViewStream
{
private readonly string _name, _stream_exp;
private StreamOffloadRequestor _requestor = new StreamOffloadRequestor();
public AddToCloudStream(string name, params string[] lines)
: this(name, 0, lines)
{
}
public AddToCloudStream(string name, int persist, params string[] lines)
{
_name = name;
_stream_exp = getExpression(lines);
_requestor.add_to_cloud(_name, _stream_exp, persist == 1);
mySetRealSource(new TableStream("cloud_streams"));
}
static string getExpression(params string[] lines) {
for (int i = 0; i < lines.Length; i++) {
// unescape quotes
lines[i] = lines[i].Replace("\\\"", "\"");
}
return string.Join("\n", lines);
}
}
}
@@ -1,22 +1,31 @@
using BCG;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using mbts.config;
using bts.utils;
using System.Threading.Tasks;
using Bts.DataStreamExpressions; using Bts.DataStreamExpressions;
using System.Collections.Generic;
using BCG.Utilities.General; using BCG.Utilities.General;
using bts;
namespace BCG { namespace BCG
{
// listens to the requests posted on "cloud_streams" request table, and // listens to the requests posted on "cloud_streams" request table, and
// if unfulfilled starts a ViewPubsliher // if unfulfilled starts a ViewPubsliher
public class RemoteQueryManager : IWorker { public class CloudStreamManager : IWorker
public class StreamPubUnit { {
public class StreamPubUnit
{
public string name; public string name;
public string stream_expression; public string stream_expression;
public ViewStream stream; public ViewStream stream;
public bool persist; public bool persist;
public ViewPublisher view_publisher; public ViewPublisher view_publisher;
public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_) { public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_)
{
name = name_; name = name_;
stream_expression = stream_expression_; stream_expression = stream_expression_;
stream = stream_; stream = stream_;
@@ -24,7 +33,7 @@ namespace BCG {
view_publisher = view_publisher_; view_publisher = view_publisher_;
} }
} }
static ViewStream? build_stream(string stream_expression) static ViewStream build_stream(string stream_expression)
{ {
try try
{ {
@@ -32,7 +41,9 @@ namespace BCG {
var rv = exp.getView(new Dictionary<string, StatementExpression>(0)); var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
return rv; return rv;
} catch (Exception ex) { }
catch (Exception ex)
{
bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}"); bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
return null; return null;
} }
@@ -41,37 +52,34 @@ namespace BCG {
_stream_pub_map = new Dictionary<string, StreamPubUnit>(); _stream_pub_map = new Dictionary<string, StreamPubUnit>();
private ViewStream _stream = new TableStream("cloud_streams"); private ViewStream _stream = new TableStream("cloud_streams");
private PubTableBroker _broker = new PubTableBroker(); private PubTableBroker _broker = new PubTableBroker();
private ManagedSubscriber _sub = ManagedSubscriber.instance;
public CloudStreamManager()
public RemoteQueryManager() { {
_stream.onChange += _on_str; _stream.onChange += _on_str;
_sub.subscribe("RemoteRequest", (string tag, byte[] data) => {
RemoteRequest req = RemoteRequest.construct_from_bytes(data);
_spawn_remote_stream(req.published_table_name, req.stream_expression, false);
});
} }
private void _on_str(object obj) private void _on_str(object obj)
{ {
if (obj is not DataPacket dp) { DataPacket dp = obj as DataPacket;
return;
}
foreach (IBlueRow r in dp.newRows) foreach (IBlueRow r in dp.newRows)
{ {
string name = r.get<string>("name"); string name = r.get<string>("name");
bool persist = r.get<bool>("persist"); bool persist = r.get<bool>("persist");
string stream_exp = r.get<string>("stream_exp"); string stream_exp = r.get<string>("stream_exp");
if (!_stream_pub_map.ContainsKey(name)) { if (!_stream_pub_map.ContainsKey(name))
_spawn_remote_stream(name, stream_exp, persist); {
_assign(name, stream_exp, persist);
} }
} }
} }
private void _spawn_remote_stream(string name, string stream_expression, bool persist) { private void _assign(string name, string stream_expression, bool persist)
{
bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}"); bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
ViewStream stream = build_stream(stream_expression); ViewStream stream = build_stream(stream_expression);
_expose_to_ws(name, stream); _expose_to_ws(name, stream);
if (persist) { if (persist)
{
stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000); // 10MBs stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000); // 10MBs
} }
ViewPublisher view_pub = new ViewPublisher(name, _broker, stream); ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
@@ -79,12 +87,14 @@ namespace BCG {
_stream_pub_map.Add(name, u); _stream_pub_map.Add(name, u);
} }
private void _expose_to_ws(string name, ViewStream stream) { private void _expose_to_ws(string name, ViewStream stream)
{
// TODO: john // TODO: john
} }
public void doWork() { public void doWork()
{
} }
} }
} }
@@ -1,48 +0,0 @@
using BCG;
using bts;
using System;
namespace BCG {
// this is a stream so we can run it from the DSQA (since dsqa can only load streams)
[DataStreamUserRespresentation("remoteQuery")]
public class RemoteQueryStream : ViewStream {
private readonly string _name, _stream_exp;
private PublisherCore _publisher_core = new PublisherCore();
private ViewStream _source_str;
public RemoteQueryStream(params string[] lines) {
_stream_exp = get_expression(lines);
_name = Guid.NewGuid().ToString();
_name = $"remotereq_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds().ToString()}";
//_name.Replace("-", "_");
}
protected override void myOnFirstSubscriber() {
RemoteRequest rem_req = new RemoteRequest(_stream_exp, _name);
byte[] rem_req_bytes = rem_req.serialize();
_publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
_source_str = new TableStream(_name);
_source_str.onChange += new ChangeDelegate(_handle_data);
}
protected override void myOnLastSubscriber() {
//maybe there should be a way to turn remote requests off ?
_source_str.onChange -= new ChangeDelegate(_handle_data);
}
static string get_expression(params string[] lines) {
for (int i = 0; i < lines.Length; i++) {
// unescape quotes
lines[i] = lines[i].Replace("\\\"", "\"");
}
return string.Join("\n", lines);
}
private void _handle_data(object o) {
myFireEvent(o);
}
}
}
@@ -1,31 +0,0 @@
namespace bts {
using System.IO;
using BCG;
using System.IO;
using System.Threading.Tasks.Dataflow;
public class RemoteRequest {
public readonly string stream_expression;
public readonly string published_table_name;
public RemoteRequest(string stream_expression_, string published_table_name_) {
stream_expression = stream_expression_;
published_table_name = published_table_name_;
}
public byte[] serialize() {
var str = new MemoryStream();
var bw = new BinaryWriter(str);
bw.write_string_cpp(stream_expression);
bw.write_string_cpp(published_table_name);
return str.ToArray();
}
public static RemoteRequest construct_from_bytes(byte[] ser_bytes) {
BinaryReader br = new BinaryReader(new MemoryStream(ser_bytes));
string stream_expression_ = br.read_string_cpp();
string published_table_name_ = br.read_string_cpp();
return new RemoteRequest(stream_expression_, published_table_name_);
}
}
}
@@ -1,54 +1,59 @@
// using BCG; using BCG;
// using bts; using System.Linq;
// using System.Linq; namespace BCG
{
public class StreamOffloadRequestor
{
public class OffloadOp
{
string name;
string stream_exp;
}
private IBlueTableSchema _schema;
private IBlueRowCollection _rows;
private PubTableBroker _broker = new PubTableBroker();
ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
public StreamOffloadRequestor()
{
_schema = new BlueTableSchema("cloud_streams",
new ColumnInfo("id", 0, typeof(int)),
new ColumnInfo("name", 1, typeof(string)),
new ColumnInfo("persist", 2, typeof(bool)),
new ColumnInfo("stream_exp", 3, typeof(string)));
_existing_offloads_vs.onChange += _on_str;
}
private void _on_str(object obj)
{
DataPacket dp = obj as DataPacket;
_rows = dp.newRows;
// this is needed so the broker knows the state of the table
_broker.createTable(_rows);
}
public void add_to_cloud(string name, string stream_exp, bool persist)
{
// check if it exists
if (_rows == null)
{
BlueTable table = new BlueTable(_schema);
IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
table.addRow(nr);
_broker.createTable(table);
// create table
}
else
{
if (exists(name))
return;
IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
_broker.addRow(nr);
}
}
// namespace BCG { bool exists(string name)
// // used on the client {
// public class StreamOffloadRequestor { return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
// public class OffloadOp { }
// string name; }
// string stream_exp; }
// }
// private IBlueTableSchema _schema;
// private IBlueRowCollection _rows;
// private PubTableBroker _broker = new PubTableBroker();
// ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
// public StreamOffloadRequestor() {
// _schema = new BlueTableSchema("cloud_streams",
// new ColumnInfo("id", 0, typeof(int)),
// new ColumnInfo("name", 1, typeof(string)),
// new ColumnInfo("persist", 2, typeof(bool)),
// new ColumnInfo("stream_exp", 3, typeof(string)));
// _existing_offloads_vs.onChange += _on_str;
// }
// private void _on_str(object obj) {
// DataPacket dp = obj as DataPacket;
// _rows = dp.newRows;
// // this is needed so the broker knows the state of the table
// _broker.create_table(_rows);
// }
// public void add_to_cloud(string name, string stream_exp, bool persist) {
// // check if it exists
// if (_rows == null) {
// BlueTable table = new BlueTable(_schema);
// IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
// table.addRow(nr);
// _broker.create_table(table);
// // create table
// } else {
// if (exists(name))
// return;
// IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
// // TODO: use make_batch_changes instead
// Log.assert(false, "Fix addRow");
// //_broker.addRow(nr);
// }
// }
// bool exists(string name) {
// return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
// }
// }
// }
@@ -1,14 +1,17 @@
using System; using System;
using System.Collections; using System.Collections;
namespace BCG { namespace BCG
internal class AddColumnsRow : IBlueRow { {
internal class AddColumnsRow : IBlueRow
{
private IBlueTableSchema mySchema; private IBlueTableSchema mySchema;
private IBlueRow myRow; private IBlueRow myRow;
//private object[] myCachedValues; //private object[] myCachedValues;
#region Constructors #region Constructors
public AddColumnsRow(IBlueTableSchema schema, IBlueRow row) { public AddColumnsRow (IBlueTableSchema schema, IBlueRow row)
{
mySchema = schema; mySchema = schema;
myRow = row; myRow = row;
//myCachedValues = new object[schema.columnNames.Length]; //myCachedValues = new object[schema.columnNames.Length];
@@ -16,78 +19,85 @@ namespace BCG {
#endregion #endregion
#region IBlueRow members #region IBlueRow members
public IBlueTableSchema schema { get { return mySchema; } } public IBlueTableSchema schema{get{return mySchema;}}
#endregion #endregion
#region ICollection members #region ICollection members
public int Count { get { return mySchema.columnNames.Length; } } public int Count {get{return mySchema.columnNames.Length;}}
public bool IsSynchronized { get { return false; } } public bool IsSynchronized {get{return false;}}
public object SyncRoot { get { return this; } } public object SyncRoot {get{return this;}}
public void CopyTo(Array array, int index) { public void CopyTo (Array array, int index)
{
int i = index; int i = index;
foreach (object o in this as IEnumerable) foreach (object o in this as IEnumerable)
array.SetValue(o, i++); array.SetValue (o, i++);
} }
#endregion #endregion
#region IDictionary members #region IDictionary members
public bool IsFixedSize { get { return true; } } public bool IsFixedSize {get{return true;}}
public bool IsReadOnly { get { return true; } } public bool IsReadOnly {get{return true;}}
/// <summary> /// <summary>
/// The key can either be the string corrsponding to the column name or /// The key can either be the string corrsponding to the column name or
/// the order of the column /// the order of the column
/// </summary> /// </summary>
public object this[object key] { public object this [object key]
get { {
get
{
if (key is string) if (key is string)
return this[(string)key]; return this[(string)key];
return this[(int)key]; return this[(int)key];
} }
set { throw new NotSupportedException("The IBlueRow is readonly"); } set {throw new NotSupportedException ("The IBlueRow is readonly");}
} }
public object this[int index] { public object this[int index]
get { {
get
{
//if (myCachedValues[index] == null) //if (myCachedValues[index] == null)
// myCachedValues[index] = mySchema[index].isConstant ? mySchema[index].colValue : myRow[index]; // myCachedValues[index] = mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
//return myCachedValues[index]; //return myCachedValues[index];
return mySchema[index].isConstant ? DBNull.Value : myRow[index]; return mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
} }
} }
public object this[string colName] { get { return this[mySchema[colName].colNo]; } } public object this[string colName] {get{return this[mySchema[colName].colNo];}}
public ICollection Keys { get { return mySchema.columnNames; } } public ICollection Keys{get{return mySchema.columnNames;}}
public ICollection Values { public ICollection Values
get { {
get
{
object[] retVal = new object[mySchema.columnNames.Length]; object[] retVal = new object[mySchema.columnNames.Length];
CopyTo(retVal, 0); CopyTo (retVal, 0);
return retVal; return retVal;
} }
} }
public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); } public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");}
public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); } public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
public bool Contains(object key) { return mySchema[(string)key] != null; } public bool Contains (object key){return mySchema[(string)key] != null;}
/// <summary> /// <summary>
/// Note, the default enumerator is the IDictionary's enumerator, so if user needs to /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
/// iterate through the values, typecast this into an IEnumerable /// iterate through the values, typecast this into an IEnumerable
/// </summary> /// </summary>
/// <returns></returns> /// <returns></returns>
public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); } public IDictionaryEnumerator GetEnumerator() {return new BlueRowDictionaryEnumerator(this);}
IEnumerator IEnumerable.GetEnumerator () {return new BlueRowEnumerator(this);}
IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); } public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");}
public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
#endregion #endregion
} }
@@ -1,34 +1,6 @@
using System; namespace BCG;
namespace BCG { public class AddColumnsStream : MapStream {
public class AddColumnsStream : ViewStream { public AddColumnsStream(ViewStream vs, string columnDef)
public static Tuple<string, Type, object> parse_exp(string exp) { : base(vs, new ColumnAdder(columnDef)) { }
string[] toks = exp.Split(':'); }
string col_name = toks[0];
string col_value = toks[1];
Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
return Tuple.Create(col_name, tup.Item1, tup.Item2);
}
private Tuple<string, Type, object> _new_col;
private IBlueTableSchema _out_schema;
public AddColumnsStream(ViewStream vs, string col_def) {
_new_col = parse_exp(col_def);
ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
mySetRealSource(rs);
}
private IBlueTableSchema _get_schema(IBlueTableSchema input_schema) {
if (_out_schema == null)
_out_schema = BlueDataUtility.add_column(input_schema, _new_col.Item1, _new_col.Item2);
return _out_schema;
}
private IBlueRow _map(IBlueRow r) {
MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
rv.setValue(_new_col.Item1, _new_col.Item3);
return rv;
}
}
}
@@ -1,24 +1,23 @@
using bts;
using System; using System;
using System.Collections; using System.Collections;
namespace BCG { namespace BCG
{
/// <summary> /// <summary>
/// Provides a stream that with distinct rows with respect to the columns /// Provides a stream that with distinct rows with respect to the columns
/// the user specifies in the argument /// the user specifies in the argument
/// </summary> /// </summary>
[DataStreamUserRespresentation("distinct")] [DataStreamUserRespresentation ("distinct")]
public class DistinctRowStream : ViewStream { public class DistinctRowStream : ViewStream
public DistinctRowStream(ViewStream stream, params string[] cols) { {
var g_by_eval = new DistinctRowEvaluator(cols); public DistinctRowStream (ViewStream stream, params string[] cols)
ViewStream r = new GroupByStream(stream, g_by_eval); {
var proj_exp = String.Empty; ArrayList groupByCols = new ArrayList ();
foreach (var c in cols) foreach (string col in cols)
proj_exp += $"{c} "; foreach (string c in col.Split (' '))
proj_exp = proj_exp.Trim(); groupByCols.Add (c);
r = new NewProjectStream(r, proj_exp); mySetRealSource (new GroupByStream (stream,
mySetRealSource(r); new DistinctRowEvaluator (groupByCols.ToArray (typeof(string)) as string[]), false));
} }
} }
} }
@@ -0,0 +1,49 @@
// using System;
// using System.Collections.Generic;
// using System.Text;
// using System.IO;
// using System.Runtime.Serialization.Formatters.Binary;
// namespace BCG
// {
// public class DumpLoaderStream : ViewStream, ISideEffectStream
// {
// //wtf
// //public class Exception : System.Exception
// //{public Exception(string msg) : base(msg) { }}
// public DumpLoaderStream(string fileName, int secondsBetweenPolls)
// {
// mySetRealSource(
// new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls,
// ViewStreamDatabase.instance));
// }
// public DumpLoaderStream(string fileName)
// {
// if (fileName == null)
// throw new LoaderStream.Exception("File name cannot be null");
// for (int i = 0; i < 5; i++)
// try
// {
// using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
// {
// IClientDeserializer deser = new ClientUniversalDeserializer();
// byte[] the_ops;
// Array the_rows;
// deser.deserialize(new Stream[] { fs }, out the_ops, out the_rows);
// mySetRealSource(new BlueRowCollectionStaticStream((IBlueRowCollection)the_rows.GetValue(0)));
// }
// break;
// }
// catch (IOException)
// {
// System.Threading.Thread.Sleep(50);
// }
// }
// }
// }
@@ -1,25 +1,48 @@
using BCG; using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
namespace bts { using System.Runtime.Serialization.Formatters.Binary;
// This is the other side of the SaverStream, its "loads" the stream and fires once
namespace BCG
{
[DataStreamUserRespresentation("loader")] [DataStreamUserRespresentation("loader")]
public class LoaderStream : ViewStream { public class LoaderStream : ViewStream, ISideEffectStream
{
public class Exception : System.Exception
{public Exception(string msg) : base(msg) { }}
private string _filename; public LoaderStream(string fileName, int secondsBetweenPolls)
private IStreamMsgDeserilizer _deser = new StreamIODeserializationController(); {
mySetRealSource(
new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls,
public LoaderStream(string filename) { ViewStreamDatabase.instance));
_filename = filename;
} }
protected override void myOnFirstSubscriber() { public LoaderStream(string fileName)
byte[] ser_msg = StreamSerializationUtils.read_from_file(_filename); {
StreamIOMessage msg = _deser.deserialize(ser_msg); if (fileName == null)
Log.assert(msg.op == StreamIOMessage.StreamOp.CREATE); throw new LoaderStream.Exception("File name cannot be null");
IBlueRowCollection rows = msg.create_message.rows;
DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket); for (int i = 0; i < 5; i++)
myFireEvent(dp); try
{
using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
{
BinaryFormatter bf = new BinaryFormatter();
#pragma warning disable SYSLIB0011
BlueTable bt = (BlueTable)bf.Deserialize(fs);
#pragma warning restore SYSLIB0011
mySetRealSource(new BlueRowCollectionStaticStream(bt));
}
break;
}
catch (IOException)
{
System.Threading.Thread.Sleep(50);
}
} }
} }
} }
@@ -5,11 +5,9 @@ using System.Text;
using System.IO; using System.IO;
using Google.Protobuf; using Google.Protobuf;
using System.Globalization; using System.Globalization;
using BCG;
namespace bts { namespace BCG {
// Writes all the udpates to a stream in a file or multiples files dependong on the rollover size (as opposed to Saver // Writes all the udpates to a stream in a file.
// which just writes one update, as a create)
// File naming: // File naming:
// token_epoch_yyyyMMdd_HHmmss // token_epoch_yyyyMMdd_HHmmss
// File location: directory(passed)\token\ // File location: directory(passed)\token\
@@ -26,7 +24,7 @@ namespace bts {
private ViewStream _source_stream; private ViewStream _source_stream;
private string _token; private string _token;
private FileStream _out_fs; private FileStream _out_fs;
private IStreamIOMsgSerializer _serializer;// = new UniversalStreamIOSerializer(); private IBlueDataSerializer _serializer;
private string _file_name = String.Empty; private string _file_name = String.Empty;
private long _rollover_size; private long _rollover_size;
private readonly string _directory_path; private readonly string _directory_path;
@@ -95,19 +93,20 @@ namespace bts {
void _on_change(object new_value) { void _on_change(object new_value) {
DataPacket dp = new_value as DataPacket; DataPacket dp = new_value as DataPacket;
byte[] serialized_update = null; byte[] serialized_update = null;
bool first_write = _serializer == null || _n_updates_in_current_file == 0; bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
_serializer = _serializer ?? new UniversalStreamIOSerializer(); _serializer = _serializer ?? new UniversalSerializer(dp.schema);
if (first_write) { if (_write_create) {
// first file always starts with create // new files always starts with create
serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows))); _serializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out serialized_update);
} else } else
serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows))); _serializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out serialized_update);
BinaryWriter bw = new BinaryWriter(_out_fs); BinaryWriter bw = new BinaryWriter(_out_fs);
long utc_epoch = epoch_now(); long utc_epoch = epoch_now();
//time, length of data, data //time, length of data, data
//bw.Write(utc_epoch); bw.Write(utc_epoch);
//bw.Write(serialized_update.Length); bw.Write(serialized_update.Length);
bw.Write(serialized_update); bw.Write(serialized_update);
_out_fs.Flush(); _out_fs.Flush();
myFireEvent(dp); myFireEvent(dp);
@@ -0,0 +1,190 @@
// using System;
// using System.Collections.Generic;
// using System.Linq;
// using System.Text;
// using System.IO;
// using System.Runtime.Serialization.Formatters.Binary;
// using BCG.Utilities.General;
// using bts.collections;
// using System.Reflection.Metadata;
// namespace BCG {
// public class PlaybackStream : ViewStream, IWorker {
// private readonly PlaybackType _type;
// private BlueTable _current_table;
// private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
// // these are the packets already downloaded from the current file
// private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
// //private IEnumerable<FileStream> _file_streams = new List<FileStream>();
// private IEnumerable<string> _file_names;
// private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
// private DateTime _playback_t0, _playback_t1;
// private bool _all_files_read = false;
// SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
// public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
// _type = PlaybackType.Streaming;
// _playback_t0 = PersistorStream.parse_datetime(datetime_start);
// _playback_t1 = PersistorStream.parse_datetime(datetime_end);
// string token_directory_name = $"{directory_name}\\{token}";
// bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
// List<string> files_in_range = new List<string>();
// if (Directory.Exists(token_directory_name)) {
// _file_names =
// Directory.GetFiles(token_directory_name).Where(f => {
// DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
// return file_time > _playback_t0 && file_time < _playback_t1;
// });
// }
// }
// protected override void myOnFirstSubscriber() {
// FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
// _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
// PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
// if (snapshot_packet != null) {
// foreach (PlaybackPacket p in _snapshot_data.packets) {
// byte[] ops;
// Array row_array;
// _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
// for (int i = 0; i < ops.Length; i++) {
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
// byte op = ops[i];
// if (op == StreamOpCode.create) {
// IBlueRowCollection full_table = rows_args[0];
// _current_table = new BlueTable(full_table);
// } else if (op == StreamOpCode.multipleRemoveAdd) {
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
// BlueDataUtility.absorbChanges(_current_table, dels, adds);
// } else {
// bts.Log.assert(false, $"Oprator: {op} is not expected");
// }
// if (p.idx == snapshot_packet.idx) {
// myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
// break;
// }
// }
// }
// }
// }
// private void _load_packets_from_current_file() {
// int curr_idx = _current_fs.Item1;
// FileStream curr_fs = _current_fs.Item2;
// _packets = new List<PlaybackPacket>();
// _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
// }
// private bool current_file_all_replayed {
// get {
// bool rv = true;
// _packets.ForEach(p => rv = rv && p.replayed);
// return rv;
// }
// }
// protected override void myOnLastSubscriber() {
// var fs = _current_fs.Item2;
// if (fs != null) {
// fs.Close();
// }
// ViewStreamDatabase.instance.removeWorker(this);
// }
// // public event PlaybackStreamProgress onProgress;
// // public event Action onFileExhausted;
// private PlaybackPacket _locate_packet_to_replay() {
// foreach (var p in _packets) {
// // returns the first non replayed packet
// if (!p.replayed)
// return p;
// }
// //else
// {
// double secondsElapsed = new BinaryReader(myFileReader).ReadDouble();
// myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed));
// if (onProgress != null)
// onProgress(myFileReader.Position, myFileReader.Length);
// }
// }
// #region IWorker Members
// public void doWork() {
// if (_all_files_read)
// return;
// if (current_file_all_replayed) {
// _load_next_file();
// }
// PlaybackPacket curr_packet = _locate_packet_to_replay();
// if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway
// byte[] ops;
// Array row_array;
// // what a stupid interface!
// _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
// _fire_packet(ops, row_array);
// curr_packet.replay_timestamp = PersistorStream.epoch_now();
// } else {
// PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
// // this is the time we need to fire (replay) this packet
// long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
// long now_epoch = PersistorStream.epoch_now();
// // fire when we are past the
// if (now_epoch >= time_to_fire) {
// byte[] ops;
// Array row_array;
// _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
// _fire_packet(ops, row_array);
// curr_packet.replay_timestamp = PersistorStream.epoch_now();
// }
// }
// }
// private void _fire_packet(byte[] ops, Array row_array) {
// for (int i = 0; i < ops.Length; i++) {
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
// byte op = ops[i];
// if (op == StreamOpCode.create) {
// IBlueRowCollection full_table = rows_args[0];
// _current_table = new BlueTable(full_table);
// //_current_table.addRows(full_table);
// myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
// (myLastValue as DataPacket).newRows));
// } else if (op == StreamOpCode.multipleRemoveAdd) {
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
// BlueDataUtility.absorbChanges(_current_table, dels, adds);
// myFireEvent(new DataPacket(_current_table, adds, dels));
// } else {
// bts.Log.assert(false, $"Oprator: {op} is not expected");
// }
// }
// }
// private void _load_next_file() {
// var new_idx = _current_fs.Item1 + 1;
// if (new_idx >= _file_names.Count()) {
// _all_files_read = true;
// bts.Log.Gen.info("All files read");
// return;
// }
// var prev_fs = _current_fs.Item2;
// if (prev_fs != null)
// prev_fs.Close();
// var new_filename = _file_names.ElementAt(new_idx);
// bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
// var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
// _current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
// _load_packets_from_current_file();
// }
// #endregion
// }
// }
@@ -0,0 +1,68 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
namespace BCG
{
public class RecorderStream : ViewStream
{
private ViewStream mySource;
private DateTime myStartTime;
private FileStream myOutFile;
private string myFileName;
private IBlueDataSerializer mySerializer;
public RecorderStream(ViewStream src, string recordFile)
{
mySource = src;
myFileName = recordFile;
myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss"));
}
protected override void myOnFirstSubscriber()
{
myStartTime = DateTime.Now;
mySource.onChange += new ChangeDelegate(mySource_onChange);
myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
}
protected override void myOnLastSubscriber()
{
mySource.onChange -= mySource_onChange;
myOutFile.Close();
}
void mySource_onChange(object newValue)
{
DataPacket dp = (DataPacket)newValue;
BlueTable toWrite = new BlueTable(dp.schema);
if (mySerializer == null)
{
mySerializer = new UniversalSerializer(dp.schema);
}
foreach (IBlueRow r in dp.newRows)
toWrite.addRow(r);
byte[] result;
if (myLastValue == null)
mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
else
mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
BinaryWriter bw = new BinaryWriter(myOutFile);
//write down what time this happened
bw.Write((DateTime.Now - myStartTime).TotalSeconds);
bw.Write(result.Length);
bw.Write(result);
myOutFile.Flush();
myFireEvent(dp);
}
}
}
@@ -1,37 +1,64 @@
using System; // using System;
using System.Collections.Generic; // using System.Collections.Generic;
using System.Text; // using System.Text;
using System.IO; // using System.IO;
using BCG;
namespace bts { // using System.Runtime.Serialization.Formatters.Binary;
// this saves the updates as a Create on the same file, each update calls -save, which
// reqrites the whole data to the packet. The other side of this stream is the Loader
[DataStreamUserRespresentation("saver")]
public class SaverStream : ViewStream {
private ViewStream _stream;
private string _filename;
private IStreamIOMsgSerializer _serializer = new StreamIOSerializationController();
public SaverStream(ViewStream vs, string filename) { // namespace BCG
_stream = vs; // {
_filename = filename; // [DataStreamUserRespresentation("saver")]
} // public class SaverStream : ViewStream, ISideEffectStream
// {
// private ViewStream myStream;
// private string myFileName;
protected override void myOnFirstSubscriber() { _stream.onChange += new ChangeDelegate(_handle_changes); } // public class Exception : System.Exception
// {
// public Exception(string msg) : base(msg) { }
// }
void _handle_changes(object dp_obj) { // public SaverStream(ViewStream vs, string fileName)
DataPacket dp = dp_obj as DataPacket; // {
_save(dp); // if (fileName == null)
myFireEvent(dp_obj); // throw new SaverStream.Exception("File name cannot be null");
} // myStream = vs;
// myFileName = fileName;
protected override void myOnLastSubscriber() { _stream.onChange -= new ChangeDelegate(_handle_changes); } // }
protected void _save(DataPacket dp) { // protected override void myOnFirstSubscriber()
StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows); // { myStream.onChange += new ChangeDelegate(myHandleChanges); }
byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
StreamSerializationUtils.write_to_file(str_bytes, _filename); // void myHandleChanges(object newValue)
} // {
} // DataPacket dp = newValue as DataPacket;
} // mySave(dp);
// myFireEvent(newValue);
// }
// protected override void myOnLastSubscriber()
// { myStream.onChange -= new ChangeDelegate(myHandleChanges); }
// protected void mySave(DataPacket dp)
// {
// for (int i = 0; i < 5 ; i++)
// try
// {
// using (FileStream fs = new FileStream(myFileName, FileMode.Create, FileAccess.Write))
// {
// BinaryFormatter bf = new BinaryFormatter();
// BlueTable t = new BlueTable(dp.schema);
// foreach (IBlueRow r in dp.newRows) // to guarantee table is serializable
// t.addRow(new BlueRow(r));
// bf.Serialize(fs, t);
// }
// break;
// }
// catch (IOException)
// {
// System.Threading.Thread.Sleep(5);
// }
// }
// }
// }
@@ -1,12 +1,12 @@
// using System; using System;
// using System.Collections.Generic; using System.Collections.Generic;
// using System.Linq; using System.Linq;
// using System.Text; using System.Text;
// using System.IO; using System.IO;
// using System.Runtime.Serialization.Formatters.Binary; using System.Runtime.Serialization.Formatters.Binary;
// using BCG.Utilities.General; using BCG.Utilities.General;
// using bts.collections; using bts.collections;
// using System.Reflection.Metadata; using System.Reflection.Metadata;
namespace BCG { namespace BCG {
@@ -15,115 +15,115 @@ namespace BCG {
FirstOfDay, FirstOfDay,
LastOfDay LastOfDay
}; };
// private BlueTable _current_table; private BlueTable _current_table;
// private IClientDeserializer _deserilializer = new ClientUniversalDeserializer(); private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
// public DateTime _snapshot_time; public DateTime _snapshot_time;
// public string _snapshot_file; public string _snapshot_file;
// public IEnumerable<PlaybackPacket> _packets; public IEnumerable<PlaybackPacket> _packets;
// public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) { public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) {
// DateTime dt; DateTime dt;
// var parsed = DateTime.TryParse(snapshot_date, out dt); var parsed = DateTime.TryParse(snapshot_date, out dt);
// dt = dt.Date; dt = dt.Date;
// bts.Log.assert(parsed); bts.Log.assert(parsed);
// SpecificSnapshot specific_snapshot_enum; SpecificSnapshot specific_snapshot_enum;
// parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum); parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum);
// bts.Log.assert(parsed); bts.Log.assert(parsed);
// string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum); string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum);
// ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token); ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token);
// mySetRealSource(rs); mySetRealSource(rs);
// } }
// public SnapshotStream(string snapshot_dt, string directory_name, string token) { public SnapshotStream(string snapshot_dt, string directory_name, string token) {
// _snapshot_time = PersistorStream.parse_datetime(snapshot_dt); _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
// string token_directory_name = $"{directory_name}\\{token}"; string token_directory_name = $"{directory_name}\\{token}";
// _snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time); _snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time);
// } }
// protected override void myOnFirstSubscriber() { protected override void myOnFirstSubscriber() {
// FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read); FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read);
// _packets = StreamPersistorUtils.load_packets_from_file(fs); _packets = StreamPersistorUtils.load_packets_from_file(fs);
// PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time); PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time);
// if (snapshot_packet != null) { if (snapshot_packet != null) {
// foreach (PlaybackPacket p in _packets) { foreach (PlaybackPacket p in _packets) {
// byte[] ops; byte[] ops;
// Array row_array; Array row_array;
// _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array); _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
// for (int i = 0; i < ops.Length; i++) { for (int i = 0; i < ops.Length; i++) {
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[]; IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
// byte op = ops[i]; byte op = ops[i];
// if (op == StreamOpCode.create) { if (op == StreamOpCode.create) {
// IBlueRowCollection full_table = rows_args[0]; IBlueRowCollection full_table = rows_args[0];
// _current_table = new BlueTable(full_table); _current_table = new BlueTable(full_table);
// } else if (op == StreamOpCode.multipleRemoveAdd) { } else if (op == StreamOpCode.multipleRemoveAdd) {
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[]; //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection; IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection; IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
// BlueDataUtility.absorbChanges(_current_table, dels, adds); BlueDataUtility.absorbChanges(_current_table, dels, adds);
// } else { } else {
// bts.Log.assert(false, $"Oprator: {op} is not expected"); bts.Log.assert(false, $"Oprator: {op} is not expected");
// } }
// if (p.idx == snapshot_packet.idx) { if (p.idx == snapshot_packet.idx) {
// myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema))); myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
// break; break;
// } }
// } }
// } }
// } }
// } }
// private void _absorb_packet(byte[] ops, Array row_array) { private void _absorb_packet(byte[] ops, Array row_array) {
// for (int i = 0; i < ops.Length; i++) { for (int i = 0; i < ops.Length; i++) {
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[]; IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
// byte op = ops[i]; byte op = ops[i];
// if (op == StreamOpCode.create) { if (op == StreamOpCode.create) {
// IBlueRowCollection full_table = rows_args[0]; IBlueRowCollection full_table = rows_args[0];
// _current_table = new BlueTable(full_table); _current_table = new BlueTable(full_table);
// } else if (op == StreamOpCode.multipleRemoveAdd) { } else if (op == StreamOpCode.multipleRemoveAdd) {
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[]; //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection; IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection; IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
// BlueDataUtility.absorbChanges(_current_table, dels, adds); BlueDataUtility.absorbChanges(_current_table, dels, adds);
// } else { } else {
// bts.Log.assert(false, $"Operator: {op} is not expected"); bts.Log.assert(false, $"Operator: {op} is not expected");
// } }
// } }
// } }
// protected override void myOnLastSubscriber() { protected override void myOnLastSubscriber() {
// } }
// // public event SnapshotStream.Progress onProgress; // public event SnapshotStream.Progress onProgress;
// // public event Action onFileExhausted; // public event Action onFileExhausted;
// private PlaybackPacket _locate_packet_to_replay() { private PlaybackPacket _locate_packet_to_replay() {
// foreach (var p in _packets) { foreach (var p in _packets) {
// // returns the first non replayed packet // returns the first non replayed packet
// if (!p.replayed) if (!p.replayed)
// return p; return p;
// } }
// return null; return null;
// } }
// private void _fire_packet(byte[] ops, Array row_array) { private void _fire_packet(byte[] ops, Array row_array) {
// for (int i = 0; i < ops.Length; i++) { for (int i = 0; i < ops.Length; i++) {
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[]; IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
// byte op = ops[i]; byte op = ops[i];
// if (op == StreamOpCode.create) { if (op == StreamOpCode.create) {
// IBlueRowCollection full_table = rows_args[0]; IBlueRowCollection full_table = rows_args[0];
// _current_table = new BlueTable(full_table); _current_table = new BlueTable(full_table);
// //_current_table.addRows(full_table); //_current_table.addRows(full_table);
// myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) : myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
// (myLastValue as DataPacket).newRows)); (myLastValue as DataPacket).newRows));
// } else if (op == StreamOpCode.multipleRemoveAdd) { } else if (op == StreamOpCode.multipleRemoveAdd) {
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[]; //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection; IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection; IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
// BlueDataUtility.absorbChanges(_current_table, dels, adds); BlueDataUtility.absorbChanges(_current_table, dels, adds);
// myFireEvent(new DataPacket(_current_table, adds, dels)); myFireEvent(new DataPacket(_current_table, adds, dels));
// } else { } else {
// bts.Log.assert(false, $"Oprator: {op} is not expected"); bts.Log.assert(false, $"Oprator: {op} is not expected");
// } }
// } }
// } }
} }
} }
@@ -3,7 +3,6 @@ using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Globalization; using System.Globalization;
using System.IO; using System.IO;
using bts;
namespace BCG { namespace BCG {
@@ -1,10 +1,16 @@
using BCG;
using System;
using System.Data;
using System.Collections;
using BCG.Utilities;
using System.Text.RegularExpressions;
using Bts.DataStream; using Bts.DataStream;
using mbts;
using mbts.spark;
using mbts.config; using mbts.config;
using System;
using System.Text.RegularExpressions;
namespace BCG.StreamWorld.Addition namespace BCG.StreamWorld.Addition
{ {
[DataStreamUserRespresentation (false)] [DataStreamUserRespresentation (false)]
@@ -145,6 +151,7 @@ namespace BCG
private string myFilterString; private string myFilterString;
private BCG.StreamWorld.Addition.FilterStream myFilterStream; private BCG.StreamWorld.Addition.FilterStream myFilterStream;
private bool myUseNewFilter = Configuration.get<DataStreamConfig>().UseNewFilter; private bool myUseNewFilter = Configuration.get<DataStreamConfig>().UseNewFilter;
private HighPerformanceTimer _timer = new HighPerformanceTimer();
#region constructors #region constructors
public delegate bool OnFilter(IBlueRow r); public delegate bool OnFilter(IBlueRow r);
@@ -219,6 +226,7 @@ namespace BCG
private void myHandleCallbacks(object newValue) private void myHandleCallbacks(object newValue)
{ {
_timer.start();
DataPacket retVal; DataPacket retVal;
DataPacket arrivedPacket = newValue as DataPacket; DataPacket arrivedPacket = newValue as DataPacket;
myLastReceivedRows = arrivedPacket.newRows; myLastReceivedRows = arrivedPacket.newRows;
@@ -247,6 +255,7 @@ namespace BCG
if (myFilter.filter(r)) if (myFilter.filter(r))
myLastFiredRows.addRow(r); myLastFiredRows.addRow(r);
} }
_timer.stop();
myFireEvent(retVal); myFireEvent(retVal);
} }
@@ -1,24 +0,0 @@
namespace bts.SilverPine {
using BCG;
using System;
public class NullFilterStream : ViewStream {
public NullFilterStream(ViewStream parent_str, string col_name)
: this(parent_str, col_name, true) { }
public NullFilterStream(ViewStream parent_str, string col_name, bool filter_out_doublenans = false) {
ViewStream rs = new FilterStream(parent_str,
r => {
bool is_dbnull = r.is_null(col_name);
if (is_dbnull) {
return false;
}
if (r.schema[col_name].type_code() == TypeCode.Double) {
return !Double.IsNaN(r.get<double>(col_name));
}
return true;
});
mySetRealSource(rs);
}
}
}
@@ -1,66 +1,37 @@
using System; using System;
using System.Text.RegularExpressions; using System.Collections.Generic;
using System.Linq;
using System.Text;
using BCG.Utilities.General; using BCG.Utilities.General;
namespace BCG { namespace BCG
public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable { {
public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable
public static TimeSpan parse_duration(string input) { {
int days = 0, hours = 0, minutes = 0, seconds = 0;
// Regular expression to match the pattern (e.g., "1d", "1h", "15m", "45s")
Regex regex = new Regex(@"(\d+)([dhms])");
MatchCollection matches = regex.Matches(input);
foreach (Match match in matches) {
int value = int.Parse(match.Groups[1].Value);
string unit = match.Groups[2].Value;
switch (unit) {
case "d":
days = value;
break;
case "h":
hours = value;
break;
case "m":
minutes = value;
break;
case "s":
seconds = value;
break;
}
}
return new TimeSpan(days, hours, minutes, seconds);
}
#region constructors #region constructors
public TimeBasedFilter(string col_name, string row_time_to_live)
: this(col_name, parse_duration(row_time_to_live)) {
}
public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive) public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive)
: this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) { : this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */)
{
} }
public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency)
: this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency) { : this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency)
{
} }
public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive) public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive)
: this(vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) { : this (vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */)
{
} }
public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) { public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency)
myVsd = vsd; {
_time_col = rowTimeColumnName; myVsd = vsd;
_ttl = rowTimeToLive; myRowTimeColumnName = rowTimeColumnName;
myFilterFrequency = filterFrequency; myRowTimeToLive = rowTimeToLive;
myNextFilter = DateTime.MinValue; myFilterFrequency = filterFrequency;
myNextFilter = DateTime.MinValue;
// add worker // add worker
myVsd.addWorker(this); myVsd.addWorker(this);
@@ -70,25 +41,32 @@ namespace BCG {
public event FilterChangeDelegate onFilterChange; public event FilterChangeDelegate onFilterChange;
#region IStreamFilter #region IStreamFilter
public bool filter(IBlueRow row) { public bool filter(IBlueRow row)
DateTime now = DateTime.Now; {
DateTime now = DateTime.Now;
DateTime timeStamp = (DateTime)row[myRowTimeColumnName];
DateTime row_time = row.get_datetime(_time_col, DateTime.Now); if (now > timeStamp)
if (now > row_time) { {
TimeSpan span = now - row_time; TimeSpan span = now - timeStamp;
if (span >= _ttl) {
if (span >= myRowTimeToLive)
{
return (false); return (false);
} }
} }
return (true); return (true);
} }
#endregion #endregion
#region IWorker #region IWorker
public void doWork() { public void doWork()
{
DateTime now = DateTime.Now; DateTime now = DateTime.Now;
if (now > myNextFilter) { if (now > myNextFilter)
{
// fire event // fire event
onFilterChange(); onFilterChange();
@@ -98,7 +76,8 @@ namespace BCG {
#endregion #endregion
#region IDisposable #region IDisposable
public void Dispose() { public void Dispose()
{
// remove worker // remove worker
myVsd.removeWorker(this); myVsd.removeWorker(this);
} }
@@ -106,20 +85,10 @@ namespace BCG {
#region state #region state
private ViewStreamDatabase myVsd; private ViewStreamDatabase myVsd;
private string _time_col; private string myRowTimeColumnName;
private TimeSpan _ttl; private TimeSpan myRowTimeToLive;
private TimeSpan myFilterFrequency; private TimeSpan myFilterFrequency;
private DateTime myNextFilter; private DateTime myNextFilter;
#endregion #endregion
} }
[DataStreamUserRespresentation("timefilter")]
public class TimeBasedFilterStream : ViewStream {
public TimeBasedFilterStream(ViewStream base_stream, string time_col, string time_span)
:
this(ViewStreamDatabase.instance, base_stream, time_col, TimeBasedFilter.parse_duration(time_span)) { }
public TimeBasedFilterStream(ViewStreamDatabase vsd, ViewStream base_stream, string time_col, TimeSpan time_span) {
TimeBasedFilter filter = new TimeBasedFilter(vsd, time_col, time_span);
mySetRealSource(new FilterStream(base_stream, filter));
}
}
} }
@@ -1,102 +1,102 @@
// using System; using System;
// using System.Collections.Generic; using System.Collections.Generic;
// using System.Linq; using System.Linq;
// using System.Text; using System.Text;
// namespace BCG namespace BCG
// { {
// public class AverageGroupsStream : ViewStream public class AverageGroupsStream : ViewStream
// { {
// public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn) public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
// { {
// myColToAverage = colToAverage; myColToAverage = colToAverage;
// myInput = input; myInput = input;
// myKeyColumn = keyColumn; myKeyColumn = keyColumn;
// } }
// private ViewStream myInput; private ViewStream myInput;
// private string myColToAverage; private string myColToAverage;
// private BlueTableSchema mySchema; private BlueTableSchema mySchema;
// private string myKeyColumn; private string myKeyColumn;
// protected override void myOnFirstSubscriber() protected override void myOnFirstSubscriber()
// { {
// myInput.onChange += new ChangeDelegate(myInput_onChange); myInput.onChange += new ChangeDelegate(myInput_onChange);
// } }
// protected override void myOnLastSubscriber() protected override void myOnLastSubscriber()
// { {
// myInput.onChange -= myInput_onChange; myInput.onChange -= myInput_onChange;
// } }
// private void myInput_onChange(object newValue) private void myInput_onChange(object newValue)
// { {
// if (mySchema == null) if (mySchema == null)
// myBuildSchema(((DataPacket)newValue).schema); myBuildSchema(((DataPacket)newValue).schema);
// IBlueRowCollection newRows = myEvaluate((DataPacket)newValue); IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);
// if (myLastValue == null) if (myLastValue == null)
// myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema))); myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
// else else
// { {
// IBlueRowCollection added, deleted; IBlueRowCollection added, deleted;
// BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added); BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
// myFireEvent(new DataPacket(newRows, added, deleted)); myFireEvent(new DataPacket(newRows, added, deleted));
// } }
// } }
// private void myBuildSchema(IBlueTableSchema schema) private void myBuildSchema(IBlueTableSchema schema)
// { {
// List<ColumnInfo> columns = new List<ColumnInfo>(); List<ColumnInfo> columns = new List<ColumnInfo>();
// foreach (ColumnInfo ci in schema.columnInfo) foreach (ColumnInfo ci in schema.columnInfo)
// columns.Add(ci); columns.Add(ci);
// columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double))); columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
// mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray()); mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
// } }
// private IBlueRowCollection myEvaluate(DataPacket packet) private IBlueRowCollection myEvaluate(DataPacket packet)
// { {
// IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>(); IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>();
// IDictionary<string, int> colToCount = new Dictionary<string, int>(); IDictionary<string, int> colToCount = new Dictionary<string, int>();
// IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>(); IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>();
// foreach (IBlueRow r in packet.newRows) foreach (IBlueRow r in packet.newRows)
// { {
// double thisVal = Convert.ToDouble(r[myColToAverage]); double thisVal = Convert.ToDouble(r[myColToAverage]);
// string keyVal = r[myKeyColumn].ToString(); string keyVal = r[myKeyColumn].ToString();
// if (!colToCount.ContainsKey(keyVal)) if (!colToCount.ContainsKey(keyVal))
// { {
// colToRunningTotal[keyVal] = thisVal; colToRunningTotal[keyVal] = thisVal;
// colToCount[keyVal] = 1; colToCount[keyVal] = 1;
// } }
// else else
// { {
// colToRunningTotal[keyVal] += thisVal; colToRunningTotal[keyVal] += thisVal;
// colToCount[keyVal] += 1; colToCount[keyVal] += 1;
// } }
// colToLastRow[keyVal] = r; colToLastRow[keyVal] = r;
// } }
// BlueTable result = new BlueTable(mySchema); BlueTable result = new BlueTable(mySchema);
// foreach (string key in colToCount.Keys) foreach (string key in colToCount.Keys)
// { {
// object[] values = new object[mySchema.columnInfo.Length]; object[] values = new object[mySchema.columnInfo.Length];
// colToLastRow[key].CopyTo(values, 0); colToLastRow[key].CopyTo(values, 0);
// values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0; values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
// result.addRow(new BlueRow(mySchema, values)); result.addRow(new BlueRow(mySchema, values));
// } }
// return result; return result;
// } }
// } }
// } }
@@ -1,31 +0,0 @@
namespace bts.SilverPine {
using BCG;
using System;
using System.Collections.Generic;
using System.Linq;
[DataStreamUserRespresentation("Average")]
public class AverageNonIncGroupsStream : ViewStream {
public AverageNonIncGroupsStream(ViewStream parent_stream, string inp_col_name, params string[] group_by_cols) {
NonIncGroupByStream rs = new NonIncGroupByStream(parent_stream, () => group_by_cols,
(g_rows) => _find_mean(inp_col_name, g_rows));
mySetRealSource(rs);
}
private List<IBlueRow> _find_mean(string inp_col_name, List<IBlueRow> g_rows) {
IBlueRow inp_row = g_rows.ElementAt(0);
IEnumerable<double> inp_vals = BlueDataUtility.get_colvalues<double>(g_rows, inp_col_name);
double sum = 0.0;
inp_vals.ToList().ForEach(d => sum += d);
string out_col_name = $"average_{inp_col_name}";
double average = sum / inp_vals.Count();
IBlueTableSchema schema_out = inp_row.schema.add_column(TypeCode.Double, out_col_name);
MutableBlueRow out_row = new MutableBlueRow(schema_out, inp_row);
out_row.setValue(out_col_name, average);
List<IBlueRow> rv = new List<IBlueRow>();
rv.Add(out_row);
return rv;
}
}
}
@@ -1,91 +0,0 @@
namespace bts.SilverPine {
using BCG;
using System;
using System.Collections.Generic;
using System.Linq;
using ZstdSharp.Unsafe;
public enum GroupByBundleOp {
Mean,
Median,
Mode,
StdDev,
Max,
Min,
First,
Last,
Sum
}
public class GroupByBundle {
public readonly List<Tuple<string, GroupByBundleOp>> operations;
public static GroupByBundle parse(string text) {
// text is inp_col: op, inp_col: op, inp_col:op
List<Tuple<string, GroupByBundleOp>> rv = new List<Tuple<string, GroupByBundleOp>>();
string[] calcs = text.Split(',');
foreach (string c in calcs) {
string[] toks = c.Split(':');
string inp_col = toks[0].Trim();
string op_txt = toks[1].Trim();
GroupByBundleOp op = Enum.Parse<GroupByBundleOp>(op_txt, true);
rv.Add(Tuple.Create(inp_col, op));
}
return new GroupByBundle(rv);
}
public GroupByBundle(List<Tuple<string, GroupByBundleOp>> operations_) {
operations = operations_;
}
public static IBlueRow group_by(List<IBlueRow> rows, GroupByBundle parameters) {
if (rows.Count() == 0) {
return null;
}
IBlueRow ir = rows.ElementAt(0);
IBlueTableSchema in_schema = ir.schema;
IBlueTableSchema out_schema = in_schema;
IDictionary<string, Tuple<string, GroupByBundleOp>> out_col_map =
new Dictionary<string, Tuple<string, GroupByBundleOp>>();
foreach (var t in parameters.operations) {
string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
out_col_map[out_col_name] = t;
}
MutableBlueRow or = new MutableBlueRow(out_schema, ir);
foreach (var kv in out_col_map) {
string out_col_name = kv.Key;
string in_col_name = kv.Value.Item1;
GroupByBundleOp op = kv.Value.Item2;
IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
double comp_res = compute_op(col_vals, op);
or.setValue(out_col_name, comp_res);
}
return or;
}
public static double compute_op(IEnumerable<double> col_vals, GroupByBundleOp op) {
switch (op) {
case GroupByBundleOp.Max:
return col_vals.Max<double>();
case GroupByBundleOp.Min:
return col_vals.Min<double>();
case GroupByBundleOp.Mean:
return col_vals.Sum() / col_vals.Count();
case GroupByBundleOp.Sum:
return col_vals.Sum();
case GroupByBundleOp.First:
return col_vals.First();
case GroupByBundleOp.Last:
return col_vals.Last();
default:
// TODO
Log.assert(false);
return Double.NaN;
}
}
}
}
@@ -4,14 +4,18 @@ using BCG.Utilities;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
namespace BCG { namespace BCG
public class GroupByStream : MethodDrivenStream { {
private class MyFuncGroupByEvaluator : IGroupByEvaluator { public class GroupByStream : MethodDrivenStream
{
private class MyFuncGroupByEvaluator : IGroupByEvaluator
{
private string[] _groupby_cols; private string[] _groupby_cols;
private Func<DataPacket, IBlueRow> _groupby_func; private Func<DataPacket, IBlueRow> _groupby_func;
public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func) { public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
{
_groupby_cols = groupby_cols.ToArray(); _groupby_cols = groupby_cols.ToArray();
_groupby_func = groupby_func; _groupby_func = groupby_func;
} }
public IBlueRow evaluate(DataPacket packet) { return _groupby_func(packet); } public IBlueRow evaluate(DataPacket packet) { return _groupby_func(packet); }
@@ -25,7 +29,7 @@ namespace BCG {
/// Serves as a quick way to access all rows belonging to the same group /// Serves as a quick way to access all rows belonging to the same group
/// </summary> /// </summary>
private bool myIdGuard = true; private bool myIdGuard = true;
private RowDictionaryTable myBookKeepingTable; private RowDictionaryTable myBookKeepingTable ;
private IGroupByEvaluator myGroupByEvaluator; private IGroupByEvaluator myGroupByEvaluator;
private IBlueTableSchema mySchema; //input schema, as output schema can differ (IGroupByEvaluator's private IBlueTableSchema mySchema; //input schema, as output schema can differ (IGroupByEvaluator's
//can map a schema to another //can map a schema to another
@@ -33,98 +37,112 @@ namespace BCG {
private int myIdGenerator = 0; private int myIdGenerator = 0;
public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func) public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
: this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func)) { } : this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func))
{ }
public GroupByStream(ViewStream stream, IGroupByEvaluator groupByEvaluator) public GroupByStream (ViewStream stream, IGroupByEvaluator groupByEvaluator)
: this(stream, groupByEvaluator, true) { } : this (stream, groupByEvaluator, true)
{}
public GroupByStream(ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard) public GroupByStream (ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard)
: base(stream) { : base (stream)
{
myGroupByEvaluator = groupbyEvaluator; myGroupByEvaluator = groupbyEvaluator;
myIdGuard = idGuard; myIdGuard = idGuard;
} }
protected override void myHandleFirstCall() { protected override void myHandleFirstCall ()
{
mySchema = myLastArrivedPacket.schema; mySchema = myLastArrivedPacket.schema;
myOutputSchema = myGroupByEvaluator.getOutputSchema(mySchema); myOutputSchema = myGroupByEvaluator.getOutputSchema (mySchema);
myBookKeepingTable = new RowDictionaryTable(myLastArrivedPacket.newRows, myBookKeepingTable = new RowDictionaryTable (myLastArrivedPacket.newRows,
myGroupByEvaluator.groupByColumns); myGroupByEvaluator.groupByColumns);
GroupByOutputTable temp = new GroupByOutputTable(myOutputSchema); GroupByOutputTable temp = new GroupByOutputTable (myOutputSchema);
foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups()) { foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups())
IBlueRow newRow = myEvaluate(new DataPacket(rowGroup, {
rowGroup, new BlueTable(mySchema))); IBlueRow newRow = myEvaluate (new DataPacket (rowGroup,
rowGroup, new BlueTable (mySchema)));
if (newRow != null) if (newRow != null)
temp.addRow(newRow, myGetGroupString(rowGroup)); temp.addRow (newRow, myGetGroupString (rowGroup));
} }
myLastFiredPacket = new DataPacket(temp, new BlueTable(temp), new BlueTable(temp.schema)); myLastFiredPacket = new DataPacket (temp, new BlueTable(temp), new BlueTable (temp.schema));
} }
/// <summary> /// <summary>
/// Adding need not to anything as deletion already takes care /// Adding need not to anything as deletion already takes care
/// of addition /// of addition
/// </summary> /// </summary>
protected override void myHandleIncrementalAdd() { } protected override void myHandleIncrementalAdd() {}
/// <summary> /// <summary>
/// Both deletes and adds are handled here to make the intermediate results not /// Both deletes and adds are handled here to make the intermediate results not
/// appear in the spitted fired DataPacket (note this is done only in GroupByStream /// appear in the spitted fired DataPacket (note this is done only in GroupByStream
/// because of the nature of it) /// because of the nature of it)
/// </summary> /// </summary>
protected override void myHandleIncrementalDelete() { protected override void myHandleIncrementalDelete()
{
myFireSwitch = false; myFireSwitch = false;
RowDictionaryTable delGroups, addGroups; RowDictionaryTable delGroups, addGroups;
//form groups for deleted and added rows //form groups for deleted and added rows
myInitIncrementalChangeData(out delGroups, out addGroups); myInitIncrementalChangeData (out delGroups, out addGroups);
ArrayList handledAddGroups = new ArrayList(); ArrayList handledAddGroups = new ArrayList ();
string groupRepString; string groupRepString;
foreach (IBlueRowCollection delGroup in delGroups.getGroups()) { foreach (IBlueRowCollection delGroup in delGroups.getGroups ())
{
// BlueEventLog.logAssert (delGroup.Count > 0); // BlueEventLog.logAssert (delGroup.Count > 0);
groupRepString = myGetGroupString(delGroup); groupRepString = myGetGroupString (delGroup);
IBlueRowCollection addGroup = addGroups.getGroup(groupRepString) == null ? IBlueRowCollection addGroup = addGroups.getGroup (groupRepString) == null ?
new BlueTable(mySchema) : addGroups.getGroup(groupRepString); new BlueTable (mySchema) : addGroups.getGroup (groupRepString);
handledAddGroups.Add(addGroup); handledAddGroups.Add (addGroup);
IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString) == null ? IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString) == null ?
new BlueTable(mySchema) : myBookKeepingTable.getGroup(groupRepString); new BlueTable (mySchema) : myBookKeepingTable.getGroup (groupRepString);
IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup, delGroup)); IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup, delGroup));
myFireSwitch |= myMergeRow(newRow, groupRepString); myFireSwitch |= myMergeRow (newRow, groupRepString);
} }
foreach (IBlueRowCollection addGroup in addGroups.getGroups()) { foreach (IBlueRowCollection addGroup in addGroups.getGroups ())
{
// BlueEventLog.logAssert (addGroup.Count > 0); // BlueEventLog.logAssert (addGroup.Count > 0);
if (!handledAddGroups.Contains(addGroup)) { if (!handledAddGroups.Contains (addGroup))
{
//this is the case when there are rows added to a group and none //this is the case when there are rows added to a group and none
//have been removed, hence it 'escapes' from being dealt with in //have been removed, hence it 'escapes' from being dealt with in
//the loop above //the loop above
groupRepString = myGetGroupString(addGroup); groupRepString = myGetGroupString (addGroup);
IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString); IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString);
// BlueEventLog.logAssert (aggGroup != null && aggGroup.Count > 0); // BlueEventLog.logAssert (aggGroup != null && aggGroup.Count > 0);
IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup, IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup,
new BlueTable(aggGroup.schema))); new BlueTable (aggGroup.schema)));
myFireSwitch |= myMergeRow(newRow, groupRepString); myFireSwitch |= myMergeRow (newRow, groupRepString);
} }
} }
} }
private bool myMergeRow(IBlueRow newRow, string groupRepString) { private bool myMergeRow (IBlueRow newRow, string groupRepString)
IBlueRow existingRow = ((GroupByOutputTable)myLastFiredPacket.newRows).getRow(groupRepString); {
IBlueRow existingRow = ((GroupByOutputTable) myLastFiredPacket.newRows).getRow (groupRepString);
//BlueEventLog.logAssert (newRow != null || existingRow != null); //BlueEventLog.logAssert (newRow != null || existingRow != null);
if (existingRow == null && newRow == null) if (existingRow == null && newRow == null)
return false; return false;
else if (existingRow != null && newRow != null && BlueDataUtility.compute_equality(existingRow, newRow)) else if (existingRow != null && newRow != null && BlueDataUtility.computeRowEquality (existingRow, newRow))
return false; return false;
else if (existingRow == null) { else if (existingRow == null)
{
//row added for the first time //row added for the first time
((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString); ((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString);
((BlueTable)myLastFiredPacket.addedRows).addRow(newRow); ((BlueTable) myLastFiredPacket.addedRows).addRow (newRow);
return true; return true;
} else { }
((GroupByOutputTable)myLastFiredPacket.newRows).removeRow(groupRepString); else
((BlueTable)myLastFiredPacket.deletedRows).addRow(existingRow); {
if (newRow != null) { ((GroupByOutputTable) myLastFiredPacket.newRows).removeRow (groupRepString);
((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString); ((BlueTable) myLastFiredPacket.deletedRows).addRow (existingRow);
((BlueTable)myLastFiredPacket.addedRows).addRow(newRow); if (newRow != null)
{
((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString);
((BlueTable) myLastFiredPacket.addedRows).addRow (newRow);
} }
return true; return true;
} }
@@ -136,68 +154,77 @@ namespace BCG {
/// </summary> /// </summary>
/// <param name="delGroups"></param> /// <param name="delGroups"></param>
/// <param name="addGroups"></param> /// <param name="addGroups"></param>
private void myInitIncrementalChangeData(out RowDictionaryTable delGroups, out RowDictionaryTable addGroups) { private void myInitIncrementalChangeData (out RowDictionaryTable delGroups, out RowDictionaryTable addGroups)
{
delGroups = delGroups =
new RowDictionaryTable(myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns); new RowDictionaryTable (myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
addGroups = addGroups =
new RowDictionaryTable(myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns); new RowDictionaryTable (myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns);
foreach (IBlueRow deletedRow in myLastArrivedPacket.deletedRows) foreach (IBlueRow deletedRow in myLastArrivedPacket.deletedRows)
myBookKeepingTable.removeRow(deletedRow); myBookKeepingTable.removeRow (deletedRow);
foreach (IBlueRow addedRow in myLastArrivedPacket.addedRows) foreach (IBlueRow addedRow in myLastArrivedPacket.addedRows)
myBookKeepingTable.addRow(addedRow); myBookKeepingTable.addRow (addedRow);
myLastFiredPacket.addedRows = new BlueTable(myOutputSchema); myLastFiredPacket.addedRows = new BlueTable (myOutputSchema);
myLastFiredPacket.deletedRows = new BlueTable(myOutputSchema); myLastFiredPacket.deletedRows = new BlueTable (myOutputSchema);
} }
protected override void myOnTableClear() { protected override void myOnTableClear ()
{
if (myBookKeepingTable.Count == 0) if (myBookKeepingTable.Count == 0)
return; //ie there arent any rows, so no need to optimize the 'cleansing' process return; //ie there arent any rows, so no need to optimize the 'cleansing' process
myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows); myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows);
myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema); myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
(myLastFiredPacket.newRows as GroupByOutputTable).clear(); (myLastFiredPacket.newRows as GroupByOutputTable).clear ();
myBookKeepingTable.clear(); myBookKeepingTable.clear ();
myGroupByEvaluator.clear(); myGroupByEvaluator.clear ();
} }
private IBlueRow myEvaluate(DataPacket dp) { private IBlueRow myEvaluate (DataPacket dp)
IBlueRow evalRow = myGroupByEvaluator.evaluate(dp); {
return evalRow == null ? null : IBlueRow evalRow = myGroupByEvaluator.evaluate (dp);
return evalRow == null ? null :
myIdGuard ? myIdGuard ?
BlueDataUtility.getNewReplacedRow(evalRow, myIdGenerator++, "id") : evalRow; BlueDataUtility.getNewReplacedRow (evalRow, myIdGenerator++, "id") : evalRow;
} }
private string myGetGroupString(IBlueRowCollection group) { private string myGetGroupString (IBlueRowCollection group)
return BlueDataUtility.rowToString(BlueDataUtility.extractRow(group, 0), myGroupByEvaluator.groupByColumns); {
return BlueDataUtility.rowToString(BlueDataUtility.extractRow (group, 0), myGroupByEvaluator.groupByColumns);
} }
protected override void myOnLastSubscriber() { protected override void myOnLastSubscriber ()
{
//the packets are initialzed to null, so that if the subscription count goes to 0, and then this stream //the packets are initialzed to null, so that if the subscription count goes to 0, and then this stream
//gets resubscribed, we start the process of building data again, and treat the callback as the first //gets resubscribed, we start the process of building data again, and treat the callback as the first
//callback. //callback.
myInit(); myInit ();
myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler); myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
} }
protected override void myInit() { protected override void myInit ()
base.myInit(); {
if (myBookKeepingTable != null) { base.myInit ();
if (myBookKeepingTable != null)
{
//to avoid calling init before the first callback //to avoid calling init before the first callback
myBookKeepingTable.clear(); myBookKeepingTable.clear ();
myGroupByEvaluator.clear(); myGroupByEvaluator.clear ();
} }
} }
public static string getRepString(DataPacket dp, string[] colNames) { public static string getRepString (DataPacket dp, string[] colNames)
return BlueDataUtility.rowToString(getRepRow(dp), colNames); {
return BlueDataUtility.rowToString (getRepRow (dp), colNames);
} }
public static IBlueRow getRepRow(DataPacket dp) { public static IBlueRow getRepRow (DataPacket dp)
{
if (dp.newRows.Count > 0) if (dp.newRows.Count > 0)
return BlueDataUtility.extractRow(dp.newRows, 0); return BlueDataUtility.extractRow (dp.newRows, 0);
else if (dp.addedRows.Count > 0) else if (dp.addedRows.Count > 0)
return BlueDataUtility.extractRow(dp.addedRows, 0); return BlueDataUtility.extractRow (dp.addedRows, 0);
else if (dp.deletedRows.Count > 0) else if (dp.deletedRows.Count > 0)
return BlueDataUtility.extractRow(dp.deletedRows, 0); return BlueDataUtility.extractRow (dp.deletedRows, 0);
// BlueEventLog.logAssert (false, "Illegal Data Packet"); // BlueEventLog.logAssert (false, "Illegal Data Packet");
return null; return null;
} }
@@ -210,21 +237,22 @@ namespace BCG {
/// <param name="newValue"></param> /// <param name="newValue"></param>
/// <param name="isAdd"></param> /// <param name="isAdd"></param>
/// <returns></returns> /// <returns></returns>
public static object incrementalAdd(object lastValue, object newValue, bool isAdd) { public static object incrementalAdd (object lastValue, object newValue, bool isAdd)
if (lastValue == null || lastValue.Equals(DBNull.Value)) {
if (lastValue == null || lastValue.Equals (DBNull.Value))
return newValue; return newValue;
// BlueEventLog.logAssert (lastValue.GetType().Equals (typeof(Int32)) && newValue.GetType().Equals (typeof(Int32)) || // BlueEventLog.logAssert (lastValue.GetType().Equals (typeof(Int32)) && newValue.GetType().Equals (typeof(Int32)) ||
// lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) || // lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
// lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal))); // lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
Object retVal = null; Object retVal = null;
if (newValue.GetType().Equals(typeof(int))) if (newValue.GetType().Equals (typeof(int)))
retVal = isAdd ? (int)lastValue + (int)newValue : (int)lastValue - (int)newValue; retVal = isAdd ? (int)lastValue + (int)newValue : (int)lastValue - (int)newValue;
else if (newValue.GetType().Equals(typeof(Double))) else if (newValue.GetType().Equals (typeof(Double)))
retVal = isAdd ? (Double)lastValue + (Double)newValue : (Double)lastValue - (Double)newValue; retVal = isAdd ? (Double)lastValue + (Double)newValue : (Double)lastValue - (Double)newValue;
else if (lastValue.GetType().Equals(typeof(Decimal))) else if (lastValue.GetType().Equals (typeof(Decimal)))
retVal = isAdd ? (Decimal)lastValue + (Decimal)newValue : (Decimal)lastValue - (Decimal)newValue; retVal = isAdd ? (Decimal)lastValue + (Decimal)newValue : (Decimal)lastValue - (Decimal)newValue;
else { } else{}
// BlueEventLog.logAssert (false, "Illegal type"); // BlueEventLog.logAssert (false, "Illegal type");
return retVal; return retVal;
} }
@@ -22,8 +22,8 @@ namespace BCG {
IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn); IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);
if (maxRow != null) { if (maxRow != null) {
bts.Log.assert(currMaxRow != null); bts.Log.assert(currMaxRow != null);
bts.Log.assert(BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) <= 0); bts.Log.assert(BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) <= 0);
if (BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) == 0) { if (BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) == 0) {
maxRow = BlueDataUtility.computeMaxRow(dp.newRows, myOutputColumn); maxRow = BlueDataUtility.computeMaxRow(dp.newRows, myOutputColumn);
if (maxRow == null) if (maxRow == null)
myOutputTable.removeRow(hashString); myOutputTable.removeRow(hashString);
@@ -34,7 +34,7 @@ namespace BCG {
maxRow = BlueDataUtility.computeMaxRow(dp.addedRows, myOutputColumn); maxRow = BlueDataUtility.computeMaxRow(dp.addedRows, myOutputColumn);
// if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true, // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
// otherwise if maxRow > currMaxRow then its replaced // otherwise if maxRow > currMaxRow then its replaced
if (maxRow != null && (currMaxRow == null || BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) > 0)) if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) > 0))
myOutputTable.addRow(maxRow, hashString); myOutputTable.addRow(maxRow, hashString);
return myProject(myOutputTable.getRow(hashString)); return myProject(myOutputTable.getRow(hashString));
} }
@@ -10,6 +10,6 @@ namespace BCG {
} }
public MaxStream(ViewStream stream, string outputCol) public MaxStream(ViewStream stream, string outputCol)
: this(new AddColumnsStream(stream, "fake_col: 0"), outputCol, new string[] { "fake_col" }) { } : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
} }
} }
@@ -18,7 +18,7 @@ namespace BCG {
if (minRow != null) { if (minRow != null) {
// BlueEventLog.logAssert (currMinRow != null); // BlueEventLog.logAssert (currMinRow != null);
// BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0); // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
if (BlueDataUtility.compare_rows(minRow, currMinRow, myOutputColumn) == 0) { if (BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) == 0) {
minRow = BlueDataUtility.computeMinRow(dp.newRows, myOutputColumn); minRow = BlueDataUtility.computeMinRow(dp.newRows, myOutputColumn);
if (minRow == null) if (minRow == null)
myOutputTable.removeRow(stringRep); myOutputTable.removeRow(stringRep);
@@ -27,7 +27,7 @@ namespace BCG {
} }
} }
minRow = BlueDataUtility.computeMinRow(dp.addedRows, myOutputColumn); minRow = BlueDataUtility.computeMinRow(dp.addedRows, myOutputColumn);
if (minRow != null && (currMinRow == null || BlueDataUtility.compare_rows(minRow, currMinRow, myOutputColumn) < 0)) if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) < 0))
myOutputTable.addRow(minRow, stringRep); myOutputTable.addRow(minRow, stringRep);
return myOutputTable.getRow(stringRep); return myOutputTable.getRow(stringRep);
} }
@@ -1,108 +1,53 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics;
using System.Linq; using System.Linq;
using System.Text.Json; using System.Text;
using System.Threading.Tasks;
using mbts;
using mbts.spark;
using bts; namespace BCG
using bts.SilverPine; {
public class NonIncGroupByStream : ViewStream
namespace BCG { {
[DataStreamUserRespresentation("aggregate")]
public class NonIncGroupByStream : ViewStream {
private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator {
Func<string[]> _groupby_cols_func;
Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
_groupby_cols_func = groupby_cols_func;
_evaluate_func = evaluate_func;
}
public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
BlueTable rv = null;
foreach (var kv in grouped_rows) {
List<IBlueRow> g_rows = kv.Value;
List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
if (out_rows != null && out_rows.Count > 0) {
rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
}
rv.addRows(out_rows);
}
return rv;
}
public string[] get_group_by_cols() {
return _groupby_cols_func();
}
}
private ViewStream _main_stream; private ViewStream _main_stream;
private INonIncGroupByEvaluator _groupby_func; private INonIncGroupByEvaluator _groupby_func;
private BlueRowHasher _row_hasher; private BlueRowHasher _row_hasher;
private string[] _groupby_cols; private string[] _groupby_cols;
private Stopwatch _timer = new(); private HighPerformanceTimer _timer = new HighPerformanceTimer();
private int _n_total_fires = 0, _n_expensive_fires = 0; private int _n_total_fires = 0, _n_expensive_fires = 0;
public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) { public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
{
_main_stream = main_stream; _main_stream = main_stream;
_groupby_func = groupby_func; _groupby_func = groupby_func;
_groupby_cols = _groupby_func.get_group_by_cols(); _groupby_cols = _groupby_func.get_group_by_cols();
_row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length); _row_hasher = new BlueRowHasher(_groupby_cols.Length);
} }
public NonIncGroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
: this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col = null) {
GroupByBundle std_group_by = GroupByBundle.parse(standard_group_by_text);
Func<string[]> groupby_cols_func = () => {
return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
};
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
if (!String.IsNullOrEmpty(optional_sort_col))
group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
IBlueRow rv = GroupByBundle.group_by(group_rows, std_group_by);
return new IBlueRow[] { rv }.ToList();
};
ViewStream rs = new NonIncGroupByStream(parent_str, groupby_cols_func, evaluate_func);
mySetRealSource(rs);
}
public NonIncGroupByStream(ViewStream parent_str, string standard_group_by_text) :
this(parent_str, String.Empty, standard_group_by_text) {
}
protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; } protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; } protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
private void _main_stream_onchange(object nv) { private void _main_stream_onchange(object nv)
{
DataPacket dp = nv as DataPacket; DataPacket dp = nv as DataPacket;
///grouping together ///grouping together
_timer.Restart(); _timer.start();
IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ? IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } : _timer.stop();
BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher); // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
_timer.Stop();
var groupElapsedMs = _timer.ElapsedMilliseconds;
if (groupElapsedMs >= 50) {
Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
}
// Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
/// reducing /// reducing
_timer.Restart(); _timer.start();
IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows); IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
_timer.Stop(); _timer.stop();
var applyElapsedMs = _timer.ElapsedMilliseconds; _n_total_fires++;
_n_total_fires++; if (_timer.lastIntervalTime > 50)
if (applyElapsedMs >= 50) { {
_n_expensive_fires++; _n_expensive_fires++;
Log.DataFlow.info( // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to evaluate {0} groups {1} , ratio {2}/{3}", grouped_rows.Count, _timer.lastIntervalTime, _n_expensive_fires, _n_total_fires));
$"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}"); }
} myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
// TODO, }
if (total_state != null)
myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
}
} }
} }
@@ -27,8 +27,7 @@ namespace BCG
string prependString = isLeft ? "left" : "right"; string prependString = isLeft ? "left" : "right";
return String.Concat (prependString, colName).GetHashCode (); return String.Concat (prependString, colName).GetHashCode ();
} }
public override bool Equals (object? obj) => public override bool Equals (object obj) {return GetHashCode () == obj.GetHashCode ();}
GetHashCode() == obj?.GetHashCode();
} }
@@ -10,9 +10,9 @@ namespace BCG
{ {
public class JoinStream : ViewStream public class JoinStream : ViewStream
{ {
protected InternalJoinStream? myIjs; protected InternalJoinStream myIjs;
protected ProjectStream? myPs; protected ProjectStream myPs;
protected ColumnMap? myLeftCm, myRightCm; protected ColumnMap myLeftCm, myRightCm;
protected bool myIsProject = true; protected bool myIsProject = true;
private bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin; private bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin;
@@ -1,16 +1,15 @@
using bts; using System;
using bts.collections;
using mbts;
using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics;
using System.Linq; using System.Linq;
using System.Security.Cryptography; using System.Text;
using System.Threading.Tasks;
using mbts;
using mbts.spark;
namespace BCG { namespace BCG
public class NonIncJoinBookkeeper : IDisposable { {
public class NonIncJoinBookkeeper : IDisposable
{
public Action on_new_data; public Action on_new_data;
private readonly NonIncJoinSpec _join_spec; private readonly NonIncJoinSpec _join_spec;
private readonly int _join_position; private readonly int _join_position;
@@ -27,82 +26,54 @@ namespace BCG {
private int[] _join_col_indices = null, private int[] _join_col_indices = null,
_data_col_indices = null; _data_col_indices = null;
public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) { public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher)
{
_join_spec = the_join_spec; _join_spec = the_join_spec;
_join_position = join_position; _join_position = join_position;
_start_index_in_output = start_index_in_output; _start_index_in_output = start_index_in_output;
_row_hasher = row_hasher; _row_hasher = row_hasher;
_is_left_most = _join_position == 0; _is_left_most = _join_position == 0;
_is_tag_along = _join_spec.join_col_names.Length == 0; _is_tag_along = _join_spec.join_col_names.Length == 0;
}
private bool _filled_missing_data = false;
private void _fill_missing_data(DataPacket dp) {
if (_filled_missing_data)
return;
Func<string[], bool> null_or_empty = (str_arr) => {
if (str_arr == null || str_arr.Length == 0)
return true;
foreach (string s in str_arr) {
if (!String.IsNullOrEmpty(s)) {
return false;
}
}
return true;
};
Func<string[], string[]> remove_bad_cols = (col_names) => {
HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
List<string> cols_list = new List<string>(col_names);
cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
return cols_list.ToArray();
};
if (null_or_empty(_join_spec.data_col_names)) {
_join_spec.data_col_names = dp.schema.columnNames;
}
_join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names);
if (null_or_empty(_join_spec.data_col_names)) {
_join_spec.data_col_names = dp.schema.columnNames;
}
if (null_or_empty(_join_spec.projected_col_names)) {
_join_spec.projected_col_names = _join_spec.data_col_names;
}
}
public void init() {
_join_spec.stream.onChange += _on_stream_cb; _join_spec.stream.onChange += _on_stream_cb;
} }
public void _on_stream_cb(object dp_obj) { public void _on_stream_cb(object dp_obj)
Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}"); {
var timer = Stopwatch.StartNew(); HighPerformanceTimer timer = new HighPerformanceTimer();
timer.start();
DataPacket dp = (DataPacket)dp_obj; DataPacket dp = (DataPacket)dp_obj;
_fill_missing_data(dp); if (_schema == null)
if (_schema == null) { {
_schema = dp.schema; _schema = dp.schema;
_join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names); _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
_data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names); _data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
} }
if (_is_left_most) { if (_is_left_most)
{
_row_list.Clear(); _row_list.Clear();
foreach (IBlueRow r in dp.newRows) foreach (IBlueRow r in dp.newRows)
_row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r)); _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));
} else { }
else
{
if (_is_tag_along) if (_is_tag_along)
_solitary_row = BlueDataUtility.extractRow(dp.newRows, 0); _solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
else { else
{
_row_map.Clear(); _row_map.Clear();
foreach (IBlueRow r in dp.newRows) foreach (IBlueRow r in dp.newRows)
_row_map[_row_hasher.get_hash(r, _join_col_indices)] = r; _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
} }
} }
timer.Stop(); timer.stop();
_n_total_fires++; _n_total_fires++;
if (timer.ElapsedMilliseconds >= 50) { if (timer.lastIntervalTime > 50)
{
_n_expensive_fires++; _n_expensive_fires++;
Log.Gen.info($"NonIncJoinBookkeepingData._on_stream_cb {dp.newRows.Count} rows {_join_col_indices.Length} " string log_msg = String.Format("NonIncJoinBookkeepingData._on_stream_cb {0} rows {1} key cols {2} join pos time {3} ratio {4}/{5}", dp.newRows.Count, _join_col_indices.Length, _join_position, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires);
+ $"key cols {_join_position} join pos time {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}"); // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
} }
if (on_new_data != null) if (on_new_data != null)
on_new_data(); on_new_data();
@@ -110,25 +81,31 @@ namespace BCG {
public bool is_ready { get { return _schema != null; } } public bool is_ready { get { return _schema != null; } }
public void constribute_cols_to_schema(List<ColumnInfo> cis) { public void constribute_cols_to_schema(List<ColumnInfo> cis)
{
for (int i = 0; i < _join_spec.data_col_names.Length; i++) for (int i = 0; i < _join_spec.data_col_names.Length; i++)
cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType)); cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
} }
public List<Pair<int, IBlueRow>> get_list() { return _row_list; } public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
public void copy_master_row_to_output_row(int row_index, object[] output_row) { public void copy_master_row_to_output_row(int row_index, object[] output_row)
{
IBlueRow this_row = _row_list[row_index].second; IBlueRow this_row = _row_list[row_index].second;
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output); BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
} }
public void copy_matching_row_to_output_row(int row_hash, object[] output_row) { public void copy_matching_row_to_output_row(int row_hash, object[] output_row)
if (_is_tag_along) { {
if (_is_tag_along)
{
if (_solitary_row != null) if (_solitary_row != null)
BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output); BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
else else
copy_dbnull_to_output_row(output_row); copy_dbnull_to_output_row(output_row);
} else { }
else
{
IBlueRow this_row = null; IBlueRow this_row = null;
if (_row_map.TryGetValue(row_hash, out this_row)) if (_row_map.TryGetValue(row_hash, out this_row))
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output); BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
@@ -137,7 +114,8 @@ namespace BCG {
} }
} }
private void copy_dbnull_to_output_row(object[] output_row) { private void copy_dbnull_to_output_row(object[] output_row)
{
for (int i = 0; i < _data_col_indices.Length; i++) for (int i = 0; i < _data_col_indices.Length; i++)
output_row[i + _start_index_in_output] = DBNull.Value; output_row[i + _start_index_in_output] = DBNull.Value;
} }
@@ -3,63 +3,63 @@ using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Text; using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
using mbts; using mbts;
namespace BCG { namespace BCG
// this a collection of data, theres no joining logic in here {
public class NonIncJoinSpec { public class NonIncJoinSpec
{
public ViewStream stream; public ViewStream stream;
// join col names are the names on which the streams are joined (for equality),
// data col names are the names in the original stream
// projected col names are their new names, ie if its x:l_x, then x is in the
// data_col_names and l_x in projected cols
public string[] join_col_names, data_col_names, projected_col_names; public string[] join_col_names, data_col_names, projected_col_names;
public NonIncJoinSpec() { } public NonIncJoinSpec() { }
public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names) public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
: this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { } : this (the_stream, the_join_col_names, the_data_col_names, the_data_col_names)
{ }
public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names) { public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names)
{
stream = the_stream; stream = the_stream;
join_col_names = the_join_col_names; join_col_names = the_join_col_names;
data_col_names = the_data_col_names; data_col_names = the_data_col_names;
projected_col_names = the_projected_col_names; projected_col_names = the_projected_col_names;
} }
public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs) { public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs)
{
List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>(); List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
foreach (Triplet<ViewStream, string, string> r in join_specs) { foreach (Triplet<ViewStream, string, string> r in join_specs)
string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' '); {
string[] keys = String.IsNullOrEmpty (r.second) ? new string[] {} : r.second.Split(' ');
string[] visible_cols = r.third.Split(' '); string[] visible_cols = r.third.Split(' ');
List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>(); List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
foreach (string s in visible_cols) { foreach (string s in visible_cols)
{
string[] toks = s.Split(':'); string[] toks = s.Split(':');
if (toks.Length == 1) { if (toks.Length == 1)
{
data_col_names.Add(toks[0]); data_col_names.Add(toks[0]);
mapped_col_names.Add(toks[0]); mapped_col_names.Add(toks[0]);
} else if (toks.Length == 2) { }
else if (toks.Length == 2)
{
data_col_names.Add(toks[0]); data_col_names.Add(toks[0]);
mapped_col_names.Add(toks[1]); mapped_col_names.Add(toks[1]);
} }
} }
rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray())); rv.Add (new NonIncJoinSpec (r.first, keys, data_col_names.ToArray (), mapped_col_names.ToArray()));
} }
return rv.ToArray(); return rv.ToArray ();
}
public override string ToString() {
Func<string[], string> arr_txt = (a) => {
string rv = "";
foreach (string s in a) {
rv += "{s} ";
}
rv = rv.Trim();
return rv;
};
string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}";
return rv;
} }
} }
//[DataStreamUserRespresentation("ninctest")]
//public class NonIncJoinTestStream : ViewStream
//{
// public NonIncJoinTestStream (string key_cols, string data_cols)
// {
// NonIncJoinSpec.from_triplet(new Triplet<ViewStream, string, string>(null, key_cols, data_cols));
// }
//}
} }
@@ -1,14 +1,16 @@
using bts; using System;
using mbts;
using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using System.Linq;
using System.Text;
using System.Threading.Tasks;
using mbts;
using mbts.spark;
namespace BCG { namespace BCG
{
[DataStreamUserRespresentation("fastJoin")] [DataStreamUserRespresentation("fastJoin")]
public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker { public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker
{
private readonly NonIncJoinSpec[] _join_specs; private readonly NonIncJoinSpec[] _join_specs;
private readonly INonIncJoinInlineMapper _output_mapper; private readonly INonIncJoinInlineMapper _output_mapper;
private readonly BlueRowHasher _row_hasher; private readonly BlueRowHasher _row_hasher;
@@ -26,19 +28,24 @@ namespace BCG {
{ {
new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj), new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj) new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
}) { } })
{ }
public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs) public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
: this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { } : this(NonIncJoinSpec.from_triplet(join_specs), null, 0)
{ }
public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms, public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
params Triplet<ViewStream, string, string>[] join_specs) params Triplet<ViewStream, string, string>[] join_specs)
: this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { } : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms)
{ }
public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
: this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { } : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms)
{ }
public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) { public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
{
_join_specs = join_specs; _join_specs = join_specs;
_output_mapper = output_mapper; _output_mapper = output_mapper;
_row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length); _row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
@@ -46,11 +53,12 @@ namespace BCG {
_vsd = vsd; _vsd = vsd;
} }
protected override void myOnFirstSubscriber() { protected override void myOnFirstSubscriber()
{
int data_col_no = 1; int data_col_no = 1;
for (int jp = 0; jp < _join_specs.Length; jp++) { for (int jp = 0; jp < _join_specs.Length; jp++)
{
NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher); NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
data_col_no += _join_specs[jp].data_col_names.Length; data_col_no += _join_specs[jp].data_col_names.Length;
s.on_new_data += _stream_callback; s.on_new_data += _stream_callback;
_book_keeping_data.Add(s); _book_keeping_data.Add(s);
@@ -58,11 +66,10 @@ namespace BCG {
if (_output_mapper != null) if (_output_mapper != null)
_output_mapper.onMapperChange += _on_mapper_change; _output_mapper.onMapperChange += _on_mapper_change;
_vsd.addWorker(this); _vsd.addWorker(this);
foreach (var bkd in _book_keeping_data)
bkd.init();
} }
protected override void myOnLastSubscriber() { protected override void myOnLastSubscriber()
{
foreach (NonIncJoinBookkeeper d in _book_keeping_data) foreach (NonIncJoinBookkeeper d in _book_keeping_data)
d.Dispose(); d.Dispose();
if (_output_mapper != null) if (_output_mapper != null)
@@ -70,26 +77,30 @@ namespace BCG {
_vsd.removeWorker(this); _vsd.removeWorker(this);
} }
private void _stream_callback() { private void _stream_callback()
Log.Gen.info("NonIncJoinStream._stream_callback received callback"); {
_is_dirty = true; _is_dirty = true;
doWork(); doWork();
} }
private void _on_mapper_change() { private void _on_mapper_change()
{
_is_dirty = true; _is_dirty = true;
doWork(); doWork();
} }
private bool _is_ready() { private bool _is_ready()
{
bool rv = true; bool rv = true;
foreach (NonIncJoinBookkeeper d in _book_keeping_data) foreach (NonIncJoinBookkeeper d in _book_keeping_data)
rv &= d.is_ready; rv &= d.is_ready;
return rv; return rv;
} }
private void _construct_schema() { private void _construct_schema()
if (_output_schema == null) { {
if (_output_schema == null)
{
List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) }); List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
foreach (NonIncJoinBookkeeper d in _book_keeping_data) foreach (NonIncJoinBookkeeper d in _book_keeping_data)
d.constribute_cols_to_schema(cis); d.constribute_cols_to_schema(cis);
@@ -98,23 +109,28 @@ namespace BCG {
} }
private int _n_total_fires = 0, _n_expensive_fires = 0; private int _n_total_fires = 0, _n_expensive_fires = 0;
private void _form_and_fire() { private void _form_and_fire()
Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}"); {
if (_is_ready()) { if (_is_ready())
{
_construct_schema(); _construct_schema();
var timer = Stopwatch.StartNew(); HighPerformanceTimer timer = new HighPerformanceTimer();
timer.start();
IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list(); IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
List<object[]> output_rows = new List<object[]>(); List<object[]> output_rows = new List<object[]>();
for (int i = 0; i < left_row_list.Count; i++) { for (int i = 0; i < left_row_list.Count; i++)
{
object[] or = new object[_output_schema.columnInfo.Length]; object[] or = new object[_output_schema.columnInfo.Length];
or[0] = i; or[0] = i;
output_rows.Add(or); output_rows.Add(or);
} }
for (int ji = 0; ji < _book_keeping_data.Count; ji++) { for (int ji = 0; ji < _book_keeping_data.Count; ji++)
{
NonIncJoinBookkeeper bk_data = _book_keeping_data[ji]; NonIncJoinBookkeeper bk_data = _book_keeping_data[ji];
for (int i = 0; i < left_row_list.Count; i++) { for (int i = 0; i < left_row_list.Count; i++)
{
if (ji == 0) if (ji == 0)
bk_data.copy_master_row_to_output_row(i, output_rows[i]); bk_data.copy_master_row_to_output_row(i, output_rows[i]);
else else
@@ -122,38 +138,44 @@ namespace BCG {
} }
} }
IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows)); IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
timer.Stop(); timer.stop();
_n_total_fires++; _n_total_fires++;
//if (timer.ElapsedMilliseconds >= 250) { if (timer.lastIntervalTime >= 250)
_n_expensive_fires++; {
string log_msg = $"Join and map time for {_book_keeping_data.Count} " _n_expensive_fires++;
+ $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}"; string log_msg = String.Format("Join and map time for {0} tables {1} ratio {2}/{3}", _book_keeping_data.Count, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires);
Log.Gen.info(log_msg); // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
//} }
var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket); myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
myFireEvent(dp_out);
//myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
} }
} }
private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) { private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows)
if (_output_mapper != null) { {
if (_output_mapper != null)
{
HighPerformanceTimer timer = new HighPerformanceTimer();
IBlueRowCollection output_rows; IBlueRowCollection output_rows;
var timer = Stopwatch.StartNew(); timer.start();
output_rows = _output_mapper.map_rows(unmapped_rows); output_rows = _output_mapper.map_rows(unmapped_rows);
timer.Stop(); timer.stop();
if (timer.ElapsedMilliseconds >= 50) { if (timer.lastIntervalTime >= 50)
Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}"); {
// string log_msg = String.Format("Time to map {0} rows into {1} rows {2}", unmapped_rows.Count, output_rows.Count, timer.lastIntervalTime);
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
} }
return output_rows; return output_rows;
} else { }
else
{
return unmapped_rows; return unmapped_rows;
} }
} }
public void doWork() { public void doWork()
Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}"); {
if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) { if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty)
{
_is_dirty = false; _is_dirty = false;
_last_fire_time = DateTime.Now; _last_fire_time = DateTime.Now;
_form_and_fire(); _form_and_fire();
@@ -1,68 +0,0 @@
namespace bts {
using BCG;
using BCG.Utilities.General;
using System;
using System.Diagnostics;
using ZstdSharp.Unsafe;
// functon that joins the 2 rows and returns the outputput row. The function
// can return a null which means the 2 rows dont need to be joined
using JoinFunc = System.Func<BCG.IBlueRow, BCG.IBlueRow, BCG.IBlueRow>;
public class ProximityJoinStream : ViewStream, IWorker {
private ViewStream _lvs, _rvs;
private JoinFunc _join_func;
private bool _dirty = false;
private Stopwatch _stopwatch = new Stopwatch();
private ViewStreamDatabase _vsd = ViewStreamDatabase.instance;
public ProximityJoinStream(ViewStream lvs, ViewStream rvs, JoinFunc join_func) {
_lvs = lvs;
_rvs = rvs;
_join_func = join_func;
}
protected override void myOnFirstSubscriber() {
_lvs.onChange += _handle_data;
_rvs.onChange += _handle_data;
_vsd.addWorker(this);
}
protected override void myOnLastSubscriber() {
_lvs.onChange -= _handle_data;
_rvs.onChange -= _handle_data;
_vsd.removeWorker(this);
}
private void _handle_data(object dp_obj) {
_dirty = true;
doWork();
}
public void doWork() {
Func<bool> all_data_revcd = () => {
return _lvs.rows != null && _rvs.rows != null;
};
_stopwatch.Start();
if (_dirty && all_data_revcd()) {
_dirty = false;
BlueTable rv = null;
foreach (IBlueRow lr in _lvs.rows) {
foreach (IBlueRow rr in _rvs.rows) {
IBlueRow resr = _join_func(lr, rr);
if (resr == null)
continue;
rv = rv ?? new BlueTable(resr.schema);
rv.addRow(resr);
}
}
_stopwatch.Stop();
Log.Gen.info($"ProximityJoinStream left_count: {_lvs.rows.Count} right_count: {_rvs.rows.Count} result_count: {rv.Count} time: {_stopwatch.ElapsedMilliseconds} msecs");
var dp = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket);
myFireEvent(dp);
}
}
}
}
@@ -1,47 +0,0 @@
namespace bts.SilverPine {
using BCG;
using System;
using System.CodeDom;
using System.Configuration;
using System.Reactive.Disposables;
public class TimeProximityJoinStream : ViewStream {
private int _id = 0;
public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
string pre_join_period, string post_join_period) :
this(ls, l_time_col, rs, r_time_col, TimeBasedFilter.parse_duration(pre_join_period),
TimeBasedFilter.parse_duration(post_join_period)) { }
public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
TimeSpan pre_join_period, TimeSpan post_join_period) {
Func<IBlueRow, IBlueRow, IBlueRow> create_row = (lr, rr) => {
var id = 0;
lock (this) {
id = _id++;
}
IBlueRow rv = BlueDataUtility.create_composite_row(lr, rr, id);
return rv;
};
Func<IBlueRow, IBlueRow, IBlueRow> join_func = (lr, rr) => {
DateTime ldt = lr.get_datetime(l_time_col);
DateTime rdt = rr.get_datetime(r_time_col);
TimeSpan diff = ldt - rdt;
if (ldt < rdt) {
if (diff.Duration() > pre_join_period) {
return null;
}
} else {
if (diff > post_join_period)
return null;
}
return create_row(lr, rr);
};
ViewStream rstr = new ProximityJoinStream(ls, rs, join_func);
mySetRealSource(rstr);
}
}
}
@@ -1,7 +1,11 @@
using System; using System;
using System.Collections; using System.Collections;
using System.Collections.Generic; using System.Collections.Generic;
using System.Text;
using BCG.Utilities.General;
using BCG.Utilities;
using mbts; using mbts;
using mbts.spark;
using Spark.Core.Collections; using Spark.Core.Collections;
namespace BCG namespace BCG
@@ -222,8 +226,7 @@ namespace BCG
#region IDictionary Members #region IDictionary Members
public void Add(object key, object? value) => public void Add(object key, object value) { { throw new NotSupportedException("IBlueRow is readonly"); } }
throw new NotSupportedException("IBlueRow is readonly");
public void Clear() { throw new NotSupportedException("IBlueRow is readonly"); } public void Clear() { throw new NotSupportedException("IBlueRow is readonly"); }
@@ -253,9 +256,10 @@ namespace BCG
} }
} }
public object? this[object key] { public object this[object key]
get => key is string str ? this[str] : this[(int)key]; {
set => throw new NotSupportedException("IBlueRow is readonly"); get { return key is string ? this[key.ToString()] : this[(int)key]; }
set { throw new NotSupportedException("IBlueRow is readonly"); }
} }
#endregion #endregion
@@ -290,6 +294,7 @@ namespace BCG
protected Dictionary<int, IBlueRow> myLeftUnmatchedRowsAdds = new Dictionary<int, IBlueRow>(), protected Dictionary<int, IBlueRow> myLeftUnmatchedRowsAdds = new Dictionary<int, IBlueRow>(),
myLeftUnmatchedRowsDels = new Dictionary<int, IBlueRow>(); myLeftUnmatchedRowsDels = new Dictionary<int, IBlueRow>();
private bool myManageUnmappedLeftRows = false; private bool myManageUnmappedLeftRows = false;
protected HighPerformanceTimer myTimer = new HighPerformanceTimer();
protected string myLeftProj = String.Empty, myRightProj = String.Empty; protected string myLeftProj = String.Empty, myRightProj = String.Empty;
protected IBlueTableSchema myLeftSchema, myRightSchema; protected IBlueTableSchema myLeftSchema, myRightSchema;
protected JoinProjectHelper myJoinProjectHelper; protected JoinProjectHelper myJoinProjectHelper;
@@ -356,6 +361,7 @@ namespace BCG
private void myHandleStreamChanges(DataPacket dp, bool isLeft) private void myHandleStreamChanges(DataPacket dp, bool isLeft)
{ {
myTimer.start();
myInitSchema(dp.schema, isLeft); myInitSchema(dp.schema, isLeft);
BlueTable dels = null, adds = null; BlueTable dels = null, adds = null;
if (myState != null) if (myState != null)
@@ -365,6 +371,7 @@ namespace BCG
} }
myDeleteRows(dp.deletedRows, isLeft, myState, dels); myDeleteRows(dp.deletedRows, isLeft, myState, dels);
myAddRows(dp.addedRows, isLeft, myState, adds); myAddRows(dp.addedRows, isLeft, myState, adds);
myTimer.stop();
if (myState != null) if (myState != null)
{ {
myFireEvent(new DataPacket(myState, adds, dels)); myFireEvent(new DataPacket(myState, adds, dels));
@@ -91,8 +91,7 @@ namespace BCG
} }
} }
public void Add (object key, object? value) => public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");}
throw new NotSupportedException ("The IBlueRow is readonly");
public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");} public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
@@ -1,96 +1,51 @@
using System; using System;
using System.Collections; using System.Collections;
using System.Collections.Generic;
using Amazon.Runtime.Internal.Util;
using BCG; using BCG;
using BCG.Utilities; using BCG.Utilities;
using bts; namespace BCG
{
namespace BCG { public class LiteralStream : ViewStream
public class LiteralStream : ViewStream { {
public LiteralStream(string tableName, string schema, string value)
public static List<Tuple<ColumnInfo, object>> parse_columns(string key_value_pairs) { {
List<Tuple<ColumnInfo, object>> rv = new List<Tuple<ColumnInfo, object>>(); string [] columns = schema.Split(' ');
string[] pairs = key_value_pairs.Split(' ');
foreach (string p in pairs) {
string[] kvp = p.Split(':');
string col_name = kvp[0];
string val = kvp[1];
var tup = BlueDataUtility.guess_type(val);
var ci = new ColumnInfo(col_name, rv.Count, tup.Item1);
rv.Add(Tuple.Create(ci, tup.Item2));
}
return rv;
}
//literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
public LiteralStream(string table_name, string rows) {
Func<string, int, IBlueRow> get_row = (json_row, id) => {
Dictionary<string, object> map = new Dictionary<string, object>();
string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
//Logger.assert(col_vals.Length == 2);
foreach (string t in toks) {
var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
string col = col_vals[0];
string val = col_vals[1];
map.Add(col, val);
}
map["id"] = id;
return new BlueRow(map);
};
string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
BlueTable table = null;
foreach (string json_string in row_arr) {
IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
get_row(json_string, table == null ? 0 : table.Count);
table = table ?? new BlueTable(r.schema);
table.addRow(r);
}
ViewStream str = new BlueTableStream(table);
str = new SchemaGuesserStream(str);
mySetRealSource(str);
}
public LiteralStream(string tableName, string schema, string value) {
string[] columns = schema.Split(' ');
ColumnInfo[] ci = new ColumnInfo[columns.Length + 1]; ColumnInfo [] ci = new ColumnInfo[columns.Length + 1];
ci[0] = new ColumnInfo("id", 0, typeof(int)); ci[0] = new ColumnInfo("id",0,typeof(int));
int n = 1; int n = 1;
foreach (string c in columns) { foreach (string c in columns)
string[] columnAndType = c.Split('/'); {
string [] columnAndType = c.Split('/');
if (columnAndType.Length == 1) if (columnAndType.Length == 1)
ci[n] = new ColumnInfo(c, n, typeof(double)); ci[n] = new ColumnInfo(c,n,typeof(double));
else else
ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true)); ci[n] = new ColumnInfo(columnAndType[0],n,Type.GetType("System." + columnAndType[1],true));
n++; n++;
} }
BlueTable t = new BlueTable(new BlueTableSchema(tableName, ci)); BlueTable t = new BlueTable(new BlueTableSchema(tableName,ci));
string[] rows = value.Split(';'); string [] rows = value.Split(';');
n = 0; n = 0;
foreach (string r in rows) foreach (string r in rows)
try { try
if (r == string.Empty) {
if(r == string.Empty)
continue; continue;
t.addRow( t.addRow(
BlueDataUtility.createRowFromCSVFileRow(r, t.schema, ++n)); BlueDataUtility.createRowFromCSVFileRow (r, t.schema,++n));
} catch (Exception e) { }
catch (Exception e)
{
throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message)); throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message));
} }
mySetRealSource(new BlueRowCollectionStaticStream(t)); mySetRealSource(new BlueRowCollectionStaticStream(t));
} }
// public LiteralStream(string schema, string value) : this("literal", schema, value) { } public LiteralStream(string schema, string value) : this ("literal",schema, value)
{}
// public LiteralStream(string value) {
// }
} }
} }
@@ -1,24 +1,24 @@
// using System; using System;
// namespace BCG namespace BCG
// { {
// public class DynamicUntypedToTypedMapper : IDynamicStreamMapper public class DynamicUntypedToTypedMapper : IDynamicStreamMapper
// { {
// IBlueTableSchema mySchema; IBlueTableSchema mySchema;
// public IBlueRow map (IBlueRow row) public IBlueRow map (IBlueRow row)
// { {
// if (mySchema == null) if (mySchema == null)
// { {
// ColumnInfo[] cis = new ColumnInfo [row.schema.columnInfo.Length]; ColumnInfo[] cis = new ColumnInfo [row.schema.columnInfo.Length];
// int i = 0; int i = 0;
// foreach (ColumnInfo ci in row.schema.columnInfo) foreach (ColumnInfo ci in row.schema.columnInfo)
// cis[i++] = ci.colType == null ? cis[i++] = ci.colType == null ?
// new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType()) : new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType()) :
// new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType(), ci.colValue); new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType(), ci.colValue);
// mySchema = new BlueTableSchema (row.schema.tableName, cis); mySchema = new BlueTableSchema (row.schema.tableName, cis);
// } }
// return new BlueRow (mySchema, row); return new BlueRow (mySchema, row);
// } }
// } }
// } }
@@ -1,72 +0,0 @@
using BCG;
using MongoDB.Driver.Core.WireProtocol.Messages;
using System;
using System.Collections.Generic;
using System.Linq;
// the first arg is all the rows, the second arg is the index of the row that needs mapping, and
// the func returns the mapped row (ie mapped version of the index row)
using GlobalMap = System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow>;
namespace bts {
// In GlobalMap, each row is mapped with the full view of all rows, ie the map functions for a row
// can use the state fo all rows.
public class GlobalMapStream : ViewStream {
private ViewStream _base_stream;
private readonly string _sort_col_name, _group_by_col;
private GlobalMap _map_func;
public GlobalMapStream(ViewStream base_stream, string sort_col_name, string group_by_col, GlobalMap map_func) {
_base_stream = base_stream;
_sort_col_name = sort_col_name;
_group_by_col = group_by_col;
_map_func = map_func;
}
public GlobalMapStream(ViewStream base_stream, string sort_col_name, GlobalMap map_func) :
this(base_stream, sort_col_name, null, map_func) {
}
protected override void myOnFirstSubscriber() {
_base_stream.onChange += _handle_stream_update;
}
protected override void myOnLastSubscriber() {
_base_stream.onChange -= _handle_stream_update;
}
void _handle_stream_update(object dp_obj) {
DataPacket dp = dp_obj as DataPacket;
IDictionary<int, List<IBlueRow>> groups = !String.IsNullOrEmpty(_group_by_col) ?
BlueDataUtility.group_together(dp.newRows, new string[] { _group_by_col })
:
new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } };
BlueTable out_table = null;
foreach (var kv in groups) {
List<IBlueRow> group_rows = kv.Value;
group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, _sort_col_name));
uint i = 0;
foreach (IBlueRow r in group_rows) {
IBlueRow nr = _map_func(group_rows, i++);
out_table = out_table ?? new BlueTable(nr.schema);
out_table.addRow(nr);
}
}
// List<IBlueRow> rows = dp.newRows.to_ienum().ToList();
// rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r1, r2, _sort_col_name));
// uint i = 0;
// BlueTable out_table = null;
// foreach (IBlueRow r in rows) {
// IBlueRow nr = _map_func(rows, i++);
// out_table = out_table ?? new BlueTable(nr.schema);
// out_table.addRow(nr);
// }
DataPacket dp_out = BlueDataUtility.build_dp_inc(out_table, myLastValue as DataPacket);
myFireEvent(dp_out);
}
}
}
@@ -1,10 +1,15 @@
using System;
using System.Data;
using System.Collections;
using Bts.DataStream; using Bts.DataStream;
using mbts;
using mbts.spark;
using mbts.config; using mbts.config;
using System; namespace BCG
{
namespace BCG {
/// <summary> /// <summary>
@@ -16,45 +21,24 @@ namespace BCG {
[DataStreamUserRespresentation (false)] [DataStreamUserRespresentation (false)]
public class MapStream : MethodDrivenStream public class MapStream : MethodDrivenStream
{ {
private HighPerformanceTimer t = new HighPerformanceTimer ();
protected IStreamMapper myMapper; protected IStreamMapper myMapper;
protected IChangeableStreamMapper myChangeableMapper = null; protected IChangeableStreamMapper myChangeableMapper = null;
private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap; private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;
public class FuncWrapper : IStreamMapper { public MapStream(ViewStream vs, IStreamMapper mapper) : base (vs)
private Func<IBlueTableSchema, IBlueTableSchema> _schema_map_func; {
Func<IBlueRow, IBlueRow> _row_map_func;
public FuncWrapper(Func<IBlueTableSchema, IBlueTableSchema> schema_map_func,
Func<IBlueRow, IBlueRow> row_map_func) {
_schema_map_func = schema_map_func;
_row_map_func = row_map_func;
}
public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) {
return _schema_map_func(input_schema);
}
public IBlueRow map(IBlueRow row) {
return _row_map_func(row);
}
}
public MapStream(ViewStream vs, IStreamMapper mapper) : base(vs) {
if (myUseNewMap) if (myUseNewMap)
mySetRealSource(new SuperMapStream(vs, mapper)); mySetRealSource (new SuperMapStream (vs, mapper));
else else
myMapper = mapper; myMapper = mapper;
} }
public MapStream(ViewStream vs, IChangeableStreamMapper mapper) : this (vs,(IStreamMapper)mapper)
public MapStream(ViewStream vs, Func<IBlueTableSchema, IBlueTableSchema> schema_map_func, {
Func<IBlueRow, IBlueRow> row_map_func)
:
this(vs, new FuncWrapper(schema_map_func, row_map_func)) {
}
public MapStream(ViewStream vs, IChangeableStreamMapper mapper) : this(vs, (IStreamMapper)mapper) {
if (myUseNewMap) if (myUseNewMap)
mySetRealSource(new SuperMapStream(vs, mapper)); mySetRealSource(new SuperMapStream(vs, mapper));
else { else
{
mapper.onMapperChange += new MapperChangeDelegate(myMapperChanged); mapper.onMapperChange += new MapperChangeDelegate(myMapperChanged);
myChangeableMapper = mapper; myChangeableMapper = mapper;
} }
@@ -63,68 +47,77 @@ namespace BCG {
/// <summary> /// <summary>
/// Gets or sets the value of the mapper. Setting the value will result in re /// Gets or sets the value of the mapper. Setting the value will result in re
/// </summary> /// </summary>
private void myMapperChanged() { private void myMapperChanged ()
{
if (myLastArrivedPacket != null) if (myLastArrivedPacket != null)
myChangeHandler(new DataPacket(myLastArrivedPacket.newRows, myChangeHandler (new DataPacket (myLastArrivedPacket.newRows,
myLastArrivedPacket.newRows, myLastArrivedPacket.newRows,
myLastArrivedPacket.newRows)); myLastArrivedPacket.newRows));
} }
protected override void myHandleFirstCall() { protected override void myHandleFirstCall ()
{
IBlueTableSchema outputSchema = myMapper.getOutputSchema(myLastArrivedPacket.newRows.schema);
myLastFiredPacket = new DataPacket(); IBlueTableSchema outputSchema = myMapper.getOutputSchema (myLastArrivedPacket.newRows.schema);
myLastFiredPacket.newRows = new BlueTable(outputSchema); myLastFiredPacket = new DataPacket ();
myLastFiredPacket.deletedRows = new BlueTable(outputSchema); myLastFiredPacket.newRows = new BlueTable (outputSchema);
myLastFiredPacket.deletedRows = new BlueTable (outputSchema);
foreach (IBlueRow row in myLastArrivedPacket.newRows) foreach (IBlueRow row in myLastArrivedPacket.newRows)
((BlueTable)(myLastFiredPacket.newRows)).addRow(myMapper.map(row)); ((BlueTable)(myLastFiredPacket.newRows)).addRow (myMapper.map(row));
myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows); myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows);
} }
protected override void myHandleIncrementalAdd() { protected override void myHandleIncrementalAdd()
{
//t.start (); //t.start ();
myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows.schema); myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.newRows.schema);
foreach (IBlueRow row in myLastArrivedPacket.addedRows) { foreach (IBlueRow row in myLastArrivedPacket.addedRows)
IBlueRow newRow = myMapper.map(row); {
(myLastFiredPacket.newRows as BlueTable).addRow(newRow); IBlueRow newRow = myMapper.map (row);
(myLastFiredPacket.addedRows as BlueTable).addRow(newRow); (myLastFiredPacket.newRows as BlueTable).addRow (newRow);
(myLastFiredPacket.addedRows as BlueTable).addRow (newRow);
} }
//t.stop (); //t.stop ();
//System.Diagnostics.Trace.WriteLine ("adding " + myLastFiredPacket.addedRows.Count.ToString () + " : " + t.lastIntervalTime); //System.Diagnostics.Trace.WriteLine ("adding " + myLastFiredPacket.addedRows.Count.ToString () + " : " + t.lastIntervalTime);
//System.Diagnostics.Trace.WriteLine (" "); //System.Diagnostics.Trace.WriteLine (" ");
} }
protected override void myHandleIncrementalDelete() { protected override void myHandleIncrementalDelete()
{
//t.start (); //t.start ();
myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows.schema); myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema);
foreach (IBlueRow row in myLastArrivedPacket.deletedRows) { foreach (IBlueRow row in myLastArrivedPacket.deletedRows)
{
IBlueRow delRow = (myLastFiredPacket.newRows as BlueTable)[(int)row["id"]]; IBlueRow delRow = (myLastFiredPacket.newRows as BlueTable)[(int)row["id"]];
(myLastFiredPacket.deletedRows as BlueTable).addRow(delRow); (myLastFiredPacket.deletedRows as BlueTable).addRow (delRow);
(myLastFiredPacket.newRows as BlueTable).deleteRow(row); (myLastFiredPacket.newRows as BlueTable).deleteRow (row);
} }
//t.stop (); //t.stop ();
//System.Diagnostics.Trace.WriteLine ("deleting " + myLastFiredPacket.deletedRows.Count.ToString () + " : " + t.lastIntervalTime); //System.Diagnostics.Trace.WriteLine ("deleting " + myLastFiredPacket.deletedRows.Count.ToString () + " : " + t.lastIntervalTime);
} }
protected override void myOnFirstSubscriber()
protected override void myOnFirstSubscriber() { {
myStream.onChange += new DataStream.ChangeDelegate(myChangeHandler); myStream.onChange += new DataStream.ChangeDelegate (myChangeHandler);
} }
protected override void myOnLastSubscriber() { protected override void myOnLastSubscriber ()
myInit(); {
myInit ();
if (myStream != null) if (myStream != null)
myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler); myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
} }
protected override void myOnTableClear() { protected override void myOnTableClear ()
{
myLastFiredPacket.deletedRows = myLastFiredPacket.newRows; myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
myLastFiredPacket.newRows = new BlueTable(myLastFiredPacket.deletedRows.schema); myLastFiredPacket.newRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema); myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
} }
} }
/* /*
@@ -1,13 +1,12 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Text; using System.Text;
using BCG.Utilities; using BCG.Utilities;
using BCG.Utilities.General; using BCG.Utilities.General;
using mbts; using mbts;
namespace BCG { namespace BCG
{
/// <summary> /// <summary>
/// SuperMap gives the user the ability /// SuperMap gives the user the ability
/// to map one row to one or more rows /// to map one row to one or more rows
@@ -18,11 +17,13 @@ namespace BCG {
/// can chage itself and raise an event causing /// can chage itself and raise an event causing
/// a remapping to happen /// a remapping to happen
/// </summary> /// </summary>
public class SuperMapStream : ViewStream { public class SuperMapStream : ViewStream
{
#region super mapper that wraps around stream mapper #region super mapper that wraps around stream mapper
private class MyMappingWrapper : ISuperStreamMapper { private class MyMappingWrapper : ISuperStreamMapper
{
private IStreamMapper myMapper; private IStreamMapper myMapper;
public MyMappingWrapper(IStreamMapper mapper) { myMapper = mapper; } public MyMappingWrapper(IStreamMapper mapper) { myMapper = mapper; }
#region ISuperStreamMapper Members #region ISuperStreamMapper Members
@@ -31,7 +32,8 @@ namespace BCG {
public IBlueRow[] map(IBlueRow inputRow) { return new IBlueRow[] { myMapper.map(inputRow) }; } public IBlueRow[] map(IBlueRow inputRow) { return new IBlueRow[] { myMapper.map(inputRow) }; }
public event MapperChangeDelegate onMapperChange { public event MapperChangeDelegate onMapperChange
{
add { } add { }
remove { } remove { }
} }
@@ -40,18 +42,21 @@ namespace BCG {
} }
#endregion #endregion
private class MyFuncWrapper : ISuperStreamMapper { private class MyFuncWrapper : ISuperStreamMapper
{
private Func<IBlueTableSchema, IBlueTableSchema> _get_schema; private Func<IBlueTableSchema, IBlueTableSchema> _get_schema;
private Func<IBlueRow, IEnumerable<IBlueRow>> _map_rows; private Func<IBlueRow, IEnumerable<IBlueRow>> _map_rows;
public MyFuncWrapper(Func<IBlueTableSchema, IBlueTableSchema> get_schema, public MyFuncWrapper(Func<IBlueTableSchema, IBlueTableSchema> get_schema,
Func<IBlueRow, IEnumerable<IBlueRow>> map_rows) { Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
{
_get_schema = get_schema; _get_schema = get_schema;
_map_rows = map_rows; _map_rows = map_rows;
} }
public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return _get_schema(schema); } public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return _get_schema(schema); }
public IBlueRow[] map(IBlueRow r) { public IBlueRow[] map(IBlueRow r)
{
IEnumerable<IBlueRow> rows = _map_rows(r); IEnumerable<IBlueRow> rows = _map_rows(r);
return rows == null ? new IBlueRow[] { } : new List<IBlueRow>(rows).ToArray(); return rows == null ? new IBlueRow[] { } : new List<IBlueRow>(rows).ToArray();
} }
@@ -71,15 +76,19 @@ namespace BCG {
public SuperMapStream(ViewStream vs, Func<IBlueTableSchema, IBlueTableSchema> get_schema, public SuperMapStream(ViewStream vs, Func<IBlueTableSchema, IBlueTableSchema> get_schema,
Func<IBlueRow, IEnumerable<IBlueRow>> map_rows) Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
: this(vs, new MyFuncWrapper(get_schema, map_rows)) { } : this(vs, new MyFuncWrapper(get_schema, map_rows))
{ }
public SuperMapStream(ViewStream vs, IBlueTableSchema out_schema, Func<IBlueRow, IEnumerable<IBlueRow>> map_rows) public SuperMapStream(ViewStream vs, IBlueTableSchema out_schema, Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
: this(vs, s => out_schema, map_rows) { } : this(vs, s => out_schema, map_rows)
{ }
public SuperMapStream(ViewStream vs, IStreamMapper mapper) public SuperMapStream(ViewStream vs, IStreamMapper mapper)
: this(vs, new MyMappingWrapper(mapper)) { } : this(vs, new MyMappingWrapper(mapper))
{ }
public SuperMapStream(ViewStream vs, ISuperStreamMapper mapper) { public SuperMapStream(ViewStream vs, ISuperStreamMapper mapper)
{
myStream = vs; myStream = vs;
myMapper = mapper; myMapper = mapper;
myMapper.onMapperChange += new MapperChangeDelegate(myHandleMapChange); myMapper.onMapperChange += new MapperChangeDelegate(myHandleMapChange);
@@ -87,7 +96,8 @@ namespace BCG {
protected override void myOnFirstSubscriber() { myStream.onChange += new ChangeDelegate(myHandleData); } protected override void myOnFirstSubscriber() { myStream.onChange += new ChangeDelegate(myHandleData); }
protected override void myOnLastSubscriber() { protected override void myOnLastSubscriber()
{
myStream.onChange -= new ChangeDelegate(myHandleData); myStream.onChange -= new ChangeDelegate(myHandleData);
myRows = null; myRows = null;
myBaseRows = null; myBaseRows = null;
@@ -95,7 +105,8 @@ namespace BCG {
//myMapper.onMapperChange -= new MapperChangeDelegate(myHandleMapChange); //myMapper.onMapperChange -= new MapperChangeDelegate(myHandleMapChange);
} }
private void myHandleData(object newValue) { private void myHandleData(object newValue)
{
DataPacket idp = newValue as DataPacket; DataPacket idp = newValue as DataPacket;
//HighPerformanceTimer timer = new HighPerformanceTimer(); //HighPerformanceTimer timer = new HighPerformanceTimer();
//timer.start(); //timer.start();
@@ -106,19 +117,24 @@ namespace BCG {
BlueTable dels = new BlueTable(myRows.schema), BlueTable dels = new BlueTable(myRows.schema),
adds = new BlueTable(myRows.schema); adds = new BlueTable(myRows.schema);
foreach (IBlueRow r in dp.deletedRows) { foreach (IBlueRow r in dp.deletedRows)
{
List<IBlueRow> mappedRows = myMap[(int)r["id"]]; List<IBlueRow> mappedRows = myMap[(int)r["id"]];
foreach (IBlueRow mr in mappedRows) { foreach (IBlueRow mr in mappedRows)
{
dels.addRow(mr); dels.addRow(mr);
myRows.deleteRow(mr); myRows.deleteRow(mr);
} }
myMap.Remove((int)r["id"]); myMap.Remove((int)r["id"]);
} }
foreach (IBlueRow r in dp.addedRows) { foreach (IBlueRow r in dp.addedRows)
{
IBlueRow[] mappedRows = myMapper.map(r); IBlueRow[] mappedRows = myMapper.map(r);
foreach (IBlueRow mr in mappedRows) { foreach (IBlueRow mr in mappedRows)
{
IBlueRow idChangedMapRow = generate_ids ? BlueDataUtility.getNewReplacedRow(mr, myID++, "id") : mr; IBlueRow idChangedMapRow = generate_ids ? BlueDataUtility.getNewReplacedRow(mr, myID++, "id") : mr;
if (!myMap.ContainsKey((int)r["id"])) { if (!myMap.ContainsKey((int)r["id"]))
{
myMap.Add((int)r["id"], new List<IBlueRow>()); myMap.Add((int)r["id"], new List<IBlueRow>());
} }
myMap[(int)r["id"]].Add(idChangedMapRow); myMap[(int)r["id"]].Add(idChangedMapRow);
@@ -129,12 +145,13 @@ namespace BCG {
//timer.stop(); //timer.stop();
// if (timer.lastIntervalTime > 20) // if (timer.lastIntervalTime > 20)
// {}//mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("SuperMapper:myHandlerData for mapper {0} took {1} msecs", myMapper.GetType().ToString(), timer.lastIntervalTime)); // {}//mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("SuperMapper:myHandlerData for mapper {0} took {1} msecs", myMapper.GetType().ToString(), timer.lastIntervalTime));
myFireEvent(new DataPacket(myRows, adds, dels)); // myFireEvent(new DataPacket(myRows, adds, dels));
} }
private void myHandleMapChange() { private void myHandleMapChange()
{
if (myBaseRows != null) if (myBaseRows != null)
myHandleData(new DataPacket(myBaseRows, new BlueTable(myBaseRows), new BlueTable(myBaseRows))); myHandleData(new DataPacket(myBaseRows, new BlueTable (myBaseRows), new BlueTable (myBaseRows)));
} }
} }
} }
@@ -1,142 +0,0 @@
using BCG;
using System;
using System.IO;
using MongoDB.Bson;
using BCG.Utilities.General;
using System.Linq;
using System.Collections.Generic;
using System.Runtime.Intrinsics.X86;
namespace bts {
// listens to the underlying stream, and writes each seriialized packet in a
// seperate file
[DataStreamUserRespresentation("dp_recorder")]
public class DataPacketRecorderStream : ViewStream {
private ViewStream _base_stream;
private readonly string _folder;
StreamIOSerializationController _serializer = new StreamIOSerializationController();
private UInt32 _n_updates = 0;
public DataPacketRecorderStream(ViewStream base_stream, string folder) {
_base_stream = base_stream;
_folder = folder;
}
protected override void myOnFirstSubscriber() {
_base_stream.onChange += _handle_update;
}
protected override void myOnLastSubscriber() {
_base_stream.onChange -= _handle_update;
}
private void _handle_update(object dp_obj) {
DataPacket dp = dp_obj as DataPacket;
byte[] ser_data = null;
if (_n_updates == 0) {
StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
ser_data = _serializer.serialize(new StreamIOMessage(cr_msg));
} else {
StreamIOMessage.MultipleRemoveAddMessage mra = new StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows);
ser_data = _serializer.serialize(new StreamIOMessage(mra));
}
Log.assert(ser_data != null);
_n_updates++;
string file_name = $"{_folder}\\{_n_updates}.bin";
_write_to_file(ser_data, file_name);
myFireEvent(dp);
}
private static void _write_to_file(byte[] data, string file_name) {
using (FileStream fs = new FileStream(file_name, FileMode.Create)) {
// Write the byte array to the file
fs.Write(data, 0, data.Length);
}
}
}
[DataStreamUserRespresentation("dp_replay")]
public class DataPacketReplayStream : ViewStream, IWorker {
private ViewStreamDatabase _vsd;
private readonly string _folder_path;
private readonly TimeSpan _fire_period;
IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
private int _file_ptr = 0;
private List<Tuple<string, byte[]>> _file_data = new List<Tuple<string, byte[]>>();
private BlueTable _state = null;
public DataPacketReplayStream(string folder_path, string fire_period)
:
this(ViewStreamDatabase.instance, folder_path, TimeBasedFilter.parse_duration(fire_period)) {
}
public DataPacketReplayStream(ViewStreamDatabase vsd, string folder_path, TimeSpan fire_period) {
_vsd = vsd;
_folder_path = folder_path;
_fire_period = fire_period;
}
protected override void myOnFirstSubscriber() {
MasterWorker.getInstance(false).addWorker(this, _fire_period);
_load_all_files();
}
protected override void myOnLastSubscriber() {
MasterWorker.getInstance(false).removeWorker(this);
}
public void doWork() {
if (_file_ptr >= _file_data.Count())
// we can set _file_ptr to 0 and keep going as well
return;
var curr_d = _file_data[_file_ptr++];
var hext_txt = StreamSerializationUtils.to_hex(curr_d.Item2);
StreamIOMessage msg = _deser.deserialize(curr_d.Item2);
Log.Gen.info($"DataPacketReplayStream.doWork packet no : {_file_ptr} file : {curr_d.Item1} data: {hext_txt}");
switch (msg.op) {
case StreamIOMessage.StreamOp.CREATE: {
_state = new BlueTable(msg.create_message.rows);
break;
}
case StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD: {
var mra = msg.multiple_remove_add_message;
foreach (IBlueRow r in mra.removed_rows)
_state.deleteRow(r);
foreach (IBlueRow r in mra.added_rows)
_state.addRow(r);
break;
}
}
var dp = BlueDataUtility.build_dp_inc(_state, myLastValue == null ? null : myLastValue as DataPacket, false);
myFireEvent(dp);
}
private void _load_all_files() {
Func<string, string> extract_file_name = (string full_path) => {
string fn = full_path.Split("\\").Last();
fn = fn.Split(".").First();
return fn;
};
string[] bin_files = Directory.GetFiles(_folder_path, "*.bin");
// Iterate over the .bin files and perform an action
foreach (string file in bin_files) {
byte[] file_data = File.ReadAllBytes(file);
_file_data.Add(Tuple.Create(file, file_data));
}
_file_data.Sort((Tuple<string, byte[]> t1, Tuple<string, byte[]> t2) => {
string fn1 = extract_file_name(t1.Item1);
string fn2 = extract_file_name(t2.Item1);
int fn_i1 = Int32.Parse(fn1);
int fn_i2 = Int32.Parse(fn2);
if (fn_i1 < fn_i2)
return -1;
else if (fn_i1 == fn_i2)
return 0;
else if (fn_i1 > fn_i2)
return 1;
return 0;
});
}
}
}
@@ -1,227 +0,0 @@
using BCG;
using BCG.Utilities.General;
using System.Collections.Generic;
//using MongoDB.Driver;
using System;
using System.IO;
using System.Linq;
namespace bts
{
public class PubSubTestingStream : ViewStream, IWorker
{
public class PayloadMessage
{
public int msg_id = 0;
public string guid;
public Int64 travel_epoch_ms = 0;
public byte[] payload = null;
public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte)
{
msg_id = msg_id_args;
travel_epoch_ms = send_epoch_args;
payload = new byte[payload_size];
for (int i = 0; i < payload_size; i++)
{
payload[i] = payload_byte;
}
}
private PayloadMessage() { }
public PayloadMessage(int msg_id, int payload_size, byte payload_byte) :
this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte)
{ }
public byte[] serialize()
{
var str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
bw.Write(msg_id);
bw.Write(guid);
bw.Write(travel_epoch_ms);
bw.Write(payload.Length);
bw.Write(payload);
str.Position = 0;
var bytes = str.ToArray();
return bytes;
}
public static PayloadMessage deserialize(byte[] ser_bytes)
{
var str = new MemoryStream(ser_bytes);
BinaryReader br = new BinaryReader(str);
int msg_id = br.ReadInt32();
string guid = br.ReadString();
Int64 epoch = br.ReadInt64();
int len = br.ReadInt32();
var payload = br.ReadBytes(len);
return new PayloadMessage()
{
msg_id = msg_id,
guid = guid,
travel_epoch_ms = epoch,
payload = payload
};
}
}
//public void
private readonly int _msg_size;
private readonly TimeSpan _publish_period;
private DateTime _last_publish_time = DateTime.MinValue;
private PublisherCore _pub_core = new PublisherCore();
private BlueTable _table;
private int _n_pubs = 0;
private readonly int _max_pubs;
private int _n_updates = 0;
private bool _continue_sub_thread = true;
private ViewStreamDatabase _vsd;
private bool _fire_dirty = false;
private PayloadMessage _last_sent_msg, _last_recvd_msg;
private readonly int _payload_sz;
private const byte _payload_byte = (byte)'A';
private readonly string _tag_name;
public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd)
{
_vsd = vsd;
_payload_sz = payload_size;
var schema = BlueDataUtility.form_schema("PubSubTest",
Tuple.Create("id", TypeCode.Int32),
Tuple.Create("n_pubs", TypeCode.Int32),
Tuple.Create("n_recvs", TypeCode.Int32),
Tuple.Create("last_sent_id", TypeCode.Int32),
Tuple.Create("last_recv_id", TypeCode.Int32),
Tuple.Create("last_sent_guid", TypeCode.String),
Tuple.Create("last_recv_guid", TypeCode.String),
Tuple.Create("match", TypeCode.Boolean),
Tuple.Create("echo_time", TypeCode.Int32));
_table = new BlueTable(schema);
_publish_period = TimeSpan.FromMilliseconds(publish_period_msecs);
_msg_size = payload_size;
_max_pubs = max_pubs;
_tag_name = tag_name;
}
public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size, string tag_name)
: this(publish_period_seconds, max_pubs, msg_size, tag_name, ViewStreamDatabase.instance) { }
protected override void myOnFirstSubscriber()
{
_vsd.addWorker(this);
System.Threading.Thread th = new System.Threading.Thread(_subscribe_thread_func);
th.IsBackground = true;
th.Start();
}
protected override void myOnLastSubscriber()
{
_vsd.removeWorker(this);
_continue_sub_thread = false;
}
void _log_updates(IEnumerable<PubSubServerMessage> updates)
{
var idx = 0;
foreach (var msg in updates)
{
bts.Log.Gen.info($"PubSubTesStream._subscribe_thread_func idx: {idx} msg: {msg}");
idx++;
}
}
private void _subscribe_thread_func()
{
// subscription is done in another thread
SubscriberCore sub_core = new SubscriberCore();
sub_core.subscribe(_tag_name);
while (_continue_sub_thread)
{
// TODO recv_id
var updates = sub_core.get_updates(TimeSpan.FromMilliseconds(250));
if (updates == null || updates.Count() == 0)
continue;
_log_updates(updates);
_n_updates += updates.Count();
if (updates.Count() > 0)
{
var server_msg = updates.Last();
if (server_msg.data != null && server_msg.data.Length > 0)
{
_last_recvd_msg = PayloadMessage.deserialize(server_msg.data);
_last_recvd_msg.travel_epoch_ms = epoch_now();
}
else
{
// why are we getting empty or null data ? (server is sending data regardless?)
}
_fire_dirty = true;
}
}
}
private void _publish_next_packet()
{
if (_last_sent_msg == null)
{
_last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte)
{
guid = Guid.NewGuid().ToString()
};
}
else
{
_last_sent_msg.msg_id++;
_last_sent_msg.guid = Guid.NewGuid().ToString();
_last_sent_msg.travel_epoch_ms = epoch_now();
}
var ser_bytes = _last_sent_msg.serialize();
_n_pubs++;
_pub_core.publish("Blob", _tag_name, ser_bytes);
_fire_dirty = true;
}
private void _fire_dp()
{
object last_sent_id = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.msg_id;
object last_recv_id = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.msg_id;
object echo_time = DBNull.Value;
object last_sent_guid = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.guid;
object last_recv_guid = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.guid;
if (_last_sent_msg != null && _last_recvd_msg != null)
{
echo_time = _last_recvd_msg.travel_epoch_ms - _last_sent_msg.travel_epoch_ms;
}
var match = last_sent_id.Equals(last_recv_id) && last_sent_guid.Equals(last_recv_guid);
IBlueRow nr = new BlueRow(_table.schema, 0, _n_pubs, _n_updates,
last_sent_id, last_recv_id, last_sent_guid, last_recv_guid, match, echo_time);
BlueTable table = new BlueTable(_table.schema);
table.addRow(nr);
var dp = BlueDataUtility.build_dp_inc(table, myLastValue as DataPacket, true);
myFireEvent(dp);
}
public void doWork()
{
if (DateTime.Now - _last_publish_time > _publish_period)
if (_n_pubs <= _max_pubs)
{
_publish_next_packet();
_last_publish_time = DateTime.Now;
}
// publish
// TODO: send id
if (_fire_dirty)
{
_fire_dirty = false;
_fire_dp();
}
}
public static long epoch_now()
{
return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
}
}
}
@@ -1,223 +0,0 @@
using BCG;
using BCG.Utilities.General;
using System.Collections.Generic;
using System;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using bts.api;
namespace bts
{
[DataStreamUserRespresentation("pss_echo")]
public class PubsubEchoTestStream : ViewStream, IWorker
{
public class PayloadMessage
{
public readonly string msg_id;
public readonly Int64 timestamp = 0;
public readonly byte[] payload = null;
public static byte[] create_data(int payload_size, byte payload_byte)
{
var rv = new byte[payload_size];
for (int i = 0; i < payload_size; i++)
{
rv[i] = payload_byte;
}
return rv;
}
// public PayloadMessage(string guid_, Int64 timestamp_, int payload_size, byte payload_byte)
// {
// guid = guid_;
// timestamp = timestamp_;
// payload = new byte[payload_size];
// for (int i = 0; i < payload_size; i++)
// {
// payload[i] = payload_byte;
// }
// }
public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_)
{
msg_id = guid_;
timestamp = timestamp_;
payload = payload_;
}
private PayloadMessage() { }
public byte[] serialize()
{
var str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
bw.Write(msg_id);
bw.Write(timestamp);
bw.Write(payload.Length);
bw.Write(payload);
str.Position = 0;
var bytes = str.ToArray();
return bytes;
}
public static PayloadMessage deserialize(byte[] ser_bytes)
{
var str = new MemoryStream(ser_bytes);
BinaryReader br = new BinaryReader(str);
string guid = br.ReadString();
Int64 timestamp_ = br.ReadInt64();
int len = br.ReadInt32();
var payload = br.ReadBytes(len);
return new PayloadMessage(guid, timestamp_, payload);
}
}
//public void
private readonly int _msg_size;
private readonly TimeSpan _publish_period;
private DateTime _last_publish_time = DateTime.MinValue;
private PublisherCore _pub_core;
private ISubscriberCore _sub_core;
private BlueTable _table;
private int _n_pubs = 0;
private readonly int _max_pubs;
private int _n_updates = 0;
private ViewStreamDatabase _vsd;
private PayloadMessage _last_sent_msg, _last_recvd_msg;
private const byte _payload_byte = (byte)'A';
private readonly string _tag_name;
private readonly int _payload_sz;
private readonly IPubSubRouter _router;
private enum IOState
{
SEND = 0,
RECV = 1
};
IOState _state = IOState.SEND;
public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd)
{
var route = new PubSubRoute(server_url, ".*");
_router = new PubSubRouter(route);
_vsd = vsd;
_payload_sz = payload_size;
var schema = BlueDataUtility.form_schema("PubSubTest",
Tuple.Create("id", TypeCode.Int32),
Tuple.Create("sent_id", TypeCode.String),
Tuple.Create("sent_time", TypeCode.Int64),
Tuple.Create("recv_id", TypeCode.String),
Tuple.Create("recv_time", TypeCode.Int64),
Tuple.Create("match", TypeCode.Boolean),
Tuple.Create("echo_time", TypeCode.Int32));
_table = new BlueTable(schema);
_publish_period = publish_period;
_tag_name = tag_name;
}
public PubsubEchoTestStream(string server_url, int publish_period_msecs, int payload_size, string tag_name)
: this(server_url, TimeSpan.FromMilliseconds(publish_period_msecs), payload_size, tag_name, ViewStreamDatabase.instance) { }
protected override void myOnFirstSubscriber()
{
_pub_core = new PublisherCore(_router);
_sub_core = new SubscriberCore(_router);
_sub_core.subscribe(_tag_name);
_vsd.addWorker(this);
}
protected override void myOnLastSubscriber()
{
_vsd.removeWorker(this);
_sub_core.unsubscribe(_tag_name);
}
private void _fire_dp_after_send()
{
// as soon as the data is published, its reflected in the stream state
Log.assert(_last_sent_msg != null);
BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value,
DBNull.Value, DBNull.Value);
_table.addRow(r);
var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
myFireEvent(dp_out);
}
private void _fire_dp_after_recv()
{
Log.assert(_last_recvd_msg != null);
// update the last publish row
int id = _table.Count - 1;
IBlueRow last_row = _table[id];
_table.deleteRow(last_row);
var recv_ts = epoch_now();
BlueRow nr = new BlueRow(_table.schema, id, _last_sent_msg.msg_id, _last_sent_msg.timestamp, _last_recvd_msg.msg_id, recv_ts,
String.Compare(_last_recvd_msg.msg_id, _last_sent_msg.msg_id) == 0, recv_ts - _last_recvd_msg.timestamp);
_table.addRow(nr);
var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
myFireEvent(dp_out);
}
private string _get_next_id()
{
return _table.Count.ToString();
}
private void _publish_next_packet()
{
var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte);
_last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes);
var ser_bytes = _last_sent_msg.serialize();
_n_pubs++;
_pub_core.publish("Blob", _tag_name, ser_bytes);
_last_publish_time = DateTime.Now;
}
public void doWork()
{
Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) =>
{
foreach (var u in updates)
{
var recv_msg = PayloadMessage.deserialize(u.data);
if (recv_msg.msg_id == _last_sent_msg.msg_id)
{
return recv_msg;
}
}
return null;
};
switch (_state)
{
case IOState.SEND:
if (DateTime.Now - _last_publish_time >= _publish_period)
{
_publish_next_packet();
_last_publish_time = DateTime.Now;
_fire_dp_after_send();
_state = IOState.RECV;
return;
}
break;
case IOState.RECV:
var updates = _sub_core.get_updates(TimeSpan.Zero);
updates = updates.Where(u => u.tag == _tag_name);
var recv_msg = find_packet(updates);
if (recv_msg != null)
{
_last_recvd_msg = recv_msg;
_fire_dp_after_recv();
_state = IOState.SEND;
break;
}
break;
}
}
public static long epoch_now()
{
return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
}
}
}
@@ -1,58 +0,0 @@
namespace bts {
using BCG;
using BCG.Utilities.General;
using System;
public class SimpleTCPEchoStream : ViewStream, IWorker {
private string _ip;
private int _port;
private SimpleTCPClient _tcp_client;
private IBlueTableSchema _schema = BlueDataUtility.form_schema("SimpleTCPEchoStream",
Tuple.Create("id", TypeCode.Int32),
Tuple.Create("event", TypeCode.String));
private BlueTable _table;
private SyncQueue<string> _action_queue = new SyncQueue<string>();
public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms)
: this(ip, port, msg_sz, send_wait_period_ms, Int32.MaxValue) { }
public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms, int max_n_msgs) {
_ip = ip;
_port = port;
_table = new BlueTable(_schema);
_tcp_client = new SimpleTCPClient(_ip, _port, Convert.ToUInt32(msg_sz), send_wait_period_ms, max_n_msgs);
}
protected override void myOnFirstSubscriber() {
ViewStreamDatabase.instance.addWorker(this);
_tcp_client.register_listener((msg) => { _on_msg(msg); });
_tcp_client.start();
}
private void _on_msg(string msg) {
_action_queue.enqueue(msg);
}
public void doWork() {
if (_action_queue.getCount() == 0)
return;
BlueTable added_rows = new BlueTable(_schema);
var sz = _action_queue.getCount();
for (int i = 0; i < sz; i++) {
string msg = _action_queue.dequeue();
if (msg == null)
break;
IBlueRow nr = new BlueRow(_schema, _table.Count, msg);
_table.addRow(nr);
added_rows.addRow(nr);
}
if (added_rows.Count == 0)
return;
DataPacket dp = new DataPacket(_table, added_rows, new BlueTable(_schema));
myFireEvent(dp);
}
}
}
@@ -1,77 +0,0 @@
using System.Net.Sockets;
using System.Net;
using BCG;
using System;
using System.Collections.Generic;
using MongoDB.Bson;
using BCG.Utilities.General;
using System.Linq;
namespace bts {
// this class, a little specialized, takes the first snapshot thats fired
// from the base stream sorts them by a given column, and fires one row at a time
public class SnapshotReplayStream : ViewStream, IWorker {
private ViewStreamDatabase _vsd;
private ViewStream _snapshot_stream;
private readonly string _sort_col;
private readonly int _buffer_size;
private TimeSpan _fire_period;
private int _in_row_ptr = 0;
private List<IBlueRow> _in_rows = new List<IBlueRow>();
private CircularBufferTable _out_table;
public SnapshotReplayStream(ViewStream base_str, string sort_col, int buffer_size, string fire_period)
: this(ViewStreamDatabase.instance, base_str, sort_col, buffer_size, TimeBasedFilter.parse_duration(fire_period)) { }
public SnapshotReplayStream(ViewStreamDatabase vsd, ViewStream base_str, string sort_col, int buffer_size, TimeSpan fire_period) {
_vsd = vsd;
_snapshot_stream = base_str;
_sort_col = sort_col;
_buffer_size = buffer_size;
_fire_period = fire_period;
}
protected override void myOnFirstSubscriber() {
_snapshot_stream.onChange += _handle_stream_update;
MasterWorker.getInstance(false).addWorker(this, _fire_period);
}
protected override void myOnLastSubscriber() {
base.myOnLastSubscriber();
}
private void _handle_stream_update(object dp_obj) {
DataPacket dp = dp_obj as DataPacket;
if (_in_rows.Count > 0)
return;
_out_table = new CircularBufferTable(dp.schema, _buffer_size);
// _in_rows is filled up only once
foreach (IBlueRow r in dp.newRows) {
_in_rows.Add(r);
}
_in_rows.Sort((IBlueRow r1, IBlueRow r2) => {
return BlueDataUtility.compare_rows(r1, r2, _sort_col);
});
}
public void doWork() {
if (_in_row_ptr >= _in_rows.Count) {
// we can loop around or not here
return;
}
if (_in_rows.Count > 0) {
// fire row
IBlueRow curr_row = _in_rows.ElementAt(_in_row_ptr++);
_out_table.add_row(curr_row);
DataPacket dp = BlueDataUtility.build_dp_inc(_out_table,
myLastValue == null ? null : myLastValue as DataPacket, false);
myFireEvent(dp);
}
}
}
}
@@ -1,809 +0,0 @@
using bts;
using System;
using System.Collections.Generic;
using System.IO;
using System.Reflection;
namespace BCG {
/// <summary>
/// This stream (not for prod) provides a way to run tests - that could otherwise be run in NUnits.
/// (truth is stream rpovide a RDE for quickly coding and running, so it was eaiser to run tests here)
/// </summary>
[DataStreamUserRespresentation("tests")]
public class TesStream : ViewStream {
static byte[] create_byte_array() {
Random random = new Random();
var sz = random.Next(1, 1000);// % 1000;
// Generate a random byte
byte randomByte = (byte)random.Next(byte.MinValue, byte.MaxValue + 1);
byte[] rv = new byte[sz];
for (int i = 0; i < sz; i++) {
rv[i] = (byte)random.Next(byte.MinValue, byte.MaxValue + 1);
}
return rv;
}
private IBlueTableSchema _schema;
class TestResult {
public string test_name = MethodBase.GetCurrentMethod()?.Name
?? throw new InvalidOperationException("Failed to get test name from method");
public bool test_result;
}
private List<Func<TestResult>> _tests = new
List<Func<TestResult>>();
public TesStream() {
_schema = BlueDataUtility.form_schema("Tests",
Tuple.Create("id", TypeCode.Int32),
Tuple.Create("test", TypeCode.String),
Tuple.Create("result", TypeCode.Boolean),
Tuple.Create("exception", TypeCode.String)
);
// register tests here
_tests.Add(new Func<TestResult>(_run_sub_msg_test));
_tests.Add(new Func<TestResult>(_run_pub_msg_test));
_tests.Add(new Func<TestResult>(_run_pubsub_server_msg_test));
_tests.Add(new Func<TestResult>(_run_pubsub_client_msg_test));
_tests.Add(new Func<TestResult>(_run_columninfo_serialization_test));
// _tests.Add(new Func<TestResult>(_run_schema_serialization_test));
// _tests.Add(new Func<TestResult>(_row_serialization_tests_no_dbnulls));
// _tests.Add(new Func<TestResult>(_row_serialization_tests_with_dbnulls));
//_tests.Add(new Func<TestResult>(_row_serialization_tests_all_types));
// _tests.Add(new Func<TestResult>(_run_full_stream_serialization_test));
// _tests.Add(new Func<TestResult>(_run_full_stream_serialization_with_controller_test));
// _tests.Add(new Func<TestResult>(_run_pubsub_server_stream_tag_handler_test));
// _tests.Add(new Func<TestResult>(_run_single_tag_data_test_clean));
//_tests.Add(new Func<TestResult>(_cpp_data_test));
//_tests.Add(new Func<TestResult>(_write_stream_updates_to_files));
_tests.Add(new Func<TestResult>(_deserialize_pub_msg_test));
}
protected override void myOnFirstSubscriber() {
BlueTable rows = new BlueTable(_schema);
// u the tests here
foreach (var test in _tests) {
string ex_msg = String.Empty;
TestResult tr = new TestResult() {
test_name = test.Method.Name,
test_result = false
};
try {
tr = test();
} catch (Exception ex) {
ex_msg = ex.Message + "\n" + ex.StackTrace;
}
IBlueRow r = new BlueRow(_schema, rows.Count, tr.test_name, tr.test_result, ex_msg);
rows.addRow(r);
}
var dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket, true);
myFireEvent(dp);
}
protected override void myOnLastSubscriber() {
}
private TestResult _run_sub_msg_test() {
bts.SubscribeMsg sub_msg = new bts.SubscribeMsg() {
connection = Guid.NewGuid().ToString(),
subscribing_app = Guid.NewGuid().ToString(),
tags = new string[] { "one", "two", "three" }
};
MemoryStream str = new MemoryStream();
sub_msg.serialize_into_str(str);
str.Position = 0;
bts.SubscribeMsg sub_msg_ = new bts.SubscribeMsg();
sub_msg_.consruct_deserialize(str);
bool res = sub_msg.Equals(sub_msg_);
return new TestResult {
test_name = "SubscribeMsg",
test_result = res
};
}
private TestResult _run_pub_msg_test() {
var pub_msg = new bts.PublishMsg() {
connection = Guid.NewGuid().ToString(),
publishing_app = Guid.NewGuid().ToString(),
tag = Guid.NewGuid().ToString(),
tag_type = "DS_",
is_compressed = false,
publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
data_to_publish = create_byte_array(),
};
MemoryStream str = new MemoryStream();
pub_msg.serialize_into_str(str);
str.Position = 0;
var pub_msg_ = new bts.PublishMsg();
pub_msg_.consruct_deserialize(str);
bool res = pub_msg.Equals(pub_msg_);
return new TestResult {
test_name = "PublishMsg",
test_result = res
};
}
private TestResult _run_pubsub_server_msg_test() {
var msg = new bts.PubSubServerMessage() {
tag = Guid.NewGuid().ToString(),
tag_type = Guid.NewGuid().ToString(),
data = create_byte_array()
};
MemoryStream str = new MemoryStream();
msg.serialize_into_str(str);
str.Position = 0;
var msg_ = new bts.PubSubServerMessage();
msg_.consruct_deserialize(str);
bool res = msg.Equals(msg_);
return new TestResult {
test_name = "PubSubServerMessage",
test_result = res
};
}
private TestResult _run_pubsub_client_msg_test() {
PubSubClientMessage msg = new PubSubClientMessage() {
msg_type = PubSubMessageType.SUBSCRIBE,
sub_msg = new bts.SubscribeMsg() {
connection = Guid.NewGuid().ToString(),
subscribing_app = Guid.NewGuid().ToString(),
tags = new string[] { "one", "two", "three" }
}
};
var str = new MemoryStream();
msg.serialize_into_str(str);
str.Position = 0;
var msg_ = new PubSubClientMessage();
msg_.consruct_deserialize(str);
var res = msg.Equals(msg_);
return new TestResult {
test_name = "PubSubClientMessage",
test_result = res
};
}
private TestResult _run_columninfo_serialization_test() {
Func<ColumnInfo, bool> test_ci = (ci) => {
MemoryStream str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
StreamSerializationUtils.serialize_column_info(ci, ref bw);
str.Position = 0;
//var ci_txt = BlueDataUtility.to_string(ci);
//var ci_ser =
//bts.Log.Gen.info("ci: {}")
BinaryReader br = new BinaryReader(str);
var ci_prime = StreamSerializationUtils.deserialize_column_info(ref br);
return BlueDataUtility.compute_equality(ci, ci_prime);
};
int i = 0;
foreach (Type t in BlueDataUtility.valid_column_types()) {
ColumnInfo ci = new ColumnInfo(Guid.NewGuid().ToString(),
i++, t);
var res = test_ci(ci);
if (!res) {
return new TestResult {
test_name = "ColumnInfoSerialization",
test_result = false
};
}
}
return new TestResult {
test_name = "ColumnInfoSerialization",
test_result = true
};
}
private TestResult _run_schema_serialization_test() {
var table_name = "test_table";
var excluded_types = new HashSet<Type>() { typeof(DateTime), typeof(Decimal) };
IBlueTableSchema schema = create_schema_with_all_types(excluded_types);
MemoryStream str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
StreamSerializationUtils.serialize_schema(schema, ref bw);
_write_to_file(str, @"c:\Temp\tests\schema.bin");
var hex_txt = StreamSerializationUtils.to_hex(str);
bts.Log.Gen.info($"_run_schema_serialization_test ser: {hex_txt}");
str.Position = 0;
BinaryReader br = new BinaryReader(str);
var schema_prime = StreamSerializationUtils.deserialize_schema(ref br);
var res = BlueDataUtility.compute_equality(schema, schema_prime);
return new TestResult {
test_name = "SchemaSerialization",
test_result = res
};
}
private TestResult _run_datetime_seriliaztion_test() {
DateTime dt = DateTime.Now;
DateTimeOffset dt_off = new DateTimeOffset(dt);
long dt_epoch_ms = dt_off.ToUnixTimeMilliseconds();
DateTimeOffset dt_off_pr = DateTimeOffset.FromUnixTimeMilliseconds(dt_epoch_ms);
DateTime dt_p = dt_off_pr.DateTime;
var passes = dt.Equals(dt_p);
return new TestResult {
test_name = "_run_datetime_seriliaztion_test",
test_result = false
};
}
private TestResult _run_full_stream_serialization_test() {
// create message
var schema = create_schema_with_all_types();
BlueTable cr_rows = new BlueTable(schema);
// 100 rows
int n_rows = 100;
for (int i = 0; i < n_rows; i++) {
var nr = generate_random_row(schema, i);
cr_rows.addRow(nr);
}
MemoryStream str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
bts.StreamIOMessage stream_io_msg = new bts.StreamIOMessage(cr_msg);
UniversalStreamIOSerializer ser = new UniversalStreamIOSerializer();
ser.serialize(stream_io_msg, ref bw);
str.Position = 0;
BinaryReader br = new System.IO.BinaryReader(str);
UniversalStreamIODeserializer deser = new bts.UniversalStreamIODeserializer();
var stream_io_msg_prime = deser.deserialize(ref br);
bool passes = stream_io_msg.Equals(stream_io_msg_prime);
var n_dels = 50;
BlueTable del_rows = new BlueTable(cr_rows.schema);
foreach (IBlueRow r in cr_rows) {
del_rows.addRow(r);
if (del_rows.Count == n_dels)
break;
}
IBlueRowCollection added_rows = del_rows;
bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
str = new MemoryStream();
bw = new BinaryWriter(str);
stream_io_msg = new bts.StreamIOMessage(mra_msg);
ser.serialize(stream_io_msg, ref bw);
bw.BaseStream.Position = 0;
br = new BinaryReader(str);
//deser = new UniversalStreamIODeserializer();
stream_io_msg_prime = deser.deserialize(ref br);
passes = passes && stream_io_msg.Equals(stream_io_msg_prime);
return new TestResult {
test_name = "UniversalSerialization",
test_result = passes
};
}
private TestResult _run_full_stream_serialization_with_controller_test() {
// create message
var schema = create_schema_with_all_types();
BlueTable cr_rows = new BlueTable(schema);
// 100 rows
int n_rows = 100;
for (int i = 0; i < n_rows; i++) {
var nr = generate_random_row(schema, i);
cr_rows.addRow(nr);
}
bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
bts.StreamIOMessage stream_io_msg = new bts.StreamIOMessage(cr_msg);
IStreamIOMsgSerializer ser = new StreamIOSerializationController();
var ser_bytes = ser.serialize(stream_io_msg);
IStreamMsgDeserilizer deser = new bts.StreamIODeserializationController();
var stream_io_msg_prime = deser.deserialize(ser_bytes);
bool passes = stream_io_msg.Equals(stream_io_msg_prime);
//mult remove adds
// remove all rows and add 50 more
BlueTable del_rows = new BlueTable(cr_rows);
BlueTable added_rows = new BlueTable(cr_rows.schema);
var n_adds = 50;
for (int i = 0; i < n_adds; i++) {
var nr = generate_random_row(schema, i);
added_rows.addRow(nr);
}
bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
stream_io_msg = new bts.StreamIOMessage(mra_msg);
ser_bytes = ser.serialize(stream_io_msg);
var ser_bytes_hex = StreamSerializationUtils.to_hex(ser_bytes);
Log.Gen.info($"ser_bytes: {ser_bytes_hex}");
stream_io_msg_prime = deser.deserialize(ser_bytes);
passes = passes && stream_io_msg.Equals(stream_io_msg_prime);
return new TestResult {
test_name = "UniversalSerializationController",
test_result = passes
};
}
private TestResult _row_serialization_tests_no_dbnulls() {
// IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
// new Tuple<string, TypeCode>("id", TypeCode.Int32),
// new Tuple<string, TypeCode>("i1", TypeCode.Int32),
// new Tuple<string, TypeCode>("d1", TypeCode.Double),
// new Tuple<string, TypeCode>("string_1", TypeCode.String));
// bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
// IBlueRow r = new BlueRow(schema, 0, 100, 123.45, "test_string");
var schema = create_schema_with_all_types(new HashSet<Type>() { typeof(DateTime), typeof(Decimal) });
bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
MemoryStream str = new MemoryStream();
IBlueRow r = generate_random_row(schema, 0, false);
BinaryWriter bw = new BinaryWriter(str);
row_ser.serialize_row(r, 0, ref bw);
_write_to_file(str, @"C:\Temp\tests\row.bin");
str.Position = 0;
byte[] raw_byte_array = str.ToArray();
var hex_txt = StreamSerializationUtils.to_hex(raw_byte_array);
Log.Gen.info($"TestStream._row_serialization_tests_no_dbnulls Row bytes: {hex_txt}");
BinaryReader br = new BinaryReader(str);
bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
IBlueRow r_prime = des.deserialize_row(ref br);
string row_str = r_prime.to_string();
Log.Gen.info($"TestStream._row_serialization_tests_no_dbnulls r_prime: {row_str}");
var passes = BlueDataUtility.compute_equality(r, r_prime);
return new TestResult {
test_name = "UniversalRowSerialization_NonNull",
test_result = passes
};
}
private TestResult _deserialize_pub_msg_test() {
var pub_msg_ser = _read_from_file(@"C:\Temp\tests\pubsub_raw_msgs\publish_msg.bin");
PubSubClientMessage ps_client_msg = new PubSubClientMessage();
ps_client_msg.consruct_deserialize(pub_msg_ser);
bool passes = ps_client_msg.msg_type == PubSubMessageType.PUBLISH;
return new TestResult {
test_name = "_deserialize_pub_msg_test",
test_result = passes
};
}
private TestResult _cpp_data_test() {
//create and write schema
IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
new Tuple<string, TypeCode>("id", TypeCode.Int32),
new Tuple<string, TypeCode>("i_1", TypeCode.Int32),
new Tuple<string, TypeCode>("i_2", TypeCode.Int32),
new Tuple<string, TypeCode>("d_1", TypeCode.Double),
new Tuple<string, TypeCode>("string_1", TypeCode.String)
);
bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
UniversalRowDeserializer row_deser = new UniversalRowDeserializer(schema);
StreamIOSerializationController ser_controller = new StreamIOSerializationController();
StreamIODeserializationController deser_controller = new StreamIODeserializationController();
IBlueTableSchema schema_p = null;
var schema_file = @"c:\Temp\tests\schema.bin";
var row_file = @"C:\Temp\tests\row.bin";
var create_op_file = @"C:\Temp\tests\create_op.bin";
var mra_op_file = @"C:\Temp\tests\mra_op.bin";
IBlueRow row = new BlueRow(schema, 0, 100, 200, 123.45, "test_string");
//serialize schema, write it to a file, read it back, deserialize it and match schemas
byte[] ser_schema_bytes = StreamSerializationUtils.serialize_schema(schema);
_write_to_file(ser_schema_bytes, schema_file);
byte[] schema_file_bytes = _read_from_file(schema_file);
schema_p = StreamSerializationUtils.deserialize_schema(schema_file_bytes);
bool passes = BlueDataUtility.compute_equality(schema, schema_p);
//serialize row, write it to a file, read it back, deserialize it and match schemas
byte[] ser_row_bytes = row_ser.serialize_row(row);
_write_to_file(ser_row_bytes, row_file);
byte[] row_file_bytes = _read_from_file(row_file);
var row_file_bytes_txt = StreamSerializationUtils.to_hex(row_file_bytes);
Log.Gen.info($"TestsStream._cpp_data_test row_file_bytes_txt: {row_file_bytes_txt}");
IBlueRow row_p = row_deser.deserialize_row(row_file_bytes);
passes = passes && BlueDataUtility.compute_equality(row, row_p);
{
// serialize a create operation
int n_rows = 100;
BlueTable table = new BlueTable(schema);
for (int i = 0; i < n_rows; i++) {
IBlueRow nr = new BlueRow(schema, i, 100, 200, 123.45, "test_string");
table.addRow(nr);
}
StreamIOMessage cr_msg = new StreamIOMessage(new StreamIOMessage.CreateMessage(table));
//StreamIOSerializationController ser_controller = new StreamIOSerializationController();
byte[] create_ser_bytes = ser_controller.serialize(cr_msg);
_write_to_file(create_ser_bytes, create_op_file);
// read it and make sure its right
byte[] create_op_file_bytes = _read_from_file(create_op_file);
//StreamIODeserializationController deser_controller = new StreamIODeserializationController();
var cr_msg_p = deser_controller.deserialize(create_op_file_bytes);
_write_to_file(create_ser_bytes, mra_op_file);
passes = passes && cr_msg.Equals(cr_msg_p);
}
//multiple remove adds
{
// remove all rows and add them again
int n_rows = 100;
BlueTable table = new BlueTable(schema);
for (int i = 0; i < n_rows; i++) {
IBlueRow nr = new BlueRow(schema, i, 100, 200, 123.45, "test_string");
table.addRow(nr);
}
StreamIOMessage mra_msg = new StreamIOMessage(new StreamIOMessage.MultipleRemoveAddMessage(table, table));
//StreamIOSerializationController ser_controller = new StreamIOSerializationController();
byte[] mra_ser_bytes = ser_controller.serialize(mra_msg);
_write_to_file(mra_ser_bytes, mra_op_file);
// reasd it and make sure it matches
byte[] mra_op_file_bytes = _read_from_file(mra_op_file);
//StreamIODeserializationController deser_controller = new StreamIODeserializationController();
StreamIOMessage mra_msg_p = deser_controller.deserialize(mra_op_file_bytes);
passes = passes && mra_msg.Equals(mra_msg_p);
}
return new TestResult {
test_name = "_cpp_data_test",
test_result = passes
};
}
private TestResult _write_stream_updates_to_files() {
IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
new Tuple<string, TypeCode>("id", TypeCode.Int32),
new Tuple<string, TypeCode>("i_1", TypeCode.Int32),
new Tuple<string, TypeCode>("i_2", TypeCode.Int32),
new Tuple<string, TypeCode>("d_1", TypeCode.Double),
new Tuple<string, TypeCode>("string_1", TypeCode.String)
);
var packets_path = @"C:\Temp\tests\stream_packets\";
Func<long> now_epochs = () => {
DateTimeOffset now = DateTimeOffset.UtcNow;
// Convert to Unix time in milliseconds
long unix_time_milliseconds = now.ToUnixTimeMilliseconds();
return unix_time_milliseconds;
};
Func<int, bool, string> curr_file_path = (id, create) => {
//var epoch_ms = now_epochs();
var file_path = $"{packets_path}{id}.bin";
if (create)
file_path = $"{packets_path}create_{id}.bin";
return file_path;
};
// write 1000 stream updates
{
int init_sz = 100;
BlueTable table = new BlueTable(schema);
StreamIOSerializationController ser_controller = new StreamIOSerializationController();
// create table of 100 rows
for (int i = 0; i < init_sz; i++) {
IBlueRow nr = new BlueRow(schema, i, i * 100, i * 200, i * 123.45, "test_string");
table.addRow(nr);
}
// write create msg
StreamIOMessage cr_msg = new StreamIOMessage(new StreamIOMessage.CreateMessage(table));
byte[] create_ser_bytes = ser_controller.serialize(cr_msg);
var create_file_path = curr_file_path(table.Count, true);
_write_to_file(create_ser_bytes, create_file_path);
// add 1000 rows and write an io msg for each update and write it to a file
int n_updates = 1000;
int period_ms = 50;
// write a create
for (int i = 0; i < n_updates; i++) {
// addition
BlueTable adds = new BlueTable(schema);
int id = table.Count;
IBlueRow nr = new BlueRow(schema, id, i * 100, i * 200, i * 123.45, "test_string");
adds.addRow(nr);
BlueTable dels = new BlueTable(schema);
table.addRow(nr);
StreamIOMessage mra_msg = new StreamIOMessage(new StreamIOMessage.MultipleRemoveAddMessage(dels, adds));
byte[] mra_ser_bytes = ser_controller.serialize(mra_msg);
var mra_file_path = curr_file_path(id, false);
_write_to_file(mra_ser_bytes, mra_file_path);
System.Threading.Thread.Sleep(TimeSpan.FromMilliseconds(period_ms));
}
// now start writing increments and add 1 row at a time
}
return new TestResult {
test_name = "_write_updates_as_files",
test_result = true
};
}
private TestResult _row_serialization_tests_with_dbnulls() {
IBlueTableSchema schema = BlueDataUtility.form_schema("test_table",
new Tuple<string, TypeCode>("id", TypeCode.Int32),
new Tuple<string, TypeCode>("i1", TypeCode.Int32),
new Tuple<string, TypeCode>("d1", TypeCode.Double),
new Tuple<string, TypeCode>("string_1", TypeCode.String));
bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
IBlueRow r = new BlueRow(schema, 0, DBNull.Value, 123.45, "test_string");
MemoryStream str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
row_ser.serialize_row(r, 0, ref bw);
str.Position = 0;
BinaryReader br = new BinaryReader(str);
bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
IBlueRow r_prime = des.deserialize_row(ref br);
var passes = BlueDataUtility.compute_equality(r, r_prime);
return new TestResult {
test_name = "UniversalRowSerialization_WithDBNulls",
test_result = passes
};
}
private TestResult _row_serialization_tests_all_types() {
var schema = create_schema_with_all_types();
bts.UniversalRowSerializer row_ser = new bts.UniversalRowSerializer(schema);
IBlueRow r = generate_random_row(schema, 0, false);
MemoryStream str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
row_ser.serialize_row(r, 0, ref bw);
_write_to_file(str, @"C:\Temp\tests\row.bin");
str.Position = 0;
BinaryReader br = new BinaryReader(str);
bts.UniversalRowDeserializer des = new UniversalRowDeserializer(schema);
IBlueRow r_prime = des.deserialize_row(ref br);
var passes = BlueDataUtility.compute_equality(r, r_prime);
return new TestResult {
test_name = "UniversalRowSerialization_AllTypes",
test_result = passes
};
}
private static TestResult _run_pubsub_server_stream_tag_handler_test() {
// create message
var schema = create_schema_with_all_types();
var tag = "DS_" + schema.tableName;
BlueTable cr_rows = new BlueTable(schema);
// 100 rows
int n_rows = 100;
for (int i = 0; i < n_rows; i++) {
var nr = generate_random_row(schema, i);
cr_rows.addRow(nr);
}
PubsubServerSingleStreamTagHandler handler = new PubsubServerSingleStreamTagHandler();
bts.StreamIOMessage.CreateMessage cr_msg = new bts.StreamIOMessage.CreateMessage(cr_rows);
bts.StreamIOMessage str_io_msg = new bts.StreamIOMessage(cr_msg);
IStreamIOMsgSerializer ser = new StreamIOSerializationController();
byte[] msg_bytes = ser.serialize(str_io_msg);
handler.update(tag, msg_bytes);
//now get it back for the handler
byte[] content = handler.get_content(tag);
IStreamMsgDeserilizer deser = new StreamIODeserializationController();
bts.StreamIOMessage str_io_msg_p = deser.deserialize(content);
// test here
var passes = str_io_msg.Equals(str_io_msg_p);
// delete all rows
BlueTable del_rows = new BlueTable(cr_rows);
cr_rows.clear();
// added 50 new rows
BlueTable added_rows = new BlueTable(del_rows.schema);
int new_state_sz = 50;
for (int i = 0; i < new_state_sz; i++) {
var nr = generate_random_row(schema, i);
added_rows.addRow(nr);
}
cr_rows.addRows(added_rows);
// the sequence of operations are ops, ser, update, get_content, deser, maake sure the get_content state is good
// 1. generate op
bts.StreamIOMessage.MultipleRemoveAddMessage mra = new bts.StreamIOMessage.MultipleRemoveAddMessage(del_rows, added_rows);
str_io_msg = new bts.StreamIOMessage(mra);
// 2. serialize it into bytes
msg_bytes = ser.serialize(str_io_msg);
// 3. send it to handler
handler.update(tag, msg_bytes);
// 4. get the state from the handler
byte[] content_p = handler.get_content(tag);
// 5. deserialize the state into a stream op
str_io_msg_p = deser.deserialize(content_p);
// 6. check to make sure the state's rows are good
passes = passes && str_io_msg_p.op == bts.StreamIOMessage.StreamOp.CREATE &&
BlueDataUtility.compute_equality(cr_rows, str_io_msg_p.create_message.rows);
return new TestResult {
test_result = passes,
test_name = "run_pubsub_server_stream_tag_handler_test"
};
}
private static TestResult _run_single_tag_data_test_clean() {
// create message
var test_name = MethodBase.GetCurrentMethod().Name;
var schema = create_schema_with_all_types();
var tag = "DS_" + schema.tableName;
BlueTable table = new BlueTable(schema);
IStreamIOMsgSerializer serializer = new StreamIOSerializationController();
IStreamMsgDeserilizer deser = new StreamIODeserializationController();
bts.SingleTagData st_data = new bts.SingleTagData(tag, DSTagManager.DataStreamTagType);
// create with 100 rows
int n_rows = 100;
bool is_succ = true;
for (int i = 0; i < n_rows; i++) {
var nr = generate_random_row(schema, i);
// perform the same action SingletagData, and verify the states match
PublishMsg pub_msg = build_pub_msg(nr, null, table.Count == 0, ref serializer);
table.addRow(nr);
st_data.handle_publish(pub_msg);
var table_p = get_rows_from_single_tag_state(ref st_data, ref deser);
is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p);
}
// change all rows
for (int i = 0; i < n_rows; i++) {
var nr = generate_random_row(schema, i);
var dr = table[i];
table.deleteRow(dr);
table.addRow(nr);
PublishMsg pub_msg = build_pub_msg(nr, dr, false, ref serializer);
st_data.handle_publish(pub_msg);
var table_p = get_rows_from_single_tag_state(ref st_data, ref deser);
is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p);
}
// delete all rows
var aux_table = new BlueTable(table);
foreach (IBlueRow r in aux_table) {
table.deleteRow(r);
PublishMsg pub_msg = build_pub_msg(null, r, false, ref serializer);
st_data.handle_publish(pub_msg);
var table_p = get_rows_from_single_tag_state(ref st_data, ref deser);
is_succ = is_succ && BlueDataUtility.compute_equality(table, table_p);
}
return new TestResult() {
test_result = is_succ,
test_name = test_name
};
}
#region Helper functions
private static void _write_to_file(byte[] ser_bytes, string file_name) {
_write_to_file(new MemoryStream(ser_bytes), file_name);
}
private static void _write_to_file(MemoryStream str, string file_name) {
if (String.IsNullOrEmpty(file_name))
return;
byte[] str_bytes = str.ToArray();
using (FileStream fs = new FileStream(file_name, FileMode.Create)) {
// Write the byte array to the file
fs.Write(str_bytes, 0, str_bytes.Length);
}
}
private static byte[] _read_from_file(string file_path) {
var rv = File.ReadAllBytes(file_path);
return rv;
}
public static IBlueRowCollection get_rows_from_single_tag_state(ref SingleTagData st_data, ref IStreamMsgDeserilizer deser) {
var content = st_data.get_current_content();
var msg_p = deser.deserialize(content);
Log.assert(msg_p.op == bts.StreamIOMessage.StreamOp.CREATE);
return msg_p.create_message.rows;
}
public static PublishMsg build_pub_msg(IBlueRow added_row, IBlueRow removed_row, bool is_create,
ref IStreamIOMsgSerializer serializer) {
Func<IBlueTableSchema> get_schema = () => {
return (added_row != null ? added_row : removed_row).schema;
};
bts.StreamIOMessage str_io_msg = null;
if (is_create) {
Log.assert(added_row != null);
var aux_new = new BlueTable(added_row.schema);
aux_new.addRow(added_row);
var cr_msg = new bts.StreamIOMessage.CreateMessage(aux_new);
str_io_msg = new bts.StreamIOMessage(cr_msg);
} else {
var schema = get_schema();
BlueTable added_table = new BlueTable(schema);
if (added_row != null)
added_table.addRow(added_row);
BlueTable rem_table = new BlueTable(schema);
if (removed_row != null)
rem_table.addRow(removed_row);
bts.StreamIOMessage.MultipleRemoveAddMessage mra_msg =
new bts.StreamIOMessage.MultipleRemoveAddMessage(rem_table, added_table);
str_io_msg = new bts.StreamIOMessage(mra_msg);
}
byte[] msg_bytes = serializer.serialize(str_io_msg);
PublishMsg pub_msg = new PublishMsg() {
connection = "test_connection",
tag = $"DS_{get_schema()}",
tag_type = "DS",
data_to_publish = msg_bytes
};
return pub_msg;
}
public static IBlueTableSchema create_schema_with_all_types(HashSet<Type> excluded_col_map = null) {
excluded_col_map = excluded_col_map == null ? new HashSet<Type>() : excluded_col_map;
var table_name = "test_table";
BlueTableSchema schema = new BlueTableSchema();
List<ColumnInfo> cis = new List<ColumnInfo>();
foreach (Type t in BlueDataUtility.valid_column_types()) {
if (t == null || t == typeof(DateTime) || excluded_col_map.Contains(t))
continue;
var col_name = t.ToString() + "_" + cis.Count.ToString(); // Guid.NewGuid().ToString();
ColumnInfo ci = new ColumnInfo(col_name, cis.Count, t);
cis.Add(ci);
}
cis.Add(new ColumnInfo("id", cis.Count, typeof(int)));
schema = new BlueTableSchema(table_name, cis.ToArray());
return schema;
}
public static IBlueRow generate_random_row(IBlueTableSchema schema, int id, bool include_dbnulls = true) {
Random rnd = new Random();
object[] row_vals = new object[schema.columnInfo.Length];
foreach (ColumnInfo ci in schema.columnInfo) {
if (ci.colName == "id") {
row_vals[ci.colNo] = id;
continue;
}
object v = get_random_object(ci.type_code());
if (include_dbnulls && rnd.NextSingle() % 4 == 0)
v = DBNull.Value;
row_vals[ci.colNo] = v;
}
return new BlueRow(schema, row_vals);
}
public static object get_random_object(TypeCode type_code) {
Random random = new Random();
switch (type_code) {
case TypeCode.Boolean:
return random.Next(2) == 0; // Random boolean value (true or false)
case TypeCode.Byte:
return (byte)random.Next(byte.MinValue, byte.MaxValue + 1); // Random byte value
case TypeCode.Char:
return (char)random.Next(65, 65 + 27); // Random char value
case TypeCode.Decimal:
Decimal dec = 100.123M;
return dec;
case TypeCode.DateTime:
return DateTime.Now;
case TypeCode.Double:
return random.NextDouble() * (double.MaxValue - double.MinValue) + double.MinValue; // Random double value
case TypeCode.Int16:
return (short)random.Next(short.MinValue, short.MaxValue + 1); // Random short value
case TypeCode.Int32:
return random.Next(int.MinValue, int.MaxValue); // Random int value
case TypeCode.Int64:
return random.NextInt64(long.MinValue, long.MaxValue); // Random long value
case TypeCode.SByte:
return (sbyte)random.Next(sbyte.MinValue, sbyte.MaxValue + 1); // Random sbyte value
case TypeCode.Single:
return (float)(random.NextDouble() * (float.MaxValue - float.MinValue) + float.MinValue); // Random float value
case TypeCode.String:
return Guid.NewGuid().ToString();
case TypeCode.UInt16:
return (ushort)random.Next(ushort.MinValue, ushort.MaxValue + 1); // Random ushort value
case TypeCode.UInt32:
return (uint)random.Next(int.MinValue, int.MaxValue) + (uint)int.MaxValue; // Random uint value
case TypeCode.UInt64:
byte[] ulongBuffer = new byte[8];
random.NextBytes(ulongBuffer);
return BitConverter.ToUInt64(ulongBuffer, 0);
default:
throw new ArgumentException($"Unsupported TypeCode: {type_code}");
return null;
}
}
#endregion
}
}
@@ -1,31 +0,0 @@
namespace BCG {
using System;
public class TickingTestStream : ViewStream {
public TickingTestStream(int ticking_period_msecs)
:
this(ViewStreamDatabase.instance, TimeSpan.FromSeconds(ticking_period_msecs)) { }
public TickingTestStream(ViewStreamDatabase vsd, TimeSpan ticking_period_msecs) {
ViewStream vs = new DisplayableMillisecondStream(Convert.ToInt32(ticking_period_msecs.TotalMilliseconds), 10);
vs = new MathStream(vs, "x=id*123 y=x*x");
mySetRealSource(vs);
//vs.onChange += _on_str_update;
}
// private void _on_str_update(object dp_obj) {
// DataPacket dp = dp_obj as DataPacket;
// IBlueTableSchema out_schema = BlueDataUtility.add_column(dp.schema, "time_txt", typeof(string));
// BlueTable state = new BlueTable(out_schema);
// foreach (IBlueRow r in dp.newRows) {
// MutableBlueRow nr = new MutableBlueRow(out_schema, r);
// DateTime time = r.get_datetime("time");
// var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
// nr.setValue("time_txt", time_txt);
// state.addRow(nr);
// }
// var dp_out = BlueDataUtility.build_dp_inc(state, myLastValue == null ? null : myLastValue as DataPacket);
// myFireEvent(dp_out);
// }
}
}
@@ -1,8 +1,9 @@
using Google.Protobuf.WellKnownTypes;
using System; using System;
using BCG.Utilities.General;
using BCG; namespace BCG {
namespace bts.SilverPine {
// Keeps adding changing rows to the stream, hence preserving the history // Keeps adding changing rows to the stream, hence preserving the history
// and creating a time series, for example this can be used to make bbo updates // and creating a time series, for example this can be used to make bbo updates
// into a time series of prices/qty // into a time series of prices/qty
@@ -40,6 +41,7 @@ namespace bts.SilverPine {
} }
_table.addRows(added_rows); _table.addRows(added_rows);
myFireEvent(new DataPacket(_table, added_rows, new BlueTable(_table.schema))); myFireEvent(new DataPacket(_table, added_rows, new BlueTable(_table.schema)));
} }
} }
@@ -1,194 +0,0 @@
namespace Bts.DataStream;
using BCG;
using bts;
using Bts.DataStreamCore;
using mbts;
using MongoDB.Bson;
using MongoDB.Driver;
using Spark.Core.Disposables;
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Reactive.Subjects;
using System.Threading;
using System.Threading.Tasks;
public class MongoCollectionStream : ViewStream, IObservable<DataPacket> {
readonly string collectionName;
readonly string databaseName;
readonly string url;
readonly object syncRoot = new();
readonly CancellationTokenSource cts = new();
readonly Subject<DataPacket> subject = new();
bool hasStarted = false;
IBlueRowFactory<BsonDocument>? rowFactory;
MongoClient? clientBacking;
IMongoDatabase? databaseBacking;
IMongoCollection<BsonDocument>? collectionBacking = null;
BlueTableImmutable? table = null;
public MongoCollectionStream(string collectionName)
: this(
collectionName,
Environment.GetEnvironmentVariable("MONGO_DB_NAME") ?? string.Empty,
Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) {
}
public MongoCollectionStream(string collectionName, string databaseName)
: this(
collectionName,
databaseName,
Environment.GetEnvironmentVariable("MONGO_DB_URL") ?? string.Empty) {
}
public MongoCollectionStream(string collectionName, string databaseName, string url)
: this(collectionName, databaseName, url, null) { }
internal MongoCollectionStream(
string collectionName, string databaseName, string url, IBlueRowFactory<BsonDocument>? rowFactory) {
this.collectionName = collectionName;
this.databaseName = databaseName;
this.url = url;
this.rowFactory = rowFactory;
mySetRealSource(new ObservableViewStream(this));
}
MongoClient client => clientBacking ??= new(url);
IMongoDatabase database => databaseBacking ??= client.GetDatabase(databaseName);
IMongoCollection<BsonDocument>? collection {
get => Volatile.Read(ref collectionBacking);
set => Volatile.Write(ref collectionBacking, value);
}
public IDisposable Subscribe(IObserver<DataPacket> observer) {
lock (syncRoot) {
if (hasStarted) {
if (table is not null) {
// The first update has already been fired. Seed observer w/ current state.
observer.OnNext(new DataPacket(table, table, new BlueTable(table.schema)));
}
return subject.Subscribe(observer);
}
hasStarted = true;
}
var subscription = subject.Subscribe(observer);
// Retrieve collection snapshot to bootstrap sequence
initTable().ignore();
// Watch changes
Task.Run(watchChanges);
return Disposable.Create(Dispose);
void Dispose() {
Log.Gen.info("Disposing MongoDbChangeObservable subscription");
subscription.Dispose();
cts.Cancel();
cts.Dispose();
}
}
protected virtual IBlueRowFactory<BsonDocument> getRowFactory(IEnumerable<BsonDocument> docs) =>
MongoSchemaAdaptor.InferSchema(docs);
async Task watchChanges() {
Log.Gen.info(
$"MongoCollectionStream. Watching changes. thread: {Environment.CurrentManagedThreadId}");
while (collection is null) {
await Task.Delay(100);
}
Log.notnull(collection);
using var cursor = collection.Watch(new ChangeStreamOptions {
FullDocument = ChangeStreamFullDocumentOption.UpdateLookup
});
try {
while (await cursor.MoveNextAsync(cts.Token)) {
/// Ignore while waiting for init.
lock (syncRoot) {
/// Note: <see cref="rowFactory"/> and <see cref="table"/> are initialized together
/// in the <see cref="initTable"/> fn since both depend on the schema, which is
/// inferred from the first incoming BsonDocument(s).
if (table is null || rowFactory is null) {
continue;
}
}
var bd = table.ToDataPacketBuilder();
foreach (var ch in cursor.Current) {
var opDescJson = ch.OperationDescription?.ToJson() ?? string.Empty;
var updDescJson = ch.UpdateDescription?.ToJson() ?? string.Empty;
Log.Gen.info(
$"MongoCollectionStream.watchChanges. Doc changed. type: {ch.OperationType} " +
$"changes:\n\t\t{updDescJson}");
switch (ch.OperationType) {
case ChangeStreamOperationType.Insert: {
bd.AddOrUpdate(Row());
break;
}
case ChangeStreamOperationType.Update: {
bd.Remove(OldRow().Id());
bd.AddOrUpdate(Row());
break;
}
case ChangeStreamOperationType.Replace: {
bd.Remove(OldRow().Id());
bd.AddOrUpdate(Row());
break;
}
case ChangeStreamOperationType.Delete: {
bd.Remove(OldRow().Id());
break;
}
default:
Log.assert(false, $"Unsupported op type '{ch.OperationType}'");
break;
}
IBlueRow Row() =>
rowFactory.GetBlueRow(ch.FullDocument);
IBlueRow OldRow() =>
rowFactory.GetBlueRow(ch.FullDocumentBeforeChange);
}
var dp = bd.ToDataPacket();
if (!dp.HasChanges()) {
Log.Gen.warn(
$"MongoCollectionStream. Ignoring update because it didn't change any rows.");
continue;
}
lock (syncRoot) {
table = (BlueTableImmutable)dp.newRows;
subject.OnNext(dp);
}
}
} catch (TaskCanceledException ex) {
Log.Gen.warn(
$"MongoCollectionStream. 'Watch' task cancelled (this is expected if subscription was disposed). {ex}");
}
}
[MemberNotNull(nameof(collection))]
async Task initTable() {
collection = database.GetCollection<BsonDocument>(collectionName);
var cursor = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
var docList = await cursor.ToListAsync();
rowFactory ??= getRowFactory(docList);
var firstTable = new BlueTableImmutable(rowFactory.Schema).ToDataPacketBuilder();
foreach (var doc in docList) {
var row = rowFactory.GetBlueRow(doc);
firstTable.AddOrUpdate(row);
}
var dp = firstTable.ToDataPacket();
lock (syncRoot) {
table = (BlueTableImmutable)dp.newRows;
subject.OnNext(dp);
}
}
}
@@ -1,133 +1,32 @@
using BCG;
using BCG.Utilities.General;
using MongoDB.Bson;
using MongoDB.Driver;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using MongoDB.Driver;
using System.Threading.Tasks; using MongoDB.Bson;
using System.Threading; using BCG;
using BCG.Utilities.General;
namespace bts { using ZstdSharp.Unsafe;
// [DataStreamUserRespresentation("mongodb_sql")] using System.Runtime.CompilerServices;
// public class MongoDBSQLStream : ViewStream, IWorker { using mbts.spark;
// private ViewStreamDatabase _vsd;
// private readonly string _connection_string, _db_name, _collection_name;
// private MongoClient _client;
// // make this a sorted table
// private BlueTable _table;
// // Access the database namespace BCG {
// private IMongoDatabase _database; [DataStreamUserRespresentation("mongodb_sql")]
// private IDictionary<string, int> _id_map = new Dictionary<string, int>(); public class MongoDBSQLStream : ViewStream, IWorker {
// // mongodb://192.168.1.80:27017
// public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
// : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
// }
// public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
// bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
// $"db_name: {db_name}");
// _connection_string = connection_string;
// _db_name = db_name;
// _collection_name = collection_name;
// _vsd = vsd;
// _client = new MongoClient(_connection_string);
// _database = _client.GetDatabase(_db_name);
// }
// protected override void myOnFirstSubscriber() {
// MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
// }
// protected override void myOnLastSubscriber() {
// MasterWorker.getInstance(false).removeWorker(this);
// }
// public void doWork() {
// _read_mogodb();
// }
// private void _read_mogodb() {
// var timer = Stopwatch.StartNew();
// // Access the collection
// IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
// // Fetch all documents from the collection
// var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
// _table = _table == null ? null : new BlueTable(_table.schema);
// // Iterate through the documents and print values
// foreach (var document in documents.ToEnumerable()) {
// bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
// if (_table == null) {
// _init_table(document);
// }
// _add_row(document);
// }
// timer.Stop();
// Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms");
// timer.Restart();
// var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
// timer.Stop();
// Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms");
// if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
// return;
// myFireEvent(dp);
// }
// private void _init_table(BsonDocument doc_row) {
// bts.Log.assert(_table == null);
// List<ColumnInfo> cis = new List<ColumnInfo>();
// cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
// foreach (var n in doc_row.Names) {
// cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
// }
// IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
// _table = new BlueTable(schema);
// }
// private void _add_row(BsonDocument doc) {
// var key_val = doc["_id"];
// var key = key_val.ToString();
// int id;
// if (!_id_map.TryGetValue(key, out id)) {
// id = _id_map.Count;
// _id_map.Add(key, id);
// }
// MutableBlueRow nr = new MutableBlueRow(_table.schema);
// nr.setValue("id", id);
// foreach (var n in doc.Names) {
// var val = doc[n].ToString();
// nr.setValue(n, val);
// }
// _table.addRow(nr);
// }
// }
[DataStreamUserRespresentation("mongodb")]
public class MongoDBStream : ViewStream, IWorker {
private ViewStreamDatabase _vsd; private ViewStreamDatabase _vsd;
private readonly string _connection_string, _db_name, _collection_name; private readonly string _connection_string, _db_name, _collection_name;
private readonly TimeSpan _poll_period;
private MongoClient _client; private MongoClient _client;
private BlueTable _table; private BlueTable _table;
private SyncQueue<DataPacket> _table_queue = new SyncQueue<DataPacket>();
private BlockingQueue _work_queue = new BlockingQueue();
// Access the database // Access the database
private IMongoDatabase _database; private IMongoDatabase _database;
private IDictionary<string, int> _id_map = new Dictionary<string, int>(); private IDictionary<string, int> _id_map = new Dictionary<string, int>();
// mongodb://192.168.1.80:27017 // mongodb://192.168.1.80:27017
public MongoDBStream(string connection_string, string db_name, string collection_name, string poll_period) public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
: this(connection_string, db_name, collection_name, TimeBasedFilter.parse_duration(poll_period), ViewStreamDatabase.instance) { : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
} }
public MongoDBStream(string connection_string, string db_name, string collection_name, TimeSpan poll_period, ViewStreamDatabase vsd) { public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " + bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
$"db_name: {db_name}"); $"db_name: {db_name}");
_connection_string = connection_string; _connection_string = connection_string;
@@ -135,34 +34,24 @@ namespace bts {
_collection_name = collection_name; _collection_name = collection_name;
_vsd = vsd; _vsd = vsd;
_client = new MongoClient(_connection_string); _client = new MongoClient(_connection_string);
_poll_period = poll_period;
_database = _client.GetDatabase(_db_name); _database = _client.GetDatabase(_db_name);
} }
protected override void myOnFirstSubscriber() { protected override void myOnFirstSubscriber() {
_vsd.addWorker(this); MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
//MasterWorker.getInstance(false).addWorker(this, _poll_period);
} }
protected override void myOnLastSubscriber() { protected override void myOnLastSubscriber() {
_vsd.removeWorker(this); MasterWorker.getInstance(false).removeWorker(this);
//MasterWorker.getInstance(false).removeWorker(this);
} }
private DateTime _last_poll_time = DateTime.MinValue;
public void doWork() { public void doWork() {
var now = DateTime.Now; _read_mogodb();
if (now - _last_poll_time > _poll_period) {
ThreadPool.QueueUserWorkItem(_ => _read_mogodb());
_last_poll_time = DateTime.Now;
}
var dp = _table_queue.dequeue(0);
if (dp != null)
myFireEvent(dp);
} }
private void _read_mogodb() { private void _read_mogodb() {
var timer = Stopwatch.StartNew(); HighPerformanceTimer timer = new HighPerformanceTimer();
timer.start();
// Access the collection // Access the collection
IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name); IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
@@ -171,22 +60,117 @@ namespace bts {
_table = _table == null ? null : new BlueTable(_table.schema); _table = _table == null ? null : new BlueTable(_table.schema);
// Iterate through the documents and print values // Iterate through the documents and print values
foreach (var document in documents.ToEnumerable()) { foreach (var document in documents.ToEnumerable()) {
//bts.Log.Gen.info($"document: {document} json: {document.ToJson()}"); bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
if (_table == null) {
_init_table(document);
}
_add_row(document);
}
timer.stop();
bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
timer.start();
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
timer.stop();
bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
return;
myFireEvent(dp);
}
private void _init_table(BsonDocument doc_row) {
bts.Log.assert(_table == null);
List<ColumnInfo> cis = new List<ColumnInfo>();
cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
foreach (var n in doc_row.Names) {
cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
}
IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
_table = new BlueTable(schema);
}
private void _add_row(BsonDocument doc) {
var key_val = doc["_id"];
var key = key_val.ToString();
int id;
if (!_id_map.TryGetValue(key, out id)) {
id = _id_map.Count;
_id_map.Add(key, id);
}
MutableBlueRow nr = new MutableBlueRow(_table.schema);
nr.setValue("id", id);
foreach (var n in doc.Names) {
var val = doc[n].ToString();
nr.setValue(n, val);
}
_table.addRow(nr);
}
}
[DataStreamUserRespresentation("mongodb")]
public class MongoDBStream : ViewStream, IWorker {
private ViewStreamDatabase _vsd;
private readonly string _connection_string, _db_name, _collection_name;
private MongoClient _client;
private BlueTable _table;
// Access the database
private IMongoDatabase _database;
private IDictionary<string, int> _id_map = new Dictionary<string, int>();
// mongodb://192.168.1.80:27017
public MongoDBStream(string connection_string, string db_name, string collection_name)
: this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
}
public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
$"db_name: {db_name}");
_connection_string = connection_string;
_db_name = db_name;
_collection_name = collection_name;
_vsd = vsd;
_client = new MongoClient(_connection_string);
_database = _client.GetDatabase(_db_name);
}
protected override void myOnFirstSubscriber() {
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
}
protected override void myOnLastSubscriber() {
MasterWorker.getInstance(false).removeWorker(this);
}
public void doWork() {
_read_mogodb();
}
private void _read_mogodb() {
HighPerformanceTimer timer = new HighPerformanceTimer();
timer.start();
// Access the collection
IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
// Fetch all documents from the collection
var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
_table = _table == null ? null : new BlueTable(_table.schema);
// Iterate through the documents and print values
foreach (var document in documents.ToEnumerable()) {
bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
if (_table == null) { if (_table == null) {
_init_table(document); _init_table(document);
} }
_add_doc(document); _add_doc(document);
} }
timer.Stop(); timer.stop();
Log.Gen.info($"MongoDBStream time to read {_collection_name}: {timer.ElapsedMilliseconds}ms"); bts.Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.lastIntervalTime} msecs");
timer.Restart(); timer.start();
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true); var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
timer.Stop(); timer.stop();
Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms"); ; bts.Log.Gen.info($"MongoDBStream time to compare tables :{timer.lastIntervalTime} msecs");
if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0) if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
return; return;
_table_queue.enqueue(dp); myFireEvent(dp);
//myFireEvent(dp);
} }
private void _init_table(BsonDocument doc_row) { private void _init_table(BsonDocument doc_row) {
@@ -225,46 +209,4 @@ namespace bts {
} }
} }
// [DataStreamUserRespresentation("mongodb_flat")] }
// public class MongoDBFlatStream : ViewStream {
// public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration)
// :
// this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { }
// public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) {
// ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd);
// rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val");
// rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601);
// //rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration);
// mySetRealSource(rs);
// }
// }
// [DataStreamUserRespresentation("newsreplay")]
// public class NewsReplayStream : ViewStream {
// public NewsReplayStream(string connection_string, string db_name, string collection_name,
// int buffer_sz, string fire_period) :
// this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { }
// public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name,
// int buffer_sz, TimeSpan fire_period) {
// ViewStream r;
// r = new MongoDBStream(connection_string, db_name, collection_name, vsd);
// r = new KeyValueToRelationalStream(r,
// "doc_id",
// "key",
// "value");
// r = new DateStandardizationStream(r,
// "created_at",
// "ISO8601");
// r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period);
// mySetRealSource(r);
// }
// }
}
@@ -1,63 +0,0 @@
namespace Bts.DataStream;
using MongoDB.Bson;
using System;
public static class MongoDbExtensions {
public static BsonArray GetArray(this BsonDocument doc, string elementName) =>
doc[elementName].AsBsonArray;
public static BsonArray? GetArrayOrDefault(this BsonDocument doc, string elementName) =>
doc.TryGetValue(elementName, out var val) && val.IsBsonArray ?
val.AsBsonArray : null;
public static string GetString(this BsonDocument doc, string elementName) =>
doc[elementName].AsString;
public static string? GetStringOrDefault(this BsonDocument doc, string elementName) =>
doc.TryGetValue(elementName, out var val) && val.IsString ?
val.AsString : null;
public static int GetInt32(this BsonDocument doc, string elementName) =>
doc[elementName].AsInt32;
public static int? GetInt32OrDefault(this BsonDocument doc, string elementName) =>
doc.TryGetValue(elementName, out var val) && val.IsInt32 ?
val.AsInt32 : null;
public static long GetInt64(this BsonDocument doc, string elementName) =>
doc[elementName].AsInt64;
public static long? GetInt64OrDefault(this BsonDocument doc, string elementName) =>
doc.TryGetValue(elementName, out var val) && val.IsInt64 ?
val.AsInt64 : null;
public static double GetDouble(this BsonDocument doc, string elementName) =>
doc[elementName].AsDouble;
public static double? GetDoubleOrDefault(this BsonDocument doc, string elementName) =>
doc.TryGetValue(elementName, out var val) && val.IsDouble ?
val.AsDouble : null;
public static DateTime GetDateTime(this BsonDocument doc, string elementName) =>
doc[elementName].ToUniversalTime();
public static DateTime? GetDateTimeOrDefault(this BsonDocument doc, string elementName) =>
doc.TryGetValue(elementName, out var val) && val.IsValidDateTime ?
val.ToUniversalTime() : null;
public static DateTime GetTimestamp(this BsonDocument doc, string elementName) =>
doc[elementName].ToUniversalTime();
public static DateTime? GetTimestampOrDefault(this BsonDocument doc, string elementName) =>
doc.TryGetValue(elementName, out var val) && val.IsValidDateTime ?
val.ToUniversalTime() : null;
public static ObjectId GetObjectId(this BsonDocument doc, string elementName) =>
doc[elementName].AsObjectId;
public static ObjectId? GetObjectIdOrDefault(this BsonDocument doc, string elementName) =>
doc.TryGetValue(elementName, out var val) && val.IsObjectId ?
val.AsObjectId : null;
}
@@ -1,55 +0,0 @@
namespace Bts.DataStream;
using bts;
using MongoDB.Bson;
using System;
using System.Collections.Generic;
public static class MongoSchemaAdaptor {
public static BlueTableSchemaAdaptor<BsonDocument> InferSchema(IEnumerable<BsonDocument> docs) {
var schemaAdaptor = new BlueTableSchemaAdaptor<BsonDocument>("MongoCollectionStream");
foreach (var doc in docs) {
foreach (var el in doc) {
if (schemaAdaptor.Contains(el.Name)) {
continue;
}
switch (el.Value.BsonType) {
case BsonType.Array:
schemaAdaptor.AddColumn(el.Name, d => d.GetArrayOrDefault(el.Name)?.ToString() ?? string.Empty);
break;
case BsonType.String:
schemaAdaptor.AddColumn(el.Name, d => d.GetStringOrDefault(el.Name) ?? string.Empty);
break;
case BsonType.Int32:
schemaAdaptor.AddColumn(el.Name, d => d.GetInt32OrDefault(el.Name));
break;
case BsonType.Int64:
schemaAdaptor.AddColumn(el.Name, d => d.GetInt64OrDefault(el.Name));
break;
case BsonType.Double:
schemaAdaptor.AddColumn(el.Name, d => d.GetDoubleOrDefault(el.Name));
break;
case BsonType.DateTime:
schemaAdaptor.AddColumn(el.Name, d => d.GetDateTimeOrDefault(el.Name));
break;
case BsonType.Timestamp:
schemaAdaptor.AddColumn(el.Name, d => d.GetTimestampOrDefault(el.Name));
break;
case BsonType.ObjectId:
schemaAdaptor.AddColumn(el.Name, d => d.GetObjectIdOrDefault(el.Name)?.ToString());
break;
case BsonType.Null:
Log.Gen.warn(
$"Ignoring column '{el.Name}' because at least one element is null");
break;
default:
throw new InvalidOperationException(
$"Unsupported BsonType '{el.Value.BsonType}'");
}
}
}
return schemaAdaptor;
}
}
@@ -1,202 +0,0 @@
namespace bts {
using BCG;
using ProtoBuf;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
public class ProjectData {
// schema from the base stream
public readonly IBlueTableSchema in_schema;
// this is the projected schema
public readonly IBlueTableSchema out_schema;
// projected cols to the input cols, for cols who's names are changed
public readonly IDictionary<string, string> mapped_to_orig_col_map = new Dictionary<string, string>();
public ProjectData(IBlueTableSchema in_schema_, IBlueTableSchema out_schema_,
IDictionary<string, string> mapped_to_orig_col_map_) {
in_schema = in_schema_;
out_schema = out_schema_;
mapped_to_orig_col_map = mapped_to_orig_col_map_;
}
}
public class NewProjectRow : IBlueRow {
#region IBlueRow members
private ProjectData _project_data;
private IBlueRow _row;
public NewProjectRow(ProjectData project_data, IBlueRow row) {
_project_data = project_data;
_row = row;
}
public IBlueTableSchema schema { get { return _project_data.out_schema; } }
#endregion
#region ICollection members
public int Count { get { return schema.columnNames.Length; } }
public bool IsSynchronized { get { return false; } }
public object SyncRoot { get { return this; } }
public void CopyTo(Array array, int index) {
int i = index;
foreach (object o in this as IEnumerable)
array.SetValue(o, i++);
}
#endregion
#region IDictionary members
public bool IsFixedSize { get { return true; } }
public bool IsReadOnly { get { return true; } }
public object this[object key] {
get {
if (key is string)
return this[(string)key];
return this[(int)key];
}
set { throw new NotSupportedException("The IBlueRow is readonly"); }
}
public object this[int index] {
get {
return this[schema.columnInfo[index].colName];
}
}
public object this[string col_name] {
get {
string mapped_to_col = null;
if (_project_data.mapped_to_orig_col_map.TryGetValue(col_name, out mapped_to_col)) {
return _row[mapped_to_col];
}
return _row[col_name];
}
}
public ICollection Keys { get { return schema.columnNames; } }
public ICollection Values {
get {
object[] retVal = new object[schema.columnNames.Length];
CopyTo(retVal, 0);
return retVal;
}
}
public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); }
public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); }
public bool Contains(object key) { return schema[(string)key] != null; }
/// <summary>
/// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
/// iterate through the values, typecast this into an IEnumerable
/// </summary>
/// <returns></returns>
public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); }
IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); }
public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
#endregion
}
public class NewProjectStream : ViewStream, IStreamMapper {
private ViewStream _stream;
private string _project_expression;
private ProjectData _project_data = null;
//public ProjectData
public NewProjectStream(ViewStream vs, string project_expression) {
_stream = vs;
_project_expression = project_expression;
_stream = new MapStream(vs, this);
mySetRealSource(_stream);
}
public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) {
_project_data = _create_project_data(input_schema, _project_expression);
return _project_data.out_schema;
}
public IBlueRow map(IBlueRow row) {
return new NewProjectRow(_project_data, row);
}
private static ProjectData _create_project_data(IBlueTableSchema input_schema,
string project_expression) {
project_expression = project_expression.Trim();
string[] project_toks = project_expression.Split(' ');
HashSet<string> removed_cols = new HashSet<string>();
Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>();
Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
// by default every col will show up, so if we have the projection as (a:a1)
// it will convert a to a1,
// examples: if a schema has cols a b c
// project(parent_str, "a:aa ") -> aa b c
// project(parent_str, "a:aa !c") -> aa b
// project(parent_str, "a:aa !") -> aa
// project(parent_str, "a:aa b = (aa*aa)!") -> aa b
bool hide_all = false;
foreach (string p_tok in project_toks) {
if (p_tok.StartsWith("!")) {
if (p_tok.Length == 1) {
// if it is a "blanket "!"=> hide all cols except the one that
// are stated explicitly
hide_all = true;
} else {
string removed_col = p_tok.Substring(1);
removed_cols.Add(removed_col);
}
} else if (p_tok.Contains(":")) {
string[] map_toks = p_tok.Split(":");
string orig_col = map_toks[0];
string mapped_col = map_toks[1];
mapped_to_orig_map.Add(mapped_col, orig_col);
orig_to_mapped.Add(orig_col, mapped_col);
} else {
if (input_schema.columnNames.Contains(p_tok)) {
string orig_col = p_tok;
string mapped_col = orig_col;
orig_to_mapped.Add(orig_col, mapped_col);
}
}
}
// make the new schema
List<ColumnInfo> new_cols = new List<ColumnInfo>();
foreach (ColumnInfo ci in input_schema.columnInfo) {
if (ci.colName == "id") {
ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
new_cols.Add(nc);
continue;
}
if (removed_cols.Contains(ci.colName)) {
continue;
} else if (orig_to_mapped.ContainsKey(ci.colName)) {
string mapped_name = orig_to_mapped[ci.colName];
ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
new_cols.Add(nc);
} else {
if (hide_all)
continue;
//if (ci.colName == "id" || removed_cols.Count > 0) {
// unless columns are explicitely hidden, all columns show up
ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
new_cols.Add(nc);
//}
}
}
IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
return new ProjectData(input_schema, new_schema, mapped_to_orig_map);
}
}
}
@@ -1,17 +1,20 @@
namespace BCG { namespace BCG {
using mbts.spark;
using Spark.Core.Collections; using Spark.Core.Collections;
using System;
using System.Collections; using System.Collections;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
[DataStreamUserRespresentation("oldproject")]
public class ProjectStream : ViewStream public class ProjectStream : ViewStream
{ {
private ProjectHelper myHelper; private ProjectHelper myHelper;
private string myColExp, myTableName; private string myColExp, myTableName;
private ViewStream myVs; private ViewStream myVs;
private IDictionary<string, string> myDic; private IDictionary<string, string> myDic;
private HighPerformanceTimer _timer = new();
/// <summary> /// <summary>
/// </summary> /// </summary>
/// <param name="vs">stream to susbsribe to as the input stream</param> /// <param name="vs">stream to susbsribe to as the input stream</param>
@@ -67,8 +70,10 @@ namespace BCG {
DataPacket dp = newValue as DataPacket; DataPacket dp = newValue as DataPacket;
if (myHelper != null) if (myHelper != null)
{ {
_timer.start();
IBlueRowCollection dels, adds; IBlueRowCollection dels, adds;
BlueDataUtility.getTrueDeletesAndAdds (dp.deletedRows, dp.addedRows, out dels, out adds, myHelper.relevantOrigCols); BlueDataUtility.getTrueDeletesAndAdds (dp.deletedRows, dp.addedRows, out dels, out adds, myHelper.relevantOrigCols);
_timer.stop();
if (dels.Count + adds.Count > 0) if (dels.Count + adds.Count > 0)
myFireEvent (new DataPacket (new ProjectedTable (dp.newRows, myHelper), myFireEvent (new DataPacket (new ProjectedTable (dp.newRows, myHelper),
new ProjectedTable (adds, myHelper), new ProjectedTable (adds, myHelper),
@@ -1,122 +0,0 @@
using System;
using System.Data;
using System.Data.SQLite;
using System.Diagnostics;
using System.Collections;
using BCG;
using System.Linq;
using System.Data.Entity;
using System.Collections.Generic;
using System.Data.SQLite;
namespace bts.SilverPine {
public static class SQLiteHelper {
public static string location = "C:\\Users\\m_tah\\development\\data\\";
public static string db_name = "stream_data.db";
public static DataTable execute_select_query(string connection_string, string query) {
// Create a new DataTable to store the results
DataTable dataTable = new DataTable();
// Create and open a SQLite connection using the provided connection string
using (SQLiteConnection connection = new SQLiteConnection(connection_string)) {
try {
connection.Open();
// Create a SQLite command with the provided query
using (SQLiteCommand command = new SQLiteCommand(query, connection)) {
// Use SQLiteDataAdapter to fill the DataTable
using (SQLiteDataAdapter adapter = new SQLiteDataAdapter(command)) {
adapter.Fill(dataTable);
}
}
} catch (Exception ex) {
Console.WriteLine("An error occurred: " + ex.Message);
}
}
// Return the populated DataTable
return dataTable;
}
public static List<Dictionary<string, object>> ExecuteBulkRead(string connectionString, string query) {
var result = new List<Dictionary<string, object>>();
using (var connection = new SQLiteConnection(connectionString)) {
connection.Open();
// Begin a transaction to optimize read speed
using (var transaction = connection.BeginTransaction())
using (var command = new SQLiteCommand(query, connection, transaction))
using (var reader = command.ExecuteReader()) {
while (reader.Read()) {
var row = new Dictionary<string, object>();
for (int i = 0; i < reader.FieldCount; i++) {
string columnName = reader.GetName(i);
object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
row[columnName] = columnValue;
}
result.Add(row);
}
// Commit the transaction
transaction.Commit();
}
}
return result;
}
public static List<Dictionary<string, object>> get_table(string connectionString, string query) {
var result = new List<Dictionary<string, object>>();
using (var connection = new SQLiteConnection(connectionString)) {
try {
connection.Open();
using (var command = new SQLiteCommand(query, connection))
using (var reader = command.ExecuteReader()) {
while (reader.Read()) {
var row = new Dictionary<string, object>();
for (int i = 0; i < reader.FieldCount; i++) {
string columnName = reader.GetName(i);
object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
row[columnName] = columnValue;
}
result.Add(row);
}
}
} catch (Exception ex) {
Console.WriteLine("An error occurred: " + ex.Message);
}
}
return result;
}
public static IEnumerable<Dictionary<string, object>> get_table_2(string connectionString, string query) {
using (var connection = new SQLiteConnection(connectionString)) {
connection.Open();
using (var command = new SQLiteCommand(query, connection))
using (var reader = command.ExecuteReader()) {
while (reader.Read()) {
var row = new Dictionary<string, object>();
for (int i = 0; i < reader.FieldCount; i++) {
string columnName = reader.GetName(i);
object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
row[columnName] = columnValue;
}
yield return row;
}
}
}
}
}
}
@@ -1,73 +0,0 @@
namespace bts.SilverPine {
using System;
using System.Data;
using System.Data.SQLite;
using System.Diagnostics;
using System.Collections;
using BCG;
[DataStreamUserRespresentation("SQLite")]
public class SQLiteStream : ViewStream {
private readonly string _connection_string, _query;
private Stopwatch _stopwatch = new Stopwatch();
private SQLiteStream(string connection_string, string query, bool noop) {
_connection_string = connection_string;
_query = query;
}
public SQLiteStream(string db_name, string query)
:
this($"Data Source={SQLiteHelper.location}{db_name};Version=3", query, true) { }
public SQLiteStream(string query)
:
this(SQLiteHelper.db_name, query) {
}
private void _read_from_sqlite_and_fire() {
_stopwatch.Start();
DataTable dt = SQLiteHelper.execute_select_query(_connection_string, _query);
_stopwatch.Stop();
Log.Gen.info($"SQLiteStream.ExecuteSelectQuery time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
_stopwatch.Restart();
#region perf measures
// SQLiteHelper.get_table(_connection_string, _query);
// _stopwatch.Stop();
// Log.Gen.info($"SQLiteStream.get_table time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
// _stopwatch.Restart();
// SQLiteHelper.ExecuteBulkRead(_connection_string, _query);
// _stopwatch.Stop();
// Log.Gen.info($"SQLiteStream.ExecuteBulkRead time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
// _stopwatch.Restart();
// SQLiteHelper.get_table_2(_connection_string, _query);
// _stopwatch.Stop();
// Log.Gen.info($"SQLiteStream.get_table_2 time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
// _stopwatch.Restart();
#endregion
IBlueRowCollection rows = BCG.BlueDataUtility.toIBlueRowCollection(dt);
//IBlueRowCollection rows = new DataTableWrapperTable(dt);
_stopwatch.Stop();
Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to load into IBlueRowCollection: {_stopwatch.ElapsedMilliseconds} msecs");
DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
myFireEvent(dp);
}
protected override void myOnFirstSubscriber() {
_read_from_sqlite_and_fire();
}
protected override void myOnLastSubscriber() {
// this value will be cleared by the base class
// but in this particular stream this behaviour
// is not usefull since any new subscribers
// would force the query to be recreated
//myLastDP = myLastValue;
}
}
}
@@ -3,15 +3,12 @@ using System.Collections.Generic;
using System.Text; using System.Text;
using System.IO; using System.IO;
using BCG.Utilities; using BCG.Utilities;
using System.Diagnostics;
using Microsoft.Data.Analysis;
using Parquet;
using Parquet.Data;
namespace BCG
namespace BCG { {
[DataStreamUserRespresentation("CSVSaver")] [DataStreamUserRespresentation("CSVSaver")]
public class CSVSaverStream : ViewStream, ISideEffectStream { public class CSVSaverStream : ViewStream, ISideEffectStream
{
private IBlueTableSchema myResultSchema = new BlueTableSchema("CSVResult", new ColumnInfo("id", 0, typeof(int)), private IBlueTableSchema myResultSchema = new BlueTableSchema("CSVResult", new ColumnInfo("id", 0, typeof(int)),
new ColumnInfo("fileName", 1, typeof(string)), new ColumnInfo("fileName", 1, typeof(string)),
new ColumnInfo("nWrites", 2, typeof(int)), new ColumnInfo("nWrites", 2, typeof(int)),
@@ -23,9 +20,11 @@ namespace BCG {
private int myNWrites = 0, myNMaxWrites = Int32.MaxValue; private int myNWrites = 0, myNMaxWrites = Int32.MaxValue;
public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName) public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName)
: this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false) { } : this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false)
{ }
public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot) { public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot)
{
myStream = vs; myStream = vs;
myFullyQualifiedFileName = fullyQualifiedFileName; myFullyQualifiedFileName = fullyQualifiedFileName;
myOverwrite = overwrite; myOverwrite = overwrite;
@@ -37,14 +36,18 @@ namespace BCG {
protected override void myOnLastSubscriber() { myStream.onChange -= new ChangeDelegate(myHandleChanges); } protected override void myOnLastSubscriber() { myStream.onChange -= new ChangeDelegate(myHandleChanges); }
private void myHandleChanges(object newValue) { private void myHandleChanges(object newValue)
{
IBlueRowCollection totalState = (newValue as DataPacket).newRows; IBlueRowCollection totalState = (newValue as DataPacket).newRows;
if (totalState.Count > 0 && myNWrites < myNMaxWrites) { if (totalState.Count > 0 && myNWrites < myNMaxWrites)
{
myNWrites++; myNWrites++;
int i = 0; int i = 0;
StreamWriter writer = new StreamWriter(new FileStream(myGetNewFileName(), FileMode.Create, FileAccess.Write)); StreamWriter writer = new StreamWriter(new FileStream(myGetNewFileName(), FileMode.Create, FileAccess.Write));
foreach (IBlueRow r in totalState) { foreach (IBlueRow r in totalState)
if (i++ == 0) { {
if (i++ == 0)
{
string header = String.Empty; string header = String.Empty;
foreach (string col in r.schema.columnNames) foreach (string col in r.schema.columnNames)
if (col != "id") if (col != "id")
@@ -56,17 +59,20 @@ namespace BCG {
writer.Flush(); writer.Flush();
} }
writer.Close(); writer.Close();
if (myFireResultSnapshot) { if (myFireResultSnapshot)
{
IBlueRow nr = new BlueRow(myResultSchema, 0, myFullyQualifiedFileName, myNWrites, totalState.Count); IBlueRow nr = new BlueRow(myResultSchema, 0, myFullyQualifiedFileName, myNWrites, totalState.Count);
BlueTable nRows = new BlueTable(myResultSchema); BlueTable nRows = new BlueTable(myResultSchema);
nRows.addRow(nr); nRows.addRow(nr);
myFireEvent(new DataPacket(nRows, nRows, new BlueTable(myLastValue == null ? new BlueTable(myResultSchema) : (myLastValue as DataPacket).newRows))); myFireEvent(new DataPacket(nRows, nRows, new BlueTable(myLastValue == null ? new BlueTable(myResultSchema) : (myLastValue as DataPacket).newRows)));
} else }
else
myFireEvent(newValue); myFireEvent(newValue);
} }
} }
private string myGetNewFileName() { private string myGetNewFileName()
{
string[] tags = myFullyQualifiedFileName.Split('.'); string[] tags = myFullyQualifiedFileName.Split('.');
if (myOverwrite) if (myOverwrite)
return tags[0] + ".csv"; return tags[0] + ".csv";
@@ -74,11 +80,14 @@ namespace BCG {
return tags[0] + "_" + myFormateDateTime(DateTime.Now) + ".csv"; return tags[0] + "_" + myFormateDateTime(DateTime.Now) + ".csv";
} }
private static string myGetCSVRow(IBlueRow r) { private static string myGetCSVRow(IBlueRow r)
{
string rv = String.Empty; string rv = String.Empty;
int i = 0; int i = 0;
foreach (string col in r.schema.columnNames) { foreach (string col in r.schema.columnNames)
if (col != "id") { {
if (col != "id")
{
string colText = String.Empty; string colText = String.Empty;
if (r[col] != null && r[col] != DBNull.Value) if (r[col] != null && r[col] != DBNull.Value)
colText = r[col].ToString(); colText = r[col].ToString();
@@ -90,7 +99,8 @@ namespace BCG {
return rv; return rv;
} }
private string myFormateDateTime(DateTime dt) { private string myFormateDateTime(DateTime dt)
{
return return
string.Format("{0}-{1}-{2}-{3}-{4}-{5}-{6}", string.Format("{0}-{1}-{2}-{3}-{4}-{5}-{6}",
dt.Year.ToString("0000"), dt.Year.ToString("0000"),
@@ -104,49 +114,52 @@ namespace BCG {
} }
[DataStreamUserRespresentation("CSVReader")] [DataStreamUserRespresentation("CSVReader")]
public class CSVReaderStream : ViewStream { public class CSVReaderStream : ViewStream
private readonly string _file_address; {
Stopwatch _stopwatch = new Stopwatch(); private IBlueRowCollection myRows;
public CSVReaderStream(string fileAddress) {
_file_address = fileAddress; public CSVReaderStream(string fileAddress)
{
myRows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(fileAddress);
} }
protected override void myOnFirstSubscriber() { protected override void myOnFirstSubscriber()
_stopwatch.Start(); {
DataFrame df = DataFrame.LoadCsv(_file_address); myFireEvent(new DataPacket(myRows, myRows, new BlueTable(myRows.schema)));
_stopwatch.Stop();
bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load col_count: {df.Columns.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
_stopwatch.Start();
IBlueRowCollection rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(_file_address);
_stopwatch.Stop();
bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load count: {rows.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
DataPacket dp_out = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
myFireEvent(dp_out);
} }
} }
[DataStreamUserRespresentation("CSVReaderStreamMapper")] [DataStreamUserRespresentation ("CSVReaderStreamMapper")]
public class CSVReaderStreamMapperStream : ViewStream { public class CSVReaderStreamMapperStream : ViewStream
public CSVReaderStreamMapperStream(ViewStream stream, string output_schema) { {
public CSVReaderStreamMapperStream(ViewStream stream, string output_schema)
{
mySetRealSource(new MapStream(stream, new CSVReaderStreamMapper(output_schema))); mySetRealSource(new MapStream(stream, new CSVReaderStreamMapper(output_schema)));
} }
} }
public class CSVReaderStreamMapper : IStreamMapper { public class CSVReaderStreamMapper : IStreamMapper
{
private IBlueTableSchema mySchema; private IBlueTableSchema mySchema;
private readonly string[] myOutputSchemaStringArray; private readonly string[] myOutputSchemaStringArray;
public CSVReaderStreamMapper(string outputSchema) { public CSVReaderStreamMapper(string outputSchema)
{
myOutputSchemaStringArray = outputSchema.Split(' '); myOutputSchemaStringArray = outputSchema.Split(' ');
} }
public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) { public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
if (mySchema == null) { {
if (mySchema == null)
{
ColumnInfo[] ci = inputSchema.columnInfo; ColumnInfo[] ci = inputSchema.columnInfo;
foreach (string col in myOutputSchemaStringArray) { foreach (string col in myOutputSchemaStringArray)
{
string colName = col.Substring(0, col.IndexOf('\\')).ToLower(); string colName = col.Substring(0, col.IndexOf('\\')).ToLower();
for (int i = 0; i < ci.Length; i++) { for (int i = 0; i < ci.Length; i++)
if (ci[i].colName.ToLower().Equals(colName)) { {
if (ci[i].colName.ToLower().Equals(colName))
{
Type t = myGetType(col.Substring(col.IndexOf('\\') + 1)); Type t = myGetType(col.Substring(col.IndexOf('\\') + 1));
ci[i] = new ColumnInfo(ci[i].colName, i, t); ci[i] = new ColumnInfo(ci[i].colName, i, t);
break; break;
@@ -158,52 +171,74 @@ namespace BCG {
return mySchema; return mySchema;
} }
public IBlueRow map(IBlueRow row) { public IBlueRow map(IBlueRow row)
{
object[] vals = new object[row.schema.columnInfo.Length]; object[] vals = new object[row.schema.columnInfo.Length];
row.Values.CopyTo(vals, 0); row.Values.CopyTo(vals, 0);
for (int i = 0; i < mySchema.columnInfo.Length; i++) { for (int i = 0; i < mySchema.columnInfo.Length; i++)
{
Type t = mySchema.columnInfo[i].colType; Type t = mySchema.columnInfo[i].colType;
try { try
{
vals[i] = System.Convert.ChangeType(vals[i], mySchema.columnInfo[i].colType, System.Globalization.CultureInfo.InvariantCulture); vals[i] = System.Convert.ChangeType(vals[i], mySchema.columnInfo[i].colType, System.Globalization.CultureInfo.InvariantCulture);
} catch { }
if (t.Equals(typeof(Int16))) { catch
{
if (t.Equals(typeof(Int16)))
{
Int16 result; Int16 result;
if (Int16.TryParse((string)vals[i], out result)) { if (Int16.TryParse((string)vals[i], out result))
{
vals[i] = result; vals[i] = result;
continue; continue;
} }
double r2; double r2;
if (Double.TryParse((string)vals[i], out r2)) { if (Double.TryParse((string)vals[i], out r2))
try { {
try
{
vals[i] = (Int16)r2; vals[i] = (Int16)r2;
continue; continue;
} catch { } }
catch { }
} }
} else if (t.Equals(typeof(Int32))) { }
else if (t.Equals(typeof(Int32)))
{
Int32 result; Int32 result;
if (Int32.TryParse((string)vals[i], out result)) { if (Int32.TryParse((string)vals[i], out result))
{
vals[i] = result; vals[i] = result;
continue; continue;
} }
double r2; double r2;
if (Double.TryParse((string)vals[i], out r2)) { if (Double.TryParse((string)vals[i], out r2))
try { {
try
{
vals[i] = (Int32)r2; vals[i] = (Int32)r2;
continue; continue;
} catch { } }
catch { }
} }
} else if (t.Equals(typeof(Int64))) { }
else if (t.Equals(typeof(Int64)))
{
Int64 result; Int64 result;
if (Int64.TryParse((string)vals[i], out result)) { if (Int64.TryParse((string)vals[i], out result))
{
vals[i] = result; vals[i] = result;
continue; continue;
} }
double r2; double r2;
if (Double.TryParse((string)vals[i], out r2)) { if (Double.TryParse((string)vals[i], out r2))
try { {
try
{
vals[i] = (Int64)r2; vals[i] = (Int64)r2;
continue; continue;
} catch { } }
catch { }
} }
} }
vals[i] = DBNull.Value; vals[i] = DBNull.Value;
@@ -212,7 +247,8 @@ namespace BCG {
return new BlueRow(mySchema, vals); return new BlueRow(mySchema, vals);
} }
private Type myGetType(string type) { private Type myGetType(string type)
{
type = type.ToLower(); type = type.ToLower();
if (type.Contains("int")) if (type.Contains("int"))
return typeof(int); return typeof(int);
@@ -24,6 +24,7 @@ namespace bts {
var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket); var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
myFireEvent(dp_out); myFireEvent(dp_out);
} }
@@ -69,14 +69,18 @@ public class SortRow : IComparable, IBlueRow {
/// The key can either be the string corrsponding to the column name or /// The key can either be the string corrsponding to the column name or
/// the order of the column /// the order of the column
/// </summary> /// </summary>
public object? this[object key] { public object this[object key] {
get => key is string str ? this[str] : this[(int)key]; get {
set => throw new NotSupportedException("The IBlueRow is readonly"); if (key is string)
return this[(string)key];
return this[(int)key];
}
set { throw new NotSupportedException("The IBlueRow is readonly"); }
} }
public object? this[int index] => row[index]; public object this[int index] => row[index];
public object? this[string colName] => row[colName]; public object this[string colName] => row[colName];
public ICollection Keys => row.Keys; public ICollection Keys => row.Keys;
@@ -1,41 +1,31 @@
using System; using System;
using System.Collections; using System.Collections;
using System.Reflection; using System.Reflection;
using BCG.Utilities; using BCG.Utilities;
namespace BCG { namespace BCG
// if a table needs to be fired, it can be wrapped around this {
public class BlueTableStream : ViewStream { [DataStreamUserRespresentation (false)]
private IBlueRowCollection _rows; public class BlueRowCollectionStaticStream : ViewStream
public BlueTableStream(IBlueRowCollection rows) { {
_rows = rows; public BlueRowCollectionStaticStream () {}
public BlueRowCollectionStaticStream (IBlueRowCollection rows) {loadCollection (rows);}
public void loadCollection (IBlueRowCollection rows)
{
myFireEvent (new DataPacket (rows, rows, myLastValue != null ?
new BlueTable((myLastValue as DataPacket).newRows) : new BlueTable (rows.schema)));
} }
protected override void myOnFirstSubscriber() { public void crush_with_single_row(IBlueRow r)
var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket); {
myFireEvent(dp_out);
}
}
[DataStreamUserRespresentation(false)]
public class BlueRowCollectionStaticStream : ViewStream {
public BlueRowCollectionStaticStream() { }
public BlueRowCollectionStaticStream(IBlueRowCollection rows) { loadCollection(rows); }
public void loadCollection(IBlueRowCollection rows) {
myFireEvent(new DataPacket(rows, rows, myLastValue != null ?
new BlueTable((myLastValue as DataPacket).newRows) : new BlueTable(rows.schema)));
}
public void crush_with_single_row(IBlueRow r) {
BlueTable t = new BlueTable(r.schema); BlueTable t = new BlueTable(r.schema);
t.addRow(r); t.addRow(r);
loadCollection(t); loadCollection(t);
} }
public void loadPacket(DataPacket dp) { myFireEvent(dp); } public void loadPacket (DataPacket dp) {myFireEvent (dp);}
/// <summary> /// <summary>
/// Will work with streams that take ViewStream as first argument ex project, filter, distinct etc /// Will work with streams that take ViewStream as first argument ex project, filter, distinct etc
@@ -45,16 +35,17 @@ namespace BCG {
/// <param name="streamType"></param> /// <param name="streamType"></param>
/// <param name="args"></param> /// <param name="args"></param>
/// <returns></returns> /// <returns></returns>
public static IBlueRowCollection statelessOp(IBlueRowCollection rows, Type streamType, params object[] args) { public static IBlueRowCollection statelessOp (IBlueRowCollection rows, Type streamType, params object[] args)
ArrayList totArgs = new ArrayList(args); {
BlueRowCollectionStaticStream v1 = new BlueRowCollectionStaticStream(); ArrayList totArgs = new ArrayList (args);
totArgs.Insert(0, v1); BlueRowCollectionStaticStream v1 = new BlueRowCollectionStaticStream ();
ViewStream retVal = Activator.CreateInstance(streamType, totArgs.ToArray()) as ViewStream; totArgs.Insert (0, v1);
v1.loadCollection(rows); ViewStream retVal = Activator.CreateInstance (streamType, totArgs.ToArray()) as ViewStream;
v1.loadCollection (rows);
return retVal.rows; return retVal.rows;
} }
} }
} }
@@ -22,12 +22,9 @@ namespace BCG
/// <param name="rows"></param> /// <param name="rows"></param>
public void loadCollection (IBlueRowCollection rows) public void loadCollection (IBlueRowCollection rows)
{ {
myFireEvent(new DataPacket( myFireEvent (new DataPacket (rows, new BlueTable(rows),
rows, myLastValue != null ?
new BlueTable(rows), new BlueTable((myLastValue as DataPacket).newRows) : new BlueTable (rows.schema)));
myLastValue is DataPacket dp
? new BlueTable(dp.newRows)
: new BlueTable(rows.schema)));
} }
/// <summary> /// <summary>
@@ -123,7 +123,7 @@ namespace BCG
public Guid subscription; public Guid subscription;
} }
private ThreadSafeQueue<Update> _queued_updates; private ThreadSafeQueue<Update> _queued_updates;
TargetDoWorkRelay? _do_work_relay; TargetDoWorkRelay _do_work_relay;
Guid _current_source_subscription; Guid _current_source_subscription;
Guid _current_target_subscription; Guid _current_target_subscription;
int _source_thread; int _source_thread;
@@ -3,23 +3,24 @@ using System.Collections.Generic;
using System.Text; using System.Text;
namespace BCG { namespace BCG
{
/// <summary> /// <summary>
/// If one adds annother value to this enum, be sure to update _date_conversion_to_type_map below as well /// If one adds annother value to this enum, be sure to update _date_conversion_to_type_map below as well
/// </summary> /// </summary>
public enum DateConversion { public enum DateConversion
{
ConvertDoubleToDateTime = 0, ConvertDoubleToDateTime = 0,
ChopTime, ChopTime,
UTCToCurrent, UTCToCurrent,
ConvertStringToDateTime, ConvertStringToDateTime,
ConvertDateTimeToMilliseconds, ConvertDateTimeToMilliseconds,
SecondsOfDay, SecondsOfDay
ISO8601,
EpochToDatetime
} }
[DataStreamUserRespresentation("dateStandardize")] [DataStreamUserRespresentation("dateStandardize")]
public class DateStandardizationStream : ViewStream, IStreamMapper { public class DateStandardizationStream : ViewStream, IStreamMapper
{
private IBlueTableSchema mySchema; private IBlueTableSchema mySchema;
private DateConversion myConversion; private DateConversion myConversion;
private string myConversionColName; private string myConversionColName;
@@ -30,13 +31,12 @@ namespace BCG {
{DateConversion.UTCToCurrent , typeof(DateTime)}, {DateConversion.UTCToCurrent , typeof(DateTime)},
{DateConversion.ConvertStringToDateTime , typeof(DateTime)}, {DateConversion.ConvertStringToDateTime , typeof(DateTime)},
{DateConversion.ConvertDateTimeToMilliseconds, typeof(Double)}, {DateConversion.ConvertDateTimeToMilliseconds, typeof(Double)},
{DateConversion.SecondsOfDay , typeof(Double)}, {DateConversion.SecondsOfDay , typeof(Double)}
{DateConversion.ISO8601 , typeof(DateTime)},
{DateConversion.EpochToDatetime , typeof(DateTime)}
}; };
private static DateConversion _parse(string s) { private static DateConversion _parse(string s)
{
DateConversion rv = DateConversion.ChopTime; DateConversion rv = DateConversion.ChopTime;
Enum.TryParse<DateConversion>(s, false, out rv); Enum.TryParse<DateConversion>(s, false, out rv);
return rv; return rv;
@@ -45,9 +45,11 @@ namespace BCG {
/// for DSQA /// for DSQA
/// </summary> /// </summary>
public DateStandardizationStream(ViewStream vs, string colName, string conversion) public DateStandardizationStream(ViewStream vs, string colName, string conversion)
: this(vs, colName, _parse(conversion)) { } : this(vs, colName, _parse(conversion))
{ }
public DateStandardizationStream(ViewStream vs, string colName, DateConversion conversion) { public DateStandardizationStream(ViewStream vs, string colName, DateConversion conversion)
{
myConversionColName = colName; myConversionColName = colName;
myConversion = conversion; myConversion = conversion;
mySetRealSource(new MapStream(vs, this)); mySetRealSource(new MapStream(vs, this));
@@ -55,18 +57,24 @@ namespace BCG {
#region IStreamMapper Members #region IStreamMapper Members
public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) { public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
if (mySchema == null) { {
if (mySchema == null)
{
List<ColumnInfo> cis = new List<ColumnInfo>(); List<ColumnInfo> cis = new List<ColumnInfo>();
foreach (ColumnInfo c in inputSchema.columnInfo) { foreach (ColumnInfo c in inputSchema.columnInfo)
if (c.colName == myConversionColName) { {
if (c.colName == myConversionColName)
{
Type convType; Type convType;
if (!_date_conversion_to_type_map.TryGetValue(myConversion, out convType)) { if (!_date_conversion_to_type_map.TryGetValue(myConversion, out convType))
{
//BCG.Utilities.BlueEventLog.logEvent("Please update the _date_conversion_to_type_map in DateStandardizationStream.cs.", BCG.Utilities.BlueEventLevel.Error); //BCG.Utilities.BlueEventLog.logEvent("Please update the _date_conversion_to_type_map in DateStandardizationStream.cs.", BCG.Utilities.BlueEventLevel.Error);
convType = typeof(DateTime); convType = typeof(DateTime);
} }
cis.Add(new ColumnInfo(c.colName, c.colNo, convType)); cis.Add(new ColumnInfo(c.colName, c.colNo, convType));
} else }
else
cis.Add(new ColumnInfo(c.colName, c.colNo, c.colType)); cis.Add(new ColumnInfo(c.colName, c.colNo, c.colType));
} }
mySchema = new BlueTableSchema(inputSchema.tableName, cis.ToArray()); mySchema = new BlueTableSchema(inputSchema.tableName, cis.ToArray());
@@ -78,59 +86,56 @@ namespace BCG {
#region IDynamicStreamMapper Members #region IDynamicStreamMapper Members
public IBlueRow map(IBlueRow row) { public IBlueRow map(IBlueRow row)
MutableBlueRow rv = new MutableBlueRow(getOutputSchema(row.schema), row); {
rv.setValue(myConversionColName, mapDateTime(row.get_datetime(myConversionColName), myConversion)); return BlueDataUtility.getNewReplacedRow(row, mapDateTime(row[myConversionColName], myConversion), myConversionColName);
return rv;
//return BlueDataUtility.getNewReplacedRow(row, mapDateTime(row[myConversionColName], myConversion), myConversionColName);
} }
#endregion #endregion
public static object mapDateTime(object obj_in, DateConversion conv) { public static object mapDateTime(object objIn, DateConversion conv)
if (obj_in == DBNull.Value || obj_in == null) { {
if (objIn == DBNull.Value || objIn == null)
{
return DBNull.Value; return DBNull.Value;
} else { }
switch (conv) { else
case DateConversion.ChopTime: { {
DateTime dtIn = (DateTime)obj_in; if (conv == DateConversion.ChopTime)
return dtIn.Date; {
} DateTime dtIn = (DateTime)objIn;
case DateConversion.ConvertDoubleToDateTime: { return dtIn.Date;
double dtIn = (double)obj_in; }
return DateTime.FromOADate(dtIn); else if (conv == DateConversion.ConvertDoubleToDateTime)
} {
case DateConversion.UTCToCurrent: { double dtIn = (double)objIn;
DateTime dtIn = (DateTime)obj_in; return DateTime.FromOADate(dtIn);
return dtIn.ToLocalTime(); }
} else if (conv == DateConversion.UTCToCurrent)
case DateConversion.ConvertStringToDateTime: { {
return DateTime.Parse((string)obj_in); DateTime dtIn = (DateTime)objIn;
} return dtIn.ToLocalTime();
case DateConversion.ConvertDateTimeToMilliseconds: { }
DateTime dtIn = (DateTime)obj_in; else if (conv == DateConversion.ConvertStringToDateTime)
return dtIn.TimeOfDay.TotalMilliseconds; {
} return DateTime.Parse((string)objIn);
case DateConversion.SecondsOfDay: { }
DateTime dtIn = (DateTime)obj_in; else if (conv == DateConversion.ConvertDateTimeToMilliseconds)
DateTime start_of_day = dtIn.Date; {
TimeSpan day_period = dtIn - start_of_day; DateTime dtIn = (DateTime)objIn;
return day_period.TotalSeconds; return dtIn.TimeOfDay.TotalMilliseconds;
} }
case DateConversion.ISO8601: { else if (conv == DateConversion.SecondsOfDay)
string iso8601Date = obj_in.ToString(); {
return DateTime.Parse(iso8601Date, null, System.Globalization.DateTimeStyles.RoundtripKind); DateTime dtIn = (DateTime)objIn;
} DateTime start_of_day = dtIn.Date;
case DateConversion.EpochToDatetime: { TimeSpan day_period = dtIn - start_of_day;
double d = Convert.ToDouble(obj_in); return day_period.TotalSeconds;
long unix_time_seconds = (long)d; }
DateTime epoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); else
return epoch.AddSeconds(unix_time_seconds); {
} bts.Log.Gen.info("incorrect type");
default: { return null;
bts.Log.Gen.info("incorrect type");
return DBNull.Value;
}
} }
} }
} }
@@ -1,38 +0,0 @@
namespace bts.SilverPine {
using bts;
using BCG;
using System;
[DataStreamUserRespresentation("DateTimeExpanderStream")]
public class DateTimeExpanstionStream : ViewStream {
public DateTimeExpanstionStream(ViewStream parent_stream, string inp_col) {
Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date");
rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day");
rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_day_of_month");
rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_month");
rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour");
rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_min");
rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day");
return rv;
};
ViewStream rs = new MapStream(parent_stream,
s => get_schema(s),
r => {
DateTime dt = r.get_datetime(inp_col);
IBlueTableSchema s = get_schema(r.schema);
MutableBlueRow rv = new MutableBlueRow(s, r);
rv.setValue($"{inp_col}_date", dt.Date);
rv.setValue($"{inp_col}_day_of_week", dt.DayOfWeek.ToString());
rv.setValue($"{inp_col}_day_of_month", dt.Day);
rv.setValue($"{inp_col}_month", dt.Date.Month);
rv.setValue($"{inp_col}_hour", dt.Hour);
rv.setValue($"{inp_col}_minute", dt.Minute);
rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
return rv;
});
mySetRealSource(rs);
}
}
}
@@ -1,71 +1,29 @@
using System; using System;
using BCG.Utilities.General; namespace BCG
{
public class MSecondStream : TimeUnitStream
{
public MSecondStream (int intervalInms, int size)
: base (DateTime.Now, new TimeSpan (0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance)
{}
using bts; protected override DateTime myRoundTime (DateTime dt) {return dt;}
namespace BCG {
[DataStreamUserRespresentation("milliseconds")]
public class MillisecondStream : ViewStream, IWorker {
private DateTime _last_fire_time = DateTime.MinValue;
private readonly TimeSpan _period;
private readonly int _size;
private readonly ViewStreamDatabase _vsd;
private IBlueTableSchema _schema;
private bts.CircularBufferTable _table;
public MillisecondStream(int interval_msecs, int size)
: this(ViewStreamDatabase.instance, interval_msecs, size) {
}
public MillisecondStream(ViewStreamDatabase vsd, int interval_msecs, int size) {
_vsd = vsd;
_period = TimeSpan.FromMilliseconds(interval_msecs);
_size = size;
_schema = BlueDataUtility.form_schema("Milliseconds",
Tuple.Create("id", TypeCode.Int32),
Tuple.Create("time", TypeCode.DateTime),
Tuple.Create("display_time", TypeCode.String));
_table = new CircularBufferTable(_schema, _size);
}
protected override void myOnFirstSubscriber() {
_vsd.addWorker(this);
}
protected override void myOnLastSubscriber() {
_vsd.removeWorker(this);
}
private int _id = 0;
public void doWork() {
if (DateTime.Now - _last_fire_time >= _period) {
_last_fire_time = DateTime.Now;
IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
_last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
_table.add_row(nr);
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
myFireEvent(dp);
}
}
}
public class MSecondStream : TimeUnitStream {
public MSecondStream(int intervalInms, int size)
: base(DateTime.Now, new TimeSpan(0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) { }
protected override DateTime myRoundTime(DateTime dt) { return dt; }
} }
public class DisplayableMillisecondStream : ViewStream, IStreamMapper { public class DisplayableMillisecondStream : ViewStream, IStreamMapper
private IBlueTableSchema mySchema = new BlueTableSchema("ms", {
new ColumnInfo("id", 0, typeof(int)), private IBlueTableSchema mySchema = new BlueTableSchema ("ms",
new ColumnInfo("time", 1, typeof(string))); new ColumnInfo ("id", 0, typeof(int)),
public DisplayableMillisecondStream(int intervalInMs, int sz) { new ColumnInfo ("time", 1, typeof(string)));
mySetRealSource(new MapStream(new MSecondStream(intervalInMs, sz), this)); public DisplayableMillisecondStream (int intervalInMs, int sz)
{
mySetRealSource (new MapStream (new MSecondStream (intervalInMs, sz), this));
} }
#region IStreamMapper Members #region IStreamMapper Members
public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) { public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
{
return mySchema; return mySchema;
} }
@@ -73,15 +31,15 @@ namespace BCG {
#region IDynamicStreamMapper Members #region IDynamicStreamMapper Members
public IBlueRow map(IBlueRow row) { public IBlueRow map (IBlueRow row)
DateTime time = row.get_datetime("time"); {
var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff"); return new BlueRow (mySchema, row["id"], myToString ((DateTime)row["time"]));
return new BlueRow(mySchema, row["id"], time_txt);
} }
#endregion #endregion
private string myToString(DateTime dt) { private string myToString (DateTime dt)
{
string sep = ":"; string sep = ":";
return dt.Hour + sep + dt.Minute + sep + dt.Second + sep + dt.Millisecond; return dt.Hour + sep + dt.Minute + sep + dt.Second + sep + dt.Millisecond;
} }
@@ -1,124 +0,0 @@
namespace bts.SilverPine {
using BCG;
using bts;
using bts.collections;
using System;
using System.CodeDom;
using System.Collections.Generic;
using System.Linq;
// Given 2 filters, and 2 columns, finds the different in the value of the 2 columns from the 2 rows
// so:
// f1(base_stream.rows)[col1] - f1(base_stream.rows)[col2]
[DataStreamUserRespresentation("ColDiff")]
public class ColumnDifferenceStream : ViewStream {
private readonly ViewStream _base_stream;
private readonly FilterStream.OnFilter _f1, _f2;
private readonly string _c1, _c2;
public ColumnDifferenceStream(ViewStream base_stream, FilterStream.OnFilter f1, FilterStream.OnFilter f2, string col1, string col2) {
_base_stream = base_stream;
_f1 = f1;
_f2 = f2;
_c1 = col1;
_c2 = col2;
}
public ColumnDifferenceStream(ViewStream base_stream, string f1, string f2, string col1, string col2) {
FilterStream.OnFilter f1_del = r => {
BooleanExpressionFilter rv = new BooleanExpressionFilter(f1);
return rv.filter(r);
};
FilterStream.OnFilter f2_del = r => {
BooleanExpressionFilter rv = new BooleanExpressionFilter(f2);
return rv.filter(r);
};
ViewStream rs = new ColumnDifferenceStream(base_stream, f1_del, f2_del, col1, col2);
mySetRealSource(rs);
}
// for each row, does a row[new_col_name] = row[col1] - rows[row_ffset][col2]
// we should generlize this for any math operation
public ColumnDifferenceStream(ViewStream parent_stream, string sort_col_name, int row_offset, string col1, string col2, string diff_col_name, double optional_perc_diff_filter = 0.0) {
// use GlobalMapStream
string diff_col_perc_name = $"{diff_col_name}_percentage";
System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow> g_map_func = (rows, idx) => {
int minuend_idx = Convert.ToInt32(idx);
var inp_schema = rows[0].schema;
var out_schema = inp_schema.add_column(TypeCode.Double, diff_col_name);
out_schema = out_schema.add_column(TypeCode.Double, diff_col_perc_name);
// minuend - subtrahend
IBlueRow minuend_row = rows.ElementAt(minuend_idx);
MutableBlueRow rv = new MutableBlueRow(out_schema, minuend_row);
int subtrahend_index = minuend_idx + row_offset;
double diff_val = Double.NaN, diff_val_perc = Double.NaN;
if (subtrahend_index >= 0) {
try {
IBlueRow subtrahend_row = rows.ElementAt(subtrahend_index);
diff_val = minuend_row.get<double>(col1) - subtrahend_row.get<double>(col2);
diff_val_perc = 100.0 * diff_val / subtrahend_row.get<double>(col1);
} catch (Exception ex) {
Log.assert(false, ex.Message);
}
}
rv.setValue(diff_col_name, diff_val);
rv.setValue(diff_col_perc_name, diff_val_perc);
return rv;
};
ViewStream rs = new GlobalMapStream(parent_stream, sort_col_name, g_map_func);
if (optional_perc_diff_filter != 0) {
rs = new FilterStream(rs, r => {
double diff_val_perc = r.get<double>(diff_col_perc_name);
double abs_diff_val_perc = Math.Abs(diff_val_perc);
return abs_diff_val_perc >= optional_perc_diff_filter;
});
}
mySetRealSource(rs);
}
protected override void myOnFirstSubscriber() {
_base_stream.onChange += _handle_base_stream_update;
}
protected override void myOnLastSubscriber() {
_base_stream.onChange -= _handle_base_stream_update;
}
private void _handle_base_stream_update(object dp_obj) {
// could have been done with filter and cartesian prod, and then math
DataPacket dp = dp_obj as DataPacket;
List<IBlueRow> rows = dp.newRows.to_list();
var f1_rows = rows.Where(r => _f1(r));
//Log.assert(f1_rows.Count() == 1);
var f2_rows = rows.Where(r => _f2(r));
//Log.assert(f2_rows.Count() == 1);
if (f1_rows != null && f1_rows.Count() > 0 && f2_rows != null && f2_rows.Count() > 0) {
IBlueRow f1_r = f1_rows.ElementAt(0);
IBlueRow f2_r = f2_rows.ElementAt(0);
double diff = f1_r.get<double>(_c1) - f2_r.get<double>(_c2);
IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
BlueTable out_table = new BlueTable(out_schema);
MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
out_row.setValue("diff", diff);
out_table.addRow(out_row);
DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
myFireEvent(dp_out);
} else {
IBlueRow f1_r = rows.ElementAt(0);
IBlueRow f2_r = rows.ElementAt(0);
double diff = Double.NaN;
IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
BlueTable out_table = new BlueTable(out_schema);
MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
out_row.setValue("diff", diff);
out_table.addRow(out_row);
DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
myFireEvent(dp_out);
}
}
}
}
@@ -1,51 +0,0 @@
namespace bts.SilverPine {
using BCG;
using bts;
using System.Collections.Generic;
using System.Linq;
// Given a sort order, an optional group by col, it observes the values of a column
// within that group (or the whole table, if grouping not specified) and returns all rows
// where the value changes, ie row[current][col] != row[current-1][col]
// TODO: a more general way to express a filter where row[current][col] != row[previous][col],
// or even expand it to row[current][col_a] - row[previous][col_b] < 0.5
[DataStreamUserRespresentation("ColEvol")]
public class ColumnEvolutionStream : ViewStream {
private readonly ViewStream _base_stream;
private readonly string _sort_col;
private readonly string _evol_col;
public ColumnEvolutionStream(ViewStream base_stream, string sort_col, string evolution_col) {
_base_stream = base_stream;
_sort_col = sort_col;
_evol_col = evolution_col;
}
protected override void myOnFirstSubscriber() {
_base_stream.onChange += _handle_base_stream_update;
}
protected override void myOnLastSubscriber() {
base.myOnLastSubscriber();
}
private void _handle_base_stream_update(object dp_obj) {
DataPacket dp = dp_obj as DataPacket;
List<IBlueRow> rows = dp.newRows.to_list();
rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, _sort_col));
BlueTable rv = new BlueTable(dp.schema);
IBlueRow prev_row = rows.ElementAt(0);
foreach (IBlueRow r in rows) {
var cr = BlueDataUtility.compare_rows(r, prev_row, _evol_col);
if (cr != 0) {
rv.addRow(r);
}
prev_row = r;
}
DataPacket dp_out = BlueDataUtility.build_dp(rv, myLastValue as DataPacket);
myFireEvent(dp_out);
}
}
}
@@ -8,36 +8,38 @@ namespace BCG {
private ViewStream mySrc; private ViewStream mySrc;
private IBlueTableSchema mySchema = null; private IBlueTableSchema mySchema = null;
public AlphabetizeColumnsStream(ViewStream src) { public AlphabetizeColumnsStream (ViewStream src) {
mySrc = src; mySrc = src;
} }
protected override void myOnFirstSubscriber() { protected override void myOnFirstSubscriber()
{
mySrc.onChange += new ChangeDelegate(mySrc_onChange); mySrc.onChange += new ChangeDelegate(mySrc_onChange);
} }
protected override void myOnLastSubscriber() { protected override void myOnLastSubscriber ()
mySrc.onChange -= new ChangeDelegate(mySrc_onChange); {
mySrc.onChange -= new ChangeDelegate (mySrc_onChange);
} }
private void mySrc_onChange(object newValue) { private void mySrc_onChange (object newValue) {
DataPacket dp = (DataPacket)newValue; DataPacket dp = (DataPacket)newValue;
if (mySchema == null) { if (mySchema == null) {
SortedList alphabetical = new SortedList(); SortedList alphabetical = new SortedList ();
foreach (ColumnInfo cinfo in dp.newRows.schema.columnInfo) foreach (ColumnInfo cinfo in dp.newRows.schema.columnInfo)
alphabetical.Add(cinfo.colName, cinfo); alphabetical.Add (cinfo.colName,cinfo);
ColumnInfo[] cinfos = new ColumnInfo[alphabetical.Count]; ColumnInfo[] cinfos = new ColumnInfo[alphabetical.Count];
alphabetical.Values.CopyTo(cinfos, 0); alphabetical.Values.CopyTo (cinfos,0);
for (int i = 0; i < cinfos.Length; ++i) for (int i = 0; i < cinfos.Length; ++i)
cinfos[i] = new ColumnInfo(cinfos[i].colName, i, cinfos[i].colType); cinfos[i] = new ColumnInfo (cinfos[i].colName,i,cinfos[i].colType,cinfos[i].colValue);
mySchema = new BlueTableSchema(cinfos, dp.schema.tableName); mySchema = new BlueTableSchema (cinfos,dp.schema.tableName);
} }
BlueTable BlueTable
deleted = new BlueTable(mySchema), deleted = new BlueTable (mySchema),
added = new BlueTable(mySchema), added = new BlueTable (mySchema),
newrows = new BlueTable(mySchema); newrows = new BlueTable (mySchema);
deleted.addRows(dp.deletedRows); deleted.addRows (dp.deletedRows);
added.addRows(dp.addedRows); added.addRows (dp.addedRows);
newrows.addRows(dp.newRows); newrows.addRows (dp.newRows);
myFireEvent(new DataPacket(newrows, added, deleted)); myFireEvent (new DataPacket (newrows,added,deleted));
} }
} }
} }
Loaded 100 of 1151 files, more files were not shown because too many files have changed in this diff. Show more