Compare commits

..
Author SHA1 Message Date
Mohammad Arham 518bd5771e Merge branch 'development.activ_md' of bitbucket.org:bluetradesys/spark into development.activ_md 2024-04-09 12:07:41 -04:00
Mohammad Arham 16522056ba active cme working small fixex and conf left 2024-03-19 14:17:52 -04:00
John Colvin 51178d85d3 Add Activ md libs to sparkd
WARNING: This breaks the sparkd clang build (only works with gcc)
2024-03-19 14:17:50 -04:00
John Colvin 86b50e58f4 Add Activ md libs to sparkd
WARNING: This breaks the sparkd clang build (only works with gcc)
2024-03-19 14:07:11 -04:00
Mohammad Arham 8431c05dfe active cme working small fixex and conf left 2024-03-19 13:59:41 -04:00
Mohammad Arham 18c05e2c8c fixed hardcoded values, should be config 2024-03-19 13:58:48 -04:00
Mohammad Arham 06535c0938 spelling fix 2024-03-19 13:58:48 -04:00
Mohammad Arham 0d8f01c8bb fixed get trades 2024-03-19 13:58:48 -04:00
Mohammad Arham 36f85b1e62 fixed sec spell 2024-03-19 13:58:48 -04:00
John Colvin 502a22a2d9 Add Activ md libs to sparkd
WARNING: This breaks the sparkd clang build (only works with gcc)
2024-03-19 13:58:48 -04:00
John Colvin daa49a5568 build: Upgrade gcc to v12 (explicitly) 2024-03-19 13:58:48 -04:00
Mohammad Arham 0ef371cc40 merged dev into activ 2024-03-14 08:32:53 -04:00
Mohammad Arham b23f0fc21b Merge branch 'development.activ_md' of https://bitbucket.org/bluetradesys/spark into development.activ_md 2024-02-15 11:37:26 -05:00
Mohammad Arham d0dc1b8e51 fixed hardcoded values, should be config 2024-02-15 11:35:02 -05:00
Mohammad Arham 0bbc4f737b spelling fix 2024-02-15 11:34:26 -05:00
Mohammad Arham 29fb1b427b fixed get trades 2024-02-15 11:34:08 -05:00
Mohammad Arham 9de6d3ffe6 fixed sec spell 2024-02-15 11:16:49 -05:00
Mohammad Arham a168d4da99 active cme working small fixex and conf left 2024-02-12 04:38:15 -05:00
John Colvin d65d4b3258 Add Activ md libs to sparkd
WARNING: This breaks the sparkd clang build (only works with gcc)
2024-01-23 09:56:33 -08:00
John Colvin b4771abad0 build: Upgrade gcc to v12 (explicitly) 2024-01-23 09:56:33 -08:00
791 changed files with 24256 additions and 35315 deletions

No files matched your search

-123
View File
@@ -1,123 +0,0 @@
FROM ubuntu:22.04
ARG DEBIAN_FRONTEND=noninteractive
# Note installations that take a while (more than 10s)
RUN apt-get update
RUN apt-get install -y ca-certificates
RUN apt-get install -y ccache
RUN apt-get install -y curl
RUN apt-get install -y fish
RUN apt-get install -y gdb
# git-all: ~250s
RUN apt-get install -y git-all
RUN apt-get install -y gnupg
# liblttng-ust-dev: ~30s
RUN apt-get install -y liblttng-ust-dev
RUN apt-get install -y libssh2-1-dev
RUN apt-get install -y locales
RUN apt-get install -y ninja-build
# openssh-server: ~40s
RUN apt-get install -y openssh-server
RUN apt-get install -y pkg-config
RUN apt-get install -y software-properties-common
RUN apt-get install -y sudo
RUN apt-get install -y tzdata
RUN apt-get install -y vim
RUN apt-get install -y zip
# Python: ~30s
RUN apt-get install -y \
python3-pip \
python3-venv \
&& python3 -m pip install --upgrade pip
# GCC
ARG gcc_major=13
RUN \
add-apt-repository ppa:ubuntu-toolchain-r/test \
&& apt-get update \
&& apt-get install -y g++-$gcc_major
# Clang
ARG clang_major=17
RUN \
curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key 2> /dev/null \
| gpg --dearmor - \
| tee /usr/share/keyrings/llvm-snapshot.gpg > /dev/null \
&& echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-$clang_major main" \
| tee -a "/etc/apt/sources.list.d/llvm.list" \
&& apt-get update \
&& apt-get install -y \
clang-$clang_major \
clang-format-$clang_major \
clangd-$clang_major
# CMake
RUN \
curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
| gpg --dearmor - \
| tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
&& echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
| tee /etc/apt/sources.list.d/kitware.list > /dev/null \
&& apt-get update \
&& apt-get install -y cmake
# Dotnet
RUN apt-get update && apt-get install -y dotnet7
RUN apt-get update && apt-get install -y dotnet8
# Node
ARG node_major=18
RUN mkdir -p /etc/apt/keyrings \
&& curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
| gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
&& echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$node_major.x nodistro main" \
| tee /etc/apt/sources.list.d/nodesource.list \
&& apt-get update \
&& apt-get install -y nodejs
# Emumba (deps)
RUN apt-get update && apt-get -y install rapidjson-dev
# Emumba
COPY emumba /emumba
RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
&& rm -r /emumba
# Pcap
RUN apt-get update && apt-get install -y libpcap0.8-dev
# Set up default executables (update-alternatives)
RUN \
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-$clang_major 100; \
update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-$clang_major 100; \
update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-$clang_major 100; \
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$gcc_major 100; \
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-$gcc_major 100
# Add compiler symlinks for ccache
RUN \
ln -s /usr/bin/ccache /usr/local/bin/clang; \
ln -s /usr/bin/ccache /usr/local/bin/clang++; \
ln -s /usr/bin/ccache /usr/local/bin/clang-$clang_major; \
ln -s /usr/bin/ccache /usr/local/bin/clang++-$clang_major; \
ln -s /usr/bin/ccache /usr/local/bin/gcc; \
ln -s /usr/bin/ccache /usr/local/bin/g++
RUN locale-gen en_US.UTF-8
ARG docker_user="bts"
ARG docker_uid="1000"
ARG docker_pass="bts"
RUN \
groupadd -g "$docker_uid" "$docker_user" \
&& useradd \
-m \
-u "$docker_uid" \
-g "$docker_uid" \
-G sudo \
--password "$(openssl passwd -1 "$docker_pass")" \
"$docker_user"
-58
View File
@@ -1,58 +0,0 @@
{
"dockerFile": "Dockerfile",
// Creates local dir to mount to HOME in docker
"initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
"hostRequirements": {
"memory": "4gb",
"storage": "6gb"
},
"postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
"containerUser": "bts",
"workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
"workspaceFolder": "/home/bts/spark",
"updateRemoteUserUID": true,
"shutdownAction": "none",
// Note: If running multiple containers on the same host, each must use a
// unique set of ports. Usually we just add 1000 to each port as needed. For
// example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
"runArgs": [
"--shm-size",
"2gb",
"-p",
"5000:5000",
"-p",
"5001:5001",
"-p",
"5173:5173",
"-p",
"5174:5174",
"-p",
"8000:8000",
"-p",
"8001:8001",
"-p",
"8002:8002",
"-p",
"8003:8003",
"-p",
"12345:12345",
],
"mounts": [
"source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
],
"customizations": {
"vscode": {
"settings": {
"clangd.arguments": [
"--compile-commands-dir=${workspaceFolder}/build/clang/debug"
]
},
"extensions": [
"eamodio.gitlens",
"llvm-vs-code-extensions.vscode-clangd",
"ms-vscode.cmake-tools",
"ms-vscode.cpptools"
]
}
}
}
@@ -1,4 +0,0 @@
#!/usr/bin/env bash
this_dir="$(realpath "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")")"
sudo dpkg -i "$this_dir/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"
+16 -30
View File
@@ -207,23 +207,17 @@ dotnet_naming_symbols.parameters.applicable_accessibilities = public,internal,pr
dotnet_naming_symbols.private_symbols.applicable_kinds = property,method,field,event,delegate dotnet_naming_symbols.private_symbols.applicable_kinds = property,method,field,event,delegate
dotnet_naming_symbols.private_symbols.applicable_accessibilities = protected,private dotnet_naming_symbols.private_symbols.applicable_accessibilities = protected,private
# local symbols
dotnet_naming_symbols.local_symbols.applicable_kinds = local
# parameter symbols
dotnet_naming_symbols.parameter_symbols.applicable_kinds = parameter
# Naming styles # Naming styles
dotnet_naming_style.pascal_case.required_prefix = dotnet_naming_style.snake_case.required_prefix =
dotnet_naming_style.pascal_case.required_suffix = dotnet_naming_style.snake_case.required_suffix =
dotnet_naming_style.pascal_case.word_separator = dotnet_naming_style.snake_case.word_separator = _
dotnet_naming_style.pascal_case.capitalization = pascal_case dotnet_naming_style.snake_case.capitalization = all_lower
dotnet_naming_style.camel_case.required_prefix = dotnet_naming_style.private_snake_case.required_prefix = _
dotnet_naming_style.camel_case.required_suffix = dotnet_naming_style.private_snake_case.required_suffix =
dotnet_naming_style.camel_case.word_separator = dotnet_naming_style.private_snake_case.word_separator = _
dotnet_naming_style.camel_case.capitalization = camel_case dotnet_naming_style.private_snake_case.capitalization = all_lower
dotnet_naming_style.user_types.required_prefix = dotnet_naming_style.user_types.required_prefix =
dotnet_naming_style.user_types.required_suffix = dotnet_naming_style.user_types.required_suffix =
@@ -235,10 +229,6 @@ dotnet_naming_style.interface.required_suffix =
dotnet_naming_style.interface.word_separator = dotnet_naming_style.interface.word_separator =
dotnet_naming_style.interface.capitalization = pascal_case dotnet_naming_style.interface.capitalization = pascal_case
dotnet_naming_style.local.capitalization = camel_case
dotnet_naming_style.parameter.capitalization = camel_case
# rules # rules
dotnet_naming_rule.user_types.symbols = user_types dotnet_naming_rule.user_types.symbols = user_types
dotnet_naming_rule.user_types.style = user_types dotnet_naming_rule.user_types.style = user_types
@@ -248,18 +238,14 @@ dotnet_naming_rule.interfaces.symbols = interface
dotnet_naming_rule.interfaces.style = interface_style dotnet_naming_rule.interfaces.style = interface_style
dotnet_naming_rule.interfaces.severity = suggestion dotnet_naming_rule.interfaces.severity = suggestion
dotnet_naming_rule.public_members.symbols = public_symbols dotnet_naming_rule.public_members_snake_case.symbols = public_symbols
dotnet_naming_rule.public_members.style = pascal_case dotnet_naming_rule.public_members_snake_case.style = snake_case
dotnet_naming_rule.public_members.severity = suggestion dotnet_naming_rule.public_members_snake_case.severity = suggestion
dotnet_naming_rule.private_members.symbols = private_symbols dotnet_naming_rule.private_members_snake_case.symbols = private_symbols
dotnet_naming_rule.private_members.style = camel_case dotnet_naming_rule.private_members_snake_case.style = private_snake_case
dotnet_naming_rule.private_members.severity = suggestion dotnet_naming_rule.private_members_snake_case.severity = suggestion
dotnet_naming_rule.variables_are_camel_case.severity = suggestion dotnet_naming_rule.parameters.symbols = parameters
dotnet_naming_rule.variables_are_camel_case.symbols = local_symbols dotnet_naming_rule.parameters.style = snake_case
dotnet_naming_rule.variables_are_camel_case.style = local
dotnet_naming_rule.parameters.symbols = parameter_symbols
dotnet_naming_rule.parameters.style = parameter
dotnet_naming_rule.parameters.severity = suggestion dotnet_naming_rule.parameters.severity = suggestion
-3
View File
@@ -1,3 +0,0 @@
*.cmd text=auto eol=crlf
*.cs text=auto eol=crlf
*.sh text=auto eol=lf
+2 -4
View File
@@ -8,9 +8,7 @@ cmake-build-debug/
.idea/ .idea/
.dir-locals.el .dir-locals.el
*.pyc *.pyc
**.vscode/* **.vscode
!.vscode/launch.json
!.vscode/tasks.json
.vs .vs
obj/ obj/
*.csproj.user *.csproj.user
@@ -56,6 +54,6 @@ trace_data/
/src/Spark.Admin.WebApp/src/bin/ /src/Spark.Admin.WebApp/src/bin/
**/CMakeDeps.cmake **/CMakeDeps.cmake
**/CMakeDeps_win.cmake **/CMakeDeps_win.cmake
.devcontainer/ **/.devcontainer
*.Generated.cs *.Generated.cs
*.lst *.lst
+1 -1
View File
@@ -1,6 +1,6 @@
[submodule "externals"] [submodule "externals"]
path = externals path = externals
url = https://bitbucket.org/bluetradesys/externals.git url = git@bitbucket.org:bluetradesys/externals.git
[submodule "vcpkg"] [submodule "vcpkg"]
path = vcpkg path = vcpkg
url = https://github.com/microsoft/vcpkg.git url = https://github.com/microsoft/vcpkg.git
-213
View File
@@ -1,213 +0,0 @@
{
"version": "0.2.0",
"configurations": [
{
"name": ".NET Core Attach",
"type": "coreclr",
"request": "attach"
},
{
"name": "Bts.DataStreamWebApp.Server",
"type": "coreclr",
"request": "launch",
"preLaunchTask": "Build DataStream WebApp (Debug)",
"program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
],
"env": {
"ASPNETCORE_ENVIRONMENT": "Development",
"ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
}
},
{
"name": "Bts.DataStreamWebApp.Server (Skip Build)",
"type": "coreclr",
"request": "launch",
"program": "${workspaceFolder}/build/Debug/bin/net7.0/Bts.DataStreamWebApp.Server.dll",
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/Bts.DataStreamWebApp.Server/Configurations/DevServer.json",
],
"env": {
"ASPNETCORE_ENVIRONMENT": "Development",
"ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
}
},
{
"name": "DataStreamPublisher",
"type": "coreclr",
"request": "launch",
"preLaunchTask": "Build DataStream (Debug)",
"program": "",
"linux": {
"program": "DataStreamPublisher",
},
"windows": {
"program": "DataStreamPublisher.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
],
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
"stopAtEntry": false,
"console": "integratedTerminal"
},
{
"name": "DataStreamPublisher (Skip Build)",
"type": "coreclr",
"request": "launch",
"program": "",
"linux": {
"program": "DataStreamPublisher",
},
"windows": {
"program": "DataStreamPublisher.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
],
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
"stopAtEntry": false,
"console": "integratedTerminal"
},
{
"name": "DataStreamQueryAnalyzer",
"type": "coreclr",
"request": "launch",
"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
"program": "",
"windows": {
"program": "DataStreamQueryAnalyzer.exe",
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
],
"cwd": "build/Debug/bin/net7.0-windows",
"env": {
"PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
},
},
"stopAtEntry": false,
"console": "integratedTerminal",
},
{
"name": "PubSubServer",
"type": "coreclr",
"request": "launch",
"preLaunchTask": "Build DataStream (Debug)",
"program": "",
"linux": {
"program": "PubSubServer",
},
"windows": {
"program": "PubSubServer.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
],
"cwd": "build/Debug/bin/net7.0",
"stopAtEntry": false,
"console": "integratedTerminal"
},
{
"name": "PubSubServer (Skip Build)",
"type": "coreclr",
"request": "launch",
"program": "",
"linux": {
"program": "PubSubServer",
},
"windows": {
"program": "PubSubServer.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
],
"cwd": "build/Debug/bin/net7.0",
"stopAtEntry": false,
"console": "integratedTerminal"
},
{
"name": "Spark",
"type": "coreclr",
"request": "launch",
"program": "",
"windows": {
"preLaunchTask": "Build Spark (Debug)",
"program": "Spark.exe",
"cwd": "build/Debug/bin/net7.0-windows",
},
"stopAtEntry": false,
"console": "integratedTerminal"
},
{
"name": "Spark (Parameterized)",
"type": "coreclr",
"request": "launch",
"preLaunchTask": "Build Spark (Debug)",
"program": "",
"windows": {
"program": "Spark.exe",
"args": [
"--sparkd",
"${input:sparkConfigServer}"
],
"cwd": "build/Debug/bin/net7.0-windows",
},
"stopAtEntry": false,
"console": "integratedTerminal"
},
{
"name": "Sparkd",
"type": "coreclr",
"request": "launch",
"preLaunchTask": "Build Sparkd (Debug | Clang)",
"program": "",
"linux": {
"program": "sparkd",
},
"args": [
"run"
],
"cwd": "build/sparkd-debug-clang/server",
"stopAtEntry": false,
"console": "integratedTerminal"
}
],
"inputs": [
{
"id": "dataStreamPublisherConfig",
"type": "pickString",
"description": "Publisher Configuration",
"options": [
"MarketDataPublisher",
"SimpleEchoClient",
"SimpleTestDataPublisher",
"TestDataPublisher",
],
"default": "MarketDataPublisher",
},
{
"id": "pubSubServerConfig",
"type": "pickString",
"description": "PubSub Configuration",
"options": [
"PubSubServer",
"SimpleEchoClient",
],
"default": "PubSubServer",
},
{
"id": "sparkConfigServer",
"type": "promptString",
"description": "Config Server URL",
"default": "http://localhost:8000/config",
},
]
}
-935
View File
@@ -1,935 +0,0 @@
{
"version": "2.0.0",
"tasks": [
{
"label": "Build Api Server",
"group": "build",
"type": "process",
"command": "",
"linux": {
"command": "tools/build_api_server.sh",
},
"windows": {
"command": "tools/build_api_server.cmd",
},
"args": [
"--configuration",
"${input:buildConfiguration}",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Api Server (Debug)",
"group": "build",
"type": "process",
"command": "",
"linux": {
"command": "tools/build_api_server.sh",
},
"windows": {
"command": "tools/build_api_server.cmd",
},
"args": [
"--configuration",
"Debug",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Api Server (Release)",
"group": "build",
"type": "process",
"command": "",
"windows": {
"command": "tools/build_api_server.cmd",
},
"linux": {
"command": "tools/build_api_server.sh",
},
"args": [
"--configuration",
"Release",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build DataStream",
"group": "build",
"type": "process",
"command": "",
"linux": {
"command": "tools/build_data_stream.sh",
},
"windows": {
"command": "tools/build_data_stream.cmd",
},
"args": [
"--configuration",
"${input:buildConfiguration}",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build DataStream (Debug)",
"group": "build",
"type": "process",
"command": "",
"linux": {
"command": "tools/build_data_stream.sh",
},
"windows": {
"command": "tools/build_data_stream.cmd",
},
"args": [
"--configuration",
"Debug",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build DataStream (Release)",
"group": "build",
"type": "process",
"command": "",
"linux": {
"command": "tools/build_data_stream.sh",
},
"windows": {
"command": "tools/build_data_stream.cmd",
},
"args": [
"--configuration",
"Release",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build DataStreamQueryAnalyzer",
"group": "build",
"type": "process",
"command": "",
"windows": {
"command": "tools/build_data_stream_query_analyzer.cmd",
},
"args": [
"--configuration",
"${input:buildConfiguration}",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build DataStreamQueryAnalyzer (Debug)",
"group": "build",
"type": "process",
"command": "",
"windows": {
"command": "tools/build_data_stream_query_analyzer.cmd",
},
"args": [
"--configuration",
"Debug",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build DataStreamQueryAnalyzer (Release)",
"group": "build",
"type": "process",
"command": "",
"windows": {
"command": "tools/build_data_stream_query_analyzer.cmd",
},
"args": [
"--configuration",
"Release",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build DataStream WebApp",
"group": "build",
"type": "process",
"command": "",
"linux": {
"command": "tools/build_data_stream_web_app.sh",
},
"windows": {
"command": "tools/build_data_stream_web_app.cmd",
},
"args": [
"--configuration",
"${input:buildConfiguration}",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build DataStream WebApp (Debug)",
"group": "build",
"type": "process",
"command": "",
"linux": {
"command": "tools/build_data_stream_web_app.sh",
},
"windows": {
"command": "tools/build_data_stream_web_app.cmd",
},
"args": [
"--configuration",
"Debug",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Spark",
"group": "build",
"type": "process",
"command": "",
"windows": {
"command": "tools/build_spark.cmd",
},
"args": [
"--configuration",
"${input:buildConfiguration}",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Spark (Debug)",
"group": "build",
"type": "process",
"command": "",
"windows": {
"command": "tools/build_spark.cmd",
},
"args": [
"--configuration",
"Debug",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Spark (Release)",
"group": "build",
"type": "process",
"command": "",
"windows": {
"command": "tools/build_spark.cmd",
},
"args": [
"--configuration",
"Release",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Sparkd",
"group": "build",
"type": "shell",
"command": "",
"linux": {
"command": "tools/build_sparkd.sh",
"args": [
"--preset",
"${input:sparkdCmakePreset}",
],
},
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Sparkd (Debug)",
"group": "build",
"type": "shell",
"command": "",
"linux": {
"command": "tools/build_sparkd.sh",
"args": [
"--build-type",
"Debug",
"--compiler",
"${input:sparkdCompiler}"
],
},
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Sparkd (Debug | Gcc)",
"group": "build",
"type": "shell",
"command": "",
"linux": {
"command": "tools/build_sparkd.sh",
"args": [
"--build-type",
"Debug",
"--compiler",
"gcc"
],
},
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Sparkd (Debug | Clang)",
"group": "build",
"type": "shell",
"command": "",
"linux": {
"command": "tools/build_sparkd.sh",
"args": [
"--build-type",
"Debug",
"--compiler",
"clang"
],
},
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Sparkd (Release)",
"group": "build",
"type": "shell",
"command": "",
"linux": {
"command": "tools/build_sparkd.sh",
"args": [
"--build-type",
"Release",
"--compiler",
"${input:sparkdCompiler}"
],
},
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Sparkd (Release | Gcc)",
"group": "build",
"type": "shell",
"command": "",
"linux": {
"command": "tools/build_sparkd.sh",
"args": [
"--build-type",
"Release",
"--compiler",
"gcc"
],
},
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Sparkd (Release | Clang)",
"group": "build",
"type": "shell",
"command": "",
"linux": {
"command": "tools/build_sparkd.sh",
"args": [
"--build-type",
"Release",
"--compiler",
"clang"
],
},
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Publish DataStreamWebApp",
"group": "build",
"type": "process",
"command": "",
"linux": {
"command": "tools/publish_data_stream_web_app.sh",
},
"windows": {
"command": "tools/publish_data_stream_web_app.cmd",
},
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Edit DataStream Config",
"type": "process",
"command": "",
"linux": {
"command": "editor",
"args": [
"${env:HOME}/.local/share/DataStream/conf/spark.conf"
],
},
"windows": {
"command": "vim",
"args": [
"${env:APPDATA}/bts/DataStream/conf/spark.conf"
],
},
"presentation": {
"reveal": "always",
"focus": true,
"panel": "shared",
"showReuseMessage": false,
"close": true,
},
"icon": {
"id": "wrench"
},
},
{
"label": "Edit Spark Config",
"type": "process",
"command": "",
"windows": {
"command": "vim",
"args": [
"${env:APPDATA}/bts/conf/spark.conf"
],
},
"presentation": {
"reveal": "always",
"focus": true,
"panel": "dedicated",
"showReuseMessage": false,
"close": true,
},
"icon": {
"id": "wrench"
},
},
{
"label": "Run DataStreamQueryAnalyzer",
"dependsOn": "Build DataStreamQueryAnalyzer",
"type": "process",
"command": "",
"windows": {
"command": "DataStreamQueryAnalyzer.exe",
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
],
"options": {
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
},
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run DataStreamQueryAnalyzer (Debug)",
"dependsOn": "Build DataStreamQueryAnalyzer (Debug)",
"type": "process",
"command": "",
"windows": {
"command": "DataStreamQueryAnalyzer.exe",
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
],
"options": {
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
},
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run DataStreamQueryAnalyzer (Release)",
"dependsOn": "Build DataStreamQueryAnalyzer (Release)",
"type": "process",
"command": "",
"windows": {
"command": "DataStreamQueryAnalyzer.exe",
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
],
"options": {
"cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
},
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run DataStreamPublisher",
"dependsOn": "Build DataStream",
"type": "process",
"command": "",
"linux": {
"command": "DataStreamPublisher",
},
"windows": {
"command": "DataStreamPublisher.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
],
"options": {
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run DataStreamPublisher (Debug)",
"dependsOn": "Build DataStream (Debug)",
"type": "process",
"command": "",
"linux": {
"command": "DataStreamPublisher",
},
"windows": {
"command": "DataStreamPublisher.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
],
"options": {
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run DataStreamPublisher (Release)",
"dependsOn": "Build DataStream (Release)",
"type": "process",
"command": "",
"linux": {
"command": "DataStreamPublisher",
},
"windows": {
"command": "DataStreamPublisher.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
],
"options": {
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run PubSubServer",
"dependsOn": "Build DataStream",
"type": "process",
"command": "",
"linux": {
"command": "PubSubServer",
},
"windows": {
"command": "PubSubServer.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
],
"options": {
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run PubSubServer (Debug)",
"dependsOn": "Build DataStream (Debug)",
"type": "process",
"command": "",
"linux": {
"command": "PubSubServer",
},
"windows": {
"command": "PubSubServer.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
],
"options": {
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run PubSubServer (Debug | Skip Build)",
"type": "process",
"command": "",
"linux": {
"command": "PubSubServer",
},
"windows": {
"command": "PubSubServer.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
],
"options": {
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run PubSubServer (Release)",
"dependsOn": "Build DataStream (Release)",
"type": "process",
"command": "",
"linux": {
"command": "PubSubServer",
},
"windows": {
"command": "PubSubServer.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
],
"options": {
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run PubSubServer (Release | Skip Build)",
"type": "process",
"command": "",
"linux": {
"command": "PubSubServer",
},
"windows": {
"command": "PubSubServer.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
],
"options": {
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run Spark",
"dependsOn": "Build Spark",
"type": "process",
"command": "",
"windows": {
"command": "Spark.exe",
"options": {
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
},
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
}
},
{
"label": "Run Spark (Debug)",
"dependsOn": "Build Spark (Debug)",
"type": "process",
"command": "",
"windows": {
"command": "Spark.exe",
"options": {
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
},
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
}
},
{
"label": "Run Spark (Release)",
"dependsOn": "Build Spark (Release)",
"type": "process",
"command": "",
"windows": {
"command": "Spark.exe",
"options": {
"cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
},
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
}
},
{
"label": "Run Spark Config Query",
"dependsOn": "Build Spark",
"type": "process",
"command": "",
"linux": {
"command": "Spark.Console",
},
"windows": {
"command": "Spark.Console.exe",
},
"args": [
"config",
"--query",
"${input:sparkConfigQuery}",
],
"options": {
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run Sparkd",
"dependsOn": "Build Sparkd",
"type": "process",
"command": "",
"linux": {
"command": "sparkd",
},
"args": [
"run"
],
"options": {
"cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run Sparkd Config Query",
"dependsOn": "Build Sparkd",
"type": "process",
"command": "",
"linux": {
"command": "sparkd",
},
"args": [
"config",
"--query",
"${input:sparkdConfigQuery}",
],
"options": {
"cwd": "${workspaceFolder}/build/${input:sparkdCompiler}/${input:sparkdBuildType}/server"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "View DataStreamQueryAnalyzer Logs (Last Run)",
"type": "process",
"command": "",
"windows": {
"command": "tools/logging/show_last_run.cmd",
"args": [
"DataStream",
"DataStreamQueryAnalyzer.exe",
"'Starting DataStreamQueryAnalyzer'"
]
},
"presentation": {
"reveal": "always",
"focus": true,
"panel": "shared",
"showReuseMessage": false,
"close": true,
},
"icon": {
"id": "wrench"
},
},
{
"label": "View PubSubServer Logs (Last Run)",
"type": "process",
"command": "",
"linux": {
"command": "tools/logging/show_last_run.sh",
"args": [
"${env:HOME}/.local/share/DataStream/data/logging.PubSubServer",
"Starting PubSubServer"
]
},
"windows": {
"command": "tools/logging/show_last_run.cmd",
"args": [
"DataStream",
"PubSubServer.exe",
"Starting PubSubServer"
]
},
"presentation": {
"reveal": "always",
"focus": true,
"panel": "shared",
"showReuseMessage": false,
"close": true,
},
"icon": {
"id": "wrench"
},
},
],
"inputs": [
{
"id": "buildConfiguration",
"type": "pickString",
"description": "Configuration",
"options": [
"Debug",
"Release",
],
"default": "Debug",
},
{
"id": "sparkdCmakePreset",
"type": "pickString",
"description": "Configuration",
"options": [
"sparkd-debug-clang",
"sparkd-debug-gcc",
"sparkd-release-clang",
"sparkd-release-gcc",
],
"default": "sparkd-debug-clang",
},
{
"id": "sparkdCompiler",
"type": "pickString",
"description": "Configuration",
"options": [
"clang",
"gcc",
],
"default": "sparkd-debug-clang",
},
{
"id": "dataStreamPublisherConfig",
"type": "pickString",
"description": "Publisher Configuration",
"options": [
"MarketDataPublisher",
"SimpleEchoClient",
"SimpleTestDataPublisher",
"TestDataPublisher",
],
"default": "MarketDataPublisher",
},
{
"id": "pubSubServerConfig",
"type": "pickString",
"description": "PubSub Configuration",
"options": [
"PubSubServer",
"SimpleEchoClient",
],
"default": "PubSubServer",
},
{
"id": "sparkConfigQuery",
"type": "promptString",
"description": "Config Type(s)"
},
{
"id": "sparkdConfigQuery",
"type": "promptString",
"description": "Config Type(s)"
}
]
}
+13 -12
View File
@@ -14,17 +14,12 @@ set(
CACHE CACHE
STRING STRING
"External dependencies root folder (should contain folders: lib, bin, include") "External dependencies root folder (should contain folders: lib, bin, include")
# SHARED_MEMORY_PREFIX serves as a kind of namespace so multiple applications
# can run simultaneously without fighting over the same shm files (e.g., sparkd,
# api server, etc.).
set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
set(SPARK_BIN "${EXTERNALS_ROOT}/bin") set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include") set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>") set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
set(BUILD_API_SERVER OFF CACHE BOOL "Build the api server")
set(BUILD_DOCS OFF CACHE BOOL "Build documentation") set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
set(BUILD_SDK OFF CACHE BOOL "Build the sdk") set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
if (WIN32) if (WIN32)
set(BUILD_SPARKD OFF CACHE BOOL "Build sparkd") set(BUILD_SPARKD OFF CACHE BOOL "Build sparkd")
set(BUILD_ADMIN_WEB_APP OFF CACHE BOOL "Build the Admin WebApp") set(BUILD_ADMIN_WEB_APP OFF CACHE BOOL "Build the Admin WebApp")
@@ -36,13 +31,13 @@ endif()
include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib) include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
link_directories(${SPARK_LIB}) link_directories(${SPARK_LIB})
add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}") if(BUILD_API_SERVER)
add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}") add_compile_definitions(SPARK_ENABLE_API_SERVER)
if(DEFINED APP_DATA_DIR_OVERRIDE)
add_compile_definitions(APP_DATA_DIR_OVERRIDE="${APP_DATA_DIR_OVERRIDE}")
endif() endif()
add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
if(WIN32) if(WIN32)
add_compile_definitions(SHARED_MEMORY_PREFIX="spark")
add_compile_definitions(APP_DATA_DIR="\$\{APPDATA\}/bts/")
# subdirs # subdirs
add_subdirectory(src/lib) add_subdirectory(src/lib)
else() else()
@@ -52,7 +47,13 @@ else()
set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install") set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>") set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")
if(BUILD_SPARKD) if(BUILD_API_SERVER)
add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/spark_api/")
add_compile_definitions(SHARED_MEMORY_PREFIX="spark_api")
elseif(BUILD_SPARKD)
add_compile_definitions(SHARED_MEMORY_PREFIX="sparkd")
add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/sparkdata/")
# Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
# read by their code). We override some below. # read by their code). We override some below.
add_compile_definitions(SPARK_ENABLE_CCAPI) add_compile_definitions(SPARK_ENABLE_CCAPI)
+65 -116
View File
@@ -15,13 +15,11 @@
"rhs": "Windows" "rhs": "Windows"
}, },
"generator": "Visual Studio 17 2022", "generator": "Visual Studio 17 2022",
"binaryDir": "${sourceDir}/build/${presetName}", "binaryDir": "${sourceDir}/build",
"environment": { "environment": {
"SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600" "SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
}, },
"cacheVariables": { "cacheVariables": {
"APP_DATA_DIR": "${dollar}{APPDATA}/bts",
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_DATA_DIR}",
"CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}", "CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
"CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd", "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
"CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT", "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
@@ -31,12 +29,8 @@
"CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake", "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
"EXTERNALS_ROOT": "${sourceDir}/externals/windows", "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf", "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows", "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
"EXPORT_MD_CONSUMER": { "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
"type": "BOOL",
"value": "ON"
},
"SHARED_MEMORY_PREFIX": "spark"
} }
}, },
@@ -52,8 +46,6 @@
"binaryDir": "${sourceDir}/build/${presetName}", "binaryDir": "${sourceDir}/build/${presetName}",
"cacheVariables": { "cacheVariables": {
"platform": "", "platform": "",
"APP_DATA_DIR": "${dollar}{HOME}/sparkdata",
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
"CMAKE_BUILD_WITH_INSTALL_RPATH": { "CMAKE_BUILD_WITH_INSTALL_RPATH": {
"type": "BOOL", "type": "BOOL",
"value": "ON" "value": "ON"
@@ -70,8 +62,7 @@
"BUILD_ADMIN_WEB_APP": { "BUILD_ADMIN_WEB_APP": {
"type": "BOOL", "type": "BOOL",
"value": "ON" "value": "ON"
}, }
"SHARED_MEMORY_PREFIX": "sparkd"
} }
}, },
@@ -104,72 +95,98 @@
{ {
"name": "sparkd-debug-clang", "name": "sparkd-debug-clang",
"inherits": "sparkd-debug", "inherits": "sparkd-debug",
"displayName": "Sparkd (Debug | Clang)", "displayName": "Debug (clang)",
"description": "Debug build compiled w/ clang", "description": "Debug build compiled w/ clang",
"binaryDir": "${sourceDir}/build/clang/debug",
"cacheVariables": { "cacheVariables": {
"CMAKE_C_COMPILER": "clang", "CMAKE_C_COMPILER": "clang",
"CMAKE_CXX_COMPILER": "clang++", "CMAKE_CXX_COMPILER": "clang++",
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG" "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/debug/sparkd"
} }
}, },
{ {
"name": "sparkd-debug-gcc", "name": "sparkd-debug-gcc",
"inherits": "sparkd-debug", "inherits": "sparkd-debug",
"displayName": "Sparkd (Debug | Gcc)", "displayName": "Debug (gcc)",
"description": "Debug build compiled w/ gcc", "description": "Debug build compiled w/ gcc",
"binaryDir": "${sourceDir}/build/gcc/debug",
"cacheVariables": { "cacheVariables": {
"CMAKE_C_COMPILER": "gcc", "CMAKE_C_COMPILER": "gcc-12",
"CMAKE_CXX_COMPILER": "g++", "CMAKE_CXX_COMPILER": "g++-12",
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG" "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/debug/sparkd"
} }
}, },
{ {
"name": "sparkd-release-clang", "name": "sparkd-release-clang",
"inherits": "sparkd-release", "inherits": "sparkd-release",
"displayName": "Sparkd (Release | Clang)", "displayName": "Release (clang)",
"description": "Release build compiled w/ clang", "description": "Release build compiled w/ clang",
"binaryDir": "${sourceDir}/build/clang/release",
"cacheVariables": { "cacheVariables": {
"CMAKE_C_COMPILER": "clang", "CMAKE_C_COMPILER": "clang",
"CMAKE_CXX_COMPILER": "clang++", "CMAKE_CXX_COMPILER": "clang++",
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3" "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/release/sparkd"
} }
}, },
{ {
"name": "sparkd-release-gcc", "name": "sparkd-release-gcc",
"inherits": "sparkd-release", "inherits": "sparkd-release",
"displayName": "Sparkd (Release | Gcc)", "displayName": "Release (gcc)",
"description": "Release build compiled w/ gcc", "description": "Release build compiled w/ gcc",
"binaryDir": "${sourceDir}/build/gcc/release",
"cacheVariables": { "cacheVariables": {
"CMAKE_C_COMPILER": "gcc", "CMAKE_C_COMPILER": "gcc",
"CMAKE_CXX_COMPILER": "g++", "CMAKE_CXX_COMPILER": "g++",
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers -Wno-stringop-overflow --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3" "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/release/sparkd"
}
},
{
"name": "spark-api-server-base",
"hidden": true,
"binaryDir": "${sourceDir}/build/${presetName}",
"cacheVariables": {
"BUILD_API_SERVER": {
"type": "bool",
"value": "ON"
},
"CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
} }
}, },
{ {
"name": "spark-api-server-windows", "name": "spark-api-server-windows",
"inherits": "spark", "inherits": "spark",
"displayName": "Spark Api Server (Windows)", "displayName": "Spark Api Server",
"cacheVariables": { "cacheVariables": {
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}", "BUILD_API_SERVER": {
"SHARED_MEMORY_PREFIX": "spark_api_server" "type": "bool",
"value": "ON"
}
} }
}, },
{ {
"name": "spark-api-server-linux", "name": "spark-api-server-linux",
"inherits": "sparkd", "inherits": "sparkd",
"displayName": "Spark Api Server (Linux)", "displayName": "Spark Api Server",
"generator": "Ninja Multi-Config", "generator": "Ninja Multi-Config",
"cacheVariables": { "cacheVariables": {
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
"BUILD_ADMIN_WEB_APP": { "BUILD_ADMIN_WEB_APP": {
"type": "BOOL", "type": "BOOL",
"value": "OFF" "value": "OFF"
}, },
"BUILD_API_SERVER": {
"type": "bool",
"value": "ON"
},
"BUILD_SPARKD": { "BUILD_SPARKD": {
"type": "BOOL", "type": "BOOL",
"value": "OFF" "value": "OFF"
@@ -180,11 +197,7 @@
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3", "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}", "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf", "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0", "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0"
"EXPORT_MD_CONSUMER": {
"type": "BOOL",
"value": "ON"
}
} }
}, },
@@ -207,63 +220,13 @@
}, },
{ {
"name": "data-stream-suite-windows", "name": "data-stream-suite",
"condition": {
"type": "equals",
"lhs": "${hostSystemName}",
"rhs": "Windows"
},
"inherits": "spark", "inherits": "spark",
"displayName": "Data Stream Suite (Windows)", "displayName": "Data Stream Suite",
"description": "Data Stream utils ported over from Edge", "description": "Data Stream utils ported over from Edge",
"binaryDir": "${sourceDir}/build/${presetName}", "binaryDir": "${sourceDir}/build/${presetName}",
"cacheVariables": { "cacheVariables": {
"APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream", "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf"
"APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net7.0-windows;${sourceDir}/build/$<CONFIG>/bin/net8.0",
"SHARED_MEMORY_PREFIX": "data_stream"
}
},
{
"name": "data-stream-suite-linux",
"displayName": "Data Stream Suite (Linux)",
"description": "Data Stream utils ported over from Edge",
"condition": {
"type": "equals",
"lhs": "${hostSystemName}",
"rhs": "Linux"
},
"inherits": "sparkd",
"generator": "Ninja Multi-Config",
"cacheVariables": {
"APP_DATA_DIR": "${dollar}{HOME}/.local/share/DataStream",
"APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
"BUILD_ADMIN_WEB_APP": {
"type": "BOOL",
"value": "OFF"
},
"BUILD_SDK": {
"type": "BOOL",
"value": "OFF"
},
"BUILD_SPARKD": {
"type": "BOOL",
"value": "OFF"
},
"CMAKE_C_COMPILER": "clang",
"CMAKE_CXX_COMPILER": "clang++",
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0;${sourceDir}/build/$<CONFIG>/bin/net8.0",
"EXPORT_MD_CONSUMER": {
"type": "BOOL",
"value": "ON"
},
"SHARED_MEMORY_PREFIX": "data_stream"
} }
} }
], ],
@@ -281,7 +244,7 @@
{ {
"name": "spark-debug", "name": "spark-debug",
"displayName": "Spark (Debug)", "displayName": "Debug",
"inherits": "spark", "inherits": "spark",
"configurePreset": "spark", "configurePreset": "spark",
"configuration": "Debug" "configuration": "Debug"
@@ -289,7 +252,7 @@
{ {
"name": "spark-release", "name": "spark-release",
"displayName": "Spark (Release)", "displayName": "Release",
"inherits": "spark", "inherits": "spark",
"configurePreset": "spark", "configurePreset": "spark",
"configuration": "Release" "configuration": "Release"
@@ -317,28 +280,28 @@
{ {
"name": "sparkd-debug-clang", "name": "sparkd-debug-clang",
"configurePreset": "sparkd-debug-clang", "configurePreset": "sparkd-debug-clang",
"displayName": "Sparkd (Debug | Clang)", "displayName": "Debug (clang)",
"inherits": "sparkd" "inherits": "sparkd"
}, },
{ {
"name": "sparkd-debug-gcc", "name": "sparkd-debug-gcc",
"configurePreset": "sparkd-debug-gcc", "configurePreset": "sparkd-debug-gcc",
"displayName": "Sparkd (Debug | Gcc)", "displayName": "Debug (gcc)",
"inherits": "sparkd" "inherits": "sparkd"
}, },
{ {
"name": "sparkd-release-clang", "name": "sparkd-release-clang",
"configurePreset": "sparkd-release-clang", "configurePreset": "sparkd-release-clang",
"displayName": "Sparkd (Release | Clang)", "displayName": "Release (clang)",
"inherits": "sparkd" "inherits": "sparkd"
}, },
{ {
"name": "sparkd-release-gcc", "name": "sparkd-release-gcc",
"configurePreset": "sparkd-release-gcc", "configurePreset": "sparkd-release-gcc",
"displayName": "Sparkd (Release | Gcc)", "displayName": "Release (gcc)",
"inherits": "sparkd" "inherits": "sparkd"
}, },
@@ -355,14 +318,14 @@
{ {
"name": "spark-api-server-windows-debug", "name": "spark-api-server-windows-debug",
"displayName": "Spark Api Server (Windows | Debug)", "displayName": "Spark Api Server (Debug)",
"inherits": "spark-api-server-windows", "inherits": "spark-api-server-windows",
"configuration": "Debug" "configuration": "Debug"
}, },
{ {
"name": "spark-api-server-windows-release", "name": "spark-api-server-windows-release",
"displayName": "Spark Api Server (Windows | Release)", "displayName": "Spark Api Server (Debug)",
"inherits": "spark-api-server-windows", "inherits": "spark-api-server-windows",
"configuration": "Release" "configuration": "Release"
}, },
@@ -381,14 +344,14 @@
{ {
"name": "spark-api-server-linux-debug", "name": "spark-api-server-linux-debug",
"inherits": "spark-api-server-linux", "inherits": "spark-api-server-linux",
"displayName": "Spark Api Server (Linux | Debug)", "displayName": "Spark Api Server (Debug)",
"configuration": "Debug" "configuration": "Debug"
}, },
{ {
"name": "spark-api-server-linux-release", "name": "spark-api-server-linux-release",
"inherits": "spark-api-server-linux", "inherits": "spark-api-server-linux",
"displayName": "Spark Api Server (Linux | Release)", "displayName": "Spark Api Server (Release)",
"configuration": "Release" "configuration": "Release"
}, },
@@ -408,30 +371,16 @@
}, },
{ {
"name": "data-stream-suite-windows-debug", "name": "data-stream-suite-debug",
"displayName": "Data Stream Suite (Windows | Debug)", "displayName": "DataStreamSuite (Debug)",
"configurePreset": "data-stream-suite-windows", "configurePreset": "data-stream-suite",
"configuration": "Debug" "configuration": "Debug"
}, },
{ {
"name": "data-stream-suite-windows-release", "name": "data-stream-suite-release",
"displayName": "Data Stream Suite (Windows | Release)", "displayName": "DataStreamSuite (Release)",
"configurePreset": "data-stream-suite-windows", "configurePreset": "data-stream-suite",
"configuration": "Release"
},
{
"name": "data-stream-suite-linux-debug",
"displayName": "Data Stream Suite (Linux | Debug)",
"configurePreset": "data-stream-suite-linux",
"configuration": "Debug"
},
{
"name": "data-stream-suite-linux-release",
"displayName": "Data Stream Suite (Linux | Release)",
"configurePreset": "data-stream-suite-linux",
"configuration": "Release" "configuration": "Release"
} }
] ]
Submodule
+1
Submodule docker/linux added at 56c61231f3.
+1
Submodule docker/windows added at de7d40c09d.
+45
View File
@@ -0,0 +1,45 @@
/* global use, db */
// MongoDB Playground
// To disable this template go to Settings | MongoDB | Use Default Template For Playground.
// Make sure you are connected to enable completions and to be able to run a playground.
// Use Ctrl+Space inside a snippet or a string literal to trigger completions.
// The result of the last command run in a playground is shown on the results panel.
// By default the first 20 documents will be returned with a cursor.
// Use 'console.log()' to print to the debug output.
// For more documentation on playgrounds please refer to
// https://www.mongodb.com/docs/mongodb-vscode/playgrounds/
// Select the database to use.
use('mongodbVSCodePlaygroundDB');
// Insert a few documents into the sales collection.
db.getCollection('sales').insertMany([
{ 'item': 'abc', 'price': 10, 'quantity': 2, 'date': new Date('2014-03-01T08:00:00Z') },
{ 'item': 'jkl', 'price': 20, 'quantity': 1, 'date': new Date('2014-03-01T09:00:00Z') },
{ 'item': 'xyz', 'price': 5, 'quantity': 10, 'date': new Date('2014-03-15T09:00:00Z') },
{ 'item': 'xyz', 'price': 5, 'quantity': 20, 'date': new Date('2014-04-04T11:21:39.736Z') },
{ 'item': 'abc', 'price': 10, 'quantity': 10, 'date': new Date('2014-04-04T21:23:13.331Z') },
{ 'item': 'def', 'price': 7.5, 'quantity': 5, 'date': new Date('2015-06-04T05:08:13Z') },
{ 'item': 'def', 'price': 7.5, 'quantity': 10, 'date': new Date('2015-09-10T08:43:00Z') },
{ 'item': 'abc', 'price': 10, 'quantity': 5, 'date': new Date('2016-02-06T20:20:13Z') },
]);
// Run a find command to view items sold on April 4th, 2014.
const salesOnApril4th = db.getCollection('sales').find({
date: { $gte: new Date('2014-04-04'), $lt: new Date('2014-04-05') }
}).count();
// Print a message to the output window.
console.log(`${salesOnApril4th} sales occurred in 2014.`);
// Here we run an aggregation and open a cursor to the results.
// Use '.toArray()' to exhaust the cursor to return the whole result set.
// You can use '.hasNext()/.next()' to iterate through the cursor page by page.
db.getCollection('sales').aggregate([
// Find all of the sales that occurred in 2014.
{ $match: { date: { $gte: new Date('2014-01-01'), $lt: new Date('2015-01-01') } } },
// Group the total sales for each product.
{ $group: { _id: '$item', totalSaleAmount: { $sum: { $multiply: ['$price', '$quantity'] } } } }
]);
@@ -8,6 +8,7 @@
public int Integer; public int Integer;
public double Real; public double Real;
public string Identifier; public string Identifier;
public DateTime DateTime;
public ImmutableList<IArithmeticExpression> Arguments; public ImmutableList<IArithmeticExpression> Arguments;
public ImmutableList<Assignment> Assignments; public ImmutableList<Assignment> Assignments;
public IArithmeticExpression Argument; public IArithmeticExpression Argument;
File diff suppressed because it is too large. Load diff
+34 -15
View File
@@ -1,22 +1,41 @@
using System.Diagnostics; using mbts.spark;
namespace BCG.Utilities.General; using System;
using System.Runtime;
/// <summary> namespace BCG.Utilities.General
/// Summary description for Crono. {
/// </summary> /// <summary>
public class Crono { /// Summary description for Crono.
readonly Stopwatch myTimer = new(); /// </summary>
public class Crono
{
public Crono()
{
myTimer = new HighPerformanceTimer();
}
public void start() => public void start()
myTimer.Start(); {myTimer.start();}
public void stop() => public void stop()
myTimer.Stop(); {myTimer.stop();}
public TimeSpan time => public TimeSpan time
myTimer.Elapsed; {
get
{
return new TimeSpan((long) (myTimer.totalMilliseconds * TimeSpan.TicksPerMillisecond));
}
}
public double miliseconds
{
get
{return myTimer.totalMilliseconds;}
}
private HighPerformanceTimer myTimer;
}
public double miliseconds =>
myTimer.Elapsed.TotalMilliseconds;
} }
+192 -192
View File
@@ -1,192 +1,192 @@
using System; using System;
using System.Collections; using System.Collections;
using System.Collections.Generic; using System.Collections.Generic;
using System.Collections.Concurrent; using System.Collections.Concurrent;
using System.Reflection; using System.Reflection;
using Microsoft.Win32; using Microsoft.Win32;
namespace BCG namespace BCG
{ {
public enum RegistryRoot { HKCU, HKLM } public enum RegistryRoot { HKCU, HKLM }
[Serializable] [Serializable]
public class BlueRegistry public class BlueRegistry
{ {
public const string BlueSubKey = @"Software\Blue Capital Group\"; public const string BlueSubKey = @"Software\Blue Capital Group\";
public static readonly string ApplicationPrefix; public static readonly string ApplicationPrefix;
protected string myFullPath; protected string myFullPath;
public BlueRegistry(string subPath) public BlueRegistry(string subPath)
{ {
myFullPath = string.Concat(BlueRegistry.ApplicationPrefix, @"\", subPath); myFullPath = string.Concat(BlueRegistry.ApplicationPrefix, @"\", subPath);
} }
public BlueRegistry(BlueRegistry parent, string subPath) public BlueRegistry(BlueRegistry parent, string subPath)
{ {
myFullPath = string.Concat(parent.myFullPath, @"\", subPath); myFullPath = string.Concat(parent.myFullPath, @"\", subPath);
} }
static BlueRegistry() static BlueRegistry()
{ {
string app = "Applications"; string app = "Applications";
Assembly ass = Assembly.GetEntryAssembly(); Assembly ass = Assembly.GetEntryAssembly();
if (ass != null) if (ass != null)
{ {
app = ass.GetName().Name; app = ass.GetName().Name;
object[] attrs = ass.GetCustomAttributes( object[] attrs = ass.GetCustomAttributes(
Type.GetType("System.Reflection.AssemblyTitleAttribute"), true); Type.GetType("System.Reflection.AssemblyTitleAttribute"), true);
if (attrs.Length > 0 && ((AssemblyTitleAttribute)attrs[0]).Title != string.Empty) if (attrs.Length > 0 && ((AssemblyTitleAttribute)attrs[0]).Title != string.Empty)
{ {
app = ((AssemblyTitleAttribute)attrs[0]).Title; app = ((AssemblyTitleAttribute)attrs[0]).Title;
} }
} }
ApplicationPrefix = string.Concat(BlueRegistry.BlueSubKey, app); ApplicationPrefix = string.Concat(BlueRegistry.BlueSubKey, app);
} }
public object getValue(string name) public object getValue(string name)
{ {
return getValue(RegistryRoot.HKCU, name); return getValue(RegistryRoot.HKCU, name);
} }
public object getValue(string name, object defaultValue) public object getValue(string name, object defaultValue)
{ {
return getValue(RegistryRoot.HKCU, name, defaultValue); return getValue(RegistryRoot.HKCU, name, defaultValue);
} }
public void setValue(string name, object value) public void setValue(string name, object value)
{ {
setValue(RegistryRoot.HKCU, name, value); setValue(RegistryRoot.HKCU, name, value);
} }
public void deleteValue(string name) public void deleteValue(string name)
{ {
deleteValue(RegistryRoot.HKCU, name); deleteValue(RegistryRoot.HKCU, name);
} }
private RegistryKey myGetRegistryKey(RegistryRoot root) private RegistryKey myGetRegistryKey(RegistryRoot root)
{ {
if (root == RegistryRoot.HKCU) if (root == RegistryRoot.HKCU)
return Registry.CurrentUser.CreateSubKey(myFullPath); return Registry.CurrentUser.CreateSubKey(myFullPath);
else else
return Registry.LocalMachine.CreateSubKey(myFullPath); return Registry.LocalMachine.CreateSubKey(myFullPath);
} }
#region Generic Methods #region Generic Methods
public virtual void setValue(RegistryRoot root, string name, object value) internal virtual void setValue(RegistryRoot root, string name, object value)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
rk.SetValue(name, value); rk.SetValue(name, value);
} }
} }
internal virtual object getValue(RegistryRoot root, string name) internal virtual object getValue(RegistryRoot root, string name)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
return rk.GetValue(name); return rk.GetValue(name);
} }
} }
internal virtual void deleteValue(RegistryRoot root, string name) internal virtual void deleteValue(RegistryRoot root, string name)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
rk.DeleteValue(name); rk.DeleteValue(name);
} }
} }
public virtual object getValue(RegistryRoot root, string name, object defaultValue) internal virtual object getValue(RegistryRoot root, string name, object defaultValue)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
return rk.GetValue(name, defaultValue); return rk.GetValue(name, defaultValue);
} }
} }
#endregion #endregion
#region Double #region Double
internal virtual void setDouble(RegistryRoot root, string name, double value) internal virtual void setDouble(RegistryRoot root, string name, double value)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
DoubleRegistryHandler.setValue(rk, name, value); DoubleRegistryHandler.setValue(rk, name, value);
} }
} }
internal virtual double getDouble(RegistryRoot root, string name) internal virtual double getDouble(RegistryRoot root, string name)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
return DoubleRegistryHandler.getValue(rk, name); return DoubleRegistryHandler.getValue(rk, name);
} }
} }
#endregion #endregion
#region Enum #region Enum
internal virtual void setEnum(RegistryRoot root, string name, Enum value) internal virtual void setEnum(RegistryRoot root, string name, Enum value)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
EnumRegistryHandler.setValue(rk, name, value); EnumRegistryHandler.setValue(rk, name, value);
} }
} }
internal virtual Enum getEnum(RegistryRoot root, string name, Type enumType) internal virtual Enum getEnum(RegistryRoot root, string name, Type enumType)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
return EnumRegistryHandler.getValue(rk, name, enumType); return EnumRegistryHandler.getValue(rk, name, enumType);
} }
} }
#endregion #endregion
#region Array #region Array
internal virtual void setArray(RegistryRoot root, string name, Array array) internal virtual void setArray(RegistryRoot root, string name, Array array)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
ArrayRegistryHandler.setValue(rk, name, array); ArrayRegistryHandler.setValue(rk, name, array);
} }
} }
internal virtual Array getArray(RegistryRoot root, string name, Type elementType) internal virtual Array getArray(RegistryRoot root, string name, Type elementType)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
return ArrayRegistryHandler.getValue(rk, name, elementType); return ArrayRegistryHandler.getValue(rk, name, elementType);
} }
} }
#endregion #endregion
#region SortedList #region SortedList
internal virtual void setSortedList(RegistryRoot root, string name, SortedList slist) internal virtual void setSortedList(RegistryRoot root, string name, SortedList slist)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
SortedListRegistryHandler.setValue(rk, name, slist); SortedListRegistryHandler.setValue(rk, name, slist);
} }
} }
internal virtual SortedList getSortedList(RegistryRoot root, string name) internal virtual SortedList getSortedList(RegistryRoot root, string name)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
return SortedListRegistryHandler.getValue(rk, name); return SortedListRegistryHandler.getValue(rk, name);
} }
} }
internal virtual void deleteSortedList(RegistryRoot root, string name) internal virtual void deleteSortedList(RegistryRoot root, string name)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
SortedListRegistryHandler.deleteValue(rk, name); SortedListRegistryHandler.deleteValue(rk, name);
} }
} }
#endregion #endregion
} }
} }
@@ -1,9 +1,10 @@
<Project Sdk="Microsoft.NET.Sdk"> <Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup> <PropertyGroup>
<TargetFramework>net7.0</TargetFramework> <TargetFramework>net7.0-windows</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings> <ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable> <Nullable>enable</Nullable>
<UseWindowsForms>true</UseWindowsForms>
</PropertyGroup> </PropertyGroup>
</Project> </Project>
@@ -1,6 +1,4 @@
using System; using System;
using System.Runtime.Versioning;
using Microsoft.Win32; using Microsoft.Win32;
namespace BCG namespace BCG
@@ -12,7 +10,6 @@ namespace BCG
rk.SetValue (name, value.ToString ()); rk.SetValue (name, value.ToString ());
} }
[SupportedOSPlatform("windows")]
public static double getValue (RegistryKey rk, string name) public static double getValue (RegistryKey rk, string name)
{ {
object v = rk.GetValue (name); object v = rk.GetValue (name);
@@ -1,166 +1,166 @@
using System; using System;
using System.Reflection; using System.Reflection;
using System.Windows.Forms; using System.Windows.Forms;
using System.Drawing; using System.Drawing;
using System.Diagnostics; using System.Diagnostics;
using Microsoft.Win32; using Microsoft.Win32;
namespace BCG namespace BCG
{ {
public class PersistWindowState public class PersistWindowState
{ {
// event info that allows form to persist extra window state data // event info that allows form to persist extra window state data
public delegate void WindowStateHandler (object sender, BlueRegistry bluereg); public delegate void WindowStateHandler (object sender, BlueRegistry bluereg);
public event WindowStateHandler LoadState; public event WindowStateHandler LoadState;
public event WindowStateHandler SaveState; public event WindowStateHandler SaveState;
private Form myParent; private Form myParent;
private BlueRegistry myRegistry; private BlueRegistry myRegistry;
private int myNormalLeft; private int myNormalLeft;
private int myNormalTop; private int myNormalTop;
private int myNormalWidth; private int myNormalWidth;
private int myNormalHeight; private int myNormalHeight;
private bool myResizable = true; private bool myResizable = true;
private FormWindowState myWindowState; private FormWindowState myWindowState;
private bool myAllowSaveMinimized = false; private bool myAllowSaveMinimized = false;
private bool mySavePosition = true; private bool mySavePosition = true;
private bool mySaveState = true; private bool mySaveState = true;
public PersistWindowState (bool savePosition) public PersistWindowState (bool savePosition)
{ {
mySavePosition = savePosition; mySavePosition = savePosition;
} }
public PersistWindowState () public PersistWindowState ()
{ {
} }
public bool saveState public bool saveState
{ {
[DebuggerStepThrough] [DebuggerStepThrough]
set { mySaveState = value; } set { mySaveState = value; }
} }
public Form parent public Form parent
{ {
[DebuggerStepThrough] [DebuggerStepThrough]
get { return myParent; } get { return myParent; }
set set
{ {
myParent = value; myParent = value;
myRegistry = new BlueRegistry (myParent.Text); myRegistry = new BlueRegistry (myParent.Text);
// subscribe to parent form's events // subscribe to parent form's events
myParent.Closing += new System.ComponentModel.CancelEventHandler(myOnClosing); myParent.Closing += new System.ComponentModel.CancelEventHandler(myOnClosing);
myParent.Resize += new System.EventHandler(myOnResize); myParent.Resize += new System.EventHandler(myOnResize);
myParent.Move += new System.EventHandler(myOnMove); myParent.Move += new System.EventHandler(myOnMove);
myParent.Load += new System.EventHandler(myOnLoad); myParent.Load += new System.EventHandler(myOnLoad);
// get initial width and height in case form is never resized // get initial width and height in case form is never resized
myNormalWidth = myParent.Width; myNormalWidth = myParent.Width;
myNormalHeight = myParent.Height; myNormalHeight = myParent.Height;
} }
} }
public BlueRegistry blueRegistry public BlueRegistry blueRegistry
{ {
[DebuggerStepThrough] [DebuggerStepThrough]
get { return myRegistry; } get { return myRegistry; }
[DebuggerStepThrough] [DebuggerStepThrough]
set { myRegistry = value; } set { myRegistry = value; }
} }
public bool resizable public bool resizable
{ {
[DebuggerStepThrough] [DebuggerStepThrough]
set { myResizable = value; } set { myResizable = value; }
} }
public bool allowSaveMinimized public bool allowSaveMinimized
{ {
[DebuggerStepThrough] [DebuggerStepThrough]
set { myAllowSaveMinimized = value; } set { myAllowSaveMinimized = value; }
} }
private void myOnResize(object sender, System.EventArgs e) private void myOnResize(object sender, System.EventArgs e)
{ {
// save width and height // save width and height
if (myParent.WindowState == FormWindowState.Normal && myResizable) if (myParent.WindowState == FormWindowState.Normal && myResizable)
{ {
myNormalWidth = myParent.Width; myNormalWidth = myParent.Width;
myNormalHeight = myParent.Height; myNormalHeight = myParent.Height;
} }
} }
private void myOnMove(object sender, System.EventArgs e) private void myOnMove(object sender, System.EventArgs e)
{ {
// save position // save position
if(myParent.WindowState == FormWindowState.Normal) if(myParent.WindowState == FormWindowState.Normal)
{ {
myNormalLeft = myParent.Left; myNormalLeft = myParent.Left;
myNormalTop = myParent.Top; myNormalTop = myParent.Top;
} }
// save state // save state
myWindowState = myParent.WindowState; myWindowState = myParent.WindowState;
} }
private void myOnClosing(object sender, System.ComponentModel.CancelEventArgs e) private void myOnClosing(object sender, System.ComponentModel.CancelEventArgs e)
{ {
// save position, size and state // save position, size and state
if (mySavePosition) if (mySavePosition)
{ {
myRegistry.setValue (RegistryRoot.HKCU, "Left", myNormalLeft); myRegistry.setValue (RegistryRoot.HKCU, "Left", myNormalLeft);
myRegistry.setValue (RegistryRoot.HKCU, "Top", myNormalTop); myRegistry.setValue (RegistryRoot.HKCU, "Top", myNormalTop);
} }
if (myResizable) if (myResizable)
{ {
myRegistry.setValue (RegistryRoot.HKCU, "Width", myNormalWidth); myRegistry.setValue (RegistryRoot.HKCU, "Width", myNormalWidth);
myRegistry.setValue (RegistryRoot.HKCU, "Height", myNormalHeight); myRegistry.setValue (RegistryRoot.HKCU, "Height", myNormalHeight);
} }
// check if we are allowed to save the state as minimized (not normally) // check if we are allowed to save the state as minimized (not normally)
if (!myAllowSaveMinimized) if (!myAllowSaveMinimized)
{ {
if(myWindowState == FormWindowState.Minimized) if(myWindowState == FormWindowState.Minimized)
myWindowState = FormWindowState.Normal; myWindowState = FormWindowState.Normal;
} }
if (mySaveState) if (mySaveState)
myRegistry.setValue (RegistryRoot.HKCU, "WindowState", (int) myWindowState); myRegistry.setValue (RegistryRoot.HKCU, "WindowState", (int) myWindowState);
// fire SaveState event // fire SaveState event
if (SaveState != null) if (SaveState != null)
SaveState (this, myRegistry); SaveState (this, myRegistry);
} }
private void myOnLoad (object sender, System.EventArgs e) private void myOnLoad (object sender, System.EventArgs e)
{ {
if (mySavePosition) if (mySavePosition)
{ {
int left = (int) myRegistry.getValue (RegistryRoot.HKCU, "Left", myParent.Left); int left = (int) myRegistry.getValue (RegistryRoot.HKCU, "Left", myParent.Left);
int top = (int) myRegistry.getValue (RegistryRoot.HKCU, "Top", myParent.Top); int top = (int) myRegistry.getValue (RegistryRoot.HKCU, "Top", myParent.Top);
myParent.Location = new Point (left, top); myParent.Location = new Point (left, top);
} }
if (myResizable) if (myResizable)
{ {
int width = (int) myRegistry.getValue (RegistryRoot.HKCU, "Width", myParent.Width); int width = (int) myRegistry.getValue (RegistryRoot.HKCU, "Width", myParent.Width);
int height = (int) myRegistry.getValue (RegistryRoot.HKCU, "Height", myParent.Height); int height = (int) myRegistry.getValue (RegistryRoot.HKCU, "Height", myParent.Height);
myParent.Size = new Size (width, height); myParent.Size = new Size (width, height);
} }
if (mySaveState) if (mySaveState)
{ {
FormWindowState windowState = (FormWindowState) myRegistry.getValue (RegistryRoot.HKCU, "WindowState", FormWindowState windowState = (FormWindowState) myRegistry.getValue (RegistryRoot.HKCU, "WindowState",
(int) myParent.WindowState); (int) myParent.WindowState);
myParent.WindowState = windowState; myParent.WindowState = windowState;
} }
// fire LoadState event // fire LoadState event
if(LoadState != null) if(LoadState != null)
LoadState (this, myRegistry); LoadState (this, myRegistry);
} }
} }
} }
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
@@ -1,20 +0,0 @@
module.exports = {
root: true,
env: { browser: true, es2020: true },
extends: [
'eslint:recommended',
'plugin:react/recommended',
'plugin:react/jsx-runtime',
'plugin:react-hooks/recommended',
],
ignorePatterns: ['dist', '.eslintrc.cjs'],
parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
settings: { react: { version: '18.3.1' } },
plugins: ['react-refresh'],
rules: {
'react-refresh/only-export-components': [
'warn',
{ allowConstantExport: true },
],
},
}
@@ -1,24 +0,0 @@
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
pnpm-debug.log*
lerna-debug.log*
node_modules
dist
dist-ssr
*.local
# Editor directories and files
.vscode/*
!.vscode/extensions.json
.idea
.DS_Store
*.suo
*.ntvs*
*.njsproj
*.sln
*.sw?
@@ -1,11 +0,0 @@
<Project Sdk="Microsoft.VisualStudio.JavaScript.Sdk/0.5.128-alpha">
<PropertyGroup>
<StartupCommand>npm run dev</StartupCommand>
<JavaScriptTestRoot>src\</JavaScriptTestRoot>
<JavaScriptTestFramework>Jest</JavaScriptTestFramework>
<!-- Allows the build (or compile) script located on package.json to run on Build -->
<ShouldRunBuildScript>true</ShouldRunBuildScript>
<!-- Folder where production build objects will be placed -->
<PublishAssetsDirectory>$(MSBuildProjectDirectory)\distill</PublishAssetsDirectory>
</PropertyGroup>
</Project>
@@ -1,8 +0,0 @@
# React + Vite
This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
Currently, two official plugins are available:
- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh
- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh
@@ -1,13 +0,0 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<link rel="icon" type="image/ico" href="/logo.ico" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>BTS Cloud Streams</title>
</head>
<body>
<div id="root"></div>
<script type="module" src="/src/main.jsx"></script>
</body>
</html>
@@ -1,10 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<packageSources>
<clear />
<add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
</packageSources>
<disabledPackageSources>
<clear />
</disabledPackageSources>
</configuration>
File diff suppressed because it is too large. Load diff
@@ -1,38 +0,0 @@
{
"name": "bts.datastreamwebapp.client",
"private": true,
"version": "0.0.0",
"type": "module",
"scripts": {
"dev": "vite",
"build": "vite build",
"lint": "eslint . --ext js,jsx --report-unused-disable-directives --max-warnings 0",
"preview": "vite preview"
},
"dependencies": {
"@angular/material": "^18.0.4",
"@emotion/react": "^11.11.4",
"@emotion/styled": "^11.11.5",
"@fontsource/noto-sans": "^5.0.22",
"@material/data-table": "^15.0.0-canary.7f224ddd4.0",
"@mui/icons-material": "^5.15.20",
"@mui/material": "^5.15.20",
"@mui/x-data-grid": "^7.7.1",
"bulma": "^1.0.1",
"react": "^18.3.1",
"react-data-table-component": "^7.6.2",
"react-dom": "^18.3.1",
"react-use-websocket": "^4.8.1",
"rpc-websockets": "^7.11.2"
},
"devDependencies": {
"@types/react": "^18.3.3",
"@types/react-dom": "^18.3.0",
"@vitejs/plugin-react": "^4.3.1",
"eslint": "^8.57.0",
"eslint-plugin-react": "^7.34.2",
"eslint-plugin-react-hooks": "^4.6.2",
"eslint-plugin-react-refresh": "^0.4.7",
"vite": "^5.2.13"
}
}
Binary file not shown.

Before

Width:  |  Height:  |  Size: 8.2 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 107 KiB

@@ -1 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>

Before

Width:  |  Height:  |  Size: 1.5 KiB

@@ -1,28 +0,0 @@
@use "@material/checkbox";
@use "@material/icon-button";
@use "@material/data-table/data-table";
h1#main-header {
padding: 4px 8px 4px 8px;
text-align: center;
}
td:hover {
background-color: cadetblue;
}
td#cell-decimal {
text-align: right;
}
td#cell-integer {
text-align: right;
}
td#cell-string {
text-align: center;
}
td#cell-date-time {
text-align: center;
}
@@ -1,24 +0,0 @@
import { createTheme, ThemeProvider, CssBaseline } from '@mui/material';
import useMediaQuery from '@mui/material/useMediaQuery';
import QueryControl from './QueryControl';
import getTheme from './Theme';
import '@fontsource/noto-sans';
import './App.css';
const App = () => {
const prefersDarkMode = useMediaQuery('(prefers-color-scheme: dark)');
const theme = getTheme({ mode: prefersDarkMode ? 'dark' : 'light' });
return (
<ThemeProvider theme={theme}>
<CssBaseline />
<h1 id='main-header' style={{
color: theme.palette.text.primary,
fontFamily: 'Noto Sans',
}}>BTS Cloud Streams</h1>
<QueryControl websocketUrl="ws://localhost:5000/ws" theme={theme} />
</ThemeProvider>
);
};
export default App;
@@ -1,32 +0,0 @@
@use "@material/checkbox";
@use "@material/icon-button";
@use "@material/data-table/data-table";
th {
font-size: large;
padding: 15px;
}
td {
padding: 15px;
}
td:hover {
background-color: cadetblue;
}
td#cell-decimal {
text-align: right;
}
td#cell-integer {
text-align: right;
}
td#cell-string {
text-align: center;
}
td#cell-date-time {
text-align: center;
}
@@ -1,52 +0,0 @@
import PropTypes from 'prop-types';
import { DataGrid } from '@mui/x-data-grid';
function getGridColDef(col) {
return {
field: col.name,
headerName: col.name,
};
}
export default function DataTable({ columns, rows }) {
return (
<DataGrid
columns={columns.map(getGridColDef)}
rows={rows}
sx={{
borderColor: 'primary.main',
backgroundColor: 'primary.dark',
color: 'text.primary',
'& .MuiDataGrid-columnHeader': {
color: 'primary.main',
fontWeight: 700,
backgroundColor: 'primary.dark'
},
'& .MuiDataGrid-filler': {
color: 'primary.main',
fontWeight: 700,
backgroundColor: 'primary.dark'
},
'& .MuiDataGrid-cell': {
color: 'text.primary',
},
'& .MuiDataGrid-cell:hover': {
color: 'primary.main',
},
'& .MuiDataGrid-row': {
'--DataGrid-rowBorderColor': 'secondary.dark'
}
}}
/>
);
}
DataTable.propTypes = {
columns: PropTypes.arrayOf(
PropTypes.shape({
name: PropTypes.string.isRequired,
type: PropTypes.string.isRequired,
})
).isRequired,
rows: PropTypes.arrayOf(PropTypes.array).isRequired,
};
@@ -1,100 +0,0 @@
import PropTypes from 'prop-types';
import DataTable from './DataTable';
import QueryInputForm from './QueryInputForm';
import Container from '@mui/material/Container';
import React from 'react';
var rpcReqIdCounter = 1;
function subscribeReq(query) {
return {
jsonrpc: "2.0",
id: rpcReqIdCounter++,
method: "subscribe",
params: {
subscription_id: "1",
query: query
}
};
}
function unsubscribeReq() {
return {
jsonrpc: "2.0",
id: rpcReqIdCounter++,
method: "unsubscribe",
params: {
subscription_id: "1",
}
};
}
class QueryControl extends React.Component {
constructor({ websocketUrl, theme }) {
super(websocketUrl);
console.log('Connecting to WebSocket:', websocketUrl);
this.state = {
rows: null,
ws: new WebSocket(websocketUrl),
columns: null,
isRunning: false,
theme: theme,
};
this.state.ws.onopen = () => {
console.info('WebSocket open');
};
this.state.ws.onclose = () => {
console.info('WebSocket closed');
};
this.state.ws.onmessage = e => {
console.info(`Received message: ${e.data}`);
const msg = JSON.parse(e.data);
if (msg.id != null) {
let newState = { ...this.state };
newState.isRunning = msg.result.status == 'Subscribed';
this.setState(newState);
} else {
let newState = { ...this.state };
newState.rows = msg.params.all;
console.info(`Received rows: ${newState.rows}`);
newState.columns = msg.params.columns;
this.setState(newState);
}
};
this.start = (query) => {
this.state.ws.send(JSON.stringify(subscribeReq(query)));
};
this.stop = () => {
this.state.ws.send(JSON.stringify(unsubscribeReq()));
};
}
componentWillUnmount() {
if (this.state.ws) {
console.warn('Closing websocket because component is unmounting');
this.state.ws.close();
}
}
render() {
console.info('Rendering');
return (
<Container>
<QueryInputForm isRunning={this.state.isRunning} onStart={this.start} onStop={this.stop} />
{
this.state.rows == null ? <p style={{ color: this.state.theme.palette.warning.main }}>No data received</p> :
<DataTable columns={this.state.columns} rows={this.state.rows} />
}
</Container>
);
}
}
QueryControl.propTypes = {
websocketUrl: PropTypes.string.isRequired,
theme: PropTypes.object.isRequired,
};
export default QueryControl;
@@ -1,93 +0,0 @@
import { useRef } from 'react';
import Box from '@mui/material/Box';
import Button from '@mui/material/Button';
import Grid from '@mui/material/Grid';
import Paper from '@mui/material/Paper';
import PlayArrowIcon from '@mui/icons-material/PlayArrow';
import PropTypes from 'prop-types';
import StopIcon from '@mui/icons-material/Stop';
import TextField from '@mui/material/TextField';
import Tooltip from '@mui/material/Tooltip';
const QueryInputForm = ({ isRunning, onStart, onStop }) => {
let textInput = useRef(null);
const onClearClick = () => {
textInput.current.value = '';
};
const onRunClick = () => {
onStart(textInput.current.value);
};
const onStopClick = () => {
onStop();
};
const onKeyDown = (e) => {
// Detect keyboard shortcuts.
// Ctrl+Enter : Run
if (e.ctrlKey && e.key === 'Enter') {
onRunClick();
}
};
return (
<Box sx={{ p: 2 }}>
<Paper elevation={3} sx={{ color: 'primary.main' }}>
<Box sx={{ p: 2 }}>
<TextField
fullWidth
multiline
rows={10}
variant="outlined"
onKeyDown={onKeyDown}
inputRef={textInput}
placeholder="Enter Query..."
/>
</Box>
<Box sx={{ p: 2, borderTop: 1 }}>
<Grid container spacing={2} justifyContent="flex-start">
<Grid item xs>
<Tooltip title="Clear Query Text">
<Button onClick={onClearClick} sx={{ padding: '11px', color: 'text.dark' }}>
Clear
</Button>
</Tooltip>
</Grid>
<Grid item>
<Tooltip title="Run (Ctrl+Enter)">
<Button onClick={onRunClick} sx={{
color: `query_start_button.${isRunning ? 'active' : 'inactive'}.color`,
bgcolor: `query_start_button.${isRunning ? 'active' : 'inactive'}.background_color`,
padding: '11px'
}}>
<PlayArrowIcon />
</Button>
</Tooltip>
</Grid>
<Grid item>
<Tooltip title="Stop">
<Button onClick={onStopClick} sx={{
color: `query_stop_button.${isRunning ? 'inactive' : 'active'}.color`,
bgcolor: `query_stop_button.${isRunning ? 'inactive' : 'active'}.background_color`,
padding: '11px'
}}>
<StopIcon />
</Button>
</Tooltip>
</Grid>
</Grid>
</Box>
</Paper>
</Box>
);
};
QueryInputForm.propTypes = {
isRunning: PropTypes.bool.isRequired,
onStart: PropTypes.func.isRequired,
onStop: PropTypes.func.isRequired,
};
export default QueryInputForm;
@@ -1,88 +0,0 @@
import { createTheme } from '@mui/material/styles';
import PropTypes from 'prop-types';
import '@fontsource/noto-sans';
import './App.css';
const getLightPalette = () => {
return {
mode: 'light',
primary: {
light: '#6573c3',
main: '#3f51b5',
dark: '#2c387e',
contrastText: '#ffffff',
},
secondary: {
light: '#f73378',
main: '#f50057',
dark: '#ab003c',
contrastText: '#ffffff',
},
text: {
primary: '#000000c4',
secondary: '#00000086',
disabled: '#0000005b',
hint: '#0000005b',
}
};
};
const getDarkPalette = () => {
return {
mode: 'dark',
primary: {
light: '#bcc5ff',
main: '#33baf9',
dark: '#202334',
contrastText: '#78a2ac',
},
secondary: {
light: '#9fd0b8',
main: '#5eb289',
dark: '#ffffff12',
contrastText: '#c8d4f7',
},
text: {
primary: '#eaf8f8',
dark: '#eaf8f8a0'
},
query_start_button: {
active: {
color: '#33baf9',
background_color: '#ffffff40',
},
inactive: {
color: '#ffffff40',
background_color: '#ffffff00',
}
},
warning: {
main: '#c4823f'
},
query_stop_button: {
active: {
color: '#ff4444a0',
background_color: '#ffffff40',
},
inactive: {
color: '#ff444450',
background_color: '#ffffff00',
}
},
};
};
const getTheme = ({ mode }) => {
return createTheme({
typography: {
fontFamily: '"Helvetica"'
},
palette: mode == 'dark' ? getDarkPalette() : getLightPalette()
});
};
export default getTheme;
getTheme.propTypes = {
mode: PropTypes.string.isRequired,
};
@@ -1 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="35.93" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 228"><path fill="#00D8FF" d="M210.483 73.824a171.49 171.49 0 0 0-8.24-2.597c.465-1.9.893-3.777 1.273-5.621c6.238-30.281 2.16-54.676-11.769-62.708c-13.355-7.7-35.196.329-57.254 19.526a171.23 171.23 0 0 0-6.375 5.848a155.866 155.866 0 0 0-4.241-3.917C100.759 3.829 77.587-4.822 63.673 3.233C50.33 10.957 46.379 33.89 51.995 62.588a170.974 170.974 0 0 0 1.892 8.48c-3.28.932-6.445 1.924-9.474 2.98C17.309 83.498 0 98.307 0 113.668c0 15.865 18.582 31.778 46.812 41.427a145.52 145.52 0 0 0 6.921 2.165a167.467 167.467 0 0 0-2.01 9.138c-5.354 28.2-1.173 50.591 12.134 58.266c13.744 7.926 36.812-.22 59.273-19.855a145.567 145.567 0 0 0 5.342-4.923a168.064 168.064 0 0 0 6.92 6.314c21.758 18.722 43.246 26.282 56.54 18.586c13.731-7.949 18.194-32.003 12.4-61.268a145.016 145.016 0 0 0-1.535-6.842c1.62-.48 3.21-.974 4.76-1.488c29.348-9.723 48.443-25.443 48.443-41.52c0-15.417-17.868-30.326-45.517-39.844Zm-6.365 70.984c-1.4.463-2.836.91-4.3 1.345c-3.24-10.257-7.612-21.163-12.963-32.432c5.106-11 9.31-21.767 12.459-31.957c2.619.758 5.16 1.557 7.61 2.4c23.69 8.156 38.14 20.213 38.14 29.504c0 9.896-15.606 22.743-40.946 31.14Zm-10.514 20.834c2.562 12.94 2.927 24.64 1.23 33.787c-1.524 8.219-4.59 13.698-8.382 15.893c-8.067 4.67-25.32-1.4-43.927-17.412a156.726 156.726 0 0 1-6.437-5.87c7.214-7.889 14.423-17.06 21.459-27.246c12.376-1.098 24.068-2.894 34.671-5.345a134.17 134.17 0 0 1 1.386 6.193ZM87.276 214.515c-7.882 2.783-14.16 2.863-17.955.675c-8.075-4.657-11.432-22.636-6.853-46.752a156.923 156.923 0 0 1 1.869-8.499c10.486 2.32 22.093 3.988 34.498 4.994c7.084 9.967 14.501 19.128 21.976 27.15a134.668 134.668 0 0 1-4.877 4.492c-9.933 8.682-19.886 14.842-28.658 17.94ZM50.35 144.747c-12.483-4.267-22.792-9.812-29.858-15.863c-6.35-5.437-9.555-10.836-9.555-15.216c0-9.322 13.897-21.212 37.076-29.293c2.813-.98 5.757-1.905 8.812-2.773c3.204 10.42 7.406 21.315 12.477 32.332c-5.137 11.18-9.399 22.249-12.634 32.792a134.718 134.718 0 0 1-6.318-1.979Zm12.378-84.26c-4.811-24.587-1.616-43.134 6.425-47.789c8.564-4.958 27.502 2.111 47.463 19.835a144.318 144.318 0 0 1 3.841 3.545c-7.438 7.987-14.787 17.08-21.808 26.988c-12.04 1.116-23.565 2.908-34.161 5.309a160.342 160.342 0 0 1-1.76-7.887Zm110.427 27.268a347.8 347.8 0 0 0-7.785-12.803c8.168 1.033 15.994 2.404 23.343 4.08c-2.206 7.072-4.956 14.465-8.193 22.045a381.151 381.151 0 0 0-7.365-13.322Zm-45.032-43.861c5.044 5.465 10.096 11.566 15.065 18.186a322.04 322.04 0 0 0-30.257-.006c4.974-6.559 10.069-12.652 15.192-18.18ZM82.802 87.83a323.167 323.167 0 0 0-7.227 13.238c-3.184-7.553-5.909-14.98-8.134-22.152c7.304-1.634 15.093-2.97 23.209-3.984a321.524 321.524 0 0 0-7.848 12.897Zm8.081 65.352c-8.385-.936-16.291-2.203-23.593-3.793c2.26-7.3 5.045-14.885 8.298-22.6a321.187 321.187 0 0 0 7.257 13.246c2.594 4.48 5.28 8.868 8.038 13.147Zm37.542 31.03c-5.184-5.592-10.354-11.779-15.403-18.433c4.902.192 9.899.29 14.978.29c5.218 0 10.376-.117 15.453-.343c-4.985 6.774-10.018 12.97-15.028 18.486Zm52.198-57.817c3.422 7.8 6.306 15.345 8.596 22.52c-7.422 1.694-15.436 3.058-23.88 4.071a382.417 382.417 0 0 0 7.859-13.026a347.403 347.403 0 0 0 7.425-13.565Zm-16.898 8.101a358.557 358.557 0 0 1-12.281 19.815a329.4 329.4 0 0 1-23.444.823c-7.967 0-15.716-.248-23.178-.732a310.202 310.202 0 0 1-12.513-19.846h.001a307.41 307.41 0 0 1-10.923-20.627a310.278 310.278 0 0 1 10.89-20.637l-.001.001a307.318 307.318 0 0 1 12.413-19.761c7.613-.576 15.42-.876 23.31-.876H128c7.926 0 15.743.303 23.354.883a329.357 329.357 0 0 1 12.335 19.695a358.489 358.489 0 0 1 11.036 20.54a329.472 329.472 0 0 1-11 20.722Zm22.56-122.124c8.572 4.944 11.906 24.881 6.52 51.026c-.344 1.668-.73 3.367-1.15 5.09c-10.622-2.452-22.155-4.275-34.23-5.408c-7.034-10.017-14.323-19.124-21.64-27.008a160.789 160.789 0 0 1 5.888-5.4c18.9-16.447 36.564-22.941 44.612-18.3ZM128 90.808c12.625 0 22.86 10.235 22.86 22.86s-10.235 22.86-22.86 22.86s-22.86-10.235-22.86-22.86s10.235-22.86 22.86-22.86Z"></path></svg>

Before

Width:  |  Height:  |  Size: 4.0 KiB

@@ -1,51 +0,0 @@
.dark-high-contrast {
--md-sys-color-primary: rgb(251 250 255);
--md-sys-color-surface-tint: rgb(165 200 255);
--md-sys-color-on-primary: rgb(0 0 0);
--md-sys-color-primary-container: rgb(173 204 255);
--md-sys-color-on-primary-container: rgb(0 0 0);
--md-sys-color-secondary: rgb(250 250 255);
--md-sys-color-on-secondary: rgb(0 0 0);
--md-sys-color-secondary-container: rgb(159 208 255);
--md-sys-color-on-secondary-container: rgb(0 0 0);
--md-sys-color-tertiary: rgb(255 249 251);
--md-sys-color-on-tertiary: rgb(0 0 0);
--md-sys-color-tertiary-container: rgb(222 193 231);
--md-sys-color-on-tertiary-container: rgb(0 0 0);
--md-sys-color-error: rgb(255 249 249);
--md-sys-color-on-error: rgb(0 0 0);
--md-sys-color-error-container: rgb(255 186 177);
--md-sys-color-on-error-container: rgb(0 0 0);
--md-sys-color-background: rgb(17 19 24);
--md-sys-color-on-background: rgb(225 226 233);
--md-sys-color-surface: rgb(17 19 24);
--md-sys-color-on-surface: rgb(255 255 255);
--md-sys-color-surface-variant: rgb(67 71 78);
--md-sys-color-on-surface-variant: rgb(251 250 255);
--md-sys-color-outline: rgb(200 202 212);
--md-sys-color-outline-variant: rgb(200 202 212);
--md-sys-color-shadow: rgb(0 0 0);
--md-sys-color-scrim: rgb(0 0 0);
--md-sys-color-inverse-surface: rgb(225 226 233);
--md-sys-color-inverse-on-surface: rgb(0 0 0);
--md-sys-color-inverse-primary: rgb(0 42 83);
--md-sys-color-primary-fixed: rgb(219 231 255);
--md-sys-color-on-primary-fixed: rgb(0 0 0);
--md-sys-color-primary-fixed-dim: rgb(173 204 255);
--md-sys-color-on-primary-fixed-variant: rgb(0 22 49);
--md-sys-color-secondary-fixed: rgb(214 233 255);
--md-sys-color-on-secondary-fixed: rgb(0 0 0);
--md-sys-color-secondary-fixed-dim: rgb(159 208 255);
--md-sys-color-on-secondary-fixed-variant: rgb(0 24 42);
--md-sys-color-tertiary-fixed: rgb(249 222 255);
--md-sys-color-on-tertiary-fixed: rgb(0 0 0);
--md-sys-color-tertiary-fixed-dim: rgb(222 193 231);
--md-sys-color-on-tertiary-fixed-variant: rgb(33 14 42);
--md-sys-color-surface-dim: rgb(17 19 24);
--md-sys-color-surface-bright: rgb(55 57 62);
--md-sys-color-surface-container-lowest: rgb(12 14 19);
--md-sys-color-surface-container-low: rgb(25 28 32);
--md-sys-color-surface-container: rgb(29 32 36);
--md-sys-color-surface-container-high: rgb(40 42 47);
--md-sys-color-surface-container-highest: rgb(50 53 58);
}
@@ -1,51 +0,0 @@
.dark-medium-contrast {
--md-sys-color-primary: rgb(173 204 255);
--md-sys-color-surface-tint: rgb(165 200 255);
--md-sys-color-on-primary: rgb(0 22 49);
--md-sys-color-primary-container: rgb(112 146 198);
--md-sys-color-on-primary-container: rgb(0 0 0);
--md-sys-color-secondary: rgb(159 208 255);
--md-sys-color-on-secondary: rgb(0 24 42);
--md-sys-color-secondary-container: rgb(100 149 193);
--md-sys-color-on-secondary-container: rgb(0 0 0);
--md-sys-color-tertiary: rgb(222 193 231);
--md-sys-color-on-tertiary: rgb(33 14 42);
--md-sys-color-tertiary-container: rgb(162 136 171);
--md-sys-color-on-tertiary-container: rgb(0 0 0);
--md-sys-color-error: rgb(255 186 177);
--md-sys-color-on-error: rgb(55 0 1);
--md-sys-color-error-container: rgb(255 84 73);
--md-sys-color-on-error-container: rgb(0 0 0);
--md-sys-color-background: rgb(17 19 24);
--md-sys-color-on-background: rgb(225 226 233);
--md-sys-color-surface: rgb(17 19 24);
--md-sys-color-on-surface: rgb(251 250 255);
--md-sys-color-surface-variant: rgb(67 71 78);
--md-sys-color-on-surface-variant: rgb(200 202 212);
--md-sys-color-outline: rgb(160 163 171);
--md-sys-color-outline-variant: rgb(128 131 139);
--md-sys-color-shadow: rgb(0 0 0);
--md-sys-color-scrim: rgb(0 0 0);
--md-sys-color-inverse-surface: rgb(225 226 233);
--md-sys-color-inverse-on-surface: rgb(40 42 47);
--md-sys-color-inverse-primary: rgb(35 73 120);
--md-sys-color-primary-fixed: rgb(212 227 255);
--md-sys-color-on-primary-fixed: rgb(0 17 40);
--md-sys-color-primary-fixed-dim: rgb(165 200 255);
--md-sys-color-on-primary-fixed-variant: rgb(10 55 100);
--md-sys-color-secondary-fixed: rgb(206 229 255);
--md-sys-color-on-secondary-fixed: rgb(0 18 34);
--md-sys-color-secondary-fixed-dim: rgb(154 203 250);
--md-sys-color-on-secondary-fixed-variant: rgb(0 57 91);
--md-sys-color-tertiary-fixed: rgb(247 216 255);
--md-sys-color-on-tertiary-fixed: rgb(28 9 37);
--md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
--md-sys-color-on-tertiary-fixed-variant: rgb(67 46 76);
--md-sys-color-surface-dim: rgb(17 19 24);
--md-sys-color-surface-bright: rgb(55 57 62);
--md-sys-color-surface-container-lowest: rgb(12 14 19);
--md-sys-color-surface-container-low: rgb(25 28 32);
--md-sys-color-surface-container: rgb(29 32 36);
--md-sys-color-surface-container-high: rgb(40 42 47);
--md-sys-color-surface-container-highest: rgb(50 53 58);
}
@@ -1,51 +0,0 @@
.dark {
--md-sys-color-primary: rgb(165 200 255);
--md-sys-color-surface-tint: rgb(165 200 255);
--md-sys-color-on-primary: rgb(0 49 94);
--md-sys-color-primary-container: rgb(34 72 118);
--md-sys-color-on-primary-container: rgb(212 227 255);
--md-sys-color-secondary: rgb(154 203 250);
--md-sys-color-on-secondary: rgb(0 51 83);
--md-sys-color-secondary-container: rgb(12 74 114);
--md-sys-color-on-secondary-container: rgb(206 229 255);
--md-sys-color-tertiary: rgb(218 189 226);
--md-sys-color-on-tertiary: rgb(61 41 70);
--md-sys-color-tertiary-container: rgb(85 63 93);
--md-sys-color-on-tertiary-container: rgb(247 216 255);
--md-sys-color-error: rgb(255 180 171);
--md-sys-color-on-error: rgb(105 0 5);
--md-sys-color-error-container: rgb(147 0 10);
--md-sys-color-on-error-container: rgb(255 218 214);
--md-sys-color-background: rgb(17 19 24);
--md-sys-color-on-background: rgb(225 226 233);
--md-sys-color-surface: rgb(17 19 24);
--md-sys-color-on-surface: rgb(225 226 233);
--md-sys-color-surface-variant: rgb(67 71 78);
--md-sys-color-on-surface-variant: rgb(195 198 207);
--md-sys-color-outline: rgb(141 145 153);
--md-sys-color-outline-variant: rgb(67 71 78);
--md-sys-color-shadow: rgb(0 0 0);
--md-sys-color-scrim: rgb(0 0 0);
--md-sys-color-inverse-surface: rgb(225 226 233);
--md-sys-color-inverse-on-surface: rgb(46 48 53);
--md-sys-color-inverse-primary: rgb(60 96 144);
--md-sys-color-primary-fixed: rgb(212 227 255);
--md-sys-color-on-primary-fixed: rgb(0 28 58);
--md-sys-color-primary-fixed-dim: rgb(165 200 255);
--md-sys-color-on-primary-fixed-variant: rgb(34 72 118);
--md-sys-color-secondary-fixed: rgb(206 229 255);
--md-sys-color-on-secondary-fixed: rgb(0 29 50);
--md-sys-color-secondary-fixed-dim: rgb(154 203 250);
--md-sys-color-on-secondary-fixed-variant: rgb(12 74 114);
--md-sys-color-tertiary-fixed: rgb(247 216 255);
--md-sys-color-on-tertiary-fixed: rgb(39 20 48);
--md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
--md-sys-color-on-tertiary-fixed-variant: rgb(85 63 93);
--md-sys-color-surface-dim: rgb(17 19 24);
--md-sys-color-surface-bright: rgb(55 57 62);
--md-sys-color-surface-container-lowest: rgb(12 14 19);
--md-sys-color-surface-container-low: rgb(25 28 32);
--md-sys-color-surface-container: rgb(29 32 36);
--md-sys-color-surface-container-high: rgb(40 42 47);
--md-sys-color-surface-container-highest: rgb(50 53 58);
}
@@ -1,51 +0,0 @@
.light-high-contrast {
--md-sys-color-primary: rgb(0 34 70);
--md-sys-color-surface-tint: rgb(60 96 144);
--md-sys-color-on-primary: rgb(255 255 255);
--md-sys-color-primary-container: rgb(29 68 114);
--md-sys-color-on-primary-container: rgb(255 255 255);
--md-sys-color-secondary: rgb(0 36 61);
--md-sys-color-on-secondary: rgb(255 255 255);
--md-sys-color-secondary-container: rgb(3 70 110);
--md-sys-color-on-secondary-container: rgb(255 255 255);
--md-sys-color-tertiary: rgb(46 26 55);
--md-sys-color-on-tertiary: rgb(255 255 255);
--md-sys-color-tertiary-container: rgb(81 59 89);
--md-sys-color-on-tertiary-container: rgb(255 255 255);
--md-sys-color-error: rgb(78 0 2);
--md-sys-color-on-error: rgb(255 255 255);
--md-sys-color-error-container: rgb(140 0 9);
--md-sys-color-on-error-container: rgb(255 255 255);
--md-sys-color-background: rgb(249 249 255);
--md-sys-color-on-background: rgb(25 28 32);
--md-sys-color-surface: rgb(249 249 255);
--md-sys-color-on-surface: rgb(0 0 0);
--md-sys-color-surface-variant: rgb(224 226 236);
--md-sys-color-on-surface-variant: rgb(32 36 43);
--md-sys-color-outline: rgb(63 67 74);
--md-sys-color-outline-variant: rgb(63 67 74);
--md-sys-color-shadow: rgb(0 0 0);
--md-sys-color-scrim: rgb(0 0 0);
--md-sys-color-inverse-surface: rgb(46 48 53);
--md-sys-color-inverse-on-surface: rgb(255 255 255);
--md-sys-color-inverse-primary: rgb(228 236 255);
--md-sys-color-primary-fixed: rgb(29 68 114);
--md-sys-color-on-primary-fixed: rgb(255 255 255);
--md-sys-color-primary-fixed-dim: rgb(0 45 88);
--md-sys-color-on-primary-fixed-variant: rgb(255 255 255);
--md-sys-color-secondary-fixed: rgb(3 70 110);
--md-sys-color-on-secondary-fixed: rgb(255 255 255);
--md-sys-color-secondary-fixed-dim: rgb(0 47 77);
--md-sys-color-on-secondary-fixed-variant: rgb(255 255 255);
--md-sys-color-tertiary-fixed: rgb(81 59 89);
--md-sys-color-on-tertiary-fixed: rgb(255 255 255);
--md-sys-color-tertiary-fixed-dim: rgb(57 37 66);
--md-sys-color-on-tertiary-fixed-variant: rgb(255 255 255);
--md-sys-color-surface-dim: rgb(217 218 224);
--md-sys-color-surface-bright: rgb(249 249 255);
--md-sys-color-surface-container-lowest: rgb(255 255 255);
--md-sys-color-surface-container-low: rgb(243 243 250);
--md-sys-color-surface-container: rgb(237 237 244);
--md-sys-color-surface-container-high: rgb(231 232 238);
--md-sys-color-surface-container-highest: rgb(225 226 233);
}
@@ -1,51 +0,0 @@
.light-medium-contrast {
--md-sys-color-primary: rgb(29 68 114);
--md-sys-color-surface-tint: rgb(60 96 144);
--md-sys-color-on-primary: rgb(255 255 255);
--md-sys-color-primary-container: rgb(83 118 167);
--md-sys-color-on-primary-container: rgb(255 255 255);
--md-sys-color-secondary: rgb(3 70 110);
--md-sys-color-on-secondary: rgb(255 255 255);
--md-sys-color-secondary-container: rgb(71 121 163);
--md-sys-color-on-secondary-container: rgb(255 255 255);
--md-sys-color-tertiary: rgb(81 59 89);
--md-sys-color-on-tertiary: rgb(255 255 255);
--md-sys-color-tertiary-container: rgb(133 108 141);
--md-sys-color-on-tertiary-container: rgb(255 255 255);
--md-sys-color-error: rgb(140 0 9);
--md-sys-color-on-error: rgb(255 255 255);
--md-sys-color-error-container: rgb(218 52 46);
--md-sys-color-on-error-container: rgb(255 255 255);
--md-sys-color-background: rgb(249 249 255);
--md-sys-color-on-background: rgb(25 28 32);
--md-sys-color-surface: rgb(249 249 255);
--md-sys-color-on-surface: rgb(25 28 32);
--md-sys-color-surface-variant: rgb(224 226 236);
--md-sys-color-on-surface-variant: rgb(63 67 74);
--md-sys-color-outline: rgb(91 95 103);
--md-sys-color-outline-variant: rgb(119 123 131);
--md-sys-color-shadow: rgb(0 0 0);
--md-sys-color-scrim: rgb(0 0 0);
--md-sys-color-inverse-surface: rgb(46 48 53);
--md-sys-color-inverse-on-surface: rgb(240 240 247);
--md-sys-color-inverse-primary: rgb(165 200 255);
--md-sys-color-primary-fixed: rgb(83 118 167);
--md-sys-color-on-primary-fixed: rgb(255 255 255);
--md-sys-color-primary-fixed-dim: rgb(58 93 141);
--md-sys-color-on-primary-fixed-variant: rgb(255 255 255);
--md-sys-color-secondary-fixed: rgb(71 121 163);
--md-sys-color-on-secondary-fixed: rgb(255 255 255);
--md-sys-color-secondary-fixed-dim: rgb(43 96 137);
--md-sys-color-on-secondary-fixed-variant: rgb(255 255 255);
--md-sys-color-tertiary-fixed: rgb(133 108 141);
--md-sys-color-on-tertiary-fixed: rgb(255 255 255);
--md-sys-color-tertiary-fixed-dim: rgb(107 84 116);
--md-sys-color-on-tertiary-fixed-variant: rgb(255 255 255);
--md-sys-color-surface-dim: rgb(217 218 224);
--md-sys-color-surface-bright: rgb(249 249 255);
--md-sys-color-surface-container-lowest: rgb(255 255 255);
--md-sys-color-surface-container-low: rgb(243 243 250);
--md-sys-color-surface-container: rgb(237 237 244);
--md-sys-color-surface-container-high: rgb(231 232 238);
--md-sys-color-surface-container-highest: rgb(225 226 233);
}
@@ -1,51 +0,0 @@
.light {
--md-sys-color-primary: rgb(60 96 144);
--md-sys-color-surface-tint: rgb(60 96 144);
--md-sys-color-on-primary: rgb(255 255 255);
--md-sys-color-primary-container: rgb(212 227 255);
--md-sys-color-on-primary-container: rgb(0 28 58);
--md-sys-color-secondary: rgb(46 98 140);
--md-sys-color-on-secondary: rgb(255 255 255);
--md-sys-color-secondary-container: rgb(206 229 255);
--md-sys-color-on-secondary-container: rgb(0 29 50);
--md-sys-color-tertiary: rgb(110 86 118);
--md-sys-color-on-tertiary: rgb(255 255 255);
--md-sys-color-tertiary-container: rgb(247 216 255);
--md-sys-color-on-tertiary-container: rgb(39 20 48);
--md-sys-color-error: rgb(186 26 26);
--md-sys-color-on-error: rgb(255 255 255);
--md-sys-color-error-container: rgb(255 218 214);
--md-sys-color-on-error-container: rgb(65 0 2);
--md-sys-color-background: rgb(249 249 255);
--md-sys-color-on-background: rgb(25 28 32);
--md-sys-color-surface: rgb(249 249 255);
--md-sys-color-on-surface: rgb(25 28 32);
--md-sys-color-surface-variant: rgb(224 226 236);
--md-sys-color-on-surface-variant: rgb(67 71 78);
--md-sys-color-outline: rgb(116 119 127);
--md-sys-color-outline-variant: rgb(195 198 207);
--md-sys-color-shadow: rgb(0 0 0);
--md-sys-color-scrim: rgb(0 0 0);
--md-sys-color-inverse-surface: rgb(46 48 53);
--md-sys-color-inverse-on-surface: rgb(240 240 247);
--md-sys-color-inverse-primary: rgb(165 200 255);
--md-sys-color-primary-fixed: rgb(212 227 255);
--md-sys-color-on-primary-fixed: rgb(0 28 58);
--md-sys-color-primary-fixed-dim: rgb(165 200 255);
--md-sys-color-on-primary-fixed-variant: rgb(34 72 118);
--md-sys-color-secondary-fixed: rgb(206 229 255);
--md-sys-color-on-secondary-fixed: rgb(0 29 50);
--md-sys-color-secondary-fixed-dim: rgb(154 203 250);
--md-sys-color-on-secondary-fixed-variant: rgb(12 74 114);
--md-sys-color-tertiary-fixed: rgb(247 216 255);
--md-sys-color-on-tertiary-fixed: rgb(39 20 48);
--md-sys-color-tertiary-fixed-dim: rgb(218 189 226);
--md-sys-color-on-tertiary-fixed-variant: rgb(85 63 93);
--md-sys-color-surface-dim: rgb(217 218 224);
--md-sys-color-surface-bright: rgb(249 249 255);
--md-sys-color-surface-container-lowest: rgb(255 255 255);
--md-sys-color-surface-container-low: rgb(243 243 250);
--md-sys-color-surface-container: rgb(237 237 244);
--md-sys-color-surface-container-high: rgb(231 232 238);
--md-sys-color-surface-container-highest: rgb(225 226 233);
}
@@ -1,107 +0,0 @@
:root {
font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
line-height: 1.5;
font-weight: 400;
color-scheme: light dark;
color: rgba(255, 255, 255, 0.87);
background-color: #242424;
font-synthesis: none;
text-rendering: optimizeLegibility;
-webkit-font-smoothing: antialiased;
-moz-osx-font-smoothing: grayscale;
}
a {
font-weight: 500;
color: #646cff;
text-decoration: inherit;
}
a:hover {
color: #535bf2;
}
body {
margin: 0;
display: flex;
place-items: center;
min-width: 320px;
min-height: 100vh;
}
button {
border-radius: 8px;
border: 1px solid transparent;
padding: 0.6em 1.2em;
font-size: 1em;
font-weight: 500;
font-family: inherit;
background-color: #1a1a1a;
cursor: pointer;
transition: border-color 0.25s;
}
button:hover {
border-color: #646cff;
}
button:focus,
button:focus-visible {
outline: 4px auto -webkit-focus-ring-color;
}
button#query-button {
height: 5em;
vertical-align: top;
}
button#play-button {
padding: '10px 20px';
font-size: '16px';
cursor: 'pointer';
background-color: '#28a745';
color: '#fff';
border: 'none';
border-radius: '5px';
};
button#pause-button {
background-color: #ffc107;
}
button#stop-button {
background-color: #dc3545;
}
h1 {
font-size: 3.2em;
line-height: 1.1;
}
p#footer {
float: none;
clear: both;
vertical-align: bottom;
}
table#query-output {
float: none;
clear: both;
}
textarea#query-input {
height: 5em;
width: 50em;
}
@media (prefers-color-scheme: light) {
:root {
color: #213547;
background-color: #ffffff;
}
a:hover {
color: #747bff;
}
button {
background-color: #f9f9f9;
}
}
@@ -1,9 +0,0 @@
import React from 'react';
import ReactDOM from 'react-dom/client';
import App from './App';
ReactDOM.createRoot(document.getElementById('root')).render(
<React.StrictMode>
<App />
</React.StrictMode>
)
@@ -1,69 +0,0 @@
import { fileURLToPath, URL } from 'node:url';
import { defineConfig } from 'vite';
import plugin from '@vitejs/plugin-react';
import fs from 'fs';
import path from 'path';
import child_process from 'child_process';
import process, { env } from 'process';
const baseFolder =
process.env.APPDATA !== undefined && process.env.APPDATA !== ''
? `${process.env.APPDATA}/ASP.NET/https`
: `${process.env.HOME}/.aspnet/https`;
const certificateName = "bts.datastreamwebapp.client";
const certFilePath = path.join(baseFolder, `${certificateName}.pem`);
const keyFilePath = path.join(baseFolder, `${certificateName}.key`);
if (!fs.existsSync(certFilePath) || !fs.existsSync(keyFilePath)) {
if (0 !== child_process.spawnSync('dotnet', [
'dev-certs',
'https',
'--export-path',
certFilePath,
'--format',
'Pem',
'--no-password',
], { stdio: 'inherit', }).status) {
throw new Error("Could not create certificate.");
}
}
const target = env.ASPNETCORE_HTTPS_PORT
? `https://localhost:${env.ASPNETCORE_HTTPS_PORT}`
: env.ASPNETCORE_URLS
? env.ASPNETCORE_URLS.split(';')[0]
: 'https://localhost:5001';
export default defineConfig(({ $$command, mode, $$isSsrBuild, $$isPreview }) => {
return {
plugins: [plugin()],
resolve: {
alias: {
'@': fileURLToPath(new URL('./src', import.meta.url))
}
},
build: {
minify: false,
sourcemap: true,
outDir: `../../../build/Bts.DataStreamWebApp.Client/${mode}/dist`
},
server: {
proxy: {
'^/api': {
target,
secure: false
},
'^/ws': {
target,
secure: false
}
},
port: 5173,
https: {
key: fs.readFileSync(keyFilePath),
cert: fs.readFileSync(certFilePath),
}
}
};
});
@@ -1,35 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<InvariantGlobalization>true</InvariantGlobalization>
<SpaRoot>..\Bts.DataStreamWebApp.Client</SpaRoot>
<SpaProxyLaunchCommand>npm run dev</SpaProxyLaunchCommand>
<SpaProxyServerUrl>https://localhost:5173</SpaProxyServerUrl>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="CommandLineParser" Version="2.9.1" />
<PackageReference Include="Microsoft.AspNetCore.SpaProxy" Version="7.0.20" />
<PackageReference Include="Microsoft.AspNetCore.SpaServices.Extensions" Version="7.0.20" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Bts.DataStreamWebApp.Client\Bts.DataStreamWebApp.Client.esproj">
<ReferenceOutputAssembly>false</ReferenceOutputAssembly>
</ProjectReference>
<ProjectReference Include="..\DataStream\DataStream.csproj" />
<ProjectReference Include="..\SharedStreams\SharedStreams.csproj" />
<ProjectReference Include="..\..\client\spark_api\spark_api.csproj" />
<ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
<ProjectReference Include="..\..\client\spark_websocket\spark_websocket.csproj" />
</ItemGroup>
<!-- <ItemGroup>
<Content Update="wwwroot\alt-index.html" CopyToPublishDirectory="Never" />
</ItemGroup> -->
</Project>
@@ -1,106 +0,0 @@
{
"MasterWorker": {
"Interval": "00:00:00.1"
},
"ViewStreamDatabase": {
"Interval": "00:00:00.1"
},
"Sparkd": {
"Config": {
"ConfigServer": "http://localhost:8000/config"
},
"Credentials": {
"User": "bts",
"Password": ""
},
"EnableAdmin": true,
"EnableOrders": true,
"EnablePosition": true
},
"PubSub": {
"MillisecondsBetweenSubInfoUpdates": 1000,
"MillisecondsBetweenTimeSliceUpdates": 5000,
"MillisecondsBetweenNoBufferSpaceRetries": 100,
"RetriesOnNoBufferSpace": 1,
"DoItAllSecondsBetweenLogs": 0,
"HeartbeatInterval": 15000,
"MissingHeartbeatsBeforeGc": 50,
"UseManagedSockets": true,
"StateUpdatePeriod": 5000,
"StatisticsUpdatePeriod": 5000,
"DisposedConnectionCheckPeriod": 1000,
"MessageProcessingWarningThreshold": 1000,
"ServerPort": 12345,
"Routing": {
"PublisherRoutes": [
{
"Name": "default",
"ServerAddress": "bts-cloud://127.0.0.1:12345",
"TagRegExp": ".*"
}
],
"SubscriberRoutes": [
{
"Name": "default",
"ServerAddress": "bts-cloud://127.0.0.1:12345",
"TagRegExp": ".*"
}
]
},
"MaxPacketSize": 500000000,
"BlueCommMustDie": true,
"CrushableTags": [],
"TagTypeToAssemblyMap": {
"Entries": [
{
"HandlerClass": "BCG.PubSubServerTagHandler",
"Assembly": "DataStreamCore.dll"
}
]
},
"ServerQueue": "PubSubQueue",
"PortRange": {
"First": 30400,
"Last": 31700
},
"BindToRandomPort": false,
"DebugTags": "^NOTAGS$",
"LatencyReportThresholdMs": 1500,
"UseSubstations": false,
"UseDynamicClients": false,
"IsSubstation": false,
"InternallyPublishSubInfo": true,
"UseManagedClientsInNativeCode": false,
"CompressedTags": "^NOTAGS$",
"TagsToReportCompressionRatio": "^NOTAGS$",
"DisableMsmq": false,
"UsePrepublishedServerState": false,
"IpTranslation": null,
"GcActivationThreshold": 0,
"WaitPeriodAfterGc": 1000,
"TagHandlerFactory": {
"Handlers": [
{
"TagType": "DataStream",
"Assembly": "DataStreamCore",
"Class": "bts.PubsubServerStreamTagHandler"
},
{
"TagType": "System_Sub",
"Assembly": "PubSub",
"Class": "BCG.SystemSubscriptionMapPublisher"
},
{
"TagType": "System_Stats",
"Assembly": "PubSub",
"Class": "BCG.SystemStatsPublisher"
},
{
"TagType": ".*",
"Assembly": "DataStreamCore",
"Class": "bts.PubsubServerGenericDataHandler"
}
]
}
}
}
@@ -1,4 +0,0 @@
{
"content_root_path": "build/Bts.DataStreamWebApp.Client/production",
"web_root_path": "dist"
}
@@ -1,30 +0,0 @@
namespace Bts.DataStreamWebApp.Server.Controllers;
using Microsoft.AspNetCore.Mvc;
[Route("/ws")]
public class DataStreamSubscriptionController : ControllerBase {
readonly ILogger logger;
public DataStreamSubscriptionController(ILogger<DataStreamSubscriptionController> logger) {
this.logger = logger;
logger.LogInformation("Creating {ControllerType}", GetType().Name);
}
[Route("/ws")]
public async Task Get(CancellationToken cancellationToken) {
if (!HttpContext.WebSockets.IsWebSocketRequest) {
logger.LogError("Expected websocket request on path '{}'", HttpContext.Request.Path);
return;
}
logger.LogInformation("Establishing websocket connection...");
try {
using var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync();
logger.LogInformation("Websocket connection established.");
using var server = new DataStreamWebSocketServer(webSocket, logger);
await server.Run(cancellationToken);
} catch (Exception ex) {
logger.LogError("DataStreamSubscriptionController.Run threw exception: {Exception}", ex);
}
}
}
@@ -1,125 +0,0 @@
namespace Bts.DataStreamWebApp.Server.Controllers;
using BCG;
using bts.websocket_core.json_rpc;
using Bts.DataStreamCore;
using Bts.DataStreamWebApp.Server.Messages;
using System.Net.WebSockets;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
class DataStreamSubscription : IDisposable, IHostedService {
public readonly ViewStream ViewStream;
public DataPacket? LastFired;
readonly ILogger logger;
readonly WebSocket webSocket;
public DataStreamSubscription(string id, JsonRpcId requestId, ViewStream viewStream, WebSocket webSocket, ILogger logger) {
ViewStream = viewStream;
RequestId = requestId;
Id = id;
this.webSocket = webSocket;
this.logger = logger;
}
public string Id { get; }
public JsonRpcId RequestId { get; }
public SubscriptionStatus Status { get; private set; }
public void Start() {
if (Status == SubscriptionStatus.Subscribed) {
logger.LogWarning("Ignoring redundant call to 'Start'. Id = {Id} Status = {Status}", Id, Status);
return;
}
logger.LogInformation("Subscribing to ViewStream. SubscriptionId = {SubscriptionId}", Id);
Status = SubscriptionStatus.Subscribed;
ViewStream.onChange += onChange;
}
public Task StartAsync(CancellationToken cancellationToken) {
cancellationToken.ThrowIfCancellationRequested();
Start();
return Task.CompletedTask;
}
public Task Clear() {
if (LastFired is null) {
return Task.CompletedTask;
}
logger.LogInformation("Clearing subscription (firing DataPacket to delete all rows). Id = {Id} Status = {Status}", Id, Status);
return sendDataPacket("clear", new(new BlueTable(LastFired.schema), new BlueTable(LastFired.schema), LastFired.newRows));
}
public void Stop() {
if (Status == SubscriptionStatus.Unsubscribed) {
logger.LogWarning("Ignoring redundant call to 'Stop'. Id = {Id} Status = {Status}", Id, Status);
return;
}
logger.LogInformation("Unsubscribing from ViewStream. SubscriptionId = {SubscriptionId}", Id);
Status = SubscriptionStatus.Unsubscribed;
ViewStream.onChange -= onChange;
}
public Task StopAsync(CancellationToken cancellationToken) {
cancellationToken.ThrowIfCancellationRequested();
Stop();
return Task.CompletedTask;
}
public void Dispose() =>
Stop();
async void onChange(object newValue) {
if (newValue is not DataPacket dp) {
logger.LogError("Expected DataPacket in stream callback (actually received {}).", newValue?.GetType());
return;
}
if (webSocket.State != WebSocketState.Open) {
logger.LogError(
"Unsubscribing from stream early because WebSocket state = {WebSocketState}", webSocket.State);
Dispose();
return;
}
await sendDataPacket("subscribe", dp);
}
async Task sendDataPacket(string method, DataPacket dp) {
var serializerOpts = new JsonSerializerOptions {
NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals
};
serializerOpts.Converters.Add(new DataPacketJsonConverter() {
IncludeCurrent = true,
UseObjectRows = true,
});
var res = JsonRpcRequest.CreateNotification(method, dp);
var str = serialize(res, serializerOpts);
if (str is null) {
logger.LogError("Unsubscribing from stream early because we failed to serialize an outgoing DataPacket");
Dispose();
return;
}
logger.LogInformation(
"Sending DataPacket. NumRows = {NumRows} NumCols = {NumCols} MessageLength = {MessageLength}",
dp.newRows.Count,
dp.schema.columnNames.Length,
str.Length);
var bytes = Encoding.UTF8.GetBytes(str);
await webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, CancellationToken.None);
LastFired = dp;
}
string? serialize<T>(T message, JsonSerializerOptions? opts) {
opts ??= JsonSerializerOptions.Default;
try {
return JsonSerializer.Serialize(message, opts);
} catch (Exception ex) {
logger.LogError("Failed to serialize message of type {MessageType}: {Exception}", typeof(T), ex);
}
return null;
}
}
@@ -1,89 +0,0 @@
namespace Bts.DataStreamWebApp.Server;
using Bts.PubSub;
using CommandLine;
using System.Text.Json;
using System.Text.Json.Serialization;
class DataStreamWebAppServerConfig {
/// <summary>
/// The login user for sparkd
/// </summary>
[JsonPropertyName("user")]
[Option("user", HelpText = "The sparkd login user")]
public string? User { get; set; }
/// <summary>
/// The login password for sparkd
/// </summary>
[JsonPropertyName("password")]
[Option("pass", HelpText = "The sparkd login password")]
public string? Password { get; set; }
/// <summary>
/// The config server url for sparkd
/// </summary>
[JsonPropertyName("sparkd")]
[Option("sparkd", HelpText = "The sparkd config server")]
public string? Sparkd { get; set; }
[JsonPropertyName("pubsub")]
public PubSubConfig? PubSub { get; set; }
[Option("content-root-path")]
[JsonPropertyName("content_root_path")]
public string? ContentRootPath { get; set; }
[Option("web-root-path")]
[JsonPropertyName("web_root_path")]
public string? WebRootPath { get; set; }
[Option("environment-name")]
[JsonPropertyName("environment_name")]
public string? EnvironmentName { get; set; }
public DirectoryBrowserOptions? DirectoryBrowser { get; set; }
[Option(
"config",
MetaValue = "PATH",
HelpText = "Path to json file representing these options (DataStreamPublisherConfig)")]
[JsonIgnore]
public string? ConfigFile { get; set; }
public static DataStreamWebAppServerConfig Default() =>
new() {
User = "bts",
PubSub = PubSubConfig.Default(),
};
public void MergeWith(DataStreamWebAppServerConfig other) {
User ??= other.User;
Password ??= other.Password;
PubSub ??= other.PubSub;
Sparkd ??= other.Sparkd;
ContentRootPath ??= other.ContentRootPath;
WebRootPath ??= other.WebRootPath;
ConfigFile ??= other.ConfigFile;
}
/// <summary>
/// Merges options with those defined in `ConfigFile` (does not overwrite
/// non-null existing values).
/// </summary>
/// <exception cref="InvalidOperationException">
/// JsonSerializer.Deserialize returns null
/// </exception>
public void MergeWithConfigFile() {
if (ConfigFile is null) {
return;
}
var cfgString = File.ReadAllText(ConfigFile);
var parsedCfg = JsonSerializer.Deserialize<DataStreamWebAppServerConfig>(cfgString) ??
throw new InvalidOperationException($"Failed to parse json from {ConfigFile}");
MergeWith(parsedCfg);
ConfigFile = null;
}
}
@@ -1,121 +0,0 @@
namespace Bts.DataStreamWebApp.Server.Controllers;
using bts.websocket_core.json_rpc;
using Bts.DataStreamWebApp.Server.Messages;
using System.Net.WebSockets;
using System.Text;
using System.Text.Json;
using System.Text.Json.Nodes;
public class DataStreamWebSocketServer : IDisposable, IAsyncDisposable {
readonly ILogger logger;
readonly WebSocket webSocket;
readonly Dictionary<string, DataStreamSubscription> subscriptions = new();
public DataStreamWebSocketServer(WebSocket webSocket, ILogger logger) {
this.webSocket = webSocket;
this.logger = logger;
}
public async Task<int> Run(CancellationToken cancellationToken) {
logger.LogInformation("DataStreamWebSocketServer running");
await foreach (var req in JsonRpcRequestAsyncEnumerable.Create(webSocket, logger, cancellationToken)) {
await (req.Method switch {
"subscribe" => onSubscribe(req, cancellationToken),
"unsubscribe" => onUnsubscribe(req.Deserialize<UnsubscribeRequest>(), cancellationToken),
_ => Task.CompletedTask,
});
}
logger.LogInformation("DataStreamWebSocketServer stopped");
return 0;
}
/// <summary>
/// Disposes internal subscriptions. To additionally notify the client of the subscription status
/// change(s), use <see cref="DisposeAsync"/>.
/// </summary>
public void Dispose() {
GC.SuppressFinalize(this);
disposeSubscriptions();
}
/// <summary>
/// Disposes internal subscriptions (like <see cref="Dispose"/>), then sends 'subscription status change'
/// notifications to the client (unlike <see cref="Dispose"/>).
/// </summary>
public async ValueTask DisposeAsync() {
GC.SuppressFinalize(this);
var finalSubs = subscriptions.Values.Select(s => (s.RequestId, s.Id)).ToList();
disposeSubscriptions();
// Send a 'subscription status change' msg to the client for each one.
foreach (var (requestId, id) in finalSubs) {
await sendSubscriptionStatus(requestId, id, "unsubscribe", SubscriptionStatus.Unsubscribed, CancellationToken.None);
}
}
async Task onSubscribe(JsonRpcRequest<JsonNode> req, CancellationToken cancellationToken) {
ArgumentNullException.ThrowIfNull(req.Id);
ArgumentException.ThrowIfNullOrEmpty(req.Method);
if (!SubscribeRequest.ParseQuery(req, out var vs, out var subId, out var errMsg, out var errCode)) {
logger.LogError("Invalid 'subscribe' request: {Reason}", errMsg);
var rpcError = new JsonRpcError<JsonNode>() {
Code = (long)errCode,
Message = $"{errCode}: {errMsg}",
};
await send(rpcError, cancellationToken);
return;
}
if (subscriptions.TryGetValue(subId, out var oldSub)) {
oldSub.Dispose();
}
var sub = new DataStreamSubscription(subId, req.Id.Value, vs, webSocket, logger);
subscriptions[subId] = sub;
await sendSubscriptionStatus(req.Id.Value, subId, req.Method, SubscriptionStatus.Subscribed, cancellationToken);
await sub.StartAsync(cancellationToken);
}
async Task onUnsubscribe(JsonRpcRequest<UnsubscribeRequest> req, CancellationToken cancellationToken) {
ArgumentNullException.ThrowIfNull(req.Id);
ArgumentException.ThrowIfNullOrEmpty(req.Method);
var subId = req.Params?.SubscriptionId ?? string.Empty;
if (!subscriptions.TryGetValue(subId, out var sub)) {
logger.LogInformation("Ignoring '{Method}' request because no subscription was found with id {Id}.", req.Method, subId);
return;
}
logger.LogInformation("Disposing subscription");
sub.Dispose();
subscriptions.Remove(subId);
await sendSubscriptionStatus(req.Id.Value, subId, req.Method, SubscriptionStatus.Unsubscribed, cancellationToken);
}
async Task unsubscribe(JsonRpcId requestId, string subscriptionId, CancellationToken cancellationToken) {
await sendSubscriptionStatus(requestId, subscriptionId, "unsubscribe", SubscriptionStatus.Unsubscribed, cancellationToken);
}
Task sendSubscriptionStatus(JsonRpcId reqId, string subId, string method, SubscriptionStatus status, CancellationToken cancellationToken) {
logger.LogInformation("Sending {Method} ack. SubscriptionId = {SubscriptionId}", method, subId);
var response = JsonRpcResponse.Create(reqId, new SubscriptionStatusMessage() {
SubscriptionId = subId,
Status = status,
});
return send(response, cancellationToken);
}
Task send<T>(T message, CancellationToken cancellationToken) {
if (message is null) {
return Task.CompletedTask;
}
var str = JsonSerializer.Serialize(message);
var bytes = Encoding.UTF8.GetBytes(str);
logger.LogTrace("Sending data. Length = {Length}", bytes.Length);
return webSocket.SendAsync(bytes, WebSocketMessageType.Text, true, cancellationToken);
}
void disposeSubscriptions() {
logger.LogInformation("Disposing DataStreamWebSocketServer subscriptions. Count: {Count}", subscriptions.Count);
subscriptions.Values.DisposeAll();
subscriptions.Clear();
}
}
@@ -1,6 +0,0 @@
namespace Bts.DataStreamWebApp.Server.Messages;
enum ErrorCode {
None = 0,
InvalidSubscriptionRequest,
}
@@ -1,67 +0,0 @@
namespace Bts.DataStreamWebApp.Server.Messages;
using BCG;
using bts.websocket_core.json_rpc;
using Bts.DataStreamExpressions;
using System.Diagnostics.CodeAnalysis;
using System.Text.Json;
using System.Text.Json.Nodes;
using System.Text.Json.Serialization;
public class SubscribeRequest {
[JsonPropertyName("query")]
public string? Query { get; set; }
[JsonPropertyName("subscription_id")]
public string? SubscriptionId { get; set; }
internal static bool ParseQuery(JsonRpcRequest<JsonNode> req, [NotNullWhen(true)] out ViewStream? stream, [NotNullWhen(true)] out string? subscriptionId, [NotNullWhen(false)] out string? errorMessage, out ErrorCode errorCode) {
stream = null;
subscriptionId = null;
errorCode = ErrorCode.None;
var paramsJsonObj = req.Params?.AsObject();
if (paramsJsonObj is null) {
errorCode = ErrorCode.InvalidSubscriptionRequest;
errorMessage = "'params' object not found";
return false;
}
var subReq = paramsJsonObj.Deserialize<SubscribeRequest>();
if (subReq is null) {
errorCode = ErrorCode.InvalidSubscriptionRequest;
errorMessage = "'params' object invalid";
return false;
}
subscriptionId = subReq.SubscriptionId ?? string.Empty;
var query = subReq.Query;
if (query is null) {
errorCode = ErrorCode.InvalidSubscriptionRequest;
errorMessage = "'params.query' field not found (expected a string)";
return false;
}
errorMessage = null;
var exp = DataStreamExpressionsInterpreter.Evaluate(query);
if (exp is null) {
errorCode = ErrorCode.InvalidSubscriptionRequest;
errorMessage = "'params.query' expression has invalid syntax";
return false;
}
string? exceptionMsg = null;
try {
stream = exp.getView(new Dictionary<string, StatementExpression>(0));
} catch (Exception ex) {
exceptionMsg = ex.ToString();
}
if (stream is null) {
errorCode = ErrorCode.InvalidSubscriptionRequest;
errorMessage = "Failed to create ViewStream from 'params.query' expression (check for typos in arguments/typenames)";
if (exceptionMsg is not null) {
errorMessage += ". Error" + exceptionMsg;
}
return false;
}
return true;
}
}
@@ -1,18 +0,0 @@
namespace Bts.DataStreamWebApp.Server.Messages;
using System.Text.Json.Serialization;
enum SubscriptionStatus {
None,
Subscribed,
Unsubscribed,
}
class SubscriptionStatusMessage {
[JsonPropertyName("subscription_id")]
public string? SubscriptionId { get; set; }
[JsonPropertyName("status")]
[JsonConverter(typeof(JsonStringEnumConverter))]
public SubscriptionStatus Status { get; set; } = SubscriptionStatus.None;
}
@@ -1,8 +0,0 @@
namespace Bts.DataStreamWebApp.Server.Messages;
using System.Text.Json.Serialization;
public class UnsubscribeRequest {
[JsonPropertyName("subscription_id")]
public string? SubscriptionId { get; set; }
}
@@ -1,59 +0,0 @@
using Bts.DataStreamCore.Services;
using Bts.PubSub.Services;
using Spark.Api.Services;
using System.Reactive.Linq;
var builder = WebApplication.CreateBuilder(new WebApplicationOptions {
Args = args,
ApplicationName = "Bts.DataStreamWebApp.Server",
});
ConfigureApp(builder.Configuration);
ConfigureServices(builder.Services, builder.Configuration);
var app = builder.Build();
if (app.Environment.IsDevelopment()) {
app.UseDirectoryBrowser();
}
var wsOpts = new WebSocketOptions {
KeepAliveInterval = TimeSpan.FromMinutes(2)
};
app.UseWebSockets(wsOpts);
app.UseDefaultFiles();
app.UseStaticFiles();
if (app.Environment.IsDevelopment()) {
app.UseSwagger();
app.UseSwaggerUI();
}
app.UseHttpsRedirection();
app.UseAuthorization();
app.MapControllers();
try {
Console.Write($"Starting Application");
app.Run();
Console.Write($"Application exited gracefully.");
} catch (Exception ex) {
Console.Error.Write($"Application crashed: {ex}");
return 1;
}
return 0;
static void ConfigureApp(ConfigurationManager configBuilder) {
configBuilder.AddCommandLine(Environment.GetCommandLineArgs());
var cfgRoot = configBuilder.GetSection("config");
if (cfgRoot?.Value is not null) {
configBuilder.AddJsonFile(cfgRoot.Value);
}
}
static void ConfigureServices(IServiceCollection services, IConfiguration config) {
services.AddDirectoryBrowser();
services.AddControllers();
services.AddEndpointsApiExplorer();
services.AddSwaggerGen();
services.AddMasterWorker(config.GetSection("MasterWorker"));
services.AddViewStreamDatabase(config.GetSection("ViewStreamDatabase"));
services.AddPubSubClient(config.GetSection("PubSub"));
services.AddSparkd(config.GetSection("Sparkd"));
}
@@ -1,26 +0,0 @@
{
"$schema": "http://json.schemastore.org/launchsettings.json",
"profiles": {
"http": {
"commandName": "Project",
"dotnetRunMessages": true,
"launchBrowser": true,
"launchUrl": "swagger",
"applicationUrl": "http://localhost:5000",
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development",
"ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
}
},
"https": {
"commandName": "Project",
"dotnetRunMessages": true,
"launchBrowser": true,
"applicationUrl": "http://localhost:5000;https://localhost:5001",
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development",
"ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy",
}
},
},
}
@@ -1,9 +0,0 @@
{
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft.AspNetCore": "Warning"
}
},
"AllowedHosts": "*"
}
@@ -1,168 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8" />
<title></title>
<style>
table {
border: 0
}
.commslog-data {
font-family: Consolas, Courier New, Courier, monospace;
}
.commslog-server {
background-color: red;
color: white
}
.commslog-client {
background-color: green;
color: white
}
</style>
</head>
<body>
<h1>ALT WebSocket Sample Application</h1>
<p id="stateLabel">Ready to connect...</p>
<div>
<label for="connectionUrl">WebSocket Server URL:</label>
<input id="connectionUrl" />
<button id="connectButton" type="submit">Connect</button>
</div>
<p></p>
<div>
<label for="sendMessage">Message to send:</label>
<input id="sendMessage" disabled />
<button id="sendButton" type="submit" disabled>Send</button>
<button id="closeButton" disabled>Close Socket</button>
</div>
<h2>Communication Log</h2>
<table style="width: 800px">
<thead>
<tr>
<td style="width: 100px">From</td>
<td style="width: 100px">To</td>
<td>Data</td>
</tr>
</thead>
<tbody id="commsLog">
</tbody>
</table>
<script>
var connectionUrl = document.getElementById("connectionUrl");
var connectButton = document.getElementById("connectButton");
var stateLabel = document.getElementById("stateLabel");
var sendMessage = document.getElementById("sendMessage");
var sendButton = document.getElementById("sendButton");
var commsLog = document.getElementById("commsLog");
var closeButton = document.getElementById("closeButton");
var socket;
var scheme = document.location.protocol === "https:" ? "wss" : "ws";
var port = document.location.port ? (":" + document.location.port) : "";
connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
function updateState() {
function disable() {
sendMessage.disabled = true;
sendButton.disabled = true;
closeButton.disabled = true;
}
function enable() {
sendMessage.disabled = false;
sendButton.disabled = false;
closeButton.disabled = false;
}
connectionUrl.disabled = true;
connectButton.disabled = true;
if (!socket) {
disable();
} else {
switch (socket.readyState) {
case WebSocket.CLOSED:
stateLabel.innerHTML = "Closed";
disable();
connectionUrl.disabled = false;
connectButton.disabled = false;
break;
case WebSocket.CLOSING:
stateLabel.innerHTML = "Closing...";
disable();
break;
case WebSocket.CONNECTING:
stateLabel.innerHTML = "Connecting...";
disable();
break;
case WebSocket.OPEN:
stateLabel.innerHTML = "Open";
enable();
break;
default:
stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
disable();
break;
}
}
}
closeButton.onclick = function () {
if (!socket || socket.readyState !== WebSocket.OPEN) {
alert("socket not connected");
}
socket.close(1000, "Closing from client");
};
sendButton.onclick = function () {
if (!socket || socket.readyState !== WebSocket.OPEN) {
alert("socket not connected");
}
var data = sendMessage.value;
socket.send(data);
commsLog.innerHTML += '<tr>' +
'<td class="commslog-client">Client</td>' +
'<td class="commslog-server">Server</td>' +
'<td class="commslog-data">' + htmlEscape(data) + '</td></tr>';
};
connectButton.onclick = function() {
stateLabel.innerHTML = "Connecting";
socket = new WebSocket(connectionUrl.value);
socket.onopen = function (event) {
updateState();
commsLog.innerHTML += '<tr>' +
'<td colspan="3" class="commslog-data">Connection opened</td>' +
'</tr>';
};
socket.onclose = function (event) {
updateState();
commsLog.innerHTML += '<tr>' +
'<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
'</tr>';
};
socket.onerror = updateState;
socket.onmessage = function (event) {
commsLog.innerHTML += '<tr>' +
'<td class="commslog-server">Server</td>' +
'<td class="commslog-client">Client</td>' +
'<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
};
};
function htmlEscape(str) {
return str.toString()
.replace(/&/g, '&amp;')
.replace(/"/g, '&quot;')
.replace(/'/g, '&#39;')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;');
}
</script>
</body>
</html>
Binary file not shown.

Before

Width:  |  Height:  |  Size: 8.2 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.1 KiB

@@ -1,176 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8" />
<title></title>
<style>
table {
border: 0
}
.commslog-data {
font-family: Consolas, Courier New, Courier, monospace;
}
.commslog-server {
background-color: red;
color: white
}
.commslog-client {
background-color: green;
color: white
}
</style>
</head>
<body>
<h1>DataStreams via WebSocket</h1>
<p id="stateLabel">Ready to connect...</p>
<div>
<label for="connectionUrl">WebSocket Server URL:</label>
<input id="connectionUrl" />
<button id="connectButton" type="submit">Connect</button>
</div>
<p></p>
<div>
<label for="sendMessage">Query:</label>
<input id="sendMessage" disabled />
<button id="sendButton" type="submit" disabled>Send</button>
<button id="closeButton" disabled>Close Socket</button>
</div>
<h2>Communication Log</h2>
<table style="width: 800px">
<thead>
<tr>
<td style="width: 100px">From</td>
<td style="width: 100px">To</td>
<td>Data</td>
</tr>
</thead>
<tbody id="commsLog">
</tbody>
</table>
<script>
var connectionUrl = document.getElementById("connectionUrl");
var connectButton = document.getElementById("connectButton");
var stateLabel = document.getElementById("stateLabel");
var sendMessage = document.getElementById("sendMessage");
var sendButton = document.getElementById("sendButton");
var commsLog = document.getElementById("commsLog");
var closeButton = document.getElementById("closeButton");
var socket;
var scheme = document.location.protocol === "https:" ? "wss" : "ws";
var port = document.location.port ? (":" + document.location.port) : "";
connectionUrl.value = scheme + "://" + document.location.hostname + port + "/ws" ;
function updateState() {
function disable() {
sendMessage.disabled = true;
sendButton.disabled = true;
closeButton.disabled = true;
}
function enable() {
sendMessage.disabled = false;
sendButton.disabled = false;
closeButton.disabled = false;
}
connectionUrl.disabled = true;
connectButton.disabled = true;
if (!socket) {
disable();
} else {
switch (socket.readyState) {
case WebSocket.CLOSED:
stateLabel.innerHTML = "Closed";
disable();
connectionUrl.disabled = false;
connectButton.disabled = false;
break;
case WebSocket.CLOSING:
stateLabel.innerHTML = "Closing...";
disable();
break;
case WebSocket.CONNECTING:
stateLabel.innerHTML = "Connecting...";
disable();
break;
case WebSocket.OPEN:
stateLabel.innerHTML = "Open";
enable();
break;
default:
stateLabel.innerHTML = "Unknown WebSocket State: " + htmlEscape(socket.readyState);
disable();
break;
}
}
}
closeButton.onclick = function () {
if (!socket || socket.readyState !== WebSocket.OPEN) {
alert("socket not connected");
}
socket.close(1000, "Closing from client");
};
sendButton.onclick = function () {
if (!socket || socket.readyState !== WebSocket.OPEN) {
alert("socket not connected");
}
const rpcReq = {
jsonrpc: "2.0",
method: "subscribe",
params: {
query: sendMessage.value,
},
id: 1,
};
const rpcReqStr = JSON.stringify(rpcReq);
socket.send(rpcReqStr);
commsLog.innerHTML += '<tr>' +
'<td class="commslog-client">Client</td>' +
'<td class="commslog-server">Server</td>' +
'<td class="commslog-data">' + htmlEscape(rpcReqStr) + '</td></tr>';
};
connectButton.onclick = function() {
stateLabel.innerHTML = "Connecting";
socket = new WebSocket(connectionUrl.value);
socket.onopen = function (event) {
updateState();
commsLog.innerHTML += '<tr>' +
'<td colspan="3" class="commslog-data">Connection opened</td>' +
'</tr>';
};
socket.onclose = function (event) {
updateState();
commsLog.innerHTML += '<tr>' +
'<td colspan="3" class="commslog-data">Connection closed. Code: ' + htmlEscape(event.code) + '. Reason: ' + htmlEscape(event.reason) + '</td>' +
'</tr>';
};
socket.onerror = updateState;
socket.onmessage = function (event) {
commsLog.innerHTML += '<tr>' +
'<td class="commslog-server">Server</td>' +
'<td class="commslog-client">Client</td>' +
'<td class="commslog-data">' + htmlEscape(event.data) + '</td></tr>';
};
};
function htmlEscape(str) {
return str.toString()
.replace(/&/g, '&amp;')
.replace(/"/g, '&quot;')
.replace(/'/g, '&#39;')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;');
}
</script>
</body>
</html>
File diff suppressed because it is too large. Load diff
@@ -1,26 +0,0 @@
namespace Bts.DataStream;
using BCG;
using Spark.Core.Disposables;
using System;
using System.Reactive.Linq;
public static class DataStreamExtensions {
public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
source as ViewStream ?? new ObservableViewStream(source);
public static IObservable<DataPacket> AsObservable(this ViewStream source) {
return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
IDisposable Subscribe(IObserver<DataPacket> observer) {
source.onChange += OnChange;
return Disposable.Create(() => source.onChange -= OnChange);
void OnChange(object? newValue) =>
observer.OnNext(newValue as DataPacket ?? throw new ArgumentException(
$"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
}
}
}
@@ -1,29 +0,0 @@
namespace Bts.DataStream;
using BCG;
using bts;
using System;
using System.IO;
public class PandasScriptStream : ViewStream {
public PandasScriptStream(ViewStream vs, string file)
: this(vs, file, null) { }
public PandasScriptStream(ViewStream vs, string file, string? pythonExe)
: this(vs, file, pythonExe, ViewStreamDatabase.instance) { }
public PandasScriptStream(ViewStream vs, string file, string? pythonExe, ViewStreamDatabase vsd) {
mySetRealSource(new PandasStream(vs, readFile(), pythonExe, vsd));
string readFile() {
try {
return File.ReadAllText(file);
} catch (Exception ex) {
Log.Gen.error($"PandasScriptStream error reading file: {ex}");
throw;
}
}
}
}
@@ -1,130 +1,62 @@
namespace Bts.DataStream; // using System;
// using System.Diagnostics;
using BCG;
using BCG.Extensions; // using BCG;
using BCG.Utilities.General; // using BCG.Utilities.General;
using bts;
using bts.collections; // namespace bts {
using Bts.DataStreamCore; // public class PandasStream : ViewStream, IWorker {
// //C:\Users\Owner\AppData\Local\Microsoft\WindowsApps\python.exe
using Spark.Core.Disposables;
// public PandasStream(ViewStream vs, ViewStreamDatabase vsd, string pandas_code) {
using System;
using System.Collections.Generic; // }
using System.Diagnostics;
using System.IO; // public PandasStream(ViewStream vs, string pandas_code) :
using System.Linq; // this(vs, ViewStreamDatabase.instance, pandas_code) {
using System.Runtime.CompilerServices; // }
using System.Text;
public class PandasStream : ViewStream, IWorker { // protected override void myOnFirstSubscriber() {
readonly string inputFile = $"PandasStream_{Guid.NewGuid()}.csv";
readonly string outputFile = $"PandasStream_{Guid.NewGuid()}.csv"; // // string python_code = @"
readonly string pythonExe; // // import winsound
readonly ViewStreamDatabase vsd; // // frequency = 2500 # Set frequency (in Hz)
readonly CSVSaverStream inputCsvStream; // // duration = 1000 # Set duration (in milliseconds)
readonly Lazy<string> pythonCode; // // winsound.Beep(frequency, duration)
IDictionary<int, IBlueRow>? lastRowMap = null; // // ";
int inputVersion = 0; // string python_code = @"
int outputVersion = 0; // file_path = \"C:\\Temp\\test2.txt\\" # r prefix for raw string to avoid escape characters
DateTime lastFireTime = DateTime.MinValue; // with open(file_path, \"w\") as f:
TimeSpan fireThrottle = TimeSpan.FromSeconds(3); // f.write(\"Hello, world!\n\")
IDisposable? dispose; // ";
public PandasStream(ViewStream vs, string dataFrameExpression) // execute_python_code(python_code);
: this(vs, dataFrameExpression, null, ViewStreamDatabase.instance) { } // }
public PandasStream(ViewStream vs, string dataFrameExpression, string? pythonExe, ViewStreamDatabase vsd) { // static void execute_python_code(string python_code) {
pythonExe ??= Environment.GetEnvironmentVariable("PYTHON_EXE") ?? @"C:\Users\johnc\GoogleDrive\Repos\bts\spark\.venv\Scripts\python.exe"; // ProcessStartInfo start = new ProcessStartInfo();
this.pythonExe = pythonExe; // start.FileName = "python"; // Python executable
this.vsd = vsd; // start.Arguments = "-c \"" + python_code.Replace("\"", "\\\"") + "\""; // Pass code as argument
// This should fire an update when it writes to the output file // start.UseShellExecute = false;
inputCsvStream = new CSVSaverStream(vs, inputFile, true, int.MaxValue, true); // start.RedirectStandardOutput = true;
pythonCode = new(getPythonCode);
// using (Process process = Process.Start(start)) {
string getPythonCode() { // using (System.IO.StreamReader reader = process.StandardOutput) {
var imports = // string result = reader.ReadToEnd();
@"import numpy as np // Console.Write(result);
import pandas as pd // }
import sys // }
"; // }
var bd = new StringBuilder(imports);
bd.AppendLine($"df=pd.read_csv('{inputFile}')"); // protected override void myOnLastSubscriber() {
bd.AppendLine(dataFrameExpression); // }
bd.AppendLine($"df.to_csv('{outputFile}', index=False)");
return bd.ToString(); // public void doWork() {
}
} // }
protected override void myOnFirstSubscriber() { // }
vsd.addWorker(this); // }
inputCsvStream.onChange += onInputChanged;
this.dispose = Disposable.Create(dispose);
void onInputChanged(object newValue) =>
inputVersion++;
void dispose() {
inputCsvStream.onChange -= onInputChanged;
vsd.removeWorker(this);
deleteFile(inputFile);
deleteFile(outputFile);
}
void deleteFile(string file, [CallerArgumentExpression(nameof(file))] string name = "") {
if (!File.Exists(file)) {
return;
}
try {
File.Delete(file);
} catch (Exception ex) {
Log.Gen.error($"PandasStream. Error deleting {name} file '{file}': {ex}");
}
}
}
protected override void myOnLastSubscriber() =>
Disposable.dispose(ref dispose);
static void executePythonCode(string pythonCode, string? pythonExe = null) {
pythonExe ??= "python";
var start = new ProcessStartInfo {
FileName = pythonExe,
Arguments = "-c \"" + pythonCode.Replace("\"", "\\\"") + "\"",
UseShellExecute = false,
RedirectStandardOutput = true
};
using var process = Process.Start(start)
?? throw new InvalidOperationException("Failed to launch python");
process.WaitForExit();
}
void IWorker.doWork() {
if (outputVersion == inputVersion) {
return;
}
var now = DateTime.Now;
var elapsedSinceLastFire = now - lastFireTime;
if (elapsedSinceLastFire < fireThrottle) {
return;
}
outputVersion = inputVersion;
executePythonCode(pythonCode.Value, pythonExe);
var rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(outputFile, out var schema);
var rowMap = rows as IDictionary<int, IBlueRow> ?? rows.Cast<IBlueRow>().ToDictionary(r => r.Id());
var comp = BlueRowEqualityComparer.Instance;
var deletedRows = new BlueTable(schema);
var addedRows = new BlueTable(schema);
if (lastRowMap is not null) {
deletedRows.SetItems(lastRowMap.Where(oldRow =>
!(rowMap.TryGetValue(oldRow.Key, out var newRow) && comp.Equals(newRow, oldRow.Value))));
addedRows.SetItems(rowMap.Where(newRow =>
!(lastRowMap.TryGetValue(newRow.Key, out var oldRow) && comp.Equals(oldRow, newRow.Value))));
}
lastRowMap = rowMap;
lastFireTime = DateTime.Now;
myFireEvent(new DataPacket(rows, addedRows, deletedRows));
}
}
@@ -39,8 +39,8 @@ namespace BCG {
protected override void myOnFirstSubscriber() { protected override void myOnFirstSubscriber() {
MasterWorker.getInstance(false).addWorker(this, _send_period); MasterWorker.getInstance(false).addWorker(this, _send_period);
var server_rci = new RemoteConnectionInfo(_ip, _port); var server_address = new BlutoSubAddress(_ip, _port);
bts.AsyncTCPConnector_APM.connect_to(server_rci, this); bts.AsyncTCPConnector_APM.connect_to(server_address, this);
} }
protected override void myOnLastSubscriber() { protected override void myOnLastSubscriber() {
@@ -1,37 +0,0 @@
namespace Bts.DataStream;
using BCG;
using bts;
using System;
public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
readonly IObservable<DataPacket> source;
IDisposable? subscription;
public ObservableViewStream(IObservable<DataPacket> source) =>
this.source = source;
public void Dispose() {
GC.SuppressFinalize(this);
if (subscription is not null) {
subscription.Dispose();
subscription = null;
}
}
protected override void myOnFirstSubscriber() {
Log.isnull(subscription);
subscription = source.Subscribe(myFireEvent);
}
protected override void myOnLastSubscriber() {
Log.notnull(subscription);
subscription.Dispose();
subscription = null;
}
IDisposable IObservable<DataPacket>.Subscribe(IObserver<DataPacket> observer) =>
source.Subscribe(observer);
}
@@ -0,0 +1,32 @@
using BCG;
namespace BCG
{
// this is a stream so we can run it from the DSQA (since dsqa can only load streams)
[DataStreamUserRespresentation("add_to_cloud")]
public class AddToCloudStream : ViewStream
{
private readonly string _name, _stream_exp;
private StreamOffloadRequestor _requestor = new StreamOffloadRequestor();
public AddToCloudStream(string name, params string[] lines)
: this(name, 0, lines)
{
}
public AddToCloudStream(string name, int persist, params string[] lines)
{
_name = name;
_stream_exp = getExpression(lines);
_requestor.add_to_cloud(_name, _stream_exp, persist == 1);
mySetRealSource(new TableStream("cloud_streams"));
}
static string getExpression(params string[] lines) {
for (int i = 0; i < lines.Length; i++) {
// unescape quotes
lines[i] = lines[i].Replace("\\\"", "\"");
}
return string.Join("\n", lines);
}
}
}
@@ -1,24 +1,31 @@
using BCG;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using mbts.config;
using bts.utils;
using System.Threading.Tasks;
using Bts.DataStreamExpressions; using Bts.DataStreamExpressions;
using System.Collections.Generic;
using BCG.Utilities.General; using BCG.Utilities.General;
using bts;
using Bts.PubSub;
using Bts.DataStreamCore;
namespace BCG { namespace BCG
{
// listens to the requests posted on "cloud_streams" request table, and // listens to the requests posted on "cloud_streams" request table, and
// if unfulfilled starts a ViewPubsliher // if unfulfilled starts a ViewPubsliher
public class RemoteQueryManager : IWorker { public class CloudStreamManager : IWorker
public class StreamPubUnit { {
public class StreamPubUnit
{
public string name; public string name;
public string stream_expression; public string stream_expression;
public ViewStream stream; public ViewStream stream;
public bool persist; public bool persist;
public ViewPublisher view_publisher; public ViewPublisher view_publisher;
public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_) { public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_)
{
name = name_; name = name_;
stream_expression = stream_expression_; stream_expression = stream_expression_;
stream = stream_; stream = stream_;
@@ -26,7 +33,7 @@ namespace BCG {
view_publisher = view_publisher_; view_publisher = view_publisher_;
} }
} }
static ViewStream? build_stream(string stream_expression) static ViewStream build_stream(string stream_expression)
{ {
try try
{ {
@@ -34,7 +41,9 @@ namespace BCG {
var rv = exp.getView(new Dictionary<string, StatementExpression>(0)); var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
return rv; return rv;
} catch (Exception ex) { }
catch (Exception ex)
{
bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}"); bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
return null; return null;
} }
@@ -42,40 +51,35 @@ namespace BCG {
private IDictionary<string, StreamPubUnit> private IDictionary<string, StreamPubUnit>
_stream_pub_map = new Dictionary<string, StreamPubUnit>(); _stream_pub_map = new Dictionary<string, StreamPubUnit>();
private ViewStream _stream = new TableStream("cloud_streams"); private ViewStream _stream = new TableStream("cloud_streams");
readonly IPubTableBroker _broker; private PubTableBroker _broker = new PubTableBroker();
readonly IManagedSubscriber _sub;
public CloudStreamManager()
public RemoteQueryManager(IManagedSubscriber managedSubscriber, IPubTableBroker pubTableBroker) { {
_sub = managedSubscriber;
_broker = pubTableBroker;
_stream.onChange += _on_str; _stream.onChange += _on_str;
_sub.Subscribe("RemoteRequest", (string tag, byte[] data) => {
RemoteRequest req = RemoteRequest.construct_from_bytes(data);
_spawn_remote_stream(req.published_table_name, req.stream_expression, false);
});
} }
private void _on_str(object obj) private void _on_str(object obj)
{ {
if (obj is not DataPacket dp) { DataPacket dp = obj as DataPacket;
return;
}
foreach (IBlueRow r in dp.newRows) foreach (IBlueRow r in dp.newRows)
{ {
string name = r.get<string>("name"); string name = r.get<string>("name");
bool persist = r.get<bool>("persist"); bool persist = r.get<bool>("persist");
string stream_exp = r.get<string>("stream_exp"); string stream_exp = r.get<string>("stream_exp");
if (!_stream_pub_map.ContainsKey(name)) { if (!_stream_pub_map.ContainsKey(name))
_spawn_remote_stream(name, stream_exp, persist); {
_assign(name, stream_exp, persist);
} }
} }
} }
private void _spawn_remote_stream(string name, string stream_expression, bool persist) { private void _assign(string name, string stream_expression, bool persist)
{
bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}"); bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
ViewStream stream = build_stream(stream_expression); ViewStream stream = build_stream(stream_expression);
_expose_to_ws(name, stream); _expose_to_ws(name, stream);
if (persist) { if (persist)
{
stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000); // 10MBs stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000); // 10MBs
} }
ViewPublisher view_pub = new ViewPublisher(name, _broker, stream); ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
@@ -83,12 +87,14 @@ namespace BCG {
_stream_pub_map.Add(name, u); _stream_pub_map.Add(name, u);
} }
private void _expose_to_ws(string name, ViewStream stream) { private void _expose_to_ws(string name, ViewStream stream)
{
// TODO: john // TODO: john
} }
public void doWork() { public void doWork()
{
} }
} }
} }
@@ -1,48 +0,0 @@
using BCG;
using bts;
using System;
namespace BCG {
// this is a stream so we can run it from the DSQA (since dsqa can only load streams)
[DataStreamUserRespresentation("remoteQuery")]
public class RemoteQueryStream : ViewStream {
private readonly string _name, _stream_exp;
private PublisherCore _publisher_core = new PublisherCore();
private ViewStream _source_str;
public RemoteQueryStream(params string[] lines) {
_stream_exp = get_expression(lines);
_name = Guid.NewGuid().ToString();
_name = $"remotereq_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds().ToString()}";
//_name.Replace("-", "_");
}
protected override void myOnFirstSubscriber() {
RemoteRequest rem_req = new RemoteRequest(_stream_exp, _name);
byte[] rem_req_bytes = rem_req.serialize();
_publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
_source_str = new TableStream(_name);
_source_str.onChange += new ChangeDelegate(_handle_data);
}
protected override void myOnLastSubscriber() {
//maybe there should be a way to turn remote requests off ?
_source_str.onChange -= new ChangeDelegate(_handle_data);
}
static string get_expression(params string[] lines) {
for (int i = 0; i < lines.Length; i++) {
// unescape quotes
lines[i] = lines[i].Replace("\\\"", "\"");
}
return string.Join("\n", lines);
}
private void _handle_data(object o) {
myFireEvent(o);
}
}
}
@@ -1,31 +0,0 @@
namespace bts {
using System.IO;
using BCG;
using System.IO;
using System.Threading.Tasks.Dataflow;
public class RemoteRequest {
public readonly string stream_expression;
public readonly string published_table_name;
public RemoteRequest(string stream_expression_, string published_table_name_) {
stream_expression = stream_expression_;
published_table_name = published_table_name_;
}
public byte[] serialize() {
var str = new MemoryStream();
var bw = new BinaryWriter(str);
bw.write_string_cpp(stream_expression);
bw.write_string_cpp(published_table_name);
return str.ToArray();
}
public static RemoteRequest construct_from_bytes(byte[] ser_bytes) {
BinaryReader br = new BinaryReader(new MemoryStream(ser_bytes));
string stream_expression_ = br.read_string_cpp();
string published_table_name_ = br.read_string_cpp();
return new RemoteRequest(stream_expression_, published_table_name_);
}
}
}
@@ -1,54 +1,59 @@
// using BCG; using BCG;
// using bts; using System.Linq;
// using System.Linq; namespace BCG
{
public class StreamOffloadRequestor
{
public class OffloadOp
{
string name;
string stream_exp;
}
private IBlueTableSchema _schema;
private IBlueRowCollection _rows;
private PubTableBroker _broker = new PubTableBroker();
ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
public StreamOffloadRequestor()
{
_schema = new BlueTableSchema("cloud_streams",
new ColumnInfo("id", 0, typeof(int)),
new ColumnInfo("name", 1, typeof(string)),
new ColumnInfo("persist", 2, typeof(bool)),
new ColumnInfo("stream_exp", 3, typeof(string)));
_existing_offloads_vs.onChange += _on_str;
}
private void _on_str(object obj)
{
DataPacket dp = obj as DataPacket;
_rows = dp.newRows;
// this is needed so the broker knows the state of the table
_broker.createTable(_rows);
}
public void add_to_cloud(string name, string stream_exp, bool persist)
{
// check if it exists
if (_rows == null)
{
BlueTable table = new BlueTable(_schema);
IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
table.addRow(nr);
_broker.createTable(table);
// create table
}
else
{
if (exists(name))
return;
IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
_broker.addRow(nr);
}
}
// namespace BCG { bool exists(string name)
// // used on the client {
// public class StreamOffloadRequestor { return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
// public class OffloadOp { }
// string name; }
// string stream_exp; }
// }
// private IBlueTableSchema _schema;
// private IBlueRowCollection _rows;
// private PubTableBroker _broker = new PubTableBroker();
// ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
// public StreamOffloadRequestor() {
// _schema = new BlueTableSchema("cloud_streams",
// new ColumnInfo("id", 0, typeof(int)),
// new ColumnInfo("name", 1, typeof(string)),
// new ColumnInfo("persist", 2, typeof(bool)),
// new ColumnInfo("stream_exp", 3, typeof(string)));
// _existing_offloads_vs.onChange += _on_str;
// }
// private void _on_str(object obj) {
// DataPacket dp = obj as DataPacket;
// _rows = dp.newRows;
// // this is needed so the broker knows the state of the table
// _broker.create_table(_rows);
// }
// public void add_to_cloud(string name, string stream_exp, bool persist) {
// // check if it exists
// if (_rows == null) {
// BlueTable table = new BlueTable(_schema);
// IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
// table.addRow(nr);
// _broker.create_table(table);
// // create table
// } else {
// if (exists(name))
// return;
// IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
// // TODO: use make_batch_changes instead
// Log.assert(false, "Fix addRow");
// //_broker.addRow(nr);
// }
// }
// bool exists(string name) {
// return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
// }
// }
// }
@@ -1,14 +1,17 @@
using System; using System;
using System.Collections; using System.Collections;
namespace BCG { namespace BCG
internal class AddColumnsRow : IBlueRow { {
internal class AddColumnsRow : IBlueRow
{
private IBlueTableSchema mySchema; private IBlueTableSchema mySchema;
private IBlueRow myRow; private IBlueRow myRow;
//private object[] myCachedValues; //private object[] myCachedValues;
#region Constructors #region Constructors
public AddColumnsRow(IBlueTableSchema schema, IBlueRow row) { public AddColumnsRow (IBlueTableSchema schema, IBlueRow row)
{
mySchema = schema; mySchema = schema;
myRow = row; myRow = row;
//myCachedValues = new object[schema.columnNames.Length]; //myCachedValues = new object[schema.columnNames.Length];
@@ -16,78 +19,85 @@ namespace BCG {
#endregion #endregion
#region IBlueRow members #region IBlueRow members
public IBlueTableSchema schema { get { return mySchema; } } public IBlueTableSchema schema{get{return mySchema;}}
#endregion #endregion
#region ICollection members #region ICollection members
public int Count { get { return mySchema.columnNames.Length; } } public int Count {get{return mySchema.columnNames.Length;}}
public bool IsSynchronized { get { return false; } } public bool IsSynchronized {get{return false;}}
public object SyncRoot { get { return this; } } public object SyncRoot {get{return this;}}
public void CopyTo(Array array, int index) { public void CopyTo (Array array, int index)
{
int i = index; int i = index;
foreach (object o in this as IEnumerable) foreach (object o in this as IEnumerable)
array.SetValue(o, i++); array.SetValue (o, i++);
} }
#endregion #endregion
#region IDictionary members #region IDictionary members
public bool IsFixedSize { get { return true; } } public bool IsFixedSize {get{return true;}}
public bool IsReadOnly { get { return true; } } public bool IsReadOnly {get{return true;}}
/// <summary> /// <summary>
/// The key can either be the string corrsponding to the column name or /// The key can either be the string corrsponding to the column name or
/// the order of the column /// the order of the column
/// </summary> /// </summary>
public object this[object key] { public object this [object key]
get { {
get
{
if (key is string) if (key is string)
return this[(string)key]; return this[(string)key];
return this[(int)key]; return this[(int)key];
} }
set { throw new NotSupportedException("The IBlueRow is readonly"); } set {throw new NotSupportedException ("The IBlueRow is readonly");}
} }
public object this[int index] { public object this[int index]
get { {
get
{
//if (myCachedValues[index] == null) //if (myCachedValues[index] == null)
// myCachedValues[index] = mySchema[index].isConstant ? mySchema[index].colValue : myRow[index]; // myCachedValues[index] = mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
//return myCachedValues[index]; //return myCachedValues[index];
return mySchema[index].isConstant ? DBNull.Value : myRow[index]; return mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
} }
} }
public object this[string colName] { get { return this[mySchema[colName].colNo]; } } public object this[string colName] {get{return this[mySchema[colName].colNo];}}
public ICollection Keys { get { return mySchema.columnNames; } } public ICollection Keys{get{return mySchema.columnNames;}}
public ICollection Values { public ICollection Values
get { {
get
{
object[] retVal = new object[mySchema.columnNames.Length]; object[] retVal = new object[mySchema.columnNames.Length];
CopyTo(retVal, 0); CopyTo (retVal, 0);
return retVal; return retVal;
} }
} }
public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); } public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");}
public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); } public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
public bool Contains(object key) { return mySchema[(string)key] != null; } public bool Contains (object key){return mySchema[(string)key] != null;}
/// <summary> /// <summary>
/// Note, the default enumerator is the IDictionary's enumerator, so if user needs to /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
/// iterate through the values, typecast this into an IEnumerable /// iterate through the values, typecast this into an IEnumerable
/// </summary> /// </summary>
/// <returns></returns> /// <returns></returns>
public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); } public IDictionaryEnumerator GetEnumerator() {return new BlueRowDictionaryEnumerator(this);}
IEnumerator IEnumerable.GetEnumerator () {return new BlueRowEnumerator(this);}
IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); } public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");}
public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
#endregion #endregion
} }
@@ -1,34 +1,6 @@
using System; namespace BCG;
namespace BCG { public class AddColumnsStream : MapStream {
public class AddColumnsStream : ViewStream { public AddColumnsStream(ViewStream vs, string columnDef)
public static Tuple<string, Type, object> parse_exp(string exp) { : base(vs, new ColumnAdder(columnDef)) { }
string[] toks = exp.Split(':'); }
string col_name = toks[0];
string col_value = toks[1];
Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
return Tuple.Create(col_name, tup.Item1, tup.Item2);
}
private Tuple<string, Type, object> _new_col;
private IBlueTableSchema _out_schema;
public AddColumnsStream(ViewStream vs, string col_def) {
_new_col = parse_exp(col_def);
ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
mySetRealSource(rs);
}
private IBlueTableSchema _get_schema(IBlueTableSchema input_schema) {
if (_out_schema == null)
_out_schema = BlueDataUtility.add_column(input_schema, _new_col.Item1, _new_col.Item2);
return _out_schema;
}
private IBlueRow _map(IBlueRow r) {
MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
rv.setValue(_new_col.Item1, _new_col.Item3);
return rv;
}
}
}
@@ -1,24 +1,48 @@
using BCG; using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
namespace bts { using System.Runtime.Serialization.Formatters.Binary;
namespace BCG
{
[DataStreamUserRespresentation("loader")] [DataStreamUserRespresentation("loader")]
public class LoaderStream : ViewStream { public class LoaderStream : ViewStream, ISideEffectStream
{
public class Exception : System.Exception
{public Exception(string msg) : base(msg) { }}
private string _filename; public LoaderStream(string fileName, int secondsBetweenPolls)
private IStreamMsgDeserilizer _deser = new StreamIODeserializationController(); {
mySetRealSource(
new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls,
public LoaderStream(string filename) { ViewStreamDatabase.instance));
_filename = filename;
} }
protected override void myOnFirstSubscriber() { public LoaderStream(string fileName)
byte[] ser_msg = StreamSerializationUtils.read_from_file(_filename); {
StreamIOMessage msg = _deser.deserialize(ser_msg); if (fileName == null)
Log.assert(msg.op == StreamIOMessage.StreamOp.CREATE); throw new LoaderStream.Exception("File name cannot be null");
IBlueRowCollection rows = msg.create_message.rows;
DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket); for (int i = 0; i < 5; i++)
myFireEvent(dp); try
{
using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
{
BinaryFormatter bf = new BinaryFormatter();
#pragma warning disable SYSLIB0011
BlueTable bt = (BlueTable)bf.Deserialize(fs);
#pragma warning restore SYSLIB0011
mySetRealSource(new BlueRowCollectionStaticStream(bt));
}
break;
}
catch (IOException)
{
System.Threading.Thread.Sleep(50);
}
} }
} }
} }
@@ -5,7 +5,6 @@ using System.Text;
using System.IO; using System.IO;
using Google.Protobuf; using Google.Protobuf;
using System.Globalization; using System.Globalization;
using bts;
namespace BCG { namespace BCG {
// Writes all the udpates to a stream in a file. // Writes all the udpates to a stream in a file.
@@ -25,7 +24,7 @@ namespace BCG {
private ViewStream _source_stream; private ViewStream _source_stream;
private string _token; private string _token;
private FileStream _out_fs; private FileStream _out_fs;
private IStreamIOMsgSerializer _serializer;// = new UniversalStreamIOSerializer(); private IBlueDataSerializer _serializer;
private string _file_name = String.Empty; private string _file_name = String.Empty;
private long _rollover_size; private long _rollover_size;
private readonly string _directory_path; private readonly string _directory_path;
@@ -95,12 +94,13 @@ namespace BCG {
DataPacket dp = new_value as DataPacket; DataPacket dp = new_value as DataPacket;
byte[] serialized_update = null; byte[] serialized_update = null;
bool _write_create = _serializer == null || _n_updates_in_current_file == 0; bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
_serializer = _serializer ?? new UniversalStreamIOSerializer(); _serializer = _serializer ?? new UniversalSerializer(dp.schema);
if (_write_create) { if (_write_create) {
// new files always starts with create // new files always starts with create
serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows))); _serializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out serialized_update);
} else } else
serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows))); _serializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out serialized_update);
BinaryWriter bw = new BinaryWriter(_out_fs); BinaryWriter bw = new BinaryWriter(_out_fs);
long utc_epoch = epoch_now(); long utc_epoch = epoch_now();
@@ -1,68 +1,68 @@
// using System; using System;
// using System.Collections.Generic; using System.Collections.Generic;
// using System.Linq; using System.Linq;
// using System.Text; using System.Text;
// using System.IO; using System.IO;
// namespace BCG namespace BCG
// { {
// public class RecorderStream : ViewStream public class RecorderStream : ViewStream
// { {
// private ViewStream mySource; private ViewStream mySource;
// private DateTime myStartTime; private DateTime myStartTime;
// private FileStream myOutFile; private FileStream myOutFile;
// private string myFileName; private string myFileName;
// private IBlueDataSerializer mySerializer; private IBlueDataSerializer mySerializer;
public RecorderStream(ViewStream src, string recordFile)
{
mySource = src;
myFileName = recordFile;
// public RecorderStream(ViewStream src, string recordFile) myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss"));
// { }
// mySource = src;
// myFileName = recordFile;
// myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss")); protected override void myOnFirstSubscriber()
// } {
myStartTime = DateTime.Now;
mySource.onChange += new ChangeDelegate(mySource_onChange);
myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
}
protected override void myOnLastSubscriber()
{
mySource.onChange -= mySource_onChange;
myOutFile.Close();
}
// protected override void myOnFirstSubscriber() void mySource_onChange(object newValue)
// { {
// myStartTime = DateTime.Now; DataPacket dp = (DataPacket)newValue;
// mySource.onChange += new ChangeDelegate(mySource_onChange);
// myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate); BlueTable toWrite = new BlueTable(dp.schema);
// }
// protected override void myOnLastSubscriber() if (mySerializer == null)
// { {
// mySource.onChange -= mySource_onChange; mySerializer = new UniversalSerializer(dp.schema);
// myOutFile.Close(); }
// } foreach (IBlueRow r in dp.newRows)
toWrite.addRow(r);
byte[] result;
// void mySource_onChange(object newValue) if (myLastValue == null)
// { mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
// DataPacket dp = (DataPacket)newValue; else
mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
// BlueTable toWrite = new BlueTable(dp.schema); BinaryWriter bw = new BinaryWriter(myOutFile);
//write down what time this happened
bw.Write((DateTime.Now - myStartTime).TotalSeconds);
bw.Write(result.Length);
bw.Write(result);
myOutFile.Flush();
// if (mySerializer == null) myFireEvent(dp);
// { }
// mySerializer = new UniversalSerializer(dp.schema); }
// } }
// foreach (IBlueRow r in dp.newRows)
// toWrite.addRow(r);
// byte[] result;
// if (myLastValue == null)
// mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
// else
// mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
// BinaryWriter bw = new BinaryWriter(myOutFile);
// //write down what time this happened
// bw.Write((DateTime.Now - myStartTime).TotalSeconds);
// bw.Write(result.Length);
// bw.Write(result);
// myOutFile.Flush();
// myFireEvent(dp);
// }
// }
// }
@@ -1,37 +1,64 @@
using System; // using System;
using System.Collections.Generic; // using System.Collections.Generic;
using System.Text; // using System.Text;
using System.IO; // using System.IO;
using BCG;
namespace bts { // using System.Runtime.Serialization.Formatters.Binary;
// this saves the total state in the filr provided in the cons
[DataStreamUserRespresentation("saver")]
public class SaverStream : ViewStream {
private ViewStream _stream;
private string _filename;
private IStreamIOMsgSerializer _serializer = new StreamIOSerializationController();
public SaverStream(ViewStream vs, string filename) { // namespace BCG
_stream = vs; // {
_filename = filename; // [DataStreamUserRespresentation("saver")]
} // public class SaverStream : ViewStream, ISideEffectStream
// {
// private ViewStream myStream;
// private string myFileName;
protected override void myOnFirstSubscriber() { _stream.onChange += new ChangeDelegate(_handle_changes); } // public class Exception : System.Exception
// {
// public Exception(string msg) : base(msg) { }
// }
void _handle_changes(object dp_obj) { // public SaverStream(ViewStream vs, string fileName)
DataPacket dp = dp_obj as DataPacket; // {
_save(dp); // if (fileName == null)
myFireEvent(dp_obj); // throw new SaverStream.Exception("File name cannot be null");
} // myStream = vs;
// myFileName = fileName;
protected override void myOnLastSubscriber() { _stream.onChange -= new ChangeDelegate(_handle_changes); } // }
protected void _save(DataPacket dp) { // protected override void myOnFirstSubscriber()
StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows); // { myStream.onChange += new ChangeDelegate(myHandleChanges); }
byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
StreamSerializationUtils.write_to_file(str_bytes, _filename);
} // void myHandleChanges(object newValue)
} // {
} // DataPacket dp = newValue as DataPacket;
// mySave(dp);
// myFireEvent(newValue);
// }
// protected override void myOnLastSubscriber()
// { myStream.onChange -= new ChangeDelegate(myHandleChanges); }
// protected void mySave(DataPacket dp)
// {
// for (int i = 0; i < 5 ; i++)
// try
// {
// using (FileStream fs = new FileStream(myFileName, FileMode.Create, FileAccess.Write))
// {
// BinaryFormatter bf = new BinaryFormatter();
// BlueTable t = new BlueTable(dp.schema);
// foreach (IBlueRow r in dp.newRows) // to guarantee table is serializable
// t.addRow(new BlueRow(r));
// bf.Serialize(fs, t);
// }
// break;
// }
// catch (IOException)
// {
// System.Threading.Thread.Sleep(5);
// }
// }
// }
// }
@@ -1,12 +1,12 @@
// using System; using System;
// using System.Collections.Generic; using System.Collections.Generic;
// using System.Linq; using System.Linq;
// using System.Text; using System.Text;
// using System.IO; using System.IO;
// using System.Runtime.Serialization.Formatters.Binary; using System.Runtime.Serialization.Formatters.Binary;
// using BCG.Utilities.General; using BCG.Utilities.General;
// using bts.collections; using bts.collections;
// using System.Reflection.Metadata; using System.Reflection.Metadata;
namespace BCG { namespace BCG {
@@ -15,115 +15,115 @@ namespace BCG {
FirstOfDay, FirstOfDay,
LastOfDay LastOfDay
}; };
// private BlueTable _current_table; private BlueTable _current_table;
// private IClientDeserializer _deserilializer = new ClientUniversalDeserializer(); private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
// public DateTime _snapshot_time; public DateTime _snapshot_time;
// public string _snapshot_file; public string _snapshot_file;
// public IEnumerable<PlaybackPacket> _packets; public IEnumerable<PlaybackPacket> _packets;
// public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) { public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) {
// DateTime dt; DateTime dt;
// var parsed = DateTime.TryParse(snapshot_date, out dt); var parsed = DateTime.TryParse(snapshot_date, out dt);
// dt = dt.Date; dt = dt.Date;
// bts.Log.assert(parsed); bts.Log.assert(parsed);
// SpecificSnapshot specific_snapshot_enum; SpecificSnapshot specific_snapshot_enum;
// parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum); parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum);
// bts.Log.assert(parsed); bts.Log.assert(parsed);
// string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum); string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum);
// ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token); ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token);
// mySetRealSource(rs); mySetRealSource(rs);
// } }
// public SnapshotStream(string snapshot_dt, string directory_name, string token) { public SnapshotStream(string snapshot_dt, string directory_name, string token) {
// _snapshot_time = PersistorStream.parse_datetime(snapshot_dt); _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
// string token_directory_name = $"{directory_name}\\{token}"; string token_directory_name = $"{directory_name}\\{token}";
// _snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time); _snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time);
// } }
// protected override void myOnFirstSubscriber() { protected override void myOnFirstSubscriber() {
// FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read); FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read);
// _packets = StreamPersistorUtils.load_packets_from_file(fs); _packets = StreamPersistorUtils.load_packets_from_file(fs);
// PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time); PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time);
// if (snapshot_packet != null) { if (snapshot_packet != null) {
// foreach (PlaybackPacket p in _packets) { foreach (PlaybackPacket p in _packets) {
// byte[] ops; byte[] ops;
// Array row_array; Array row_array;
// _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array); _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
// for (int i = 0; i < ops.Length; i++) { for (int i = 0; i < ops.Length; i++) {
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[]; IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
// byte op = ops[i]; byte op = ops[i];
// if (op == StreamOpCode.create) { if (op == StreamOpCode.create) {
// IBlueRowCollection full_table = rows_args[0]; IBlueRowCollection full_table = rows_args[0];
// _current_table = new BlueTable(full_table); _current_table = new BlueTable(full_table);
// } else if (op == StreamOpCode.multipleRemoveAdd) { } else if (op == StreamOpCode.multipleRemoveAdd) {
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[]; //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection; IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection; IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
// BlueDataUtility.absorbChanges(_current_table, dels, adds); BlueDataUtility.absorbChanges(_current_table, dels, adds);
// } else { } else {
// bts.Log.assert(false, $"Oprator: {op} is not expected"); bts.Log.assert(false, $"Oprator: {op} is not expected");
// } }
// if (p.idx == snapshot_packet.idx) { if (p.idx == snapshot_packet.idx) {
// myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema))); myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
// break; break;
// } }
// } }
// } }
// } }
// } }
// private void _absorb_packet(byte[] ops, Array row_array) { private void _absorb_packet(byte[] ops, Array row_array) {
// for (int i = 0; i < ops.Length; i++) { for (int i = 0; i < ops.Length; i++) {
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[]; IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
// byte op = ops[i]; byte op = ops[i];
// if (op == StreamOpCode.create) { if (op == StreamOpCode.create) {
// IBlueRowCollection full_table = rows_args[0]; IBlueRowCollection full_table = rows_args[0];
// _current_table = new BlueTable(full_table); _current_table = new BlueTable(full_table);
// } else if (op == StreamOpCode.multipleRemoveAdd) { } else if (op == StreamOpCode.multipleRemoveAdd) {
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[]; //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection; IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection; IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
// BlueDataUtility.absorbChanges(_current_table, dels, adds); BlueDataUtility.absorbChanges(_current_table, dels, adds);
// } else { } else {
// bts.Log.assert(false, $"Operator: {op} is not expected"); bts.Log.assert(false, $"Operator: {op} is not expected");
// } }
// } }
// } }
// protected override void myOnLastSubscriber() { protected override void myOnLastSubscriber() {
// } }
// // public event SnapshotStream.Progress onProgress; // public event SnapshotStream.Progress onProgress;
// // public event Action onFileExhausted; // public event Action onFileExhausted;
// private PlaybackPacket _locate_packet_to_replay() { private PlaybackPacket _locate_packet_to_replay() {
// foreach (var p in _packets) { foreach (var p in _packets) {
// // returns the first non replayed packet // returns the first non replayed packet
// if (!p.replayed) if (!p.replayed)
// return p; return p;
// } }
// return null; return null;
// } }
// private void _fire_packet(byte[] ops, Array row_array) { private void _fire_packet(byte[] ops, Array row_array) {
// for (int i = 0; i < ops.Length; i++) { for (int i = 0; i < ops.Length; i++) {
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[]; IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
// byte op = ops[i]; byte op = ops[i];
// if (op == StreamOpCode.create) { if (op == StreamOpCode.create) {
// IBlueRowCollection full_table = rows_args[0]; IBlueRowCollection full_table = rows_args[0];
// _current_table = new BlueTable(full_table); _current_table = new BlueTable(full_table);
// //_current_table.addRows(full_table); //_current_table.addRows(full_table);
// myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) : myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
// (myLastValue as DataPacket).newRows)); (myLastValue as DataPacket).newRows));
// } else if (op == StreamOpCode.multipleRemoveAdd) { } else if (op == StreamOpCode.multipleRemoveAdd) {
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[]; //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection; IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection; IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
// BlueDataUtility.absorbChanges(_current_table, dels, adds); BlueDataUtility.absorbChanges(_current_table, dels, adds);
// myFireEvent(new DataPacket(_current_table, adds, dels)); myFireEvent(new DataPacket(_current_table, adds, dels));
// } else { } else {
// bts.Log.assert(false, $"Oprator: {op} is not expected"); bts.Log.assert(false, $"Oprator: {op} is not expected");
// } }
// } }
// } }
} }
} }
@@ -1,10 +1,16 @@
using BCG;
using System;
using System.Data;
using System.Collections;
using BCG.Utilities;
using System.Text.RegularExpressions;
using Bts.DataStream; using Bts.DataStream;
using mbts;
using mbts.spark;
using mbts.config; using mbts.config;
using System;
using System.Text.RegularExpressions;
namespace BCG.StreamWorld.Addition namespace BCG.StreamWorld.Addition
{ {
[DataStreamUserRespresentation (false)] [DataStreamUserRespresentation (false)]
@@ -145,6 +151,7 @@ namespace BCG
private string myFilterString; private string myFilterString;
private BCG.StreamWorld.Addition.FilterStream myFilterStream; private BCG.StreamWorld.Addition.FilterStream myFilterStream;
private bool myUseNewFilter = Configuration.get<DataStreamConfig>().UseNewFilter; private bool myUseNewFilter = Configuration.get<DataStreamConfig>().UseNewFilter;
private HighPerformanceTimer _timer = new HighPerformanceTimer();
#region constructors #region constructors
public delegate bool OnFilter(IBlueRow r); public delegate bool OnFilter(IBlueRow r);
@@ -219,6 +226,7 @@ namespace BCG
private void myHandleCallbacks(object newValue) private void myHandleCallbacks(object newValue)
{ {
_timer.start();
DataPacket retVal; DataPacket retVal;
DataPacket arrivedPacket = newValue as DataPacket; DataPacket arrivedPacket = newValue as DataPacket;
myLastReceivedRows = arrivedPacket.newRows; myLastReceivedRows = arrivedPacket.newRows;
@@ -247,6 +255,7 @@ namespace BCG
if (myFilter.filter(r)) if (myFilter.filter(r))
myLastFiredRows.addRow(r); myLastFiredRows.addRow(r);
} }
_timer.stop();
myFireEvent(retVal); myFireEvent(retVal);
} }
@@ -1,66 +1,37 @@
using System; using System;
using System.Text.RegularExpressions; using System.Collections.Generic;
using System.Linq;
using System.Text;
using BCG.Utilities.General; using BCG.Utilities.General;
namespace BCG { namespace BCG
public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable { {
public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable
public static TimeSpan parse_duration(string input) { {
int days = 0, hours = 0, minutes = 0, seconds = 0;
// Regular expression to match the pattern (e.g., "1d", "1h", "15m", "45s")
Regex regex = new Regex(@"(\d+)([dhms])");
MatchCollection matches = regex.Matches(input);
foreach (Match match in matches) {
int value = int.Parse(match.Groups[1].Value);
string unit = match.Groups[2].Value;
switch (unit) {
case "d":
days = value;
break;
case "h":
hours = value;
break;
case "m":
minutes = value;
break;
case "s":
seconds = value;
break;
}
}
return new TimeSpan(days, hours, minutes, seconds);
}
#region constructors #region constructors
public TimeBasedFilter(string col_name, string row_time_to_live)
: this(col_name, parse_duration(row_time_to_live)) {
}
public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive) public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive)
: this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) { : this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */)
{
} }
public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency)
: this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency) { : this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency)
{
} }
public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive) public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive)
: this(vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) { : this (vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */)
{
} }
public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) { public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency)
myVsd = vsd; {
_time_col = rowTimeColumnName; myVsd = vsd;
_ttl = rowTimeToLive; myRowTimeColumnName = rowTimeColumnName;
myFilterFrequency = filterFrequency; myRowTimeToLive = rowTimeToLive;
myNextFilter = DateTime.MinValue; myFilterFrequency = filterFrequency;
myNextFilter = DateTime.MinValue;
// add worker // add worker
myVsd.addWorker(this); myVsd.addWorker(this);
@@ -70,25 +41,32 @@ namespace BCG {
public event FilterChangeDelegate onFilterChange; public event FilterChangeDelegate onFilterChange;
#region IStreamFilter #region IStreamFilter
public bool filter(IBlueRow row) { public bool filter(IBlueRow row)
DateTime now = DateTime.Now; {
DateTime now = DateTime.Now;
DateTime timeStamp = (DateTime)row[myRowTimeColumnName];
DateTime row_time = row.get_datetime(_time_col, DateTime.Now); if (now > timeStamp)
if (now > row_time) { {
TimeSpan span = now - row_time; TimeSpan span = now - timeStamp;
if (span >= _ttl) {
if (span >= myRowTimeToLive)
{
return (false); return (false);
} }
} }
return (true); return (true);
} }
#endregion #endregion
#region IWorker #region IWorker
public void doWork() { public void doWork()
{
DateTime now = DateTime.Now; DateTime now = DateTime.Now;
if (now > myNextFilter) { if (now > myNextFilter)
{
// fire event // fire event
onFilterChange(); onFilterChange();
@@ -98,7 +76,8 @@ namespace BCG {
#endregion #endregion
#region IDisposable #region IDisposable
public void Dispose() { public void Dispose()
{
// remove worker // remove worker
myVsd.removeWorker(this); myVsd.removeWorker(this);
} }
@@ -106,20 +85,10 @@ namespace BCG {
#region state #region state
private ViewStreamDatabase myVsd; private ViewStreamDatabase myVsd;
private string _time_col; private string myRowTimeColumnName;
private TimeSpan _ttl; private TimeSpan myRowTimeToLive;
private TimeSpan myFilterFrequency; private TimeSpan myFilterFrequency;
private DateTime myNextFilter; private DateTime myNextFilter;
#endregion #endregion
} }
[DataStreamUserRespresentation("timefilter")]
public class TimeBasedFilterStream : ViewStream {
public TimeBasedFilterStream(ViewStream base_stream, string time_col, string time_span)
:
this(ViewStreamDatabase.instance, base_stream, time_col, TimeBasedFilter.parse_duration(time_span)) { }
public TimeBasedFilterStream(ViewStreamDatabase vsd, ViewStream base_stream, string time_col, TimeSpan time_span) {
TimeBasedFilter filter = new TimeBasedFilter(vsd, time_col, time_span);
mySetRealSource(new FilterStream(base_stream, filter));
}
}
} }
@@ -4,14 +4,18 @@ using BCG.Utilities;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
namespace BCG { namespace BCG
public class GroupByStream : MethodDrivenStream { {
private class MyFuncGroupByEvaluator : IGroupByEvaluator { public class GroupByStream : MethodDrivenStream
{
private class MyFuncGroupByEvaluator : IGroupByEvaluator
{
private string[] _groupby_cols; private string[] _groupby_cols;
private Func<DataPacket, IBlueRow> _groupby_func; private Func<DataPacket, IBlueRow> _groupby_func;
public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func) { public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
{
_groupby_cols = groupby_cols.ToArray(); _groupby_cols = groupby_cols.ToArray();
_groupby_func = groupby_func; _groupby_func = groupby_func;
} }
public IBlueRow evaluate(DataPacket packet) { return _groupby_func(packet); } public IBlueRow evaluate(DataPacket packet) { return _groupby_func(packet); }
@@ -25,7 +29,7 @@ namespace BCG {
/// Serves as a quick way to access all rows belonging to the same group /// Serves as a quick way to access all rows belonging to the same group
/// </summary> /// </summary>
private bool myIdGuard = true; private bool myIdGuard = true;
private RowDictionaryTable myBookKeepingTable; private RowDictionaryTable myBookKeepingTable ;
private IGroupByEvaluator myGroupByEvaluator; private IGroupByEvaluator myGroupByEvaluator;
private IBlueTableSchema mySchema; //input schema, as output schema can differ (IGroupByEvaluator's private IBlueTableSchema mySchema; //input schema, as output schema can differ (IGroupByEvaluator's
//can map a schema to another //can map a schema to another
@@ -33,98 +37,112 @@ namespace BCG {
private int myIdGenerator = 0; private int myIdGenerator = 0;
public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func) public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
: this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func)) { } : this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func))
{ }
public GroupByStream(ViewStream stream, IGroupByEvaluator groupByEvaluator) public GroupByStream (ViewStream stream, IGroupByEvaluator groupByEvaluator)
: this(stream, groupByEvaluator, true) { } : this (stream, groupByEvaluator, true)
{}
public GroupByStream(ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard) public GroupByStream (ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard)
: base(stream) { : base (stream)
{
myGroupByEvaluator = groupbyEvaluator; myGroupByEvaluator = groupbyEvaluator;
myIdGuard = idGuard; myIdGuard = idGuard;
} }
protected override void myHandleFirstCall() { protected override void myHandleFirstCall ()
{
mySchema = myLastArrivedPacket.schema; mySchema = myLastArrivedPacket.schema;
myOutputSchema = myGroupByEvaluator.getOutputSchema(mySchema); myOutputSchema = myGroupByEvaluator.getOutputSchema (mySchema);
myBookKeepingTable = new RowDictionaryTable(myLastArrivedPacket.newRows, myBookKeepingTable = new RowDictionaryTable (myLastArrivedPacket.newRows,
myGroupByEvaluator.groupByColumns); myGroupByEvaluator.groupByColumns);
GroupByOutputTable temp = new GroupByOutputTable(myOutputSchema); GroupByOutputTable temp = new GroupByOutputTable (myOutputSchema);
foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups()) { foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups())
IBlueRow newRow = myEvaluate(new DataPacket(rowGroup, {
rowGroup, new BlueTable(mySchema))); IBlueRow newRow = myEvaluate (new DataPacket (rowGroup,
rowGroup, new BlueTable (mySchema)));
if (newRow != null) if (newRow != null)
temp.addRow(newRow, myGetGroupString(rowGroup)); temp.addRow (newRow, myGetGroupString (rowGroup));
} }
myLastFiredPacket = new DataPacket(temp, new BlueTable(temp), new BlueTable(temp.schema)); myLastFiredPacket = new DataPacket (temp, new BlueTable(temp), new BlueTable (temp.schema));
} }
/// <summary> /// <summary>
/// Adding need not to anything as deletion already takes care /// Adding need not to anything as deletion already takes care
/// of addition /// of addition
/// </summary> /// </summary>
protected override void myHandleIncrementalAdd() { } protected override void myHandleIncrementalAdd() {}
/// <summary> /// <summary>
/// Both deletes and adds are handled here to make the intermediate results not /// Both deletes and adds are handled here to make the intermediate results not
/// appear in the spitted fired DataPacket (note this is done only in GroupByStream /// appear in the spitted fired DataPacket (note this is done only in GroupByStream
/// because of the nature of it) /// because of the nature of it)
/// </summary> /// </summary>
protected override void myHandleIncrementalDelete() { protected override void myHandleIncrementalDelete()
{
myFireSwitch = false; myFireSwitch = false;
RowDictionaryTable delGroups, addGroups; RowDictionaryTable delGroups, addGroups;
//form groups for deleted and added rows //form groups for deleted and added rows
myInitIncrementalChangeData(out delGroups, out addGroups); myInitIncrementalChangeData (out delGroups, out addGroups);
ArrayList handledAddGroups = new ArrayList(); ArrayList handledAddGroups = new ArrayList ();
string groupRepString; string groupRepString;
foreach (IBlueRowCollection delGroup in delGroups.getGroups()) { foreach (IBlueRowCollection delGroup in delGroups.getGroups ())
{
// BlueEventLog.logAssert (delGroup.Count > 0); // BlueEventLog.logAssert (delGroup.Count > 0);
groupRepString = myGetGroupString(delGroup); groupRepString = myGetGroupString (delGroup);
IBlueRowCollection addGroup = addGroups.getGroup(groupRepString) == null ? IBlueRowCollection addGroup = addGroups.getGroup (groupRepString) == null ?
new BlueTable(mySchema) : addGroups.getGroup(groupRepString); new BlueTable (mySchema) : addGroups.getGroup (groupRepString);
handledAddGroups.Add(addGroup); handledAddGroups.Add (addGroup);
IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString) == null ? IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString) == null ?
new BlueTable(mySchema) : myBookKeepingTable.getGroup(groupRepString); new BlueTable (mySchema) : myBookKeepingTable.getGroup (groupRepString);
IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup, delGroup)); IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup, delGroup));
myFireSwitch |= myMergeRow(newRow, groupRepString); myFireSwitch |= myMergeRow (newRow, groupRepString);
} }
foreach (IBlueRowCollection addGroup in addGroups.getGroups()) { foreach (IBlueRowCollection addGroup in addGroups.getGroups ())
{
// BlueEventLog.logAssert (addGroup.Count > 0); // BlueEventLog.logAssert (addGroup.Count > 0);
if (!handledAddGroups.Contains(addGroup)) { if (!handledAddGroups.Contains (addGroup))
{
//this is the case when there are rows added to a group and none //this is the case when there are rows added to a group and none
//have been removed, hence it 'escapes' from being dealt with in //have been removed, hence it 'escapes' from being dealt with in
//the loop above //the loop above
groupRepString = myGetGroupString(addGroup); groupRepString = myGetGroupString (addGroup);
IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString); IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString);
// BlueEventLog.logAssert (aggGroup != null && aggGroup.Count > 0); // BlueEventLog.logAssert (aggGroup != null && aggGroup.Count > 0);
IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup, IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup,
new BlueTable(aggGroup.schema))); new BlueTable (aggGroup.schema)));
myFireSwitch |= myMergeRow(newRow, groupRepString); myFireSwitch |= myMergeRow (newRow, groupRepString);
} }
} }
} }
private bool myMergeRow(IBlueRow newRow, string groupRepString) { private bool myMergeRow (IBlueRow newRow, string groupRepString)
IBlueRow existingRow = ((GroupByOutputTable)myLastFiredPacket.newRows).getRow(groupRepString); {
IBlueRow existingRow = ((GroupByOutputTable) myLastFiredPacket.newRows).getRow (groupRepString);
//BlueEventLog.logAssert (newRow != null || existingRow != null); //BlueEventLog.logAssert (newRow != null || existingRow != null);
if (existingRow == null && newRow == null) if (existingRow == null && newRow == null)
return false; return false;
else if (existingRow != null && newRow != null && BlueDataUtility.compute_equality(existingRow, newRow)) else if (existingRow != null && newRow != null && BlueDataUtility.computeRowEquality (existingRow, newRow))
return false; return false;
else if (existingRow == null) { else if (existingRow == null)
{
//row added for the first time //row added for the first time
((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString); ((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString);
((BlueTable)myLastFiredPacket.addedRows).addRow(newRow); ((BlueTable) myLastFiredPacket.addedRows).addRow (newRow);
return true; return true;
} else { }
((GroupByOutputTable)myLastFiredPacket.newRows).removeRow(groupRepString); else
((BlueTable)myLastFiredPacket.deletedRows).addRow(existingRow); {
if (newRow != null) { ((GroupByOutputTable) myLastFiredPacket.newRows).removeRow (groupRepString);
((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString); ((BlueTable) myLastFiredPacket.deletedRows).addRow (existingRow);
((BlueTable)myLastFiredPacket.addedRows).addRow(newRow); if (newRow != null)
{
((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString);
((BlueTable) myLastFiredPacket.addedRows).addRow (newRow);
} }
return true; return true;
} }
@@ -136,68 +154,77 @@ namespace BCG {
/// </summary> /// </summary>
/// <param name="delGroups"></param> /// <param name="delGroups"></param>
/// <param name="addGroups"></param> /// <param name="addGroups"></param>
private void myInitIncrementalChangeData(out RowDictionaryTable delGroups, out RowDictionaryTable addGroups) { private void myInitIncrementalChangeData (out RowDictionaryTable delGroups, out RowDictionaryTable addGroups)
{
delGroups = delGroups =
new RowDictionaryTable(myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns); new RowDictionaryTable (myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
addGroups = addGroups =
new RowDictionaryTable(myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns); new RowDictionaryTable (myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns);
foreach (IBlueRow deletedRow in myLastArrivedPacket.deletedRows) foreach (IBlueRow deletedRow in myLastArrivedPacket.deletedRows)
myBookKeepingTable.removeRow(deletedRow); myBookKeepingTable.removeRow (deletedRow);
foreach (IBlueRow addedRow in myLastArrivedPacket.addedRows) foreach (IBlueRow addedRow in myLastArrivedPacket.addedRows)
myBookKeepingTable.addRow(addedRow); myBookKeepingTable.addRow (addedRow);
myLastFiredPacket.addedRows = new BlueTable(myOutputSchema); myLastFiredPacket.addedRows = new BlueTable (myOutputSchema);
myLastFiredPacket.deletedRows = new BlueTable(myOutputSchema); myLastFiredPacket.deletedRows = new BlueTable (myOutputSchema);
} }
protected override void myOnTableClear() { protected override void myOnTableClear ()
{
if (myBookKeepingTable.Count == 0) if (myBookKeepingTable.Count == 0)
return; //ie there arent any rows, so no need to optimize the 'cleansing' process return; //ie there arent any rows, so no need to optimize the 'cleansing' process
myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows); myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows);
myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema); myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
(myLastFiredPacket.newRows as GroupByOutputTable).clear(); (myLastFiredPacket.newRows as GroupByOutputTable).clear ();
myBookKeepingTable.clear(); myBookKeepingTable.clear ();
myGroupByEvaluator.clear(); myGroupByEvaluator.clear ();
} }
private IBlueRow myEvaluate(DataPacket dp) { private IBlueRow myEvaluate (DataPacket dp)
IBlueRow evalRow = myGroupByEvaluator.evaluate(dp); {
return evalRow == null ? null : IBlueRow evalRow = myGroupByEvaluator.evaluate (dp);
return evalRow == null ? null :
myIdGuard ? myIdGuard ?
BlueDataUtility.getNewReplacedRow(evalRow, myIdGenerator++, "id") : evalRow; BlueDataUtility.getNewReplacedRow (evalRow, myIdGenerator++, "id") : evalRow;
} }
private string myGetGroupString(IBlueRowCollection group) { private string myGetGroupString (IBlueRowCollection group)
return BlueDataUtility.rowToString(BlueDataUtility.extractRow(group, 0), myGroupByEvaluator.groupByColumns); {
return BlueDataUtility.rowToString(BlueDataUtility.extractRow (group, 0), myGroupByEvaluator.groupByColumns);
} }
protected override void myOnLastSubscriber() { protected override void myOnLastSubscriber ()
{
//the packets are initialzed to null, so that if the subscription count goes to 0, and then this stream //the packets are initialzed to null, so that if the subscription count goes to 0, and then this stream
//gets resubscribed, we start the process of building data again, and treat the callback as the first //gets resubscribed, we start the process of building data again, and treat the callback as the first
//callback. //callback.
myInit(); myInit ();
myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler); myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
} }
protected override void myInit() { protected override void myInit ()
base.myInit(); {
if (myBookKeepingTable != null) { base.myInit ();
if (myBookKeepingTable != null)
{
//to avoid calling init before the first callback //to avoid calling init before the first callback
myBookKeepingTable.clear(); myBookKeepingTable.clear ();
myGroupByEvaluator.clear(); myGroupByEvaluator.clear ();
} }
} }
public static string getRepString(DataPacket dp, string[] colNames) { public static string getRepString (DataPacket dp, string[] colNames)
return BlueDataUtility.rowToString(getRepRow(dp), colNames); {
return BlueDataUtility.rowToString (getRepRow (dp), colNames);
} }
public static IBlueRow getRepRow(DataPacket dp) { public static IBlueRow getRepRow (DataPacket dp)
{
if (dp.newRows.Count > 0) if (dp.newRows.Count > 0)
return BlueDataUtility.extractRow(dp.newRows, 0); return BlueDataUtility.extractRow (dp.newRows, 0);
else if (dp.addedRows.Count > 0) else if (dp.addedRows.Count > 0)
return BlueDataUtility.extractRow(dp.addedRows, 0); return BlueDataUtility.extractRow (dp.addedRows, 0);
else if (dp.deletedRows.Count > 0) else if (dp.deletedRows.Count > 0)
return BlueDataUtility.extractRow(dp.deletedRows, 0); return BlueDataUtility.extractRow (dp.deletedRows, 0);
// BlueEventLog.logAssert (false, "Illegal Data Packet"); // BlueEventLog.logAssert (false, "Illegal Data Packet");
return null; return null;
} }
@@ -210,21 +237,22 @@ namespace BCG {
/// <param name="newValue"></param> /// <param name="newValue"></param>
/// <param name="isAdd"></param> /// <param name="isAdd"></param>
/// <returns></returns> /// <returns></returns>
public static object incrementalAdd(object lastValue, object newValue, bool isAdd) { public static object incrementalAdd (object lastValue, object newValue, bool isAdd)
if (lastValue == null || lastValue.Equals(DBNull.Value)) {
if (lastValue == null || lastValue.Equals (DBNull.Value))
return newValue; return newValue;
// BlueEventLog.logAssert (lastValue.GetType().Equals (typeof(Int32)) && newValue.GetType().Equals (typeof(Int32)) || // BlueEventLog.logAssert (lastValue.GetType().Equals (typeof(Int32)) && newValue.GetType().Equals (typeof(Int32)) ||
// lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) || // lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
// lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal))); // lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
Object retVal = null; Object retVal = null;
if (newValue.GetType().Equals(typeof(int))) if (newValue.GetType().Equals (typeof(int)))
retVal = isAdd ? (int)lastValue + (int)newValue : (int)lastValue - (int)newValue; retVal = isAdd ? (int)lastValue + (int)newValue : (int)lastValue - (int)newValue;
else if (newValue.GetType().Equals(typeof(Double))) else if (newValue.GetType().Equals (typeof(Double)))
retVal = isAdd ? (Double)lastValue + (Double)newValue : (Double)lastValue - (Double)newValue; retVal = isAdd ? (Double)lastValue + (Double)newValue : (Double)lastValue - (Double)newValue;
else if (lastValue.GetType().Equals(typeof(Decimal))) else if (lastValue.GetType().Equals (typeof(Decimal)))
retVal = isAdd ? (Decimal)lastValue + (Decimal)newValue : (Decimal)lastValue - (Decimal)newValue; retVal = isAdd ? (Decimal)lastValue + (Decimal)newValue : (Decimal)lastValue - (Decimal)newValue;
else { } else{}
// BlueEventLog.logAssert (false, "Illegal type"); // BlueEventLog.logAssert (false, "Illegal type");
return retVal; return retVal;
} }
@@ -22,8 +22,8 @@ namespace BCG {
IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn); IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn);
if (maxRow != null) { if (maxRow != null) {
bts.Log.assert(currMaxRow != null); bts.Log.assert(currMaxRow != null);
bts.Log.assert(BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) <= 0); bts.Log.assert(BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) <= 0);
if (BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) == 0) { if (BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) == 0) {
maxRow = BlueDataUtility.computeMaxRow(dp.newRows, myOutputColumn); maxRow = BlueDataUtility.computeMaxRow(dp.newRows, myOutputColumn);
if (maxRow == null) if (maxRow == null)
myOutputTable.removeRow(hashString); myOutputTable.removeRow(hashString);
@@ -34,7 +34,7 @@ namespace BCG {
maxRow = BlueDataUtility.computeMaxRow(dp.addedRows, myOutputColumn); maxRow = BlueDataUtility.computeMaxRow(dp.addedRows, myOutputColumn);
// if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true, // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
// otherwise if maxRow > currMaxRow then its replaced // otherwise if maxRow > currMaxRow then its replaced
if (maxRow != null && (currMaxRow == null || BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) > 0)) if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows(maxRow, currMaxRow, myOutputColumn) > 0))
myOutputTable.addRow(maxRow, hashString); myOutputTable.addRow(maxRow, hashString);
return myProject(myOutputTable.getRow(hashString)); return myProject(myOutputTable.getRow(hashString));
} }
@@ -18,7 +18,7 @@ namespace BCG {
if (minRow != null) { if (minRow != null) {
// BlueEventLog.logAssert (currMinRow != null); // BlueEventLog.logAssert (currMinRow != null);
// BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0); // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
if (BlueDataUtility.compare_rows(minRow, currMinRow, myOutputColumn) == 0) { if (BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) == 0) {
minRow = BlueDataUtility.computeMinRow(dp.newRows, myOutputColumn); minRow = BlueDataUtility.computeMinRow(dp.newRows, myOutputColumn);
if (minRow == null) if (minRow == null)
myOutputTable.removeRow(stringRep); myOutputTable.removeRow(stringRep);
@@ -27,7 +27,7 @@ namespace BCG {
} }
} }
minRow = BlueDataUtility.computeMinRow(dp.addedRows, myOutputColumn); minRow = BlueDataUtility.computeMinRow(dp.addedRows, myOutputColumn);
if (minRow != null && (currMinRow == null || BlueDataUtility.compare_rows(minRow, currMinRow, myOutputColumn) < 0)) if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows(minRow, currMinRow, myOutputColumn) < 0))
myOutputTable.addRow(minRow, stringRep); myOutputTable.addRow(minRow, stringRep);
return myOutputTable.getRow(stringRep); return myOutputTable.getRow(stringRep);
} }
@@ -1,8 +1,10 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using System.Linq;
using System.Text;
using bts; using System.Threading.Tasks;
using mbts;
using mbts.spark;
namespace BCG namespace BCG
{ {
@@ -12,7 +14,7 @@ namespace BCG
private INonIncGroupByEvaluator _groupby_func; private INonIncGroupByEvaluator _groupby_func;
private BlueRowHasher _row_hasher; private BlueRowHasher _row_hasher;
private string[] _groupby_cols; private string[] _groupby_cols;
private Stopwatch _timer = new(); private HighPerformanceTimer _timer = new HighPerformanceTimer();
private int _n_total_fires = 0, _n_expensive_fires = 0; private int _n_total_fires = 0, _n_expensive_fires = 0;
public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
@@ -31,27 +33,21 @@ namespace BCG
{ {
DataPacket dp = nv as DataPacket; DataPacket dp = nv as DataPacket;
///grouping together ///grouping together
_timer.Restart(); _timer.start();
IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher); IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
_timer.Stop(); _timer.stop();
var groupElapsedMs = _timer.ElapsedMilliseconds; // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
if (groupElapsedMs >= 50) {
Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
}
// Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
/// reducing /// reducing
_timer.Restart(); _timer.start();
IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows); IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
_timer.Stop(); _timer.stop();
var applyElapsedMs = _timer.ElapsedMilliseconds;
_n_total_fires++; _n_total_fires++;
if (applyElapsedMs >= 50) if (_timer.lastIntervalTime > 50)
{ {
_n_expensive_fires++; _n_expensive_fires++;
Log.DataFlow.info( // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to evaluate {0} groups {1} , ratio {2}/{3}", grouped_rows.Count, _timer.lastIntervalTime, _n_expensive_fires, _n_total_fires));
$"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
} }
myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows)); myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
} }
} }
} }
@@ -27,8 +27,7 @@ namespace BCG
string prependString = isLeft ? "left" : "right"; string prependString = isLeft ? "left" : "right";
return String.Concat (prependString, colName).GetHashCode (); return String.Concat (prependString, colName).GetHashCode ();
} }
public override bool Equals (object? obj) => public override bool Equals (object obj) {return GetHashCode () == obj.GetHashCode ();}
GetHashCode() == obj?.GetHashCode();
} }
@@ -10,9 +10,9 @@ namespace BCG
{ {
public class JoinStream : ViewStream public class JoinStream : ViewStream
{ {
protected InternalJoinStream? myIjs; protected InternalJoinStream myIjs;
protected ProjectStream? myPs; protected ProjectStream myPs;
protected ColumnMap? myLeftCm, myRightCm; protected ColumnMap myLeftCm, myRightCm;
protected bool myIsProject = true; protected bool myIsProject = true;
private bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin; private bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin;
@@ -1,10 +1,10 @@
using bts; using System;
using mbts;
using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using System.Linq;
using System.Text;
using System.Threading.Tasks;
using mbts;
using mbts.spark;
namespace BCG namespace BCG
{ {
@@ -40,7 +40,8 @@ namespace BCG
public void _on_stream_cb(object dp_obj) public void _on_stream_cb(object dp_obj)
{ {
var timer = Stopwatch.StartNew(); HighPerformanceTimer timer = new HighPerformanceTimer();
timer.start();
DataPacket dp = (DataPacket)dp_obj; DataPacket dp = (DataPacket)dp_obj;
if (_schema == null) if (_schema == null)
{ {
@@ -66,12 +67,13 @@ namespace BCG
_row_map[_row_hasher.get_hash(r, _join_col_indices)] = r; _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
} }
} }
timer.Stop(); timer.stop();
_n_total_fires++; _n_total_fires++;
if (timer.ElapsedMilliseconds >= 50) { if (timer.lastIntervalTime > 50)
{
_n_expensive_fires++; _n_expensive_fires++;
Log.Gen.info($"NonIncJoinBookkeepingData._on_stream_cb {dp.newRows.Count} rows {_join_col_indices.Length} " string log_msg = String.Format("NonIncJoinBookkeepingData._on_stream_cb {0} rows {1} key cols {2} join pos time {3} ratio {4}/{5}", dp.newRows.Count, _join_col_indices.Length, _join_position, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires);
+ $"key cols {_join_position} join pos time {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}"); // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
} }
if (on_new_data != null) if (on_new_data != null)
on_new_data(); on_new_data();
@@ -1,10 +1,10 @@
using bts; using System;
using mbts;
using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using System.Linq;
using System.Text;
using System.Threading.Tasks;
using mbts;
using mbts.spark;
namespace BCG namespace BCG
{ {
@@ -114,7 +114,8 @@ namespace BCG
if (_is_ready()) if (_is_ready())
{ {
_construct_schema(); _construct_schema();
var timer = Stopwatch.StartNew(); HighPerformanceTimer timer = new HighPerformanceTimer();
timer.start();
IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list(); IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
List<object[]> output_rows = new List<object[]>(); List<object[]> output_rows = new List<object[]>();
@@ -137,12 +138,13 @@ namespace BCG
} }
} }
IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows)); IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
timer.Stop(); timer.stop();
_n_total_fires++; _n_total_fires++;
if (timer.ElapsedMilliseconds >= 250) { if (timer.lastIntervalTime >= 250)
{
_n_expensive_fires++; _n_expensive_fires++;
string log_msg = $"Join and map time for {_book_keeping_data.Count} " string log_msg = String.Format("Join and map time for {0} tables {1} ratio {2}/{3}", _book_keeping_data.Count, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires);
+ $"tables {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}"; // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
} }
myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows)); myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
} }
@@ -152,12 +154,15 @@ namespace BCG
{ {
if (_output_mapper != null) if (_output_mapper != null)
{ {
HighPerformanceTimer timer = new HighPerformanceTimer();
IBlueRowCollection output_rows; IBlueRowCollection output_rows;
var timer = Stopwatch.StartNew(); timer.start();
output_rows = _output_mapper.map_rows(unmapped_rows); output_rows = _output_mapper.map_rows(unmapped_rows);
timer.Stop(); timer.stop();
if (timer.ElapsedMilliseconds >= 50) { if (timer.lastIntervalTime >= 50)
Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}"); {
// string log_msg = String.Format("Time to map {0} rows into {1} rows {2}", unmapped_rows.Count, output_rows.Count, timer.lastIntervalTime);
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
} }
return output_rows; return output_rows;
} }
Loaded 100 of 791 files, more files were not shown because too many files have changed in this diff. Show more