Compare commits

..
Author SHA1 Message Date
Mohammad Arham be9c61f435 temp stuff working 2024-07-16 12:29:23 +00:00
Mohammad Arham 1f7bacdf3b Added Onix Libraries 2024-06-25 22:37:42 +00:00
1013 changed files with 24155 additions and 59542 deletions

No files matched your search

+2 -2
View File
@@ -9,7 +9,7 @@
] ]
}, },
"fable": { "fable": {
"version": "4.24.0", "version": "4.0.6",
"commands": [ "commands": [
"fable" "fable"
] ]
@@ -27,4 +27,4 @@
] ]
} }
} }
} }
-9
View File
@@ -87,15 +87,6 @@ RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
# Pcap # Pcap
RUN apt-get update && apt-get install -y libpcap0.8-dev RUN apt-get update && apt-get install -y libpcap0.8-dev
# Databento (deps)
RUN apt-get update && apt-get -y install libzstd-dev
# Docker in Docker
RUN apt-get update && apt-get install -y \
docker.io \
docker-compose \
docker-compose-v2
# Set up default executables (update-alternatives) # Set up default executables (update-alternatives)
RUN \ RUN \
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \ update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
+23 -6
View File
@@ -1,10 +1,15 @@
{ {
"dockerFile": "Dockerfile", "dockerFile": "Dockerfile",
// Creates local dir to mount to HOME in docker
"initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
"hostRequirements": { "hostRequirements": {
"memory": "8gb", "memory": "4gb",
"storage": "10gb" "storage": "6gb"
}, },
// "postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
"containerUser": "bts", "containerUser": "bts",
"workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
"workspaceFolder": "/home/bts/spark",
"updateRemoteUserUID": true, "updateRemoteUserUID": true,
"shutdownAction": "none", "shutdownAction": "none",
// Note: If running multiple containers on the same host, each must use a // Note: If running multiple containers on the same host, each must use a
@@ -12,19 +17,31 @@
// example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc. // example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
"runArgs": [ "runArgs": [
"--shm-size", "--shm-size",
"8gb" "2gb",
"-p",
"8000:8000",
"-p",
"8001:8001",
"-p",
"8002:8002",
"-p",
"8003:8003"
], ],
"mounts": [ "mounts": [
"source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind" "source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
], ],
"customizations": { "customizations": {
"vscode": { "vscode": {
"settings": {
"clangd.arguments": [
"--compile-commands-dir=${workspaceFolder}/build/clang/debug"
]
},
"extensions": [ "extensions": [
"eamodio.gitlens", "eamodio.gitlens",
"llvm-vs-code-extensions.vscode-clangd", "llvm-vs-code-extensions.vscode-clangd",
"ms-vscode.cmake-tools", "ms-vscode.cmake-tools",
"ms-vscode.cpptools", "ms-vscode.cpptools"
"xaver.clang-format"
] ]
} }
} }
-48
View File
@@ -1,48 +0,0 @@
# This file defines the global default configuration for docker deployments.
# It is tracked in source control. For temporary modifications, please create
# a separate file called ".env.local" (it's in .gitignore, so won't be tracked).
# If a variable is defined in both files, the value in .env.local will be used.
#
# About the deployment procedure:
# We use Docker Compose to build images and launch containers for various services
# that make up our software suite:
# - sparkd
# - TODO: pubsub
# - TODO: md-publisher
# - TODO: webapp (DSQA on the web)
# - TODO: api-server (websocket based)
# From inside a devcontainer, you can launch all services in separate containers
# on your machine by running:
#
# tools/deploy.sh
#
# This script builds release versions for each service, packaged (generally) in
# self-contained directories. Next, it runs a few "docker compose" commands to
# build docker images loaded with the self-contained binaries from the previous
# step, and start containers with those images to run each service. To view the
# commands that deploy.sh runs internally (without actually running them), run:
#
# tools/deploy.sh --dry-run
#
# For additional information/help, run:
#
# tools/deploy.sh --help
#
### Sparkd ###
# The internal image for sparkd to use. For more info, view:
# src/server/sparkd/Dockerfile
SPARKD__IMAGE=sparkd
# The config dir for this service to use
SPARKD__CONFIG_DIR=src/server/sparkd/configurations/dev
# Mount a specific volume or dir to "~/sparkdata/data" inside the docker.
# Defaults to a volume named "sparkd_<config_port>" (e.g., sparkd_8000).
# SPARKD__DATA_DIR=/home/bts/sparkdata/data
SPARKD__SOURCE_DIR=install/sparkd-release-gcc
SPARKD__RUNTIME_CONF_DIR=/home/ubuntu/8000/sparkdata/conf/
SPARKD__CONFIG_PORT=8000
SPARKD__QUICKBUS_PORT=8001
SPARKD__SIM_FIX_PORT=8002
SPARKD__SHM_SIZE=4gb
# TODO: add config for other services (pubsub, md-publisher, webapp, api-server, etc.)
+2 -3
View File
@@ -1,3 +1,2 @@
*.cmd text=auto eol=crlf *.cmd eol=crlf
*.cs text=auto eol=crlf *.sh eol=lf
*.sh text=auto eol=lf
-1
View File
@@ -59,4 +59,3 @@ trace_data/
.devcontainer/ .devcontainer/
*.Generated.cs *.Generated.cs
*.lst *.lst
.env.local
+1 -1
View File
@@ -1,6 +1,6 @@
[submodule "externals"] [submodule "externals"]
path = externals path = externals
url = https://bitbucket.org/bluetradesys/externals.git url = git@bitbucket.org:bluetradesys/externals.git
[submodule "vcpkg"] [submodule "vcpkg"]
path = vcpkg path = vcpkg
url = https://github.com/microsoft/vcpkg.git url = https://github.com/microsoft/vcpkg.git
+24 -199
View File
@@ -2,10 +2,18 @@
"version": "0.2.0", "version": "0.2.0",
"configurations": [ "configurations": [
{ {
"name": ".NET Core Attach", "name": "(gdb) Launch",
"type": "coreclr", "type": "cppdbg",
"request": "attach" "request": "launch",
"program": "${workspaceFolder}/build/sparkd-debug-gcc/server/sparkd",
"args": ["run"],
"stopAtEntry": false,
"cwd": "${workspaceFolder}/build/sparkd-debug-gcc/server",
"environment": [],
"externalConsole": false,
"MIMode": "gdb",
}, },
{ {
"name": "DataStreamPublisher", "name": "DataStreamPublisher",
"type": "coreclr", "type": "coreclr",
@@ -26,55 +34,11 @@
"stopAtEntry": false, "stopAtEntry": false,
"console": "integratedTerminal" "console": "integratedTerminal"
}, },
{
"name": "(gdb) Launch",
"type": "cppdbg",
"request": "launch",
"program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
"args": [
"run"
],
"stopAtEntry": false,
"cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
"environment": [],
"externalConsole": false,
"MIMode": "gdb",
},
{
"name": "spreader tests2",
"type": "cppdbg",
"request": "launch",
"program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
"args": [
"test",
"[spreader2]"
],
"stopAtEntry": false,
"cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
"environment": [],
"externalConsole": false,
"MIMode": "gdb",
},
{
"name": "spreader tests",
"type": "cppdbg",
"request": "launch",
"program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
"args": [
"test",
"[spreader]"
],
"stopAtEntry": false,
"cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
"environment": [],
"externalConsole": false,
"MIMode": "gdb",
},
{ {
"name": "DataStreamQueryAnalyzer", "name": "DataStreamQueryAnalyzer",
"type": "coreclr", "type": "coreclr",
"request": "launch", "request": "launch",
//"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)", "preLaunchTask": "Build DataStream (Debug)",
"program": "", "program": "",
"windows": { "windows": {
"program": "DataStreamQueryAnalyzer.exe", "program": "DataStreamQueryAnalyzer.exe",
@@ -123,18 +87,6 @@
"stopAtEntry": false, "stopAtEntry": false,
"console": "integratedTerminal" "console": "integratedTerminal"
}, },
{
"name": "Spark (Skip Build)",
"type": "coreclr",
"request": "launch",
"program": "",
"windows": {
"program": "Spark.exe",
"cwd": "build/Debug/bin/net7.0-windows",
},
"stopAtEntry": false,
"console": "integratedTerminal"
},
{ {
"name": "Spark (Parameterized)", "name": "Spark (Parameterized)",
"type": "coreclr", "type": "coreclr",
@@ -152,139 +104,22 @@
"stopAtEntry": false, "stopAtEntry": false,
"console": "integratedTerminal" "console": "integratedTerminal"
}, },
{
"name": "Spark Test",
"type": "coreclr",
"request": "launch",
"program": "",
"windows": {
"preLaunchTask": "Build Spark (Debug)",
"program": "Spark.Console.exe",
"cwd": "build/Debug/bin/net7.0",
"args": [
"test",
"${input:sparkTest}"
]
},
"stopAtEntry": false,
"console": "integratedTerminal"
},
{
"name": "Spark Test (Skip Build)",
"type": "coreclr",
"request": "launch",
"program": "",
"windows": {
"program": "Spark.Console.exe",
"cwd": "build/Debug/bin/net7.0",
"args": [
"test",
"${input:sparkTest}"
]
},
"stopAtEntry": false,
"console": "integratedTerminal"
},
{ {
"name": "Sparkd", "name": "Sparkd",
"type": "cppdbg", "type": "coreclr",
"request": "launch", "request": "launch",
"preLaunchTask": "Build Sparkd (Debug | Clang)",
"program": "", "program": "",
"linux": { "linux": {
"preLaunchTask": "Build Sparkd (Debug)", "program": "sparkd",
"program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd", },
"args": [ "args": [
"run" "run"
], ],
"stopAtEntry": false, "cwd": "build/sparkd-debug-clang/server",
"cwd": "${workspaceFolder}/build/sparkd-debug-clang/server", "stopAtEntry": false,
"environment": [], "console": "integratedTerminal"
"externalConsole": false, }
"MIMode": "gdb",
"setupCommands": [
{
"description": "Enable pretty-printing for gdb",
"text": "-enable-pretty-printing",
"ignoreFailures": true
}
]
}
},
{
"name": "Sparkd (Skip Build)",
"type": "cppdbg",
"request": "launch",
"program": "",
"linux": {
"program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
"args": [
"run"
],
"stopAtEntry": false,
"cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
"environment": [],
"externalConsole": false,
"MIMode": "gdb",
"setupCommands": [
{
"description": "Enable pretty-printing for gdb",
"text": "-enable-pretty-printing",
"ignoreFailures": true
}
]
}
},
{
"name": "Sparkd Test",
"type": "cppdbg",
"request": "launch",
"program": "",
"linux": {
"preLaunchTask": "Build Sparkd (Debug)",
"program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
"args": [
"test",
"${input:sparkdTest}",
],
"stopAtEntry": false,
"cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
"environment": [],
"externalConsole": false,
"MIMode": "gdb",
"setupCommands": [
{
"description": "Enable pretty-printing for gdb",
"text": "-enable-pretty-printing",
"ignoreFailures": true
}
]
}
},
{
"name": "Sparkd Test (Skip Build)",
"type": "cppdbg",
"request": "launch",
"program": "",
"linux": {
"program": "${workspaceFolder}/build/sparkd-debug-clang/server/sparkd",
"args": [
"test",
"${input:sparkdTest}",
],
"stopAtEntry": false,
"cwd": "${workspaceFolder}/build/sparkd-debug-clang/server",
"environment": [],
"externalConsole": false,
"MIMode": "gdb",
"setupCommands": [
{
"description": "Enable pretty-printing for gdb",
"text": "-enable-pretty-printing",
"ignoreFailures": true
}
]
}
},
], ],
"inputs": [ "inputs": [
{ {
@@ -315,15 +150,5 @@
"description": "Config Server URL", "description": "Config Server URL",
"default": "http://localhost:8000/config", "default": "http://localhost:8000/config",
}, },
{
"id": "sparkTest",
"type": "promptString",
"description": "Test to run",
},
{
"id": "sparkdTest",
"type": "promptString",
"description": "Test to run",
},
] ]
} }
+20 -177
View File
@@ -189,6 +189,25 @@
"color": "terminal.ansiCyan" "color": "terminal.ansiCyan"
}, },
}, },
{
"label": "Build Sparkd -j8",
"group": "build",
"type": "shell",
"command": "",
"linux": {
"command": "tools/build_sparkd.sh",
"args": [
"--preset",
"${input:sparkdCmakePreset}",
"-j",
"8"
],
},
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{ {
"label": "Build Sparkd (Debug)", "label": "Build Sparkd (Debug)",
"group": "build", "group": "build",
@@ -526,28 +545,6 @@
"color": "terminal.ansiGreen", "color": "terminal.ansiGreen",
}, },
}, },
{
"label": "Run PubSubServer (Debug | Skip Build)",
"type": "process",
"command": "",
"linux": {
"command": "PubSubServer",
},
"windows": {
"command": "PubSubServer.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
],
"options": {
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{ {
"label": "Run PubSubServer (Release)", "label": "Run PubSubServer (Release)",
"dependsOn": "Build DataStream (Release)", "dependsOn": "Build DataStream (Release)",
@@ -571,28 +568,6 @@
"color": "terminal.ansiGreen", "color": "terminal.ansiGreen",
}, },
}, },
{
"label": "Run PubSubServer (Release | Skip Build)",
"type": "process",
"command": "",
"linux": {
"command": "PubSubServer",
},
"windows": {
"command": "PubSubServer.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
],
"options": {
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{ {
"label": "Run Spark", "label": "Run Spark",
"dependsOn": "Build Spark", "dependsOn": "Build Spark",
@@ -641,21 +616,6 @@
"color": "terminal.ansiGreen", "color": "terminal.ansiGreen",
} }
}, },
{
"label": "Run Spark (Skip Build)",
"type": "process",
"command": "",
"windows": {
"command": "Spark.exe",
"options": {
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
},
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
}
},
{ {
"label": "Run Spark Config Query", "label": "Run Spark Config Query",
"dependsOn": "Build Spark", "dependsOn": "Build Spark",
@@ -699,24 +659,6 @@
"color": "terminal.ansiGreen", "color": "terminal.ansiGreen",
}, },
}, },
{
"label": "Run Sparkd (Skip Build)",
"type": "process",
"command": "",
"linux": {
"command": "sparkd",
},
"args": [
"run"
],
"options": {
"cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{ {
"label": "Run Sparkd Config Query", "label": "Run Sparkd Config Query",
"dependsOn": "Build Sparkd", "dependsOn": "Build Sparkd",
@@ -738,65 +680,6 @@
"color": "terminal.ansiGreen", "color": "terminal.ansiGreen",
}, },
}, },
{
"label": "Test Spark (Debug)",
"dependsOn": "Build Spark (Debug)",
"type": "process",
"icon": {
"id": "coverage",
"color": "terminal.ansiYellow",
},
"command": "",
"windows": {
"command": "Spark.Console.exe",
"args": [
"test",
"${input:sparkTest}"
],
"options": {
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0",
}
},
},
{
"label": "Test Sparkd",
"dependsOn": "Build Sparkd",
"type": "process",
"command": "",
"linux": {
"command": "sparkd",
},
"args": [
"test",
"${input:sparkdTest}",
],
"options": {
"cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
},
"icon": {
"id": "coverage",
"color": "terminal.ansiYellow",
},
},
{
"label": "Test Sparkd (Skip Build)",
"type": "process",
"command": "",
"linux": {
"command": "sparkd",
},
"args": [
"test",
"${input:sparkdTest}",
],
"options": {
"cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
},
"icon": {
"id": "coverage",
"color": "terminal.ansiYellow",
},
},
{ {
"label": "View DataStreamQueryAnalyzer Logs (Last Run)", "label": "View DataStreamQueryAnalyzer Logs (Last Run)",
"type": "process", "type": "process",
@@ -820,36 +703,6 @@
"id": "wrench" "id": "wrench"
}, },
}, },
{
"label": "View PubSubServer Logs (Last Run)",
"type": "process",
"command": "",
"linux": {
"command": "tools/logging/show_last_run.sh",
"args": [
"${env:HOME}/.local/share/DataStream/data/logging.PubSubServer",
"Starting PubSubServer"
]
},
"windows": {
"command": "tools/logging/show_last_run.cmd",
"args": [
"DataStream",
"PubSubServer.exe",
"Starting PubSubServer"
]
},
"presentation": {
"reveal": "always",
"focus": true,
"panel": "shared",
"showReuseMessage": false,
"close": true,
},
"icon": {
"id": "wrench"
},
},
], ],
"inputs": [ "inputs": [
{ {
@@ -915,16 +768,6 @@
"id": "sparkdConfigQuery", "id": "sparkdConfigQuery",
"type": "promptString", "type": "promptString",
"description": "Config Type(s)" "description": "Config Type(s)"
}, }
{
"id": "sparkTest",
"type": "promptString",
"description": "Test to run",
},
{
"id": "sparkdTest",
"type": "promptString",
"description": "Test to run",
},
] ]
} }
+7 -20
View File
@@ -1,10 +1,7 @@
cmake_minimum_required(VERSION 3.12) cmake_minimum_required(VERSION 3.12)
project(spark) project(spark)
include(GNUInstallDirs)
include(${CMAKE_SOURCE_DIR}/cmake/GitVersion.cmake)
set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD 20)
get_version_from_git()
if(WIN32) if(WIN32)
set(platform "windows") set(platform "windows")
else() else()
@@ -52,6 +49,8 @@ else()
# make sparkd look for libspark.so in its directory # make sparkd look for libspark.so in its directory
set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
set(CMAKE_INSTALL_RPATH ".") set(CMAKE_INSTALL_RPATH ".")
set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")
if(BUILD_SPARKD) if(BUILD_SPARKD)
# Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
@@ -90,15 +89,9 @@ else()
add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr") add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
# ** Binance USDS Futures ** # ** Binance USDS Futures **
add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://fapi.binance.com") add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/fapi/v1/exchangeInfo") add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/fapi/v1/ticker/24hr") add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
add_compile_definitions(BINANCE_USDS_FUTURES_USER_DATA_PATH="/api/v3/userDataStream")
# ** Binance Coin Futures **
add_compile_definitions(CCAPI_BINANCE_COIN_FUTURES_URL_REST_BASE="https://dapi.binance.com")
add_compile_definitions(BINANCE_COIN_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
add_compile_definitions(BINANCE_COIN_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
# ** Deribit ** # ** Deribit **
add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument") add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
@@ -118,7 +111,6 @@ else()
# subdirs # subdirs
if (BUILD_SPARKD) if (BUILD_SPARKD)
add_subdirectory(src/lib_binance_execution)
add_subdirectory(src/server/sparkd) add_subdirectory(src/server/sparkd)
add_subdirectory(src/data_files) add_subdirectory(src/data_files)
endif() endif()
@@ -140,8 +132,6 @@ else()
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
COMMAND COMMAND
ln ARGS -sf ln ARGS -sf
${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0 ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
@@ -155,14 +145,11 @@ else()
install( install(
DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins" DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
DESTINATION "${CMAKE_INSTALL_BINDIR}") DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
install( install(
FILES FILES
"${SPARK_LIB}/libACE-5.7.5.so" "${SPARK_LIB}/libACE-5.7.5.so"
"${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0" "${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
"${SPARK_LIB}/libonixs-fix-engine.so.4.10.1" DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
"${SPARK_LIB}/libonixs-fix-engine.so.4"
"${SPARK_LIB}/libonixs-fix-engine.so"
DESTINATION "${CMAKE_INSTALL_BINDIR}")
endif() endif()
endif() endif()
+11 -3
View File
@@ -26,7 +26,6 @@
"CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd", "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
"CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT", "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
"CMAKE_CXX_STANDARD": "20", "CMAKE_CXX_STANDARD": "20",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
"CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG", "CMAKE_SHARED_LINKER_FLAGS_DEBUG": "/DEBUG",
"CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG", "CMAKE_SHARED_LINKER_FLAGS_RELEASE": "/OPT:REF /OPT:ICF /LTCG",
"CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake", "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake",
@@ -62,7 +61,7 @@
}, },
"CMAKE_CXX_STANDARD": "20", "CMAKE_CXX_STANDARD": "20",
"CMAKE_INSTALL_RPATH": ".", "CMAKE_INSTALL_RPATH": ".",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}", "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}/sparkd",
"CMAKE_EXE_LINKER_FLAGS": "-Wno-undef", "CMAKE_EXE_LINKER_FLAGS": "-Wno-undef",
"EXTERNALS_ROOT": "${sourceDir}/externals/linux", "EXTERNALS_ROOT": "${sourceDir}/externals/linux",
"BUILD_SDK": { "BUILD_SDK": {
@@ -303,7 +302,16 @@
"lhs": "${hostSystemName}", "lhs": "${hostSystemName}",
"rhs": "Linux" "rhs": "Linux"
}, },
"hidden": true "hidden": true,
"targets": [
"data_files_dir",
"admin_web_app_bundle",
"install",
"libspark",
"rapidjson_project",
"server_layout",
"sparkd"
]
}, },
{ {
-139
View File
@@ -14,34 +14,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "spark_test", "src\client\sp
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}" Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Spark.Console", "src\client\Spark.Console\Spark.Console.csproj", "{89D666E6-B500-498F-B026-4D167F68ADFC}"
EndProject EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStreamSuite", "DataStreamSuite", "{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamCore", "src\DataStreamSuite\DataStreamCore\DataStreamCore.csproj", "{EF26A475-6101-4C9C-A717-D60BB92A2302}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedStreams", "src\DataStreamSuite\SharedStreams\SharedStreams.csproj", "{911B08A3-515A-441D-84C8-8A43740B5FC1}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PubSub", "src\DataStreamSuite\PubSub\PubSub.csproj", "{14140310-2233-430A-821B-F3058A25234B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BaseUtilities", "src\DataStreamSuite\BaseUtilities\BaseUtilities.csproj", "{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExpressionCore", "src\DataStreamSuite\ExpressionCore\ExpressionCore.csproj", "{6F22DB96-6EAB-4034-99D9-287021C36F01}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ArithmeticExpressions", "src\DataStreamSuite\ArithmeticExpressions\ArithmeticExpressions.csproj", "{A5C33956-A41F-49B3-A41D-1082CBA59D02}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamExpressions", "src\DataStreamSuite\DataStreamExpressions\DataStreamExpressions.csproj", "{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "src\DataStreamSuite\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ColumnMapExpressions", "src\DataStreamSuite\ColumnMapExpressions\ColumnMapExpressions.csproj", "{B945A183-2174-4ACB-B289-8A5C546656FE}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BooleanExpressions", "src\DataStreamSuite\BooleanExpressions\BooleanExpressions.csproj", "{9F6D7DE8-185B-4C1D-B809-3314D092DF28}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "src\DataStreamSuite\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj", "{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStream", "src\DataStreamSuite\DataStream\DataStream.csproj", "{39AEA075-6457-46C0-8821-1158BE6EFA57}"
EndProject
Global Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|x64 = Debug|x64 Debug|x64 = Debug|x64
@@ -98,102 +70,6 @@ Global
{89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64 {89D666E6-B500-498F-B026-4D167F68ADFC}.Release|x64.Build.0 = Release|x64
{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64 {89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.ActiveCfg = Debug|x64
{89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64 {89D666E6-B500-498F-B026-4D167F68ADFC}.RelWithDebInfo|x64.Build.0 = Debug|x64
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.ActiveCfg = Debug|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Debug|x64.Build.0 = Debug|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.ActiveCfg = Release|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.Release|x64.Build.0 = Release|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{EF26A475-6101-4C9C-A717-D60BB92A2302}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.ActiveCfg = Debug|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Debug|x64.Build.0 = Debug|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.ActiveCfg = Release|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.Release|x64.Build.0 = Release|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{911B08A3-515A-441D-84C8-8A43740B5FC1}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.ActiveCfg = Debug|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.Debug|x64.Build.0 = Debug|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.Release|x64.ActiveCfg = Release|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.Release|x64.Build.0 = Release|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{14140310-2233-430A-821B-F3058A25234B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.ActiveCfg = Debug|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Debug|x64.Build.0 = Debug|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.ActiveCfg = Release|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.Release|x64.Build.0 = Release|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.ActiveCfg = Debug|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Debug|x64.Build.0 = Debug|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.ActiveCfg = Release|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.Release|x64.Build.0 = Release|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{6F22DB96-6EAB-4034-99D9-287021C36F01}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.ActiveCfg = Debug|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Debug|x64.Build.0 = Debug|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.ActiveCfg = Release|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.Release|x64.Build.0 = Release|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{A5C33956-A41F-49B3-A41D-1082CBA59D02}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.ActiveCfg = Debug|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Debug|x64.Build.0 = Debug|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.ActiveCfg = Release|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.Release|x64.Build.0 = Release|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.ActiveCfg = Debug|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Debug|x64.Build.0 = Debug|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.ActiveCfg = Release|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.Release|x64.Build.0 = Release|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.ActiveCfg = Debug|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.Debug|x64.Build.0 = Debug|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.ActiveCfg = Release|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.Release|x64.Build.0 = Release|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{B945A183-2174-4ACB-B289-8A5C546656FE}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.ActiveCfg = Debug|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Debug|x64.Build.0 = Debug|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.ActiveCfg = Release|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.Release|x64.Build.0 = Release|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{39AEA075-6457-46C0-8821-1158BE6EFA57}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.ActiveCfg = Debug|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Debug|x64.Build.0 = Debug|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.ActiveCfg = Release|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.Release|x64.Build.0 = Release|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{9F6D7DE8-185B-4C1D-B809-3314D092DF28}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.ActiveCfg = Debug|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Debug|x64.Build.0 = Debug|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.MinSizeRel|x64.Build.0 = Debug|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.ActiveCfg = Release|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.Release|x64.Build.0 = Release|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.ActiveCfg = Debug|Any CPU
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B}.RelWithDebInfo|x64.Build.0 = Debug|Any CPU
EndGlobalSection EndGlobalSection
GlobalSection(SolutionProperties) = preSolution GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE HideSolutionNode = FALSE
@@ -204,19 +80,4 @@ Global
GlobalSection(Performance) = preSolution GlobalSection(Performance) = preSolution
HasPerformanceSessions = true HasPerformanceSessions = true
EndGlobalSection EndGlobalSection
GlobalSection(NestedProjects) = preSolution
{C09ED521-D2DF-40BC-A76F-EB3CB870B2F6} = {267F81EF-A8AB-49FD-9F56-D7EC3C1F7C7C}
{EF26A475-6101-4C9C-A717-D60BB92A2302} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
{911B08A3-515A-441D-84C8-8A43740B5FC1} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
{14140310-2233-430A-821B-F3058A25234B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
{43146EA8-6FCC-46B8-88B0-42F9BB5E6E0A} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
{6F22DB96-6EAB-4034-99D9-287021C36F01} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
{A5C33956-A41F-49B3-A41D-1082CBA59D02} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
{2FE745C3-1EDF-4421-8CD7-A63EE0C1AE43} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
{FDA791E0-2B91-4AD5-B91D-CF920A4C4E38} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
{B945A183-2174-4ACB-B289-8A5C546656FE} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
{39AEA075-6457-46C0-8821-1158BE6EFA57} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
{9F6D7DE8-185B-4C1D-B809-3314D092DF28} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
{6BD3F6D8-B6FC-44E8-B8FE-06352F638B9B} = {C09ED521-D2DF-40BC-A76F-EB3CB870B2F6}
EndGlobalSection
EndGlobal EndGlobal
-48
View File
@@ -1,48 +0,0 @@
include_guard(GLOBAL)
function(get_version_from_git)
find_package(Git QUIET)
if(NOT Git_FOUND)
message(WARNING "Git not found")
return()
endif()
execute_process(
COMMAND ${GIT_EXECUTABLE} describe --tags --always
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
OUTPUT_VARIABLE GIT_TAG
OUTPUT_STRIP_TRAILING_WHITESPACE
RESULT_VARIABLE GIT_RESULT
)
if(NOT GIT_RESULT EQUAL 0)
message(WARNING "Failed to get git tag")
return()
endif()
execute_process(
COMMAND ${GIT_EXECUTABLE} rev-parse --short=7 HEAD
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
OUTPUT_VARIABLE GIT_COMMIT_SHORT_HASH
OUTPUT_STRIP_TRAILING_WHITESPACE
)
set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH})
set(PROJECT_VERSION_BUILD_METADATA ${GIT_COMMIT_SHORT_HASH} PARENT_SCOPE)
string(REGEX REPLACE "^v" "" CLEAN_TAG "${GIT_TAG}")
if(CLEAN_TAG MATCHES "^([0-9]+)\\.([0-9]+)\\.([0-9]+)(-.*)?$")
set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1})
set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1} PARENT_SCOPE)
set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2})
set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2} PARENT_SCOPE)
set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3})
set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3} PARENT_SCOPE)
set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}")
set(PROJECT_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}" PARENT_SCOPE)
set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}")
set(PROJECT_VERSION_FULL "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}+${GIT_COMMIT_SHORT_HASH}" PARENT_SCOPE)
else()
message(WARNING "Tag '${CLEAN_TAG}' does not match semver format")
endif()
endfunction()
-49
View File
@@ -1,49 +0,0 @@
include_guard(DIRECTORY)
function(protobuf_generate_fs)
set(_options)
set(_one_value_args TARGET_NAME OUT_DIR PROTOC)
set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
if (NOT DEFINED _ARGS_TARGET_NAME)
set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_fs_protos")
endif()
if (NOT DEFINED _ARGS_PROTOC)
set(_ARGS_PROTOC "${SPARK_BIN}/protoc")
endif()
set(_proto_include_cli_args)
foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
list(APPEND _proto_include_cli_args -I ${_dir})
endforeach()
set(_outputs)
foreach(_proto_source ${_ARGS_PROTO_SOURCES})
# get_filename_component(_abs_proto_source ${_proto_source} ABSOLUTE)
get_filename_component(_proto_source_we ${_proto_source} NAME_WE)
# get_filename_component(_proto_source_dir ${_proto_source} DIRECTORY)
list(APPEND _outputs ${_ARGS_OUT_DIR}/${_proto_source_we}.fable.fs)
endforeach()
set(_venv_exec ${CMAKE_SOURCE_DIR}/tools/venv_exec.sh)
set(_plugin_py ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/plugin.py)
set(_template_fs ${CMAKE_SOURCE_DIR}/src/lib/api/service_compiler/templates/fs/fable.fs)
add_custom_command(
OUTPUT ${_outputs}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
COMMAND ${_venv_exec}
${_ARGS_PROTOC}
--plugin=protoc-gen-service=${_plugin_py}
--service_out=lang=fs,strip_out_path=1:${_ARGS_OUT_DIR}
${_proto_include_cli_args}
${_ARGS_PROTO_SOURCES}
DEPENDS
${_venv_exec}
${_plugin_py}
${_template_fs}
${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
${_ARGS_PROTO_SOURCES}
)
add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS "${_outputs}")
endfunction()
-33
View File
@@ -1,33 +0,0 @@
include_guard(DIRECTORY)
function(protobuf_generate_js)
set(_options)
set(_one_value_args TARGET_NAME OUTPUT)
set(_multi_value_args PROTO_INCLUDE_DIRS PROTO_SOURCES)
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
if (NOT DEFINED _ARGS_TARGET_NAME)
set(_ARGS_TARGET_NAME "${PROJECT_NAME}_generate_js_protos")
endif()
set(_proto_include_cli_args)
foreach(_dir ${_ARGS_PROTO_INCLUDE_DIRS})
list(APPEND _proto_include_cli_args -p ${_dir})
endforeach()
add_custom_command(
OUTPUT ${_ARGS_OUTPUT}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
COMMAND npx
--no-install pbjs
-t json-module
-w commonjs
-o ${_ARGS_OUTPUT}
${_proto_include_cli_args}
${_ARGS_PROTO_SOURCES}
DEPENDS
${CMAKE_CURRENT_SOURCE_DIR}/package-lock.json
${_ARGS_PROTO_SOURCES}
)
add_custom_target(${_ARGS_TARGET_NAME} ALL DEPENDS ${_ARGS_OUTPUT})
endfunction()
-49
View File
@@ -1,49 +0,0 @@
include_guard(DIRECTORY)
include(${CMAKE_SOURCE_DIR}/cmake/TernaryOperator.cmake)
function(sym_link_dir)
set(_options)
set(_one_value_args TARGET_NAME SOURCE DEST COMMENT)
set(_multi_value_args)
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
add_custom_command(
OUTPUT ${_ARGS_DEST}
COMMAND ln ARGS -sf ${_ARGS_SOURCE} ${_ARGS_DEST}
)
set_ternary(_comment _ARGS_COMMENT "${_ARGS_COMMENT}" "linking dir ${_ARGS_SOURCE} -> ${_ARGS_DEST}")
add_custom_target(
${_ARGS_TARGET_NAME} ALL
DEPENDS ${_ARGS_DEST}
COMMENT "${_comment}"
)
endfunction()
function(sym_link_file)
set(_options)
set(_one_value_args SOURCE DEST INSTALL_DEST)
set(_multi_value_args)
cmake_parse_arguments(_ARGS "${_options}" "${_one_value_args}" "${_multi_value_args}" ${ARGN})
file(
CREATE_LINK
${_ARGS_SOURCE}
${_ARGS_DEST}
SYMBOLIC
)
if (DEFINED _ARGS_INSTALL_DEST)
install(
FILES
${_ARGS_SOURCE}
DESTINATION ${_ARGS_INSTALL_DEST}
)
endif()
endfunction()
function(sym_link_local_lib LIB_NAME)
set(_lib_file_name "${CMAKE_SHARED_LIBRARY_PREFIX}${LIB_NAME}.so")
file(
CREATE_LINK
"${CMAKE_BINARY_DIR}/src/lib_${LIB_NAME}/${_lib_file_name}"
"${CMAKE_CURRENT_BINARY_DIR}/${_lib_file_name}"
SYMBOLIC
)
endfunction()
-9
View File
@@ -1,9 +0,0 @@
include_guard(GLOBAL)
macro(set_ternary RESULT CONDITION TRUE_VALUE FALSE_VALUE)
if(${CONDITION})
set(${RESULT} ${TRUE_VALUE})
else()
set(${RESULT} ${FALSE_VALUE})
endif()
endmacro()
-11
View File
@@ -1,11 +0,0 @@
#ifndef BTS_SPARK_VERSION_H
#define BTS_SPARK_VERSION_H
#define BTS_SPARK_VERSION_MAJOR @PROJECT_VERSION_MAJOR@
#define BTS_SPARK_VERSION_MINOR @PROJECT_VERSION_MINOR@
#define BTS_SPARK_VERSION_PATCH @PROJECT_VERSION_PATCH@
#define BTS_SPARK_VERSION_BUILD_METADATA @PROJECT_VERSION_BUILD_METADATA@
#define BTS_SPARK_VERSION "@PROJECT_VERSION@"
#define BTS_SPARK_VERSION_FULL "@PROJECT_VERSION_FULL@"
#endif
-100
View File
@@ -1,100 +0,0 @@
services:
sparkd:
volumes:
- ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
build:
dockerfile: src/server/sparkd/Dockerfile
args:
- IMAGE=${SPARKD__IMAGE}
- SOURCE_DIR=${SPARKD__SOURCE_DIR}
- CONFIG_DIR=${SPARKD__CONFIG_DIR}
shm_size: "${SPARKD__SHM_SIZE:-512mb}"
command: >
sh -c "
echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
fi
if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
set -a
. /home/bts/sparkdata/conf/.env.local
set +a
fi
./sparkd run"
env_file:
- .env
- ${SPARKD__CONFIG_DIR}/.env
ports:
- "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
- "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
- "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
deploy:
resources:
limits:
memory: "${SPARKD__MEMORY:-4gb}"
sparkd-symbols:
volumes:
- ${SPARKD__DATA_DIR:-sparkd_data}:/home/bts/sparkdata/data
- ${SPARKD__RUNTIME_CONF_DIR:-sparkd_conf}:/home/bts/sparkdata/conf
build:
dockerfile: src/server/sparkd/Dockerfile
args:
- IMAGE=${SPARKD__IMAGE}
- SOURCE_DIR=${SPARKD__SOURCE_DIR}
- CONFIG_DIR=${SPARKD__CONFIG_DIR}
shm_size: "${SPARKD__SHM_SIZE:-512mb}"
command: >
sh -c "
echo bts | sudo -S chown -R bts:bts /home/bts/sparkdata/data
if [ -e "/home/bts/sparkdata/data/logging.sparkd/log" ]; then
mv "/home/bts/sparkdata/data/logging.sparkd/log" "/home/bts/sparkdata/data/logging.sparkd/log_$(date '+%Y%m%d-%H%M')"
fi
ls -alh /home/bts/sparkdata/conf/
cat /home/bts/sparkdata/conf/.env.local
if [ -e "/home/bts/sparkdata/conf/.env.local" ]; then
echo 'Sourcing /home/bts/sparkdata/conf/.env.local'
set -a
. /home/bts/sparkdata/conf/.env.local
set +a
fi
./sparkd run"
env_file:
- .env
- ${SPARKD__CONFIG_DIR}/.env
ports:
- "${SPARKD__CONFIG_PORT}:${SPARKD__CONFIG_PORT}"
- "${SPARKD__QUICKBUS_PORT}:${SPARKD__QUICKBUS_PORT}"
- "${SPARKD__SIM_FIX_PORT}:${SPARKD__SIM_FIX_PORT}"
deploy:
resources:
limits:
memory: "${SPARKD__MEMORY:-4gb}"
sparkd-tail:
image: bluetradingsystems/linux_dotnet7
volumes:
- type: volume
read_only: true
source: sparkd_data
target: /home/bts/sparkdata/data/logging.sparkd
volume:
subpath: logging.sparkd
command: ["tail", "-f", "/home/bts/sparkdata/data/logging.sparkd/log"]
sparkd-ping-test:
depends_on:
- sparkd
profiles:
- ping-test
build:
context: .
dockerfile_inline: |
FROM bluetradingsystems/linux_dotnet7
USER "root"
RUN apt-get update && apt-get -y install iputils-ping
command: [ "ping", "sparkd" ]
volumes:
sparkd_data:
name: "sparkd_${SPARKD__CONFIG_PORT:-8000}"
@@ -1,38 +0,0 @@
FROM ubuntu:22.04
ARG DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y pkg-config
RUN apt-get update && apt-get install -y locales
RUN apt-get update && apt-get install -y ca-certificates
RUN apt-get update && apt-get install -y libssh2-1-dev
RUN apt-get update && apt-get install -y software-properties-common
RUN apt-get update && apt-get install -y sudo
# GCC
ARG gcc_major=13
RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
&& apt-get update \
&& apt-get install -y g++-$gcc_major
RUN apt-get update \
&& apt-get install -y dotnet7
RUN locale-gen en_US.UTF-8
ARG docker_user="bts"
ARG docker_uid="1000"
ARG docker_pass="bts"
RUN \
groupadd -g "$docker_uid" "$docker_user" \
&& useradd \
-m \
-u "$docker_uid" \
-g "$docker_uid" \
-G sudo \
--password "$(openssl passwd -1 "$docker_pass")" \
"$docker_user"
USER bts
WORKDIR /home/bts
Submodule
+1
Submodule docker/linux added at 56c61231f3.
+1
Submodule docker/windows added at de7d40c09d.
-14
View File
@@ -1,20 +1,11 @@
astroid==2.7.3 astroid==2.7.3
autopep8==1.5.7 autopep8==1.5.7
blinker==1.9.0
boto3==1.37.9
botocore==1.37.9
certifi==2023.5.7 certifi==2023.5.7
charset-normalizer==3.1.0 charset-normalizer==3.1.0
click==8.1.8
colorama==0.4.4 colorama==0.4.4
Flask==3.1.0
idna==3.4 idna==3.4
isort==5.9.3 isort==5.9.3
itsdangerous==2.2.0
Jinja2==3.1.6
jmespath==1.0.1
lazy-object-proxy==1.6.0 lazy-object-proxy==1.6.0
MarkupSafe==3.0.2
mccabe==0.6.1 mccabe==0.6.1
platformdirs==2.3.0 platformdirs==2.3.0
protobuf==4.21.12 protobuf==4.21.12
@@ -23,14 +14,9 @@ pycodestyle==2.7.0
pycparser==2.20 pycparser==2.20
pylint==2.10.2 pylint==2.10.2
PyMeta3==0.5.1 PyMeta3==0.5.1
python-dateutil==2.9.0.post0
python-dotenv==1.0.1
requests==2.31.0 requests==2.31.0
s3transfer==0.11.4
six==1.17.0
termcolor==1.1.0 termcolor==1.1.0
toml==0.10.2 toml==0.10.2
urllib3==2.0.3 urllib3==2.0.3
Werkzeug==3.1.3
wrapt==1.12.1 wrapt==1.12.1
yapf==0.31.0 yapf==0.31.0
@@ -18,11 +18,8 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="HtmlAgilityPack" Version="1.11.57" /> <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
<PackageReference Include="Microsoft.Data.Analysis" Version="0.21.1" />
<PackageReference Include="MongoDB.Bson" Version="2.23.1" /> <PackageReference Include="MongoDB.Bson" Version="2.23.1" />
<PackageReference Include="MongoDB.Driver" Version="2.23.1" /> <PackageReference Include="MongoDB.Driver" Version="2.23.1" />
<PackageReference Include="Parquet.Net" Version="5.0.1" />
<PackageReference Include="System.Data.SQLite" Version="1.0.118" />
</ItemGroup> </ItemGroup>
</Project> </Project>
@@ -1,32 +0,0 @@
namespace bts.SilverPine {
using System;
using BCG;
using System.Collections.Generic;
using Bts.DataStreamExpressions;
// All this does is take in a dsl expression, parse it and fire the data that come out of running it.
// So for example, if an expresison is passed
[DataStreamUserRespresentation("dsl")]
public class DSLStream : ViewStream {
private readonly string _dsl;
public DSLStream(string dsl) {
Log.Gen.info($"DSLStream.cons dsl: {dsl}");
_dsl = dsl;
ViewStream rs = build_stream(dsl);
rs = rs ?? new LiteralStream("error", $"msg:invalid dsl \"{dsl}\"");
mySetRealSource(rs);
}
static ViewStream? build_stream(string stream_expression) {
try {
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
return rv;
} catch (Exception ex) {
Log.Gen.error($"Could not build stream: {stream_expression} ex: {ex}");
return null;
}
}
}
}
@@ -39,8 +39,8 @@ namespace BCG {
protected override void myOnFirstSubscriber() { protected override void myOnFirstSubscriber() {
MasterWorker.getInstance(false).addWorker(this, _send_period); MasterWorker.getInstance(false).addWorker(this, _send_period);
var server_rci = new RemoteConnectionInfo(_ip, _port); var server_address = new BlutoSubAddress(_ip, _port);
bts.AsyncTCPConnector_APM.connect_to(server_rci, this); bts.AsyncTCPConnector_APM.connect_to(server_address, this);
} }
protected override void myOnLastSubscriber() { protected override void myOnLastSubscriber() {
@@ -1,24 +1,23 @@
using bts;
using System; using System;
using System.Collections; using System.Collections;
namespace BCG { namespace BCG
{
/// <summary> /// <summary>
/// Provides a stream that with distinct rows with respect to the columns /// Provides a stream that with distinct rows with respect to the columns
/// the user specifies in the argument /// the user specifies in the argument
/// </summary> /// </summary>
[DataStreamUserRespresentation("distinct")] [DataStreamUserRespresentation ("distinct")]
public class DistinctRowStream : ViewStream { public class DistinctRowStream : ViewStream
public DistinctRowStream(ViewStream stream, params string[] cols) { {
var g_by_eval = new DistinctRowEvaluator(cols); public DistinctRowStream (ViewStream stream, params string[] cols)
ViewStream r = new GroupByStream(stream, g_by_eval); {
var proj_exp = String.Empty; ArrayList groupByCols = new ArrayList ();
foreach (var c in cols) foreach (string col in cols)
proj_exp += $"{c} "; foreach (string c in col.Split (' '))
proj_exp = proj_exp.Trim(); groupByCols.Add (c);
r = new NewProjectStream(r, proj_exp); mySetRealSource (new GroupByStream (stream,
mySetRealSource(r); new DistinctRowEvaluator (groupByCols.ToArray (typeof(string)) as string[]), false));
} }
} }
} }
@@ -0,0 +1,49 @@
// using System;
// using System.Collections.Generic;
// using System.Text;
// using System.IO;
// using System.Runtime.Serialization.Formatters.Binary;
// namespace BCG
// {
// public class DumpLoaderStream : ViewStream, ISideEffectStream
// {
// //wtf
// //public class Exception : System.Exception
// //{public Exception(string msg) : base(msg) { }}
// public DumpLoaderStream(string fileName, int secondsBetweenPolls)
// {
// mySetRealSource(
// new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls,
// ViewStreamDatabase.instance));
// }
// public DumpLoaderStream(string fileName)
// {
// if (fileName == null)
// throw new LoaderStream.Exception("File name cannot be null");
// for (int i = 0; i < 5; i++)
// try
// {
// using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
// {
// IClientDeserializer deser = new ClientUniversalDeserializer();
// byte[] the_ops;
// Array the_rows;
// deser.deserialize(new Stream[] { fs }, out the_ops, out the_rows);
// mySetRealSource(new BlueRowCollectionStaticStream((IBlueRowCollection)the_rows.GetValue(0)));
// }
// break;
// }
// catch (IOException)
// {
// System.Threading.Thread.Sleep(50);
// }
// }
// }
// }
@@ -1,7 +1,6 @@
using BCG; using BCG;
namespace bts { namespace bts {
// This is the other side of the SaverStream, its "loads" the stream and fires once
[DataStreamUserRespresentation("loader")] [DataStreamUserRespresentation("loader")]
public class LoaderStream : ViewStream { public class LoaderStream : ViewStream {
@@ -5,11 +5,10 @@ using System.Text;
using System.IO; using System.IO;
using Google.Protobuf; using Google.Protobuf;
using System.Globalization; using System.Globalization;
using BCG; using bts;
namespace bts { namespace BCG {
// Writes all the udpates to a stream in a file or multiples files dependong on the rollover size (as opposed to Saver // Writes all the udpates to a stream in a file.
// which just writes one update, as a create)
// File naming: // File naming:
// token_epoch_yyyyMMdd_HHmmss // token_epoch_yyyyMMdd_HHmmss
// File location: directory(passed)\token\ // File location: directory(passed)\token\
@@ -95,10 +94,10 @@ namespace bts {
void _on_change(object new_value) { void _on_change(object new_value) {
DataPacket dp = new_value as DataPacket; DataPacket dp = new_value as DataPacket;
byte[] serialized_update = null; byte[] serialized_update = null;
bool first_write = _serializer == null || _n_updates_in_current_file == 0; bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
_serializer = _serializer ?? new UniversalStreamIOSerializer(); _serializer = _serializer ?? new UniversalStreamIOSerializer();
if (first_write) { if (_write_create) {
// first file always starts with create // new files always starts with create
serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows))); serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows)));
} else } else
serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows))); serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows)));
@@ -106,8 +105,8 @@ namespace bts {
long utc_epoch = epoch_now(); long utc_epoch = epoch_now();
//time, length of data, data //time, length of data, data
//bw.Write(utc_epoch); bw.Write(utc_epoch);
//bw.Write(serialized_update.Length); bw.Write(serialized_update.Length);
bw.Write(serialized_update); bw.Write(serialized_update);
_out_fs.Flush(); _out_fs.Flush();
myFireEvent(dp); myFireEvent(dp);
@@ -0,0 +1,190 @@
// using System;
// using System.Collections.Generic;
// using System.Linq;
// using System.Text;
// using System.IO;
// using System.Runtime.Serialization.Formatters.Binary;
// using BCG.Utilities.General;
// using bts.collections;
// using System.Reflection.Metadata;
// namespace BCG {
// public class PlaybackStream : ViewStream, IWorker {
// private readonly PlaybackType _type;
// private BlueTable _current_table;
// private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
// // these are the packets already downloaded from the current file
// private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
// //private IEnumerable<FileStream> _file_streams = new List<FileStream>();
// private IEnumerable<string> _file_names;
// private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
// private DateTime _playback_t0, _playback_t1;
// private bool _all_files_read = false;
// SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
// public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
// _type = PlaybackType.Streaming;
// _playback_t0 = PersistorStream.parse_datetime(datetime_start);
// _playback_t1 = PersistorStream.parse_datetime(datetime_end);
// string token_directory_name = $"{directory_name}\\{token}";
// bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
// List<string> files_in_range = new List<string>();
// if (Directory.Exists(token_directory_name)) {
// _file_names =
// Directory.GetFiles(token_directory_name).Where(f => {
// DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
// return file_time > _playback_t0 && file_time < _playback_t1;
// });
// }
// }
// protected override void myOnFirstSubscriber() {
// FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
// _snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
// PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
// if (snapshot_packet != null) {
// foreach (PlaybackPacket p in _snapshot_data.packets) {
// byte[] ops;
// Array row_array;
// _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
// for (int i = 0; i < ops.Length; i++) {
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
// byte op = ops[i];
// if (op == StreamOpCode.create) {
// IBlueRowCollection full_table = rows_args[0];
// _current_table = new BlueTable(full_table);
// } else if (op == StreamOpCode.multipleRemoveAdd) {
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
// BlueDataUtility.absorbChanges(_current_table, dels, adds);
// } else {
// bts.Log.assert(false, $"Oprator: {op} is not expected");
// }
// if (p.idx == snapshot_packet.idx) {
// myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
// break;
// }
// }
// }
// }
// }
// private void _load_packets_from_current_file() {
// int curr_idx = _current_fs.Item1;
// FileStream curr_fs = _current_fs.Item2;
// _packets = new List<PlaybackPacket>();
// _packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
// }
// private bool current_file_all_replayed {
// get {
// bool rv = true;
// _packets.ForEach(p => rv = rv && p.replayed);
// return rv;
// }
// }
// protected override void myOnLastSubscriber() {
// var fs = _current_fs.Item2;
// if (fs != null) {
// fs.Close();
// }
// ViewStreamDatabase.instance.removeWorker(this);
// }
// // public event PlaybackStreamProgress onProgress;
// // public event Action onFileExhausted;
// private PlaybackPacket _locate_packet_to_replay() {
// foreach (var p in _packets) {
// // returns the first non replayed packet
// if (!p.replayed)
// return p;
// }
// //else
// {
// double secondsElapsed = new BinaryReader(myFileReader).ReadDouble();
// myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed));
// if (onProgress != null)
// onProgress(myFileReader.Position, myFileReader.Length);
// }
// }
// #region IWorker Members
// public void doWork() {
// if (_all_files_read)
// return;
// if (current_file_all_replayed) {
// _load_next_file();
// }
// PlaybackPacket curr_packet = _locate_packet_to_replay();
// if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway
// byte[] ops;
// Array row_array;
// // what a stupid interface!
// _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
// _fire_packet(ops, row_array);
// curr_packet.replay_timestamp = PersistorStream.epoch_now();
// } else {
// PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
// // this is the time we need to fire (replay) this packet
// long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
// long now_epoch = PersistorStream.epoch_now();
// // fire when we are past the
// if (now_epoch >= time_to_fire) {
// byte[] ops;
// Array row_array;
// _deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
// _fire_packet(ops, row_array);
// curr_packet.replay_timestamp = PersistorStream.epoch_now();
// }
// }
// }
// private void _fire_packet(byte[] ops, Array row_array) {
// for (int i = 0; i < ops.Length; i++) {
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
// byte op = ops[i];
// if (op == StreamOpCode.create) {
// IBlueRowCollection full_table = rows_args[0];
// _current_table = new BlueTable(full_table);
// //_current_table.addRows(full_table);
// myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
// (myLastValue as DataPacket).newRows));
// } else if (op == StreamOpCode.multipleRemoveAdd) {
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
// BlueDataUtility.absorbChanges(_current_table, dels, adds);
// myFireEvent(new DataPacket(_current_table, adds, dels));
// } else {
// bts.Log.assert(false, $"Oprator: {op} is not expected");
// }
// }
// }
// private void _load_next_file() {
// var new_idx = _current_fs.Item1 + 1;
// if (new_idx >= _file_names.Count()) {
// _all_files_read = true;
// bts.Log.Gen.info("All files read");
// return;
// }
// var prev_fs = _current_fs.Item2;
// if (prev_fs != null)
// prev_fs.Close();
// var new_filename = _file_names.ElementAt(new_idx);
// bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
// var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
// _current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
// _load_packets_from_current_file();
// }
// #endregion
// }
// }
@@ -0,0 +1,68 @@
// using System;
// using System.Collections.Generic;
// using System.Linq;
// using System.Text;
// using System.IO;
// namespace BCG
// {
// public class RecorderStream : ViewStream
// {
// private ViewStream mySource;
// private DateTime myStartTime;
// private FileStream myOutFile;
// private string myFileName;
// private IBlueDataSerializer mySerializer;
// public RecorderStream(ViewStream src, string recordFile)
// {
// mySource = src;
// myFileName = recordFile;
// myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss"));
// }
// protected override void myOnFirstSubscriber()
// {
// myStartTime = DateTime.Now;
// mySource.onChange += new ChangeDelegate(mySource_onChange);
// myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
// }
// protected override void myOnLastSubscriber()
// {
// mySource.onChange -= mySource_onChange;
// myOutFile.Close();
// }
// void mySource_onChange(object newValue)
// {
// DataPacket dp = (DataPacket)newValue;
// BlueTable toWrite = new BlueTable(dp.schema);
// if (mySerializer == null)
// {
// mySerializer = new UniversalSerializer(dp.schema);
// }
// foreach (IBlueRow r in dp.newRows)
// toWrite.addRow(r);
// byte[] result;
// if (myLastValue == null)
// mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
// else
// mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
// BinaryWriter bw = new BinaryWriter(myOutFile);
// //write down what time this happened
// bw.Write((DateTime.Now - myStartTime).TotalSeconds);
// bw.Write(result.Length);
// bw.Write(result);
// myOutFile.Flush();
// myFireEvent(dp);
// }
// }
// }
@@ -5,8 +5,7 @@ using System.IO;
using BCG; using BCG;
namespace bts { namespace bts {
// this saves the updates as a Create on the same file, each update calls -save, which // this saves the total state in the filr provided in the cons
// reqrites the whole data to the packet. The other side of this stream is the Loader
[DataStreamUserRespresentation("saver")] [DataStreamUserRespresentation("saver")]
public class SaverStream : ViewStream { public class SaverStream : ViewStream {
private ViewStream _stream; private ViewStream _stream;
@@ -32,6 +31,7 @@ namespace bts {
StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows); StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg)); byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
StreamSerializationUtils.write_to_file(str_bytes, _filename); StreamSerializationUtils.write_to_file(str_bytes, _filename);
} }
} }
} }
@@ -3,7 +3,6 @@ using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Globalization; using System.Globalization;
using System.IO; using System.IO;
using bts;
namespace BCG { namespace BCG {
@@ -1,24 +0,0 @@
namespace bts.SilverPine {
using BCG;
using System;
public class NullFilterStream : ViewStream {
public NullFilterStream(ViewStream parent_str, string col_name)
: this(parent_str, col_name, true) { }
public NullFilterStream(ViewStream parent_str, string col_name, bool filter_out_doublenans = false) {
ViewStream rs = new FilterStream(parent_str,
r => {
bool is_dbnull = r.is_null(col_name);
if (is_dbnull) {
return false;
}
if (r.schema[col_name].type_code() == TypeCode.Double) {
return !Double.IsNaN(r.get<double>(col_name));
}
return true;
});
mySetRealSource(rs);
}
}
}
@@ -1,102 +1,102 @@
// using System; using System;
// using System.Collections.Generic; using System.Collections.Generic;
// using System.Linq; using System.Linq;
// using System.Text; using System.Text;
// namespace BCG namespace BCG
// { {
// public class AverageGroupsStream : ViewStream public class AverageGroupsStream : ViewStream
// { {
// public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn) public AverageGroupsStream(ViewStream input, string colToAverage, string keyColumn)
// { {
// myColToAverage = colToAverage; myColToAverage = colToAverage;
// myInput = input; myInput = input;
// myKeyColumn = keyColumn; myKeyColumn = keyColumn;
// } }
// private ViewStream myInput; private ViewStream myInput;
// private string myColToAverage; private string myColToAverage;
// private BlueTableSchema mySchema; private BlueTableSchema mySchema;
// private string myKeyColumn; private string myKeyColumn;
// protected override void myOnFirstSubscriber() protected override void myOnFirstSubscriber()
// { {
// myInput.onChange += new ChangeDelegate(myInput_onChange); myInput.onChange += new ChangeDelegate(myInput_onChange);
// } }
// protected override void myOnLastSubscriber() protected override void myOnLastSubscriber()
// { {
// myInput.onChange -= myInput_onChange; myInput.onChange -= myInput_onChange;
// } }
// private void myInput_onChange(object newValue) private void myInput_onChange(object newValue)
// { {
// if (mySchema == null) if (mySchema == null)
// myBuildSchema(((DataPacket)newValue).schema); myBuildSchema(((DataPacket)newValue).schema);
// IBlueRowCollection newRows = myEvaluate((DataPacket)newValue); IBlueRowCollection newRows = myEvaluate((DataPacket)newValue);
// if (myLastValue == null) if (myLastValue == null)
// myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema))); myFireEvent(new DataPacket(newRows, newRows, new BlueTable(mySchema)));
// else else
// { {
// IBlueRowCollection added, deleted; IBlueRowCollection added, deleted;
// BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added); BlueDataUtility.computeDifferences(newRows, ((DataPacket) myLastValue).newRows, out deleted, out added);
// myFireEvent(new DataPacket(newRows, added, deleted)); myFireEvent(new DataPacket(newRows, added, deleted));
// } }
// } }
// private void myBuildSchema(IBlueTableSchema schema) private void myBuildSchema(IBlueTableSchema schema)
// { {
// List<ColumnInfo> columns = new List<ColumnInfo>(); List<ColumnInfo> columns = new List<ColumnInfo>();
// foreach (ColumnInfo ci in schema.columnInfo) foreach (ColumnInfo ci in schema.columnInfo)
// columns.Add(ci); columns.Add(ci);
// columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double))); columns.Add(new ColumnInfo("average_"+myColToAverage, columns.Count, typeof(double)));
// mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray()); mySchema = new BlueTableSchema("average_of_" + schema.tableName, columns.ToArray());
// } }
// private IBlueRowCollection myEvaluate(DataPacket packet) private IBlueRowCollection myEvaluate(DataPacket packet)
// { {
// IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>(); IDictionary<string, double> colToRunningTotal = new Dictionary<string, double>();
// IDictionary<string, int> colToCount = new Dictionary<string, int>(); IDictionary<string, int> colToCount = new Dictionary<string, int>();
// IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>(); IDictionary<string, IBlueRow> colToLastRow = new Dictionary<string, IBlueRow>();
// foreach (IBlueRow r in packet.newRows) foreach (IBlueRow r in packet.newRows)
// { {
// double thisVal = Convert.ToDouble(r[myColToAverage]); double thisVal = Convert.ToDouble(r[myColToAverage]);
// string keyVal = r[myKeyColumn].ToString(); string keyVal = r[myKeyColumn].ToString();
// if (!colToCount.ContainsKey(keyVal)) if (!colToCount.ContainsKey(keyVal))
// { {
// colToRunningTotal[keyVal] = thisVal; colToRunningTotal[keyVal] = thisVal;
// colToCount[keyVal] = 1; colToCount[keyVal] = 1;
// } }
// else else
// { {
// colToRunningTotal[keyVal] += thisVal; colToRunningTotal[keyVal] += thisVal;
// colToCount[keyVal] += 1; colToCount[keyVal] += 1;
// } }
// colToLastRow[keyVal] = r; colToLastRow[keyVal] = r;
// } }
// BlueTable result = new BlueTable(mySchema); BlueTable result = new BlueTable(mySchema);
// foreach (string key in colToCount.Keys) foreach (string key in colToCount.Keys)
// { {
// object[] values = new object[mySchema.columnInfo.Length]; object[] values = new object[mySchema.columnInfo.Length];
// colToLastRow[key].CopyTo(values, 0); colToLastRow[key].CopyTo(values, 0);
// values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0; values[values.Length - 1] = packet.newRows.Count > 0 ? colToRunningTotal[key] / packet.newRows.Count : 0.0;
// result.addRow(new BlueRow(mySchema, values)); result.addRow(new BlueRow(mySchema, values));
// } }
// return result; return result;
// } }
// } }
// } }
@@ -1,31 +0,0 @@
namespace bts.SilverPine {
using BCG;
using System;
using System.Collections.Generic;
using System.Linq;
[DataStreamUserRespresentation("Average")]
public class AverageNonIncGroupsStream : ViewStream {
public AverageNonIncGroupsStream(ViewStream parent_stream, string inp_col_name, params string[] group_by_cols) {
NonIncGroupByStream rs = new NonIncGroupByStream(parent_stream, () => group_by_cols,
(g_rows) => _find_mean(inp_col_name, g_rows));
mySetRealSource(rs);
}
private List<IBlueRow> _find_mean(string inp_col_name, List<IBlueRow> g_rows) {
IBlueRow inp_row = g_rows.ElementAt(0);
IEnumerable<double> inp_vals = BlueDataUtility.get_colvalues<double>(g_rows, inp_col_name);
double sum = 0.0;
inp_vals.ToList().ForEach(d => sum += d);
string out_col_name = $"average_{inp_col_name}";
double average = sum / inp_vals.Count();
IBlueTableSchema schema_out = inp_row.schema.add_column(TypeCode.Double, out_col_name);
MutableBlueRow out_row = new MutableBlueRow(schema_out, inp_row);
out_row.setValue(out_col_name, average);
List<IBlueRow> rv = new List<IBlueRow>();
rv.Add(out_row);
return rv;
}
}
}
@@ -1,91 +0,0 @@
namespace bts.SilverPine {
using BCG;
using System;
using System.Collections.Generic;
using System.Linq;
using ZstdSharp.Unsafe;
public enum GroupByBundleOp {
Mean,
Median,
Mode,
StdDev,
Max,
Min,
First,
Last,
Sum
}
public class GroupByBundle {
public readonly List<Tuple<string, GroupByBundleOp>> operations;
public static GroupByBundle parse(string text) {
// text is inp_col: op, inp_col: op, inp_col:op
List<Tuple<string, GroupByBundleOp>> rv = new List<Tuple<string, GroupByBundleOp>>();
string[] calcs = text.Split(',');
foreach (string c in calcs) {
string[] toks = c.Split(':');
string inp_col = toks[0].Trim();
string op_txt = toks[1].Trim();
GroupByBundleOp op = Enum.Parse<GroupByBundleOp>(op_txt, true);
rv.Add(Tuple.Create(inp_col, op));
}
return new GroupByBundle(rv);
}
public GroupByBundle(List<Tuple<string, GroupByBundleOp>> operations_) {
operations = operations_;
}
public static IBlueRow group_by(List<IBlueRow> rows, GroupByBundle parameters) {
if (rows.Count() == 0) {
return null;
}
IBlueRow ir = rows.ElementAt(0);
IBlueTableSchema in_schema = ir.schema;
IBlueTableSchema out_schema = in_schema;
IDictionary<string, Tuple<string, GroupByBundleOp>> out_col_map =
new Dictionary<string, Tuple<string, GroupByBundleOp>>();
foreach (var t in parameters.operations) {
string out_col_name = $"{t.Item1}_{t.Item2}".ToLower();
out_schema = out_schema.add_column(TypeCode.Double, out_col_name);
out_col_map[out_col_name] = t;
}
MutableBlueRow or = new MutableBlueRow(out_schema, ir);
foreach (var kv in out_col_map) {
string out_col_name = kv.Key;
string in_col_name = kv.Value.Item1;
GroupByBundleOp op = kv.Value.Item2;
IEnumerable<double> col_vals = BlueDataUtility.get_colvalues<double>(rows, in_col_name);
double comp_res = compute_op(col_vals, op);
or.setValue(out_col_name, comp_res);
}
return or;
}
public static double compute_op(IEnumerable<double> col_vals, GroupByBundleOp op) {
switch (op) {
case GroupByBundleOp.Max:
return col_vals.Max<double>();
case GroupByBundleOp.Min:
return col_vals.Min<double>();
case GroupByBundleOp.Mean:
return col_vals.Sum() / col_vals.Count();
case GroupByBundleOp.Sum:
return col_vals.Sum();
case GroupByBundleOp.First:
return col_vals.First();
case GroupByBundleOp.Last:
return col_vals.Last();
default:
// TODO
Log.assert(false);
return Double.NaN;
}
}
}
}
@@ -10,6 +10,6 @@ namespace BCG {
} }
public MaxStream(ViewStream stream, string outputCol) public MaxStream(ViewStream stream, string outputCol)
: this(new AddColumnsStream(stream, "fake_col: 0"), outputCol, new string[] { "fake_col" }) { } : this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { }
} }
} }
@@ -1,40 +1,13 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using System.Diagnostics;
using System.Linq;
using System.Text.Json;
using bts; using bts;
using bts.SilverPine;
namespace BCG { namespace BCG
[DataStreamUserRespresentation("aggregate")] {
public class NonIncGroupByStream : ViewStream { public class NonIncGroupByStream : ViewStream
private class DelNonIncGroupByEvaluator : INonIncGroupByEvaluator { {
Func<string[]> _groupby_cols_func;
Func<List<IBlueRow>, List<IBlueRow>> _evaluate_func;
public DelNonIncGroupByEvaluator(Func<string[]> groupby_cols_func,
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func) {
_groupby_cols_func = groupby_cols_func;
_evaluate_func = evaluate_func;
}
public IBlueRowCollection evaluate(IDictionary<int, List<IBlueRow>> grouped_rows) {
BlueTable rv = null;
foreach (var kv in grouped_rows) {
List<IBlueRow> g_rows = kv.Value;
List<IBlueRow> out_rows = _evaluate_func.Invoke(g_rows);
if (out_rows != null && out_rows.Count > 0) {
rv = rv ?? new BlueTable(out_rows.ElementAt(0).schema);
}
rv.addRows(out_rows);
}
return rv;
}
public string[] get_group_by_cols() {
return _groupby_cols_func();
}
}
private ViewStream _main_stream; private ViewStream _main_stream;
private INonIncGroupByEvaluator _groupby_func; private INonIncGroupByEvaluator _groupby_func;
private BlueRowHasher _row_hasher; private BlueRowHasher _row_hasher;
@@ -42,47 +15,24 @@ namespace BCG {
private Stopwatch _timer = new(); private Stopwatch _timer = new();
private int _n_total_fires = 0, _n_expensive_fires = 0; private int _n_total_fires = 0, _n_expensive_fires = 0;
public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) { public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
{
_main_stream = main_stream; _main_stream = main_stream;
_groupby_func = groupby_func; _groupby_func = groupby_func;
_groupby_cols = _groupby_func.get_group_by_cols(); _groupby_cols = _groupby_func.get_group_by_cols();
_row_hasher = new BlueRowHasher(_groupby_cols == null ? 0 : _groupby_cols.Length); _row_hasher = new BlueRowHasher(_groupby_cols.Length);
} }
public NonIncGroupByStream(ViewStream main_stream, Func<string[]> groupby_cols_func,
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func)
: this(main_stream, new DelNonIncGroupByEvaluator(groupby_cols_func, evaluate_func)) { }
public NonIncGroupByStream(ViewStream parent_str, string groupby_col, string standard_group_by_text, string optional_sort_col = null) {
GroupByBundle std_group_by = GroupByBundle.parse(standard_group_by_text);
Func<string[]> groupby_cols_func = () => {
return String.IsNullOrEmpty(groupby_col) ? null : new string[] { groupby_col };
};
Func<List<IBlueRow>, List<IBlueRow>> evaluate_func = (group_rows) => {
if (!String.IsNullOrEmpty(optional_sort_col))
group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, optional_sort_col));
IBlueRow rv = GroupByBundle.group_by(group_rows, std_group_by);
return new IBlueRow[] { rv }.ToList();
};
ViewStream rs = new NonIncGroupByStream(parent_str, groupby_cols_func, evaluate_func);
mySetRealSource(rs);
}
public NonIncGroupByStream(ViewStream parent_str, string standard_group_by_text) :
this(parent_str, String.Empty, standard_group_by_text) {
}
protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; } protected override void myOnFirstSubscriber() { _main_stream.onChange += _main_stream_onchange; }
protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; } protected override void myOnLastSubscriber() { _main_stream.onChange -= _main_stream_onchange; }
private void _main_stream_onchange(object nv) { private void _main_stream_onchange(object nv)
{
DataPacket dp = nv as DataPacket; DataPacket dp = nv as DataPacket;
///grouping together ///grouping together
_timer.Restart(); _timer.Restart();
IDictionary<int, List<IBlueRow>> grouped_rows = (_groupby_cols == null || _groupby_cols.Length == 0) ? IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } } :
BlueDataUtility.group_together(dp.newRows, _groupby_cols, ref _row_hasher);
_timer.Stop(); _timer.Stop();
var groupElapsedMs = _timer.ElapsedMilliseconds; var groupElapsedMs = _timer.ElapsedMilliseconds;
if (groupElapsedMs >= 50) { if (groupElapsedMs >= 50) {
@@ -94,15 +44,14 @@ namespace BCG {
IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows); IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
_timer.Stop(); _timer.Stop();
var applyElapsedMs = _timer.ElapsedMilliseconds; var applyElapsedMs = _timer.ElapsedMilliseconds;
_n_total_fires++; _n_total_fires++;
if (applyElapsedMs >= 50) { if (applyElapsedMs >= 50)
{
_n_expensive_fires++; _n_expensive_fires++;
Log.DataFlow.info( Log.DataFlow.info(
$"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}"); $"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
} }
// TODO, myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
if (total_state != null)
myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
} }
} }
} }
@@ -1,16 +1,15 @@
using bts; using bts;
using bts.collections;
using mbts; using mbts;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using System.Diagnostics;
using System.Linq;
using System.Security.Cryptography;
namespace BCG { namespace BCG
public class NonIncJoinBookkeeper : IDisposable { {
public class NonIncJoinBookkeeper : IDisposable
{
public Action on_new_data; public Action on_new_data;
private readonly NonIncJoinSpec _join_spec; private readonly NonIncJoinSpec _join_spec;
private readonly int _join_position; private readonly int _join_position;
@@ -27,71 +26,41 @@ namespace BCG {
private int[] _join_col_indices = null, private int[] _join_col_indices = null,
_data_col_indices = null; _data_col_indices = null;
public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher) { public NonIncJoinBookkeeper(NonIncJoinSpec the_join_spec, int join_position, int start_index_in_output, BlueRowHasher row_hasher)
{
_join_spec = the_join_spec; _join_spec = the_join_spec;
_join_position = join_position; _join_position = join_position;
_start_index_in_output = start_index_in_output; _start_index_in_output = start_index_in_output;
_row_hasher = row_hasher; _row_hasher = row_hasher;
_is_left_most = _join_position == 0; _is_left_most = _join_position == 0;
_is_tag_along = _join_spec.join_col_names.Length == 0; _is_tag_along = _join_spec.join_col_names.Length == 0;
}
private bool _filled_missing_data = false;
private void _fill_missing_data(DataPacket dp) {
if (_filled_missing_data)
return;
Func<string[], bool> null_or_empty = (str_arr) => {
if (str_arr == null || str_arr.Length == 0)
return true;
foreach (string s in str_arr) {
if (!String.IsNullOrEmpty(s)) {
return false;
}
}
return true;
};
Func<string[], string[]> remove_bad_cols = (col_names) => {
HashSet<string> schema_col_map = new HashSet<string>(dp.schema.columnNames);
List<string> cols_list = new List<string>(col_names);
cols_list.RemoveWhere(c => !schema_col_map.Contains(c));
return cols_list.ToArray();
};
if (null_or_empty(_join_spec.data_col_names)) {
_join_spec.data_col_names = dp.schema.columnNames;
}
_join_spec.data_col_names = remove_bad_cols(_join_spec.data_col_names);
if (null_or_empty(_join_spec.data_col_names)) {
_join_spec.data_col_names = dp.schema.columnNames;
}
if (null_or_empty(_join_spec.projected_col_names)) {
_join_spec.projected_col_names = _join_spec.data_col_names;
}
}
public void init() {
_join_spec.stream.onChange += _on_stream_cb; _join_spec.stream.onChange += _on_stream_cb;
} }
public void _on_stream_cb(object dp_obj) { public void _on_stream_cb(object dp_obj)
Log.Gen.info($"NonIncJoinBookkeeper._on_stream_cb dp_obj: {dp_obj}"); {
var timer = Stopwatch.StartNew(); var timer = Stopwatch.StartNew();
DataPacket dp = (DataPacket)dp_obj; DataPacket dp = (DataPacket)dp_obj;
_fill_missing_data(dp); if (_schema == null)
if (_schema == null) { {
_schema = dp.schema; _schema = dp.schema;
_join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names); _join_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.join_col_names);
_data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names); _data_col_indices = BlueDataUtility.get_column_indices(_schema, _join_spec.data_col_names);
} }
if (_is_left_most) { if (_is_left_most)
{
_row_list.Clear(); _row_list.Clear();
foreach (IBlueRow r in dp.newRows) foreach (IBlueRow r in dp.newRows)
_row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r)); _row_list.Add(new Pair<int, IBlueRow>(_row_hasher.get_hash(r, _join_col_indices), r));
} else { }
else
{
if (_is_tag_along) if (_is_tag_along)
_solitary_row = BlueDataUtility.extractRow(dp.newRows, 0); _solitary_row = BlueDataUtility.extractRow(dp.newRows, 0);
else { else
{
_row_map.Clear(); _row_map.Clear();
foreach (IBlueRow r in dp.newRows) foreach (IBlueRow r in dp.newRows)
_row_map[_row_hasher.get_hash(r, _join_col_indices)] = r; _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
@@ -110,25 +79,31 @@ namespace BCG {
public bool is_ready { get { return _schema != null; } } public bool is_ready { get { return _schema != null; } }
public void constribute_cols_to_schema(List<ColumnInfo> cis) { public void constribute_cols_to_schema(List<ColumnInfo> cis)
{
for (int i = 0; i < _join_spec.data_col_names.Length; i++) for (int i = 0; i < _join_spec.data_col_names.Length; i++)
cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType)); cis.Add(new ColumnInfo(_join_spec.projected_col_names[i], cis.Count, _schema[_join_spec.data_col_names[i]].colType));
} }
public List<Pair<int, IBlueRow>> get_list() { return _row_list; } public List<Pair<int, IBlueRow>> get_list() { return _row_list; }
public void copy_master_row_to_output_row(int row_index, object[] output_row) { public void copy_master_row_to_output_row(int row_index, object[] output_row)
{
IBlueRow this_row = _row_list[row_index].second; IBlueRow this_row = _row_list[row_index].second;
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output); BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
} }
public void copy_matching_row_to_output_row(int row_hash, object[] output_row) { public void copy_matching_row_to_output_row(int row_hash, object[] output_row)
if (_is_tag_along) { {
if (_is_tag_along)
{
if (_solitary_row != null) if (_solitary_row != null)
BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output); BlueDataUtility.copy_to_row(_solitary_row, _data_col_indices, output_row, _start_index_in_output);
else else
copy_dbnull_to_output_row(output_row); copy_dbnull_to_output_row(output_row);
} else { }
else
{
IBlueRow this_row = null; IBlueRow this_row = null;
if (_row_map.TryGetValue(row_hash, out this_row)) if (_row_map.TryGetValue(row_hash, out this_row))
BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output); BlueDataUtility.copy_to_row(this_row, _data_col_indices, output_row, _start_index_in_output);
@@ -137,7 +112,8 @@ namespace BCG {
} }
} }
private void copy_dbnull_to_output_row(object[] output_row) { private void copy_dbnull_to_output_row(object[] output_row)
{
for (int i = 0; i < _data_col_indices.Length; i++) for (int i = 0; i < _data_col_indices.Length; i++)
output_row[i + _start_index_in_output] = DBNull.Value; output_row[i + _start_index_in_output] = DBNull.Value;
} }
@@ -3,63 +3,63 @@ using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Text; using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
using mbts; using mbts;
namespace BCG { namespace BCG
// this a collection of data, theres no joining logic in here {
public class NonIncJoinSpec { public class NonIncJoinSpec
{
public ViewStream stream; public ViewStream stream;
// join col names are the names on which the streams are joined (for equality),
// data col names are the names in the original stream
// projected col names are their new names, ie if its x:l_x, then x is in the
// data_col_names and l_x in projected cols
public string[] join_col_names, data_col_names, projected_col_names; public string[] join_col_names, data_col_names, projected_col_names;
public NonIncJoinSpec() { } public NonIncJoinSpec() { }
public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names) public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names)
: this(the_stream, the_join_col_names, the_data_col_names, the_data_col_names) { } : this (the_stream, the_join_col_names, the_data_col_names, the_data_col_names)
{ }
public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names) { public NonIncJoinSpec(ViewStream the_stream, string[] the_join_col_names, string[] the_data_col_names, string[] the_projected_col_names)
{
stream = the_stream; stream = the_stream;
join_col_names = the_join_col_names; join_col_names = the_join_col_names;
data_col_names = the_data_col_names; data_col_names = the_data_col_names;
projected_col_names = the_projected_col_names; projected_col_names = the_projected_col_names;
} }
public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs) { public static NonIncJoinSpec[] from_triplet(params Triplet<ViewStream, string, string>[] join_specs)
{
List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>(); List<NonIncJoinSpec> rv = new List<NonIncJoinSpec>();
foreach (Triplet<ViewStream, string, string> r in join_specs) { foreach (Triplet<ViewStream, string, string> r in join_specs)
string[] keys = String.IsNullOrEmpty(r.second) ? new string[] { } : r.second.Split(' '); {
string[] keys = String.IsNullOrEmpty (r.second) ? new string[] {} : r.second.Split(' ');
string[] visible_cols = r.third.Split(' '); string[] visible_cols = r.third.Split(' ');
List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>(); List<string> data_col_names = new List<string>(), mapped_col_names = new List<string>();
foreach (string s in visible_cols) { foreach (string s in visible_cols)
{
string[] toks = s.Split(':'); string[] toks = s.Split(':');
if (toks.Length == 1) { if (toks.Length == 1)
{
data_col_names.Add(toks[0]); data_col_names.Add(toks[0]);
mapped_col_names.Add(toks[0]); mapped_col_names.Add(toks[0]);
} else if (toks.Length == 2) { }
else if (toks.Length == 2)
{
data_col_names.Add(toks[0]); data_col_names.Add(toks[0]);
mapped_col_names.Add(toks[1]); mapped_col_names.Add(toks[1]);
} }
} }
rv.Add(new NonIncJoinSpec(r.first, keys, data_col_names.ToArray(), mapped_col_names.ToArray())); rv.Add (new NonIncJoinSpec (r.first, keys, data_col_names.ToArray (), mapped_col_names.ToArray()));
} }
return rv.ToArray(); return rv.ToArray ();
}
public override string ToString() {
Func<string[], string> arr_txt = (a) => {
string rv = "";
foreach (string s in a) {
rv += "{s} ";
}
rv = rv.Trim();
return rv;
};
string rv = $"stream: {stream} join_col_names: {arr_txt(join_col_names)} data_col_names: {arr_txt(data_col_names)} projected_col_names: {arr_txt(projected_col_names)}";
return rv;
} }
} }
//[DataStreamUserRespresentation("ninctest")]
//public class NonIncJoinTestStream : ViewStream
//{
// public NonIncJoinTestStream (string key_cols, string data_cols)
// {
// NonIncJoinSpec.from_triplet(new Triplet<ViewStream, string, string>(null, key_cols, data_cols));
// }
//}
} }
@@ -6,9 +6,11 @@ using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using System.Diagnostics;
namespace BCG { namespace BCG
{
[DataStreamUserRespresentation("fastJoin")] [DataStreamUserRespresentation("fastJoin")]
public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker { public class NonIncJoinStream : ViewStream, BCG.Utilities.General.IWorker
{
private readonly NonIncJoinSpec[] _join_specs; private readonly NonIncJoinSpec[] _join_specs;
private readonly INonIncJoinInlineMapper _output_mapper; private readonly INonIncJoinInlineMapper _output_mapper;
private readonly BlueRowHasher _row_hasher; private readonly BlueRowHasher _row_hasher;
@@ -26,19 +28,24 @@ namespace BCG {
{ {
new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj), new Triplet<ViewStream, string, string> (left, left_join_cols, left_proj),
new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj) new Triplet<ViewStream, string, string> (right, right_join_cols, right_proj)
}) { } })
{ }
public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs) public NonIncJoinStream(params Triplet<ViewStream, string, string>[] join_specs)
: this(NonIncJoinSpec.from_triplet(join_specs), null, 0) { } : this(NonIncJoinSpec.from_triplet(join_specs), null, 0)
{ }
public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms, public NonIncJoinStream(ViewStreamDatabase vsd, INonIncJoinInlineMapper output_mapper, int throttles_ms,
params Triplet<ViewStream, string, string>[] join_specs) params Triplet<ViewStream, string, string>[] join_specs)
: this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms) { } : this(vsd, NonIncJoinSpec.from_triplet(join_specs), output_mapper, throttles_ms)
{ }
public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) public NonIncJoinStream(NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
: this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms) { } : this(ViewStreamDatabase.instance, join_specs, output_mapper, throttles_ms)
{ }
public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0) { public NonIncJoinStream(ViewStreamDatabase vsd, NonIncJoinSpec[] join_specs, INonIncJoinInlineMapper output_mapper, int throttles_ms = 0)
{
_join_specs = join_specs; _join_specs = join_specs;
_output_mapper = output_mapper; _output_mapper = output_mapper;
_row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length); _row_hasher = new BlueRowHasher(_join_specs[0].join_col_names.Length);
@@ -46,11 +53,12 @@ namespace BCG {
_vsd = vsd; _vsd = vsd;
} }
protected override void myOnFirstSubscriber() { protected override void myOnFirstSubscriber()
{
int data_col_no = 1; int data_col_no = 1;
for (int jp = 0; jp < _join_specs.Length; jp++) { for (int jp = 0; jp < _join_specs.Length; jp++)
{
NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher); NonIncJoinBookkeeper s = new NonIncJoinBookkeeper(_join_specs[jp], jp, data_col_no, _row_hasher);
Log.Gen.info($"NonIncJoinStream.myOnFirstSubscriber jp: {jp} spec: {_join_specs[jp]}");
data_col_no += _join_specs[jp].data_col_names.Length; data_col_no += _join_specs[jp].data_col_names.Length;
s.on_new_data += _stream_callback; s.on_new_data += _stream_callback;
_book_keeping_data.Add(s); _book_keeping_data.Add(s);
@@ -58,11 +66,10 @@ namespace BCG {
if (_output_mapper != null) if (_output_mapper != null)
_output_mapper.onMapperChange += _on_mapper_change; _output_mapper.onMapperChange += _on_mapper_change;
_vsd.addWorker(this); _vsd.addWorker(this);
foreach (var bkd in _book_keeping_data)
bkd.init();
} }
protected override void myOnLastSubscriber() { protected override void myOnLastSubscriber()
{
foreach (NonIncJoinBookkeeper d in _book_keeping_data) foreach (NonIncJoinBookkeeper d in _book_keeping_data)
d.Dispose(); d.Dispose();
if (_output_mapper != null) if (_output_mapper != null)
@@ -70,26 +77,30 @@ namespace BCG {
_vsd.removeWorker(this); _vsd.removeWorker(this);
} }
private void _stream_callback() { private void _stream_callback()
Log.Gen.info("NonIncJoinStream._stream_callback received callback"); {
_is_dirty = true; _is_dirty = true;
doWork(); doWork();
} }
private void _on_mapper_change() { private void _on_mapper_change()
{
_is_dirty = true; _is_dirty = true;
doWork(); doWork();
} }
private bool _is_ready() { private bool _is_ready()
{
bool rv = true; bool rv = true;
foreach (NonIncJoinBookkeeper d in _book_keeping_data) foreach (NonIncJoinBookkeeper d in _book_keeping_data)
rv &= d.is_ready; rv &= d.is_ready;
return rv; return rv;
} }
private void _construct_schema() { private void _construct_schema()
if (_output_schema == null) { {
if (_output_schema == null)
{
List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) }); List<ColumnInfo> cis = new List<ColumnInfo>(new ColumnInfo[] { new ColumnInfo("id", 0, typeof(int)) });
foreach (NonIncJoinBookkeeper d in _book_keeping_data) foreach (NonIncJoinBookkeeper d in _book_keeping_data)
d.constribute_cols_to_schema(cis); d.constribute_cols_to_schema(cis);
@@ -98,23 +109,27 @@ namespace BCG {
} }
private int _n_total_fires = 0, _n_expensive_fires = 0; private int _n_total_fires = 0, _n_expensive_fires = 0;
private void _form_and_fire() { private void _form_and_fire()
Log.Gen.info($"NonIncJoinStream._form_and_fire _is_ready(): {_is_ready()}"); {
if (_is_ready()) { if (_is_ready())
{
_construct_schema(); _construct_schema();
var timer = Stopwatch.StartNew(); var timer = Stopwatch.StartNew();
IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list(); IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
List<object[]> output_rows = new List<object[]>(); List<object[]> output_rows = new List<object[]>();
for (int i = 0; i < left_row_list.Count; i++) { for (int i = 0; i < left_row_list.Count; i++)
{
object[] or = new object[_output_schema.columnInfo.Length]; object[] or = new object[_output_schema.columnInfo.Length];
or[0] = i; or[0] = i;
output_rows.Add(or); output_rows.Add(or);
} }
for (int ji = 0; ji < _book_keeping_data.Count; ji++) { for (int ji = 0; ji < _book_keeping_data.Count; ji++)
{
NonIncJoinBookkeeper bk_data = _book_keeping_data[ji]; NonIncJoinBookkeeper bk_data = _book_keeping_data[ji];
for (int i = 0; i < left_row_list.Count; i++) { for (int i = 0; i < left_row_list.Count; i++)
{
if (ji == 0) if (ji == 0)
bk_data.copy_master_row_to_output_row(i, output_rows[i]); bk_data.copy_master_row_to_output_row(i, output_rows[i]);
else else
@@ -124,20 +139,19 @@ namespace BCG {
IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows)); IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
timer.Stop(); timer.Stop();
_n_total_fires++; _n_total_fires++;
//if (timer.ElapsedMilliseconds >= 250) { if (timer.ElapsedMilliseconds >= 250) {
_n_expensive_fires++; _n_expensive_fires++;
string log_msg = $"Join and map time for {_book_keeping_data.Count} " string log_msg = $"Join and map time for {_book_keeping_data.Count} "
+ $"tables {timer.Elapsed.TotalMilliseconds} msecs, ratio {_n_expensive_fires}/{_n_total_fires}"; + $"tables {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}";
Log.Gen.info(log_msg); }
//} myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
var dp_out = BlueDataUtility.build_dp_inc(dp_output_rows, myLastValue as DataPacket);
myFireEvent(dp_out);
//myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
} }
} }
private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows) { private IBlueRowCollection _apply_map(IBlueRowCollection unmapped_rows)
if (_output_mapper != null) { {
if (_output_mapper != null)
{
IBlueRowCollection output_rows; IBlueRowCollection output_rows;
var timer = Stopwatch.StartNew(); var timer = Stopwatch.StartNew();
output_rows = _output_mapper.map_rows(unmapped_rows); output_rows = _output_mapper.map_rows(unmapped_rows);
@@ -146,14 +160,17 @@ namespace BCG {
Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}"); Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}");
} }
return output_rows; return output_rows;
} else { }
else
{
return unmapped_rows; return unmapped_rows;
} }
} }
public void doWork() { public void doWork()
Log.Gen.info($"NonIncJoinStream.doWork _last_fire_time: {_last_fire_time} _is_dirty: {_is_dirty}"); {
if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty) { if (DateTime.Now - _last_fire_time >= _throttle_period && _is_dirty)
{
_is_dirty = false; _is_dirty = false;
_last_fire_time = DateTime.Now; _last_fire_time = DateTime.Now;
_form_and_fire(); _form_and_fire();
@@ -1,68 +0,0 @@
namespace bts {
using BCG;
using BCG.Utilities.General;
using System;
using System.Diagnostics;
using ZstdSharp.Unsafe;
// functon that joins the 2 rows and returns the outputput row. The function
// can return a null which means the 2 rows dont need to be joined
using JoinFunc = System.Func<BCG.IBlueRow, BCG.IBlueRow, BCG.IBlueRow>;
public class ProximityJoinStream : ViewStream, IWorker {
private ViewStream _lvs, _rvs;
private JoinFunc _join_func;
private bool _dirty = false;
private Stopwatch _stopwatch = new Stopwatch();
private ViewStreamDatabase _vsd = ViewStreamDatabase.instance;
public ProximityJoinStream(ViewStream lvs, ViewStream rvs, JoinFunc join_func) {
_lvs = lvs;
_rvs = rvs;
_join_func = join_func;
}
protected override void myOnFirstSubscriber() {
_lvs.onChange += _handle_data;
_rvs.onChange += _handle_data;
_vsd.addWorker(this);
}
protected override void myOnLastSubscriber() {
_lvs.onChange -= _handle_data;
_rvs.onChange -= _handle_data;
_vsd.removeWorker(this);
}
private void _handle_data(object dp_obj) {
_dirty = true;
doWork();
}
public void doWork() {
Func<bool> all_data_revcd = () => {
return _lvs.rows != null && _rvs.rows != null;
};
_stopwatch.Start();
if (_dirty && all_data_revcd()) {
_dirty = false;
BlueTable rv = null;
foreach (IBlueRow lr in _lvs.rows) {
foreach (IBlueRow rr in _rvs.rows) {
IBlueRow resr = _join_func(lr, rr);
if (resr == null)
continue;
rv = rv ?? new BlueTable(resr.schema);
rv.addRow(resr);
}
}
_stopwatch.Stop();
Log.Gen.info($"ProximityJoinStream left_count: {_lvs.rows.Count} right_count: {_rvs.rows.Count} result_count: {rv.Count} time: {_stopwatch.ElapsedMilliseconds} msecs");
var dp = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket);
myFireEvent(dp);
}
}
}
}
@@ -1,47 +0,0 @@
namespace bts.SilverPine {
using BCG;
using System;
using System.CodeDom;
using System.Configuration;
using System.Reactive.Disposables;
public class TimeProximityJoinStream : ViewStream {
private int _id = 0;
public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
string pre_join_period, string post_join_period) :
this(ls, l_time_col, rs, r_time_col, TimeBasedFilter.parse_duration(pre_join_period),
TimeBasedFilter.parse_duration(post_join_period)) { }
public TimeProximityJoinStream(ViewStream ls, string l_time_col, ViewStream rs, string r_time_col,
TimeSpan pre_join_period, TimeSpan post_join_period) {
Func<IBlueRow, IBlueRow, IBlueRow> create_row = (lr, rr) => {
var id = 0;
lock (this) {
id = _id++;
}
IBlueRow rv = BlueDataUtility.create_composite_row(lr, rr, id);
return rv;
};
Func<IBlueRow, IBlueRow, IBlueRow> join_func = (lr, rr) => {
DateTime ldt = lr.get_datetime(l_time_col);
DateTime rdt = rr.get_datetime(r_time_col);
TimeSpan diff = ldt - rdt;
if (ldt < rdt) {
if (diff.Duration() > pre_join_period) {
return null;
}
} else {
if (diff > post_join_period)
return null;
}
return create_row(lr, rr);
};
ViewStream rstr = new ProximityJoinStream(ls, rs, join_func);
mySetRealSource(rstr);
}
}
}
@@ -2,13 +2,9 @@ using System;
using System.Collections; using System.Collections;
using System.Collections.Generic; using System.Collections.Generic;
using Amazon.Runtime.Internal.Util;
using BCG; using BCG;
using BCG.Utilities; using BCG.Utilities;
using bts;
namespace BCG { namespace BCG {
public class LiteralStream : ViewStream { public class LiteralStream : ViewStream {
@@ -26,34 +22,6 @@ namespace BCG {
return rv; return rv;
} }
//literal("table1", "x:123, y:1.001, z:abc;x:123, y:1.001, z:abc")
public LiteralStream(string table_name, string rows) {
Func<string, int, IBlueRow> get_row = (json_row, id) => {
Dictionary<string, object> map = new Dictionary<string, object>();
string[] toks = json_row.Split(',', StringSplitOptions.TrimEntries);
//Logger.assert(col_vals.Length == 2);
foreach (string t in toks) {
var col_vals = t.Split(':', StringSplitOptions.TrimEntries);
string col = col_vals[0];
string val = col_vals[1];
map.Add(col, val);
}
map["id"] = id;
return new BlueRow(map);
};
string[] row_arr = rows.Split(';', StringSplitOptions.TrimEntries);
BlueTable table = null;
foreach (string json_string in row_arr) {
IBlueRow r = //BlueDataUtility.convert_json_to_row(json_string, table == null ? 0 : table.Count);
get_row(json_string, table == null ? 0 : table.Count);
table = table ?? new BlueTable(r.schema);
table.addRow(r);
}
ViewStream str = new BlueTableStream(table);
str = new SchemaGuesserStream(str);
mySetRealSource(str);
}
public LiteralStream(string tableName, string schema, string value) { public LiteralStream(string tableName, string schema, string value) {
string[] columns = schema.Split(' '); string[] columns = schema.Split(' ');
@@ -87,7 +55,7 @@ namespace BCG {
mySetRealSource(new BlueRowCollectionStaticStream(t)); mySetRealSource(new BlueRowCollectionStaticStream(t));
} }
// public LiteralStream(string schema, string value) : this("literal", schema, value) { } public LiteralStream(string schema, string value) : this("literal", schema, value) { }
// public LiteralStream(string value) { // public LiteralStream(string value) {
@@ -1,72 +0,0 @@
using BCG;
using MongoDB.Driver.Core.WireProtocol.Messages;
using System;
using System.Collections.Generic;
using System.Linq;
// the first arg is all the rows, the second arg is the index of the row that needs mapping, and
// the func returns the mapped row (ie mapped version of the index row)
using GlobalMap = System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow>;
namespace bts {
// In GlobalMap, each row is mapped with the full view of all rows, ie the map functions for a row
// can use the state fo all rows.
public class GlobalMapStream : ViewStream {
private ViewStream _base_stream;
private readonly string _sort_col_name, _group_by_col;
private GlobalMap _map_func;
public GlobalMapStream(ViewStream base_stream, string sort_col_name, string group_by_col, GlobalMap map_func) {
_base_stream = base_stream;
_sort_col_name = sort_col_name;
_group_by_col = group_by_col;
_map_func = map_func;
}
public GlobalMapStream(ViewStream base_stream, string sort_col_name, GlobalMap map_func) :
this(base_stream, sort_col_name, null, map_func) {
}
protected override void myOnFirstSubscriber() {
_base_stream.onChange += _handle_stream_update;
}
protected override void myOnLastSubscriber() {
_base_stream.onChange -= _handle_stream_update;
}
void _handle_stream_update(object dp_obj) {
DataPacket dp = dp_obj as DataPacket;
IDictionary<int, List<IBlueRow>> groups = !String.IsNullOrEmpty(_group_by_col) ?
BlueDataUtility.group_together(dp.newRows, new string[] { _group_by_col })
:
new Dictionary<int, List<IBlueRow>>() { { 0, dp.newRows.to_list() } };
BlueTable out_table = null;
foreach (var kv in groups) {
List<IBlueRow> group_rows = kv.Value;
group_rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, _sort_col_name));
uint i = 0;
foreach (IBlueRow r in group_rows) {
IBlueRow nr = _map_func(group_rows, i++);
out_table = out_table ?? new BlueTable(nr.schema);
out_table.addRow(nr);
}
}
// List<IBlueRow> rows = dp.newRows.to_ienum().ToList();
// rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r1, r2, _sort_col_name));
// uint i = 0;
// BlueTable out_table = null;
// foreach (IBlueRow r in rows) {
// IBlueRow nr = _map_func(rows, i++);
// out_table = out_table ?? new BlueTable(nr.schema);
// out_table.addRow(nr);
// }
DataPacket dp_out = BlueDataUtility.build_dp_inc(out_table, myLastValue as DataPacket);
myFireEvent(dp_out);
}
}
}
@@ -8,33 +8,26 @@ using System;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
namespace bts namespace bts {
{ public class PubSubTestingStream : ViewStream, IWorker {
public class PubSubTestingStream : ViewStream, IWorker public class PayloadMessage {
{
public class PayloadMessage
{
public int msg_id = 0; public int msg_id = 0;
public string guid; public string guid;
public Int64 travel_epoch_ms = 0; public Int64 travel_epoch_ms = 0;
public byte[] payload = null; public byte[] payload = null;
public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte) public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte) {
{
msg_id = msg_id_args; msg_id = msg_id_args;
travel_epoch_ms = send_epoch_args; travel_epoch_ms = send_epoch_args;
payload = new byte[payload_size]; payload = new byte[payload_size];
for (int i = 0; i < payload_size; i++) for (int i = 0; i < payload_size; i++) {
{
payload[i] = payload_byte; payload[i] = payload_byte;
} }
} }
private PayloadMessage() { } private PayloadMessage() { }
public PayloadMessage(int msg_id, int payload_size, byte payload_byte) : public PayloadMessage(int msg_id, int payload_size, byte payload_byte) :
this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte) this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte) { }
{ }
public byte[] serialize() public byte[] serialize() {
{
var str = new MemoryStream(); var str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str); BinaryWriter bw = new BinaryWriter(str);
bw.Write(msg_id); bw.Write(msg_id);
@@ -47,8 +40,7 @@ namespace bts
return bytes; return bytes;
} }
public static PayloadMessage deserialize(byte[] ser_bytes) public static PayloadMessage deserialize(byte[] ser_bytes) {
{
var str = new MemoryStream(ser_bytes); var str = new MemoryStream(ser_bytes);
BinaryReader br = new BinaryReader(str); BinaryReader br = new BinaryReader(str);
int msg_id = br.ReadInt32(); int msg_id = br.ReadInt32();
@@ -56,8 +48,7 @@ namespace bts
Int64 epoch = br.ReadInt64(); Int64 epoch = br.ReadInt64();
int len = br.ReadInt32(); int len = br.ReadInt32();
var payload = br.ReadBytes(len); var payload = br.ReadBytes(len);
return new PayloadMessage() return new PayloadMessage() {
{
msg_id = msg_id, msg_id = msg_id,
guid = guid, guid = guid,
travel_epoch_ms = epoch, travel_epoch_ms = epoch,
@@ -82,8 +73,7 @@ namespace bts
private readonly int _payload_sz; private readonly int _payload_sz;
private const byte _payload_byte = (byte)'A'; private const byte _payload_byte = (byte)'A';
private readonly string _tag_name; private readonly string _tag_name;
public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd) public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd) {
{
_vsd = vsd; _vsd = vsd;
_payload_sz = payload_size; _payload_sz = payload_size;
var schema = BlueDataUtility.form_schema("PubSubTest", var schema = BlueDataUtility.form_schema("PubSubTest",
@@ -106,53 +96,43 @@ namespace bts
public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size, string tag_name) public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size, string tag_name)
: this(publish_period_seconds, max_pubs, msg_size, tag_name, ViewStreamDatabase.instance) { } : this(publish_period_seconds, max_pubs, msg_size, tag_name, ViewStreamDatabase.instance) { }
protected override void myOnFirstSubscriber() protected override void myOnFirstSubscriber() {
{
_vsd.addWorker(this); _vsd.addWorker(this);
System.Threading.Thread th = new System.Threading.Thread(_subscribe_thread_func); System.Threading.Thread th = new System.Threading.Thread(_subscribe_thread_func);
th.IsBackground = true; th.IsBackground = true;
th.Start(); th.Start();
} }
protected override void myOnLastSubscriber() protected override void myOnLastSubscriber() {
{
_vsd.removeWorker(this); _vsd.removeWorker(this);
_continue_sub_thread = false; _continue_sub_thread = false;
} }
void _log_updates(IEnumerable<PubSubServerMessage> updates) void _log_updates(IEnumerable<PubSubServerMessage> updates) {
{
var idx = 0; var idx = 0;
foreach (var msg in updates) foreach (var msg in updates) {
{
bts.Log.Gen.info($"PubSubTesStream._subscribe_thread_func idx: {idx} msg: {msg}"); bts.Log.Gen.info($"PubSubTesStream._subscribe_thread_func idx: {idx} msg: {msg}");
idx++; idx++;
} }
} }
private void _subscribe_thread_func() private void _subscribe_thread_func() {
{
// subscription is done in another thread // subscription is done in another thread
SubscriberCore sub_core = new SubscriberCore(); SubscriberCore sub_core = new SubscriberCore();
sub_core.subscribe(_tag_name); sub_core.subscribe(_tag_name);
while (_continue_sub_thread) while (_continue_sub_thread) {
{
// TODO recv_id // TODO recv_id
var updates = sub_core.get_updates(TimeSpan.FromMilliseconds(250)); var updates = sub_core.get_updates(TimeSpan.FromMilliseconds(250));
if (updates == null || updates.Count() == 0) if (updates == null || updates.Count() == 0)
continue; continue;
_log_updates(updates); _log_updates(updates);
_n_updates += updates.Count(); _n_updates += updates.Count();
if (updates.Count() > 0) if (updates.Count() > 0) {
{
var server_msg = updates.Last(); var server_msg = updates.Last();
if (server_msg.data != null && server_msg.data.Length > 0) if (server_msg.data != null && server_msg.data.Length > 0) {
{
_last_recvd_msg = PayloadMessage.deserialize(server_msg.data); _last_recvd_msg = PayloadMessage.deserialize(server_msg.data);
_last_recvd_msg.travel_epoch_ms = epoch_now(); _last_recvd_msg.travel_epoch_ms = epoch_now();
} } else {
else
{
// why are we getting empty or null data ? (server is sending data regardless?) // why are we getting empty or null data ? (server is sending data regardless?)
} }
_fire_dirty = true; _fire_dirty = true;
@@ -160,36 +140,28 @@ namespace bts
} }
} }
private void _publish_next_packet() private void _publish_next_packet() {
{ if (_last_sent_msg == null) {
if (_last_sent_msg == null) _last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte) {
{
_last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte)
{
guid = Guid.NewGuid().ToString() guid = Guid.NewGuid().ToString()
}; };
} } else {
else
{
_last_sent_msg.msg_id++; _last_sent_msg.msg_id++;
_last_sent_msg.guid = Guid.NewGuid().ToString(); _last_sent_msg.guid = Guid.NewGuid().ToString();
_last_sent_msg.travel_epoch_ms = epoch_now(); _last_sent_msg.travel_epoch_ms = epoch_now();
} }
var ser_bytes = _last_sent_msg.serialize(); var ser_bytes = _last_sent_msg.serialize();
_n_pubs++; _n_pubs++;
_pub_core.publish("Blob", _tag_name, ser_bytes); _pub_core.publish(String.Empty, _tag_name, ser_bytes);
_fire_dirty = true;
} }
private void _fire_dp() private void _fire_dp() {
{
object last_sent_id = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.msg_id; object last_sent_id = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.msg_id;
object last_recv_id = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.msg_id; object last_recv_id = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.msg_id;
object echo_time = DBNull.Value; object echo_time = DBNull.Value;
object last_sent_guid = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.guid; object last_sent_guid = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.guid;
object last_recv_guid = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.guid; object last_recv_guid = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.guid;
if (_last_sent_msg != null && _last_recvd_msg != null) if (_last_sent_msg != null && _last_recvd_msg != null) {
{
echo_time = _last_recvd_msg.travel_epoch_ms - _last_sent_msg.travel_epoch_ms; echo_time = _last_recvd_msg.travel_epoch_ms - _last_sent_msg.travel_epoch_ms;
} }
var match = last_sent_id.Equals(last_recv_id) && last_sent_guid.Equals(last_recv_guid); var match = last_sent_id.Equals(last_recv_id) && last_sent_guid.Equals(last_recv_guid);
@@ -201,25 +173,21 @@ namespace bts
myFireEvent(dp); myFireEvent(dp);
} }
public void doWork() public void doWork() {
{
if (DateTime.Now - _last_publish_time > _publish_period) if (DateTime.Now - _last_publish_time > _publish_period)
if (_n_pubs <= _max_pubs) if (_n_pubs <= _max_pubs) {
{
_publish_next_packet(); _publish_next_packet();
_last_publish_time = DateTime.Now; _last_publish_time = DateTime.Now;
} }
// publish // publish
// TODO: send id // TODO: send id
if (_fire_dirty) if (_fire_dirty) {
{
_fire_dirty = false; _fire_dirty = false;
_fire_dp(); _fire_dp();
} }
} }
public static long epoch_now() public static long epoch_now() {
{
return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
} }
@@ -1,223 +0,0 @@
using BCG;
using BCG.Utilities.General;
using System.Collections.Generic;
using System;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using bts.api;
namespace bts
{
[DataStreamUserRespresentation("pss_echo")]
public class PubsubEchoTestStream : ViewStream, IWorker
{
public class PayloadMessage
{
public readonly string msg_id;
public readonly Int64 timestamp = 0;
public readonly byte[] payload = null;
public static byte[] create_data(int payload_size, byte payload_byte)
{
var rv = new byte[payload_size];
for (int i = 0; i < payload_size; i++)
{
rv[i] = payload_byte;
}
return rv;
}
// public PayloadMessage(string guid_, Int64 timestamp_, int payload_size, byte payload_byte)
// {
// guid = guid_;
// timestamp = timestamp_;
// payload = new byte[payload_size];
// for (int i = 0; i < payload_size; i++)
// {
// payload[i] = payload_byte;
// }
// }
public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_)
{
msg_id = guid_;
timestamp = timestamp_;
payload = payload_;
}
private PayloadMessage() { }
public byte[] serialize()
{
var str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
bw.Write(msg_id);
bw.Write(timestamp);
bw.Write(payload.Length);
bw.Write(payload);
str.Position = 0;
var bytes = str.ToArray();
return bytes;
}
public static PayloadMessage deserialize(byte[] ser_bytes)
{
var str = new MemoryStream(ser_bytes);
BinaryReader br = new BinaryReader(str);
string guid = br.ReadString();
Int64 timestamp_ = br.ReadInt64();
int len = br.ReadInt32();
var payload = br.ReadBytes(len);
return new PayloadMessage(guid, timestamp_, payload);
}
}
//public void
private readonly int _msg_size;
private readonly TimeSpan _publish_period;
private DateTime _last_publish_time = DateTime.MinValue;
private PublisherCore _pub_core;
private ISubscriberCore _sub_core;
private BlueTable _table;
private int _n_pubs = 0;
private readonly int _max_pubs;
private int _n_updates = 0;
private ViewStreamDatabase _vsd;
private PayloadMessage _last_sent_msg, _last_recvd_msg;
private const byte _payload_byte = (byte)'A';
private readonly string _tag_name;
private readonly int _payload_sz;
private readonly IPubSubRouter _router;
private enum IOState
{
SEND = 0,
RECV = 1
};
IOState _state = IOState.SEND;
public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd)
{
var route = new PubSubRoute(server_url, ".*");
_router = new PubSubRouter(route);
_vsd = vsd;
_payload_sz = payload_size;
var schema = BlueDataUtility.form_schema("PubSubTest",
Tuple.Create("id", TypeCode.Int32),
Tuple.Create("sent_id", TypeCode.String),
Tuple.Create("sent_time", TypeCode.Int64),
Tuple.Create("recv_id", TypeCode.String),
Tuple.Create("recv_time", TypeCode.Int64),
Tuple.Create("match", TypeCode.Boolean),
Tuple.Create("echo_time", TypeCode.Int32));
_table = new BlueTable(schema);
_publish_period = publish_period;
_tag_name = tag_name;
}
public PubsubEchoTestStream(string server_url, int publish_period_msecs, int payload_size, string tag_name)
: this(server_url, TimeSpan.FromMilliseconds(publish_period_msecs), payload_size, tag_name, ViewStreamDatabase.instance) { }
protected override void myOnFirstSubscriber()
{
_pub_core = new PublisherCore(_router);
_sub_core = new SubscriberCore(_router);
_sub_core.subscribe(_tag_name);
_vsd.addWorker(this);
}
protected override void myOnLastSubscriber()
{
_vsd.removeWorker(this);
_sub_core.unsubscribe(_tag_name);
}
private void _fire_dp_after_send()
{
// as soon as the data is published, its reflected in the stream state
Log.assert(_last_sent_msg != null);
BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value,
DBNull.Value, DBNull.Value);
_table.addRow(r);
var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
myFireEvent(dp_out);
}
private void _fire_dp_after_recv()
{
Log.assert(_last_recvd_msg != null);
// update the last publish row
int id = _table.Count - 1;
IBlueRow last_row = _table[id];
_table.deleteRow(last_row);
var recv_ts = epoch_now();
BlueRow nr = new BlueRow(_table.schema, id, _last_sent_msg.msg_id, _last_sent_msg.timestamp, _last_recvd_msg.msg_id, recv_ts,
String.Compare(_last_recvd_msg.msg_id, _last_sent_msg.msg_id) == 0, recv_ts - _last_recvd_msg.timestamp);
_table.addRow(nr);
var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
myFireEvent(dp_out);
}
private string _get_next_id()
{
return _table.Count.ToString();
}
private void _publish_next_packet()
{
var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte);
_last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes);
var ser_bytes = _last_sent_msg.serialize();
_n_pubs++;
_pub_core.publish("Blob", _tag_name, ser_bytes);
_last_publish_time = DateTime.Now;
}
public void doWork()
{
Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) =>
{
foreach (var u in updates)
{
var recv_msg = PayloadMessage.deserialize(u.data);
if (recv_msg.msg_id == _last_sent_msg.msg_id)
{
return recv_msg;
}
}
return null;
};
switch (_state)
{
case IOState.SEND:
if (DateTime.Now - _last_publish_time >= _publish_period)
{
_publish_next_packet();
_last_publish_time = DateTime.Now;
_fire_dp_after_send();
_state = IOState.RECV;
return;
}
break;
case IOState.RECV:
var updates = _sub_core.get_updates(TimeSpan.Zero);
updates = updates.Where(u => u.tag == _tag_name);
var recv_msg = find_packet(updates);
if (recv_msg != null)
{
_last_recvd_msg = recv_msg;
_fire_dp_after_recv();
_state = IOState.SEND;
break;
}
break;
}
}
public static long epoch_now()
{
return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
}
}
}
@@ -14,14 +14,11 @@ namespace bts {
private BlueTable _table; private BlueTable _table;
private SyncQueue<string> _action_queue = new SyncQueue<string>(); private SyncQueue<string> _action_queue = new SyncQueue<string>();
public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms) public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms) {
: this(ip, port, msg_sz, send_wait_period_ms, Int32.MaxValue) { }
public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms, int max_n_msgs) {
_ip = ip; _ip = ip;
_port = port; _port = port;
_table = new BlueTable(_schema); _table = new BlueTable(_schema);
_tcp_client = new SimpleTCPClient(_ip, _port, Convert.ToUInt32(msg_sz), send_wait_period_ms, max_n_msgs); _tcp_client = new SimpleTCPClient(_ip, _port, Convert.ToUInt32(msg_sz), send_wait_period_ms);
} }
protected override void myOnFirstSubscriber() { protected override void myOnFirstSubscriber() {
@@ -43,11 +43,11 @@ namespace BCG {
Tuple.Create("exception", TypeCode.String) Tuple.Create("exception", TypeCode.String)
); );
// register tests here // register tests here
_tests.Add(new Func<TestResult>(_run_sub_msg_test)); // _tests.Add(new Func<TestResult>(_run_sub_msg_test));
_tests.Add(new Func<TestResult>(_run_pub_msg_test)); // _tests.Add(new Func<TestResult>(_run_pub_msg_test));
_tests.Add(new Func<TestResult>(_run_pubsub_server_msg_test)); // _tests.Add(new Func<TestResult>(_run_pubsub_server_msg_test));
_tests.Add(new Func<TestResult>(_run_pubsub_client_msg_test)); // _tests.Add(new Func<TestResult>(_run_pubsub_client_msg_test));
_tests.Add(new Func<TestResult>(_run_columninfo_serialization_test)); // _tests.Add(new Func<TestResult>(_run_columninfo_serialization_test));
// _tests.Add(new Func<TestResult>(_run_schema_serialization_test)); // _tests.Add(new Func<TestResult>(_run_schema_serialization_test));
// _tests.Add(new Func<TestResult>(_row_serialization_tests_no_dbnulls)); // _tests.Add(new Func<TestResult>(_row_serialization_tests_no_dbnulls));
// _tests.Add(new Func<TestResult>(_row_serialization_tests_with_dbnulls)); // _tests.Add(new Func<TestResult>(_row_serialization_tests_with_dbnulls));
@@ -1,31 +0,0 @@
namespace BCG {
using System;
public class TickingTestStream : ViewStream {
public TickingTestStream(int ticking_period_msecs)
:
this(ViewStreamDatabase.instance, TimeSpan.FromSeconds(ticking_period_msecs)) { }
public TickingTestStream(ViewStreamDatabase vsd, TimeSpan ticking_period_msecs) {
ViewStream vs = new DisplayableMillisecondStream(Convert.ToInt32(ticking_period_msecs.TotalMilliseconds), 10);
vs = new MathStream(vs, "x=id*123 y=x*x");
mySetRealSource(vs);
//vs.onChange += _on_str_update;
}
// private void _on_str_update(object dp_obj) {
// DataPacket dp = dp_obj as DataPacket;
// IBlueTableSchema out_schema = BlueDataUtility.add_column(dp.schema, "time_txt", typeof(string));
// BlueTable state = new BlueTable(out_schema);
// foreach (IBlueRow r in dp.newRows) {
// MutableBlueRow nr = new MutableBlueRow(out_schema, r);
// DateTime time = r.get_datetime("time");
// var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff");
// nr.setValue("time_txt", time_txt);
// state.addRow(nr);
// }
// var dp_out = BlueDataUtility.build_dp_inc(state, myLastValue == null ? null : myLastValue as DataPacket);
// myFireEvent(dp_out);
// }
}
}
@@ -1,8 +1,9 @@
using Google.Protobuf.WellKnownTypes;
using System; using System;
using BCG.Utilities.General;
using BCG; namespace BCG {
namespace bts.SilverPine {
// Keeps adding changing rows to the stream, hence preserving the history // Keeps adding changing rows to the stream, hence preserving the history
// and creating a time series, for example this can be used to make bbo updates // and creating a time series, for example this can be used to make bbo updates
// into a time series of prices/qty // into a time series of prices/qty
@@ -40,6 +41,7 @@ namespace bts.SilverPine {
} }
_table.addRows(added_rows); _table.addRows(added_rows);
myFireEvent(new DataPacket(_table, added_rows, new BlueTable(_table.schema))); myFireEvent(new DataPacket(_table, added_rows, new BlueTable(_table.schema)));
} }
} }
@@ -7,45 +7,26 @@ namespace bts {
using System.Text.Json; using System.Text.Json;
using System.Text.Json.Nodes; using System.Text.Json.Nodes;
[DataStreamUserRespresentation("benzingaNews")] [DataStreamUserRespresentation("benzingaNews")]
public class BenzingaNewsStream : ViewStream { public class BenzingaNewStream : ViewStream {
private IBlueTableSchema _schema = null; private IBlueTableSchema _schema = null;
private int _id = 0; private int _id = 0;
// constructor takes in the raw mongo db stream, and the time to live (ttl), "7d" for example
public BenzingaNewStream(ViewStream mongodb_raw_stream, string ttl) {
private static ViewStream _pre_process_raw_kvstream(ViewStream mongo_db_str) {
ViewStream r = new KeyValueToRelationalStream(mongo_db_str, "doc_id", "key", "value");
r = new DateStandardizationStream(r, "created_at", "ISO8601");
return r;
}
public BenzingaNewsStream(string connection_string, string db_name, string collection_name, string poll_period, string ttl)
: this(new MongoDBStream(connection_string, db_name, collection_name, poll_period), ttl) {
}
// public BenzingaNewsStream()
// : this(new MongoDBStream(DataStreamPub connection_string, db_name, collection_name, poll_period), ttl) {
// }
// assumes it gets a raw mongodb_raw_kv_stream with key value pairs
public BenzingaNewsStream(ViewStream mongodb_raw_kv_stream, string ttl) {
ViewStream output_stream = _pre_process_raw_kvstream(mongodb_raw_kv_stream);
// Hiding body (with project, columns can be hidden, or renamed) // Hiding body (with project, columns can be hidden, or renamed)
output_stream = new ProjectStream(output_stream, "!body"); ViewStream output_stream = new ProjectStream(mongodb_raw_stream, "!body");
// add column news_source = 'Benzinga', so we can append other news sources // add column news_source = 'Benzinga', so we can append other news sources
output_stream = new AddColumnsStream(output_stream, @"news_source : Benzinga"); output_stream = new AddColumnsStream(output_stream, @"news_source : Benzinga");
// now SuperMapStream is used to map one row into multiple, one per symbol, // now SuperMapStream is used to map one row into multiple, one per symbol,
// (SuperMapStream takes 2 lambdas, one to get the enw schema and another to // (SuperMapStream takes 2 lambdas, one to get the enw schema and another to
// return an array of rows, per one row) // return an array of rows, per one row)
output_stream = new SuperMapStream(output_stream, output_stream = new SuperMapStream(mongodb_raw_stream,
(IBlueTableSchema s) => { (IBlueTableSchema s) => {
// this lambda returns the new schema (ie with extra cols symbol, exchange, is_primary_exchange) // this lambda returns the new schema (ie with extra cols symbol, exchange, is_primary_exchange)
if (_schema == null) { if (_schema == null) {
var out_schema = s; var out_schema = s;
out_schema = BlueDataUtility.remove_columns(out_schema, "securities"); out_schema = BlueDataUtility.remove_columns(out_schema, "securities");
out_schema = BlueDataUtility.remove_columns(out_schema, "body"); out_schema = BlueDataUtility.add_column(s, "doc_id_internal", typeof(string));
out_schema = BlueDataUtility.remove_columns(out_schema, "tags");
out_schema = BlueDataUtility.add_column(out_schema, "doc_id_internal", typeof(string));
out_schema = BlueDataUtility.add_column(out_schema, "symbol", typeof(string)); out_schema = BlueDataUtility.add_column(out_schema, "symbol", typeof(string));
out_schema = BlueDataUtility.add_column(out_schema, "exchange", typeof(string)); out_schema = BlueDataUtility.add_column(out_schema, "exchange", typeof(string));
out_schema = BlueDataUtility.add_column(out_schema, "is_primary_exchange", typeof(bool)); out_schema = BlueDataUtility.add_column(out_schema, "is_primary_exchange", typeof(bool));
@@ -63,13 +44,11 @@ namespace bts {
} }
foreach (JsonNode jnode in json_objs) { foreach (JsonNode jnode in json_objs) {
if (jnode is JsonObject json_object) { if (jnode is JsonObject json_object) {
string symbol = jnode["symbol"] == null ? String.Empty : jnode["symbol"].ToString();
string exchange = jnode["exchange"] == null ? String.Empty : jnode["exchange"].ToString();
MutableBlueRow nr = new MutableBlueRow(_schema, r); MutableBlueRow nr = new MutableBlueRow(_schema, r);
nr.setValue("id", _id++); nr.setValue("id", _id++);
nr.setValue("doc_id_internal", $"{r["doc_id"]}_{jnode["symbol"]}"); nr.setValue("doc_id_internal", $"{r["doc_id"]}_{jnode["symbol"]}");
nr.setValue("symbol", symbol); nr.setValue("symbol", jnode["symbol"]);
nr.setValue("exchange", exchange); nr.setValue("exchange", jnode["exchange"]);
nr.setValue("is_primary_exchange", Boolean.Parse(jnode["primary"].ToString())); nr.setValue("is_primary_exchange", Boolean.Parse(jnode["primary"].ToString()));
expanded_rows.Add(nr); expanded_rows.Add(nr);
} }
@@ -7,127 +7,27 @@ using MongoDB.Driver;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using System.Diagnostics;
using System.Threading.Tasks;
using System.Threading;
namespace bts { namespace bts {
// [DataStreamUserRespresentation("mongodb_sql")] [DataStreamUserRespresentation("mongodb_sql")]
// public class MongoDBSQLStream : ViewStream, IWorker { public class MongoDBSQLStream : ViewStream, IWorker {
// private ViewStreamDatabase _vsd;
// private readonly string _connection_string, _db_name, _collection_name;
// private MongoClient _client;
// // make this a sorted table
// private BlueTable _table;
// // Access the database
// private IMongoDatabase _database;
// private IDictionary<string, int> _id_map = new Dictionary<string, int>();
// // mongodb://192.168.1.80:27017
// public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
// : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
// }
// public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
// bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
// $"db_name: {db_name}");
// _connection_string = connection_string;
// _db_name = db_name;
// _collection_name = collection_name;
// _vsd = vsd;
// _client = new MongoClient(_connection_string);
// _database = _client.GetDatabase(_db_name);
// }
// protected override void myOnFirstSubscriber() {
// MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
// }
// protected override void myOnLastSubscriber() {
// MasterWorker.getInstance(false).removeWorker(this);
// }
// public void doWork() {
// _read_mogodb();
// }
// private void _read_mogodb() {
// var timer = Stopwatch.StartNew();
// // Access the collection
// IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
// // Fetch all documents from the collection
// var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
// _table = _table == null ? null : new BlueTable(_table.schema);
// // Iterate through the documents and print values
// foreach (var document in documents.ToEnumerable()) {
// bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
// if (_table == null) {
// _init_table(document);
// }
// _add_row(document);
// }
// timer.Stop();
// Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms");
// timer.Restart();
// var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
// timer.Stop();
// Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms");
// if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
// return;
// myFireEvent(dp);
// }
// private void _init_table(BsonDocument doc_row) {
// bts.Log.assert(_table == null);
// List<ColumnInfo> cis = new List<ColumnInfo>();
// cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
// foreach (var n in doc_row.Names) {
// cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
// }
// IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
// _table = new BlueTable(schema);
// }
// private void _add_row(BsonDocument doc) {
// var key_val = doc["_id"];
// var key = key_val.ToString();
// int id;
// if (!_id_map.TryGetValue(key, out id)) {
// id = _id_map.Count;
// _id_map.Add(key, id);
// }
// MutableBlueRow nr = new MutableBlueRow(_table.schema);
// nr.setValue("id", id);
// foreach (var n in doc.Names) {
// var val = doc[n].ToString();
// nr.setValue(n, val);
// }
// _table.addRow(nr);
// }
// }
[DataStreamUserRespresentation("mongodb")]
public class MongoDBStream : ViewStream, IWorker {
private ViewStreamDatabase _vsd; private ViewStreamDatabase _vsd;
private readonly string _connection_string, _db_name, _collection_name; private readonly string _connection_string, _db_name, _collection_name;
private readonly TimeSpan _poll_period;
private MongoClient _client; private MongoClient _client;
// make this a sorted table
private BlueTable _table; private BlueTable _table;
private SyncQueue<DataPacket> _table_queue = new SyncQueue<DataPacket>();
private BlockingQueue _work_queue = new BlockingQueue();
// Access the database // Access the database
private IMongoDatabase _database; private IMongoDatabase _database;
private IDictionary<string, int> _id_map = new Dictionary<string, int>(); private IDictionary<string, int> _id_map = new Dictionary<string, int>();
// mongodb://192.168.1.80:27017 // mongodb://192.168.1.80:27017
public MongoDBStream(string connection_string, string db_name, string collection_name, string poll_period) public MongoDBSQLStream(string connection_string, string db_name, string collection_name)
: this(connection_string, db_name, collection_name, TimeBasedFilter.parse_duration(poll_period), ViewStreamDatabase.instance) { : this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
} }
public MongoDBStream(string connection_string, string db_name, string collection_name, TimeSpan poll_period, ViewStreamDatabase vsd) { public MongoDBSQLStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " + bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
$"db_name: {db_name}"); $"db_name: {db_name}");
_connection_string = connection_string; _connection_string = connection_string;
@@ -135,30 +35,114 @@ namespace bts {
_collection_name = collection_name; _collection_name = collection_name;
_vsd = vsd; _vsd = vsd;
_client = new MongoClient(_connection_string); _client = new MongoClient(_connection_string);
_poll_period = poll_period;
_database = _client.GetDatabase(_db_name); _database = _client.GetDatabase(_db_name);
} }
protected override void myOnFirstSubscriber() { protected override void myOnFirstSubscriber() {
_vsd.addWorker(this); MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
//MasterWorker.getInstance(false).addWorker(this, _poll_period);
} }
protected override void myOnLastSubscriber() { protected override void myOnLastSubscriber() {
_vsd.removeWorker(this); MasterWorker.getInstance(false).removeWorker(this);
//MasterWorker.getInstance(false).removeWorker(this);
} }
private DateTime _last_poll_time = DateTime.MinValue;
public void doWork() { public void doWork() {
var now = DateTime.Now; _read_mogodb();
if (now - _last_poll_time > _poll_period) { }
ThreadPool.QueueUserWorkItem(_ => _read_mogodb());
_last_poll_time = DateTime.Now; private void _read_mogodb() {
var timer = Stopwatch.StartNew();
// Access the collection
IMongoCollection<BsonDocument> collection = _database.GetCollection<BsonDocument>(name: _collection_name);
// Fetch all documents from the collection
var documents = collection.Find(Builders<BsonDocument>.Filter.Empty).ToCursor();
_table = _table == null ? null : new BlueTable(_table.schema);
// Iterate through the documents and print values
foreach (var document in documents.ToEnumerable()) {
bts.Log.Gen.info($"document: {document} json: {document.ToJson()}");
if (_table == null) {
_init_table(document);
}
_add_row(document);
} }
var dp = _table_queue.dequeue(0); timer.Stop();
if (dp != null) Log.Gen.info($"MongoDBStream time to read {_collection_name} {timer.ElapsedMilliseconds}ms");
myFireEvent(dp); timer.Restart();
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
timer.Stop();
Log.Gen.info($"MongoDBStream time to compare tables :{timer.ElapsedMilliseconds}ms");
if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
return;
myFireEvent(dp);
}
private void _init_table(BsonDocument doc_row) {
bts.Log.assert(_table == null);
List<ColumnInfo> cis = new List<ColumnInfo>();
cis.Add((new ColumnInfo("id", 0, typeof(Int32))));
foreach (var n in doc_row.Names) {
cis.Add(new ColumnInfo(n, cis.Count, typeof(string)));
}
IBlueTableSchema schema = new BlueTableSchema(cis.ToArray(), _collection_name);
_table = new BlueTable(schema);
}
private void _add_row(BsonDocument doc) {
var key_val = doc["_id"];
var key = key_val.ToString();
int id;
if (!_id_map.TryGetValue(key, out id)) {
id = _id_map.Count;
_id_map.Add(key, id);
}
MutableBlueRow nr = new MutableBlueRow(_table.schema);
nr.setValue("id", id);
foreach (var n in doc.Names) {
var val = doc[n].ToString();
nr.setValue(n, val);
}
_table.addRow(nr);
}
}
[DataStreamUserRespresentation("mongodb")]
public class MongoDBStream : ViewStream, IWorker {
private ViewStreamDatabase _vsd;
private readonly string _connection_string, _db_name, _collection_name;
private MongoClient _client;
private BlueTable _table;
// Access the database
private IMongoDatabase _database;
private IDictionary<string, int> _id_map = new Dictionary<string, int>();
// mongodb://192.168.1.80:27017
public MongoDBStream(string connection_string, string db_name, string collection_name)
: this(connection_string, db_name, collection_name, ViewStreamDatabase.instance) {
}
public MongoDBStream(string connection_string, string db_name, string collection_name, ViewStreamDatabase vsd) {
bts.Log.Gen.info($"MongoDBStream constructed connection_string: {connection_string} " +
$"db_name: {db_name}");
_connection_string = connection_string;
_db_name = db_name;
_collection_name = collection_name;
_vsd = vsd;
_client = new MongoClient(_connection_string);
_database = _client.GetDatabase(_db_name);
}
protected override void myOnFirstSubscriber() {
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(10));
}
protected override void myOnLastSubscriber() {
MasterWorker.getInstance(false).removeWorker(this);
}
public void doWork() {
_read_mogodb();
} }
private void _read_mogodb() { private void _read_mogodb() {
@@ -182,11 +166,10 @@ namespace bts {
timer.Restart(); timer.Restart();
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true); var dp = BlueDataUtility.build_dp_inc(_table, myLastValue == null ? null : myLastValue as DataPacket, true);
timer.Stop(); timer.Stop();
Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms"); ; Log.Gen.info($"MongoDBStream time to compare tables: {timer.ElapsedMilliseconds}ms");;
if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0) if (dp.addedRows.Count == 0 && dp.deletedRows.Count == 0)
return; return;
_table_queue.enqueue(dp); myFireEvent(dp);
//myFireEvent(dp);
} }
private void _init_table(BsonDocument doc_row) { private void _init_table(BsonDocument doc_row) {
@@ -225,46 +208,46 @@ namespace bts {
} }
} }
// [DataStreamUserRespresentation("mongodb_flat")] [DataStreamUserRespresentation("mongodb_flat")]
// public class MongoDBFlatStream : ViewStream { public class MongoDBFlatStream : ViewStream {
// public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration) public MongoDBFlatStream(string connection_string, string db_name, string collection_name, string timestamp_col, string ttl_duration)
// : :
// this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { } this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, timestamp_col, TimeBasedFilter.parse_duration(ttl_duration)) { }
// public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) { public MongoDBFlatStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, string timestamp_col, TimeSpan ttl_duration) {
// ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd); ViewStream rs = new MongoDBStream(connection_string, db_name, collection_name, vsd);
// rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val"); rs = new KeyValueToRelationalStream(rs, "doc_id", "key", "val");
// rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601); rs = new DateStandardizationStream(rs, timestamp_col, DateConversion.ISO8601);
// //rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration); //rs = new TimeBasedFilterStream(vsd, rs, timestamp_col, ttl_duration);
// mySetRealSource(rs); mySetRealSource(rs);
// } }
// } }
// [DataStreamUserRespresentation("newsreplay")] [DataStreamUserRespresentation("newsreplay")]
// public class NewsReplayStream : ViewStream { public class NewsReplayStream : ViewStream {
// public NewsReplayStream(string connection_string, string db_name, string collection_name, public NewsReplayStream(string connection_string, string db_name, string collection_name,
// int buffer_sz, string fire_period) : int buffer_sz, string fire_period) :
// this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { } this(ViewStreamDatabase.instance, connection_string, db_name, collection_name, buffer_sz, TimeBasedFilter.parse_duration(fire_period)) { }
// public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name, public NewsReplayStream(ViewStreamDatabase vsd, string connection_string, string db_name, string collection_name,
// int buffer_sz, TimeSpan fire_period) { int buffer_sz, TimeSpan fire_period) {
// ViewStream r; ViewStream r;
// r = new MongoDBStream(connection_string, db_name, collection_name, vsd); r = new MongoDBStream(connection_string, db_name, collection_name, vsd);
// r = new KeyValueToRelationalStream(r, r = new KeyValueToRelationalStream(r,
// "doc_id", "doc_id",
// "key", "key",
// "value"); "value");
// r = new DateStandardizationStream(r, r = new DateStandardizationStream(r,
// "created_at", "created_at",
// "ISO8601"); "ISO8601");
// r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period); r = new SnapshotReplayStream(vsd, r, "created_at", buffer_sz, fire_period);
// mySetRealSource(r); mySetRealSource(r);
// } }
// } }
} }
@@ -6,7 +6,6 @@ namespace bts {
using System; using System;
using System.Collections; using System.Collections;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq;
public class ProjectData { public class ProjectData {
// schema from the base stream // schema from the base stream
@@ -137,24 +136,10 @@ namespace bts {
HashSet<string> removed_cols = new HashSet<string>(); HashSet<string> removed_cols = new HashSet<string>();
Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>(); Dictionary<string, string> mapped_to_orig_map = new Dictionary<string, string>();
Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>(); Dictionary<string, string> orig_to_mapped = new Dictionary<string, string>();
// by default every col will show up, so if we have the projection as (a:a1)
// it will convert a to a1,
// examples: if a schema has cols a b c
// project(parent_str, "a:aa ") -> aa b c
// project(parent_str, "a:aa !c") -> aa b
// project(parent_str, "a:aa !") -> aa
// project(parent_str, "a:aa b = (aa*aa)!") -> aa b
bool hide_all = false;
foreach (string p_tok in project_toks) { foreach (string p_tok in project_toks) {
if (p_tok.StartsWith("!")) { if (p_tok.StartsWith("!")) {
if (p_tok.Length == 1) { string removed_col = p_tok.Substring(1);
// if it is a "blanket "!"=> hide all cols except the one that removed_cols.Add(removed_col);
// are stated explicitly
hide_all = true;
} else {
string removed_col = p_tok.Substring(1);
removed_cols.Add(removed_col);
}
} else if (p_tok.Contains(":")) { } else if (p_tok.Contains(":")) {
string[] map_toks = p_tok.Split(":"); string[] map_toks = p_tok.Split(":");
string orig_col = map_toks[0]; string orig_col = map_toks[0];
@@ -162,22 +147,12 @@ namespace bts {
mapped_to_orig_map.Add(mapped_col, orig_col); mapped_to_orig_map.Add(mapped_col, orig_col);
orig_to_mapped.Add(orig_col, mapped_col); orig_to_mapped.Add(orig_col, mapped_col);
} else { } else {
if (input_schema.columnNames.Contains(p_tok)) { Log.Gen.error($"NewProjectStream unexpecetd token: {p_tok}");
string orig_col = p_tok;
string mapped_col = orig_col;
orig_to_mapped.Add(orig_col, mapped_col);
}
} }
} }
// make the new schema // make the new schema
List<ColumnInfo> new_cols = new List<ColumnInfo>(); List<ColumnInfo> new_cols = new List<ColumnInfo>();
foreach (ColumnInfo ci in input_schema.columnInfo) { foreach (ColumnInfo ci in input_schema.columnInfo) {
if (ci.colName == "id") {
ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
new_cols.Add(nc);
continue;
}
if (removed_cols.Contains(ci.colName)) { if (removed_cols.Contains(ci.colName)) {
continue; continue;
} else if (orig_to_mapped.ContainsKey(ci.colName)) { } else if (orig_to_mapped.ContainsKey(ci.colName)) {
@@ -185,13 +160,8 @@ namespace bts {
ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType); ColumnInfo nc = new ColumnInfo(mapped_name, new_cols.Count, ci.colType);
new_cols.Add(nc); new_cols.Add(nc);
} else { } else {
if (hide_all)
continue;
//if (ci.colName == "id" || removed_cols.Count > 0) {
// unless columns are explicitely hidden, all columns show up
ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType); ColumnInfo nc = new ColumnInfo(ci.colName, new_cols.Count, ci.colType);
new_cols.Add(nc); new_cols.Add(nc);
//}
} }
} }
IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray()); IBlueTableSchema new_schema = new BlueTableSchema(input_schema.tableName, new_cols.ToArray());
@@ -1,122 +0,0 @@
using System;
using System.Data;
using System.Data.SQLite;
using System.Diagnostics;
using System.Collections;
using BCG;
using System.Linq;
using System.Data.Entity;
using System.Collections.Generic;
using System.Data.SQLite;
namespace bts.SilverPine {
public static class SQLiteHelper {
public static string location = "C:\\Users\\m_tah\\development\\data\\";
public static string db_name = "stream_data.db";
public static DataTable execute_select_query(string connection_string, string query) {
// Create a new DataTable to store the results
DataTable dataTable = new DataTable();
// Create and open a SQLite connection using the provided connection string
using (SQLiteConnection connection = new SQLiteConnection(connection_string)) {
try {
connection.Open();
// Create a SQLite command with the provided query
using (SQLiteCommand command = new SQLiteCommand(query, connection)) {
// Use SQLiteDataAdapter to fill the DataTable
using (SQLiteDataAdapter adapter = new SQLiteDataAdapter(command)) {
adapter.Fill(dataTable);
}
}
} catch (Exception ex) {
Console.WriteLine("An error occurred: " + ex.Message);
}
}
// Return the populated DataTable
return dataTable;
}
public static List<Dictionary<string, object>> ExecuteBulkRead(string connectionString, string query) {
var result = new List<Dictionary<string, object>>();
using (var connection = new SQLiteConnection(connectionString)) {
connection.Open();
// Begin a transaction to optimize read speed
using (var transaction = connection.BeginTransaction())
using (var command = new SQLiteCommand(query, connection, transaction))
using (var reader = command.ExecuteReader()) {
while (reader.Read()) {
var row = new Dictionary<string, object>();
for (int i = 0; i < reader.FieldCount; i++) {
string columnName = reader.GetName(i);
object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
row[columnName] = columnValue;
}
result.Add(row);
}
// Commit the transaction
transaction.Commit();
}
}
return result;
}
public static List<Dictionary<string, object>> get_table(string connectionString, string query) {
var result = new List<Dictionary<string, object>>();
using (var connection = new SQLiteConnection(connectionString)) {
try {
connection.Open();
using (var command = new SQLiteCommand(query, connection))
using (var reader = command.ExecuteReader()) {
while (reader.Read()) {
var row = new Dictionary<string, object>();
for (int i = 0; i < reader.FieldCount; i++) {
string columnName = reader.GetName(i);
object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
row[columnName] = columnValue;
}
result.Add(row);
}
}
} catch (Exception ex) {
Console.WriteLine("An error occurred: " + ex.Message);
}
}
return result;
}
public static IEnumerable<Dictionary<string, object>> get_table_2(string connectionString, string query) {
using (var connection = new SQLiteConnection(connectionString)) {
connection.Open();
using (var command = new SQLiteCommand(query, connection))
using (var reader = command.ExecuteReader()) {
while (reader.Read()) {
var row = new Dictionary<string, object>();
for (int i = 0; i < reader.FieldCount; i++) {
string columnName = reader.GetName(i);
object columnValue = reader.IsDBNull(i) ? null : reader.GetValue(i);
row[columnName] = columnValue;
}
yield return row;
}
}
}
}
}
}
@@ -1,73 +0,0 @@
namespace bts.SilverPine {
using System;
using System.Data;
using System.Data.SQLite;
using System.Diagnostics;
using System.Collections;
using BCG;
[DataStreamUserRespresentation("SQLite")]
public class SQLiteStream : ViewStream {
private readonly string _connection_string, _query;
private Stopwatch _stopwatch = new Stopwatch();
private SQLiteStream(string connection_string, string query, bool noop) {
_connection_string = connection_string;
_query = query;
}
public SQLiteStream(string db_name, string query)
:
this($"Data Source={SQLiteHelper.location}{db_name};Version=3", query, true) { }
public SQLiteStream(string query)
:
this(SQLiteHelper.db_name, query) {
}
private void _read_from_sqlite_and_fire() {
_stopwatch.Start();
DataTable dt = SQLiteHelper.execute_select_query(_connection_string, _query);
_stopwatch.Stop();
Log.Gen.info($"SQLiteStream.ExecuteSelectQuery time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
_stopwatch.Restart();
#region perf measures
// SQLiteHelper.get_table(_connection_string, _query);
// _stopwatch.Stop();
// Log.Gen.info($"SQLiteStream.get_table time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
// _stopwatch.Restart();
// SQLiteHelper.ExecuteBulkRead(_connection_string, _query);
// _stopwatch.Stop();
// Log.Gen.info($"SQLiteStream.ExecuteBulkRead time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
// _stopwatch.Restart();
// SQLiteHelper.get_table_2(_connection_string, _query);
// _stopwatch.Stop();
// Log.Gen.info($"SQLiteStream.get_table_2 time to read db: {_stopwatch.ElapsedMilliseconds} msecs");
// _stopwatch.Restart();
#endregion
IBlueRowCollection rows = BCG.BlueDataUtility.toIBlueRowCollection(dt);
//IBlueRowCollection rows = new DataTableWrapperTable(dt);
_stopwatch.Stop();
Log.Gen.info($"SQLiteStream._read_from_sqlite_and_fire time to load into IBlueRowCollection: {_stopwatch.ElapsedMilliseconds} msecs");
DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
myFireEvent(dp);
}
protected override void myOnFirstSubscriber() {
_read_from_sqlite_and_fire();
}
protected override void myOnLastSubscriber() {
// this value will be cleared by the base class
// but in this particular stream this behaviour
// is not usefull since any new subscribers
// would force the query to be recreated
//myLastDP = myLastValue;
}
}
}
@@ -3,15 +3,12 @@ using System.Collections.Generic;
using System.Text; using System.Text;
using System.IO; using System.IO;
using BCG.Utilities; using BCG.Utilities;
using System.Diagnostics;
using Microsoft.Data.Analysis;
using Parquet;
using Parquet.Data;
namespace BCG
namespace BCG { {
[DataStreamUserRespresentation("CSVSaver")] [DataStreamUserRespresentation("CSVSaver")]
public class CSVSaverStream : ViewStream, ISideEffectStream { public class CSVSaverStream : ViewStream, ISideEffectStream
{
private IBlueTableSchema myResultSchema = new BlueTableSchema("CSVResult", new ColumnInfo("id", 0, typeof(int)), private IBlueTableSchema myResultSchema = new BlueTableSchema("CSVResult", new ColumnInfo("id", 0, typeof(int)),
new ColumnInfo("fileName", 1, typeof(string)), new ColumnInfo("fileName", 1, typeof(string)),
new ColumnInfo("nWrites", 2, typeof(int)), new ColumnInfo("nWrites", 2, typeof(int)),
@@ -23,9 +20,11 @@ namespace BCG {
private int myNWrites = 0, myNMaxWrites = Int32.MaxValue; private int myNWrites = 0, myNMaxWrites = Int32.MaxValue;
public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName) public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName)
: this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false) { } : this(vs, fullyQualifiedFileName, true, Int32.MaxValue, false)
{ }
public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot) { public CSVSaverStream(ViewStream vs, string fullyQualifiedFileName, bool overwrite, int nMaxWrites, bool fireResultSnapshot)
{
myStream = vs; myStream = vs;
myFullyQualifiedFileName = fullyQualifiedFileName; myFullyQualifiedFileName = fullyQualifiedFileName;
myOverwrite = overwrite; myOverwrite = overwrite;
@@ -37,14 +36,18 @@ namespace BCG {
protected override void myOnLastSubscriber() { myStream.onChange -= new ChangeDelegate(myHandleChanges); } protected override void myOnLastSubscriber() { myStream.onChange -= new ChangeDelegate(myHandleChanges); }
private void myHandleChanges(object newValue) { private void myHandleChanges(object newValue)
{
IBlueRowCollection totalState = (newValue as DataPacket).newRows; IBlueRowCollection totalState = (newValue as DataPacket).newRows;
if (totalState.Count > 0 && myNWrites < myNMaxWrites) { if (totalState.Count > 0 && myNWrites < myNMaxWrites)
{
myNWrites++; myNWrites++;
int i = 0; int i = 0;
StreamWriter writer = new StreamWriter(new FileStream(myGetNewFileName(), FileMode.Create, FileAccess.Write)); StreamWriter writer = new StreamWriter(new FileStream(myGetNewFileName(), FileMode.Create, FileAccess.Write));
foreach (IBlueRow r in totalState) { foreach (IBlueRow r in totalState)
if (i++ == 0) { {
if (i++ == 0)
{
string header = String.Empty; string header = String.Empty;
foreach (string col in r.schema.columnNames) foreach (string col in r.schema.columnNames)
if (col != "id") if (col != "id")
@@ -56,17 +59,20 @@ namespace BCG {
writer.Flush(); writer.Flush();
} }
writer.Close(); writer.Close();
if (myFireResultSnapshot) { if (myFireResultSnapshot)
{
IBlueRow nr = new BlueRow(myResultSchema, 0, myFullyQualifiedFileName, myNWrites, totalState.Count); IBlueRow nr = new BlueRow(myResultSchema, 0, myFullyQualifiedFileName, myNWrites, totalState.Count);
BlueTable nRows = new BlueTable(myResultSchema); BlueTable nRows = new BlueTable(myResultSchema);
nRows.addRow(nr); nRows.addRow(nr);
myFireEvent(new DataPacket(nRows, nRows, new BlueTable(myLastValue == null ? new BlueTable(myResultSchema) : (myLastValue as DataPacket).newRows))); myFireEvent(new DataPacket(nRows, nRows, new BlueTable(myLastValue == null ? new BlueTable(myResultSchema) : (myLastValue as DataPacket).newRows)));
} else }
else
myFireEvent(newValue); myFireEvent(newValue);
} }
} }
private string myGetNewFileName() { private string myGetNewFileName()
{
string[] tags = myFullyQualifiedFileName.Split('.'); string[] tags = myFullyQualifiedFileName.Split('.');
if (myOverwrite) if (myOverwrite)
return tags[0] + ".csv"; return tags[0] + ".csv";
@@ -74,11 +80,14 @@ namespace BCG {
return tags[0] + "_" + myFormateDateTime(DateTime.Now) + ".csv"; return tags[0] + "_" + myFormateDateTime(DateTime.Now) + ".csv";
} }
private static string myGetCSVRow(IBlueRow r) { private static string myGetCSVRow(IBlueRow r)
{
string rv = String.Empty; string rv = String.Empty;
int i = 0; int i = 0;
foreach (string col in r.schema.columnNames) { foreach (string col in r.schema.columnNames)
if (col != "id") { {
if (col != "id")
{
string colText = String.Empty; string colText = String.Empty;
if (r[col] != null && r[col] != DBNull.Value) if (r[col] != null && r[col] != DBNull.Value)
colText = r[col].ToString(); colText = r[col].ToString();
@@ -90,7 +99,8 @@ namespace BCG {
return rv; return rv;
} }
private string myFormateDateTime(DateTime dt) { private string myFormateDateTime(DateTime dt)
{
return return
string.Format("{0}-{1}-{2}-{3}-{4}-{5}-{6}", string.Format("{0}-{1}-{2}-{3}-{4}-{5}-{6}",
dt.Year.ToString("0000"), dt.Year.ToString("0000"),
@@ -104,49 +114,52 @@ namespace BCG {
} }
[DataStreamUserRespresentation("CSVReader")] [DataStreamUserRespresentation("CSVReader")]
public class CSVReaderStream : ViewStream { public class CSVReaderStream : ViewStream
private readonly string _file_address; {
Stopwatch _stopwatch = new Stopwatch(); private IBlueRowCollection myRows;
public CSVReaderStream(string fileAddress) {
_file_address = fileAddress; public CSVReaderStream(string fileAddress)
{
myRows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(fileAddress);
} }
protected override void myOnFirstSubscriber() { protected override void myOnFirstSubscriber()
_stopwatch.Start(); {
DataFrame df = DataFrame.LoadCsv(_file_address); myFireEvent(new DataPacket(myRows, myRows, new BlueTable(myRows.schema)));
_stopwatch.Stop();
bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load col_count: {df.Columns.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
_stopwatch.Start();
IBlueRowCollection rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(_file_address);
_stopwatch.Stop();
bts.Log.Gen.info($"CSVReaderStream.myOnFirstSubscriber stream csv load count: {rows.Count} rows, time: {_stopwatch.ElapsedMilliseconds} msecs");
DataPacket dp_out = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket);
myFireEvent(dp_out);
} }
} }
[DataStreamUserRespresentation("CSVReaderStreamMapper")] [DataStreamUserRespresentation ("CSVReaderStreamMapper")]
public class CSVReaderStreamMapperStream : ViewStream { public class CSVReaderStreamMapperStream : ViewStream
public CSVReaderStreamMapperStream(ViewStream stream, string output_schema) { {
public CSVReaderStreamMapperStream(ViewStream stream, string output_schema)
{
mySetRealSource(new MapStream(stream, new CSVReaderStreamMapper(output_schema))); mySetRealSource(new MapStream(stream, new CSVReaderStreamMapper(output_schema)));
} }
} }
public class CSVReaderStreamMapper : IStreamMapper { public class CSVReaderStreamMapper : IStreamMapper
{
private IBlueTableSchema mySchema; private IBlueTableSchema mySchema;
private readonly string[] myOutputSchemaStringArray; private readonly string[] myOutputSchemaStringArray;
public CSVReaderStreamMapper(string outputSchema) { public CSVReaderStreamMapper(string outputSchema)
{
myOutputSchemaStringArray = outputSchema.Split(' '); myOutputSchemaStringArray = outputSchema.Split(' ');
} }
public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) { public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
if (mySchema == null) { {
if (mySchema == null)
{
ColumnInfo[] ci = inputSchema.columnInfo; ColumnInfo[] ci = inputSchema.columnInfo;
foreach (string col in myOutputSchemaStringArray) { foreach (string col in myOutputSchemaStringArray)
{
string colName = col.Substring(0, col.IndexOf('\\')).ToLower(); string colName = col.Substring(0, col.IndexOf('\\')).ToLower();
for (int i = 0; i < ci.Length; i++) { for (int i = 0; i < ci.Length; i++)
if (ci[i].colName.ToLower().Equals(colName)) { {
if (ci[i].colName.ToLower().Equals(colName))
{
Type t = myGetType(col.Substring(col.IndexOf('\\') + 1)); Type t = myGetType(col.Substring(col.IndexOf('\\') + 1));
ci[i] = new ColumnInfo(ci[i].colName, i, t); ci[i] = new ColumnInfo(ci[i].colName, i, t);
break; break;
@@ -158,52 +171,74 @@ namespace BCG {
return mySchema; return mySchema;
} }
public IBlueRow map(IBlueRow row) { public IBlueRow map(IBlueRow row)
{
object[] vals = new object[row.schema.columnInfo.Length]; object[] vals = new object[row.schema.columnInfo.Length];
row.Values.CopyTo(vals, 0); row.Values.CopyTo(vals, 0);
for (int i = 0; i < mySchema.columnInfo.Length; i++) { for (int i = 0; i < mySchema.columnInfo.Length; i++)
{
Type t = mySchema.columnInfo[i].colType; Type t = mySchema.columnInfo[i].colType;
try { try
{
vals[i] = System.Convert.ChangeType(vals[i], mySchema.columnInfo[i].colType, System.Globalization.CultureInfo.InvariantCulture); vals[i] = System.Convert.ChangeType(vals[i], mySchema.columnInfo[i].colType, System.Globalization.CultureInfo.InvariantCulture);
} catch { }
if (t.Equals(typeof(Int16))) { catch
{
if (t.Equals(typeof(Int16)))
{
Int16 result; Int16 result;
if (Int16.TryParse((string)vals[i], out result)) { if (Int16.TryParse((string)vals[i], out result))
{
vals[i] = result; vals[i] = result;
continue; continue;
} }
double r2; double r2;
if (Double.TryParse((string)vals[i], out r2)) { if (Double.TryParse((string)vals[i], out r2))
try { {
try
{
vals[i] = (Int16)r2; vals[i] = (Int16)r2;
continue; continue;
} catch { } }
catch { }
} }
} else if (t.Equals(typeof(Int32))) { }
else if (t.Equals(typeof(Int32)))
{
Int32 result; Int32 result;
if (Int32.TryParse((string)vals[i], out result)) { if (Int32.TryParse((string)vals[i], out result))
{
vals[i] = result; vals[i] = result;
continue; continue;
} }
double r2; double r2;
if (Double.TryParse((string)vals[i], out r2)) { if (Double.TryParse((string)vals[i], out r2))
try { {
try
{
vals[i] = (Int32)r2; vals[i] = (Int32)r2;
continue; continue;
} catch { } }
catch { }
} }
} else if (t.Equals(typeof(Int64))) { }
else if (t.Equals(typeof(Int64)))
{
Int64 result; Int64 result;
if (Int64.TryParse((string)vals[i], out result)) { if (Int64.TryParse((string)vals[i], out result))
{
vals[i] = result; vals[i] = result;
continue; continue;
} }
double r2; double r2;
if (Double.TryParse((string)vals[i], out r2)) { if (Double.TryParse((string)vals[i], out r2))
try { {
try
{
vals[i] = (Int64)r2; vals[i] = (Int64)r2;
continue; continue;
} catch { } }
catch { }
} }
} }
vals[i] = DBNull.Value; vals[i] = DBNull.Value;
@@ -212,7 +247,8 @@ namespace BCG {
return new BlueRow(mySchema, vals); return new BlueRow(mySchema, vals);
} }
private Type myGetType(string type) { private Type myGetType(string type)
{
type = type.ToLower(); type = type.ToLower();
if (type.Contains("int")) if (type.Contains("int"))
return typeof(int); return typeof(int);
@@ -24,6 +24,7 @@ namespace bts {
var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket); var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket);
myFireEvent(dp_out); myFireEvent(dp_out);
} }
@@ -1,41 +1,31 @@
using System; using System;
using System.Collections; using System.Collections;
using System.Reflection; using System.Reflection;
using BCG.Utilities; using BCG.Utilities;
namespace BCG { namespace BCG
// if a table needs to be fired, it can be wrapped around this {
public class BlueTableStream : ViewStream { [DataStreamUserRespresentation (false)]
private IBlueRowCollection _rows; public class BlueRowCollectionStaticStream : ViewStream
public BlueTableStream(IBlueRowCollection rows) { {
_rows = rows; public BlueRowCollectionStaticStream () {}
public BlueRowCollectionStaticStream (IBlueRowCollection rows) {loadCollection (rows);}
public void loadCollection (IBlueRowCollection rows)
{
myFireEvent (new DataPacket (rows, rows, myLastValue != null ?
new BlueTable((myLastValue as DataPacket).newRows) : new BlueTable (rows.schema)));
} }
protected override void myOnFirstSubscriber() { public void crush_with_single_row(IBlueRow r)
var dp_out = BlueDataUtility.build_dp_inc(_rows, myLastValue as DataPacket); {
myFireEvent(dp_out);
}
}
[DataStreamUserRespresentation(false)]
public class BlueRowCollectionStaticStream : ViewStream {
public BlueRowCollectionStaticStream() { }
public BlueRowCollectionStaticStream(IBlueRowCollection rows) { loadCollection(rows); }
public void loadCollection(IBlueRowCollection rows) {
myFireEvent(new DataPacket(rows, rows, myLastValue != null ?
new BlueTable((myLastValue as DataPacket).newRows) : new BlueTable(rows.schema)));
}
public void crush_with_single_row(IBlueRow r) {
BlueTable t = new BlueTable(r.schema); BlueTable t = new BlueTable(r.schema);
t.addRow(r); t.addRow(r);
loadCollection(t); loadCollection(t);
} }
public void loadPacket(DataPacket dp) { myFireEvent(dp); } public void loadPacket (DataPacket dp) {myFireEvent (dp);}
/// <summary> /// <summary>
/// Will work with streams that take ViewStream as first argument ex project, filter, distinct etc /// Will work with streams that take ViewStream as first argument ex project, filter, distinct etc
@@ -45,16 +35,17 @@ namespace BCG {
/// <param name="streamType"></param> /// <param name="streamType"></param>
/// <param name="args"></param> /// <param name="args"></param>
/// <returns></returns> /// <returns></returns>
public static IBlueRowCollection statelessOp(IBlueRowCollection rows, Type streamType, params object[] args) { public static IBlueRowCollection statelessOp (IBlueRowCollection rows, Type streamType, params object[] args)
ArrayList totArgs = new ArrayList(args); {
BlueRowCollectionStaticStream v1 = new BlueRowCollectionStaticStream(); ArrayList totArgs = new ArrayList (args);
totArgs.Insert(0, v1); BlueRowCollectionStaticStream v1 = new BlueRowCollectionStaticStream ();
ViewStream retVal = Activator.CreateInstance(streamType, totArgs.ToArray()) as ViewStream; totArgs.Insert (0, v1);
v1.loadCollection(rows); ViewStream retVal = Activator.CreateInstance (streamType, totArgs.ToArray()) as ViewStream;
v1.loadCollection (rows);
return retVal.rows; return retVal.rows;
} }
} }
} }
@@ -80,7 +80,7 @@ namespace BCG {
public IBlueRow map(IBlueRow row) { public IBlueRow map(IBlueRow row) {
MutableBlueRow rv = new MutableBlueRow(getOutputSchema(row.schema), row); MutableBlueRow rv = new MutableBlueRow(getOutputSchema(row.schema), row);
rv.setValue(myConversionColName, mapDateTime(row.get_datetime(myConversionColName), myConversion)); rv.setValue(myConversionColName, mapDateTime(row[myConversionColName], myConversion));
return rv; return rv;
//return BlueDataUtility.getNewReplacedRow(row, mapDateTime(row[myConversionColName], myConversion), myConversionColName); //return BlueDataUtility.getNewReplacedRow(row, mapDateTime(row[myConversionColName], myConversion), myConversionColName);
} }
@@ -1,38 +0,0 @@
namespace bts.SilverPine {
using bts;
using BCG;
using System;
[DataStreamUserRespresentation("DateTimeExpanderStream")]
public class DateTimeExpanstionStream : ViewStream {
public DateTimeExpanstionStream(ViewStream parent_stream, string inp_col) {
Func<IBlueTableSchema, IBlueTableSchema> get_schema = (s) => {
IBlueTableSchema rv = s.add_column(System.TypeCode.DateTime, $"{inp_col}_date");
rv = rv.add_column(System.TypeCode.String, $"{inp_col}_day");
rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_day_of_month");
rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_month");
rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_hour");
rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_min");
rv = rv.add_column(System.TypeCode.Int32, $"{inp_col}_seconds_of_day");
return rv;
};
ViewStream rs = new MapStream(parent_stream,
s => get_schema(s),
r => {
DateTime dt = r.get_datetime(inp_col);
IBlueTableSchema s = get_schema(r.schema);
MutableBlueRow rv = new MutableBlueRow(s, r);
rv.setValue($"{inp_col}_date", dt.Date);
rv.setValue($"{inp_col}_day_of_week", dt.DayOfWeek.ToString());
rv.setValue($"{inp_col}_day_of_month", dt.Day);
rv.setValue($"{inp_col}_month", dt.Date.Month);
rv.setValue($"{inp_col}_hour", dt.Hour);
rv.setValue($"{inp_col}_minute", dt.Minute);
rv.setValue($"{inp_col}_seconds_of_day", (dt - dt.Date).TotalSeconds);
return rv;
});
mySetRealSource(rs);
}
}
}
@@ -1,71 +1,29 @@
using System; using System;
using BCG.Utilities.General; namespace BCG
{
public class MSecondStream : TimeUnitStream
{
public MSecondStream (int intervalInms, int size)
: base (DateTime.Now, new TimeSpan (0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance)
{}
using bts; protected override DateTime myRoundTime (DateTime dt) {return dt;}
namespace BCG {
[DataStreamUserRespresentation("milliseconds")]
public class MillisecondStream : ViewStream, IWorker {
private DateTime _last_fire_time = DateTime.MinValue;
private readonly TimeSpan _period;
private readonly int _size;
private readonly ViewStreamDatabase _vsd;
private IBlueTableSchema _schema;
private bts.CircularBufferTable _table;
public MillisecondStream(int interval_msecs, int size)
: this(ViewStreamDatabase.instance, interval_msecs, size) {
}
public MillisecondStream(ViewStreamDatabase vsd, int interval_msecs, int size) {
_vsd = vsd;
_period = TimeSpan.FromMilliseconds(interval_msecs);
_size = size;
_schema = BlueDataUtility.form_schema("Milliseconds",
Tuple.Create("id", TypeCode.Int32),
Tuple.Create("time", TypeCode.DateTime),
Tuple.Create("display_time", TypeCode.String));
_table = new CircularBufferTable(_schema, _size);
}
protected override void myOnFirstSubscriber() {
_vsd.addWorker(this);
}
protected override void myOnLastSubscriber() {
_vsd.removeWorker(this);
}
private int _id = 0;
public void doWork() {
if (DateTime.Now - _last_fire_time >= _period) {
_last_fire_time = DateTime.Now;
IBlueRow nr = new BlueRow(_schema, _id++, _last_fire_time,
_last_fire_time.ToString("yyyy-MM-dd HH:mm:ss.fff"));
_table.add_row(nr);
var dp = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
myFireEvent(dp);
}
}
}
public class MSecondStream : TimeUnitStream {
public MSecondStream(int intervalInms, int size)
: base(DateTime.Now, new TimeSpan(0, 0, 0, 0, intervalInms), size, ViewStreamDatabase.instance) { }
protected override DateTime myRoundTime(DateTime dt) { return dt; }
} }
public class DisplayableMillisecondStream : ViewStream, IStreamMapper { public class DisplayableMillisecondStream : ViewStream, IStreamMapper
private IBlueTableSchema mySchema = new BlueTableSchema("ms", {
new ColumnInfo("id", 0, typeof(int)), private IBlueTableSchema mySchema = new BlueTableSchema ("ms",
new ColumnInfo("time", 1, typeof(string))); new ColumnInfo ("id", 0, typeof(int)),
public DisplayableMillisecondStream(int intervalInMs, int sz) { new ColumnInfo ("time", 1, typeof(string)));
mySetRealSource(new MapStream(new MSecondStream(intervalInMs, sz), this)); public DisplayableMillisecondStream (int intervalInMs, int sz)
{
mySetRealSource (new MapStream (new MSecondStream (intervalInMs, sz), this));
} }
#region IStreamMapper Members #region IStreamMapper Members
public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) { public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema)
{
return mySchema; return mySchema;
} }
@@ -73,15 +31,15 @@ namespace BCG {
#region IDynamicStreamMapper Members #region IDynamicStreamMapper Members
public IBlueRow map(IBlueRow row) { public IBlueRow map (IBlueRow row)
DateTime time = row.get_datetime("time"); {
var time_txt = time.ToString("yyyy-MM-dd HH:mm:ss.fff"); return new BlueRow (mySchema, row["id"], myToString ((DateTime)row["time"]));
return new BlueRow(mySchema, row["id"], time_txt);
} }
#endregion #endregion
private string myToString(DateTime dt) { private string myToString (DateTime dt)
{
string sep = ":"; string sep = ":";
return dt.Hour + sep + dt.Minute + sep + dt.Second + sep + dt.Millisecond; return dt.Hour + sep + dt.Minute + sep + dt.Second + sep + dt.Millisecond;
} }
@@ -1,124 +0,0 @@
namespace bts.SilverPine {
using BCG;
using bts;
using bts.collections;
using System;
using System.CodeDom;
using System.Collections.Generic;
using System.Linq;
// Given 2 filters, and 2 columns, finds the different in the value of the 2 columns from the 2 rows
// so:
// f1(base_stream.rows)[col1] - f1(base_stream.rows)[col2]
[DataStreamUserRespresentation("ColDiff")]
public class ColumnDifferenceStream : ViewStream {
private readonly ViewStream _base_stream;
private readonly FilterStream.OnFilter _f1, _f2;
private readonly string _c1, _c2;
public ColumnDifferenceStream(ViewStream base_stream, FilterStream.OnFilter f1, FilterStream.OnFilter f2, string col1, string col2) {
_base_stream = base_stream;
_f1 = f1;
_f2 = f2;
_c1 = col1;
_c2 = col2;
}
public ColumnDifferenceStream(ViewStream base_stream, string f1, string f2, string col1, string col2) {
FilterStream.OnFilter f1_del = r => {
BooleanExpressionFilter rv = new BooleanExpressionFilter(f1);
return rv.filter(r);
};
FilterStream.OnFilter f2_del = r => {
BooleanExpressionFilter rv = new BooleanExpressionFilter(f2);
return rv.filter(r);
};
ViewStream rs = new ColumnDifferenceStream(base_stream, f1_del, f2_del, col1, col2);
mySetRealSource(rs);
}
// for each row, does a row[new_col_name] = row[col1] - rows[row_ffset][col2]
// we should generlize this for any math operation
public ColumnDifferenceStream(ViewStream parent_stream, string sort_col_name, int row_offset, string col1, string col2, string diff_col_name, double optional_perc_diff_filter = 0.0) {
// use GlobalMapStream
string diff_col_perc_name = $"{diff_col_name}_percentage";
System.Func<System.Collections.Generic.List<BCG.IBlueRow>, uint, BCG.IBlueRow> g_map_func = (rows, idx) => {
int minuend_idx = Convert.ToInt32(idx);
var inp_schema = rows[0].schema;
var out_schema = inp_schema.add_column(TypeCode.Double, diff_col_name);
out_schema = out_schema.add_column(TypeCode.Double, diff_col_perc_name);
// minuend - subtrahend
IBlueRow minuend_row = rows.ElementAt(minuend_idx);
MutableBlueRow rv = new MutableBlueRow(out_schema, minuend_row);
int subtrahend_index = minuend_idx + row_offset;
double diff_val = Double.NaN, diff_val_perc = Double.NaN;
if (subtrahend_index >= 0) {
try {
IBlueRow subtrahend_row = rows.ElementAt(subtrahend_index);
diff_val = minuend_row.get<double>(col1) - subtrahend_row.get<double>(col2);
diff_val_perc = 100.0 * diff_val / subtrahend_row.get<double>(col1);
} catch (Exception ex) {
Log.assert(false, ex.Message);
}
}
rv.setValue(diff_col_name, diff_val);
rv.setValue(diff_col_perc_name, diff_val_perc);
return rv;
};
ViewStream rs = new GlobalMapStream(parent_stream, sort_col_name, g_map_func);
if (optional_perc_diff_filter != 0) {
rs = new FilterStream(rs, r => {
double diff_val_perc = r.get<double>(diff_col_perc_name);
double abs_diff_val_perc = Math.Abs(diff_val_perc);
return abs_diff_val_perc >= optional_perc_diff_filter;
});
}
mySetRealSource(rs);
}
protected override void myOnFirstSubscriber() {
_base_stream.onChange += _handle_base_stream_update;
}
protected override void myOnLastSubscriber() {
_base_stream.onChange -= _handle_base_stream_update;
}
private void _handle_base_stream_update(object dp_obj) {
// could have been done with filter and cartesian prod, and then math
DataPacket dp = dp_obj as DataPacket;
List<IBlueRow> rows = dp.newRows.to_list();
var f1_rows = rows.Where(r => _f1(r));
//Log.assert(f1_rows.Count() == 1);
var f2_rows = rows.Where(r => _f2(r));
//Log.assert(f2_rows.Count() == 1);
if (f1_rows != null && f1_rows.Count() > 0 && f2_rows != null && f2_rows.Count() > 0) {
IBlueRow f1_r = f1_rows.ElementAt(0);
IBlueRow f2_r = f2_rows.ElementAt(0);
double diff = f1_r.get<double>(_c1) - f2_r.get<double>(_c2);
IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
BlueTable out_table = new BlueTable(out_schema);
MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
out_row.setValue("diff", diff);
out_table.addRow(out_row);
DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
myFireEvent(dp_out);
} else {
IBlueRow f1_r = rows.ElementAt(0);
IBlueRow f2_r = rows.ElementAt(0);
double diff = Double.NaN;
IBlueTableSchema out_schema = f1_r.schema.add_columns(Tuple.Create(typeof(double), "diff"));
BlueTable out_table = new BlueTable(out_schema);
MutableBlueRow out_row = new MutableBlueRow(out_schema, f1_r);
out_row.setValue("diff", diff);
out_table.addRow(out_row);
DataPacket dp_out = BlueDataUtility.build_dp(out_table, myLastValue as DataPacket);
myFireEvent(dp_out);
}
}
}
}
@@ -1,51 +0,0 @@
namespace bts.SilverPine {
using BCG;
using bts;
using System.Collections.Generic;
using System.Linq;
// Given a sort order, an optional group by col, it observes the values of a column
// within that group (or the whole table, if grouping not specified) and returns all rows
// where the value changes, ie row[current][col] != row[current-1][col]
// TODO: a more general way to express a filter where row[current][col] != row[previous][col],
// or even expand it to row[current][col_a] - row[previous][col_b] < 0.5
[DataStreamUserRespresentation("ColEvol")]
public class ColumnEvolutionStream : ViewStream {
private readonly ViewStream _base_stream;
private readonly string _sort_col;
private readonly string _evol_col;
public ColumnEvolutionStream(ViewStream base_stream, string sort_col, string evolution_col) {
_base_stream = base_stream;
_sort_col = sort_col;
_evol_col = evolution_col;
}
protected override void myOnFirstSubscriber() {
_base_stream.onChange += _handle_base_stream_update;
}
protected override void myOnLastSubscriber() {
base.myOnLastSubscriber();
}
private void _handle_base_stream_update(object dp_obj) {
DataPacket dp = dp_obj as DataPacket;
List<IBlueRow> rows = dp.newRows.to_list();
rows.Sort((r1, r2) => BlueDataUtility.compare_rows(r2, r1, _sort_col));
BlueTable rv = new BlueTable(dp.schema);
IBlueRow prev_row = rows.ElementAt(0);
foreach (IBlueRow r in rows) {
var cr = BlueDataUtility.compare_rows(r, prev_row, _evol_col);
if (cr != 0) {
rv.addRow(r);
}
prev_row = r;
}
DataPacket dp_out = BlueDataUtility.build_dp(rv, myLastValue as DataPacket);
myFireEvent(dp_out);
}
}
}
@@ -15,75 +15,10 @@ using mbts;
using BCG.Extensions; using BCG.Extensions;
using System.Text.Json; using System.Text.Json;
using System.Text.Json.Nodes; using System.Text.Json.Nodes;
using Newtonsoft.Json;
using BCG;
using System.Runtime.CompilerServices;
using bts;
namespace bts {
// public class CompositeRow : IBlueRow {
// private IBlueRow _lrow, _rrow;
// private BlueRow _out_row;
// public CompositeRow(IBlueRow lrow, IBlueRow rrow) {
// _lrow = lrow;
// _rrow = rrow;
// IBlueTableSchema schema = _lrow.schema;
// List<object> vals = new List<object>();
// foreach (var o in _lrow) {
// vals.Add(o);
// }
// foreach (ColumnInfo ci in _rrow.schema.columnInfo) {
// if (ci.colName == "id") {
// continue;
// }
// vals.Add(_rrow[ci.colNo]);
// schema = schema.add_column(ci.type_code(), ci.colName);
// }
// _out_row = new BlueRow(schema, vals.ToArray());
// }
// public IBlueTableSchema schema {
// get { return _out_row.schema; }
// }
// public object? this[int index] {
// get { return _out_row[index]; }
// }
// public object? this[string colName] {
// get {
// return _out_row[colName];
// }
// }
// }
}
namespace BCG { namespace BCG {
public static class BlueDataUtility { public static class BlueDataUtility {
public static IBlueRow create_composite_row(IBlueRow lr, IBlueRow rr, int id) {
IBlueTableSchema schema = lr.schema;
List<object> vals = new List<object>();
foreach (ColumnInfo ci in lr.schema.columnInfo) {
if (ci.colName == "id") {
vals.Add(id);
}
vals.Add(lr[ci.colName]);
}
foreach (ColumnInfo ci in rr.schema.columnInfo) {
if (ci.colName == "id") {
continue;
}
schema = schema.add_column(ci.type_code(), ci.colName);
vals.Add(rr[ci.colName]);
}
IBlueRow r = new BlueRow(schema, vals.ToArray());
return r;
}
#region Extensions #region Extensions
public static bool is_null(this IBlueRow r, string col_name) {
object col_val = r[col_name];
return col_val == null || DBNull.Value.Equals(col_val);
}
public static DateTime get_datetime(this IBlueRow r, string col_name, DateTime err_value = default(DateTime)) { public static DateTime get_datetime(this IBlueRow r, string col_name, DateTime err_value = default(DateTime)) {
return BlueDataUtility.get_row_datetime_value(r, col_name, err_value); return BlueDataUtility.get_row_datetime_value(r, col_name, err_value);
} }
@@ -114,17 +49,6 @@ namespace BCG {
rv.Add(r.get<T>(col)); rv.Add(r.get<T>(col));
return rv; return rv;
} }
public static IEnumerable<T> get_colvalues<T>(IEnumerable<IBlueRow> rows, string col, int start_index, int count) {
List<T> rv = new List<T>();
int i = 0;
int n_copied = 0;
foreach (IBlueRow r in rows)
if (i++ >= start_index && n_copied < count) {
rv.Add(r.get<T>(col));
n_copied++;
}
return rv;
}
public static IDictionary<T, IEnumerable<IBlueRow>> group_by<T>(IBlueRowCollection rows, string gp_col) { public static IDictionary<T, IEnumerable<IBlueRow>> group_by<T>(IBlueRowCollection rows, string gp_col) {
Dictionary<T, List<IBlueRow>> rv = new Dictionary<T, List<IBlueRow>>(); Dictionary<T, List<IBlueRow>> rv = new Dictionary<T, List<IBlueRow>>();
@@ -188,17 +112,13 @@ namespace BCG {
} }
} }
public static List<IBlueRow> to_list(this IBlueRowCollection rows) { public static IEnumerable<IBlueRow> to_ienum(this IBlueRowCollection rows) {
List<IBlueRow> rv = new List<IBlueRow>(); List<IBlueRow> rv = new List<IBlueRow>();
foreach (IBlueRow r in rows) foreach (IBlueRow r in rows)
rv.Add(r); rv.Add(r);
return rv; return rv;
} }
public static IEnumerable<IBlueRow> to_ienum(this IBlueRowCollection rows) {
return rows.to_list();
}
public static IDictionary<int, IBlueRow> to_map(this IBlueRowCollection rows) { public static IDictionary<int, IBlueRow> to_map(this IBlueRowCollection rows) {
Dictionary<int, IBlueRow> rv = new Dictionary<int, IBlueRow>(); Dictionary<int, IBlueRow> rv = new Dictionary<int, IBlueRow>();
foreach (IBlueRow r in rows) { foreach (IBlueRow r in rows) {
@@ -208,10 +128,6 @@ namespace BCG {
return rv; return rv;
} }
public static bool contains_column(this IBlueTableSchema input_schema, string col) {
return input_schema.columnNames.Contains(col);
}
public static IBlueTableSchema add_columns(this IBlueTableSchema input_schema, params Tuple<Type, string>[] cols) { public static IBlueTableSchema add_columns(this IBlueTableSchema input_schema, params Tuple<Type, string>[] cols) {
List<ColumnInfo> cis = new List<ColumnInfo>(input_schema.columnInfo); List<ColumnInfo> cis = new List<ColumnInfo>(input_schema.columnInfo);
foreach (Tuple<Type, string> c in cols) foreach (Tuple<Type, string> c in cols)
@@ -219,11 +135,6 @@ namespace BCG {
return new BlueTableSchema(input_schema.tableName, cis.ToArray()); return new BlueTableSchema(input_schema.tableName, cis.ToArray());
} }
public static IBlueTableSchema add_column(this IBlueTableSchema input_schema, TypeCode type_arg, string col_name) {
Tuple<Type, string> tup = new Tuple<Type, string>(_typecode_to_type[type_arg], col_name);
return add_columns(input_schema, tup);
}
public static IBlueRowCollection to_collection(this IBlueRow r) { public static IBlueRowCollection to_collection(this IBlueRow r) {
if (r == null) if (r == null)
return null; return null;
@@ -280,18 +191,6 @@ namespace BCG {
#region json #region json
public static Dictionary<string, string> convert_json_to_dictionary(string json_string) {
Dictionary<string, string> dictionary = JsonConvert.DeserializeObject<Dictionary<string, string>>(json_string);
return dictionary;
}
public static IBlueRow convert_json_to_row(string json_string, int id = 0) {
Dictionary<string, object> dictionary = JsonConvert.DeserializeObject<Dictionary<string, object>>(json_string);
if (!dictionary.ContainsKey("id")) {
dictionary.Add("id", id);
}
return new BlueRow(dictionary);
}
public static List<JsonNode> json_to_json_node_list(string jsonString) { public static List<JsonNode> json_to_json_node_list(string jsonString) {
var jsonNodeList = new List<JsonNode>(); var jsonNodeList = new List<JsonNode>();
bool noe = String.IsNullOrEmpty(jsonString); bool noe = String.IsNullOrEmpty(jsonString);
@@ -310,7 +209,7 @@ namespace BCG {
foreach (JsonNode jsonNode in jsonArray) { foreach (JsonNode jsonNode in jsonArray) {
jsonNodeList.Add(jsonNode); jsonNodeList.Add(jsonNode);
} }
} catch (System.Text.Json.JsonException ex) { } catch (JsonException ex) {
Console.WriteLine($"Invalid JSON: {ex.Message}"); Console.WriteLine($"Invalid JSON: {ex.Message}");
} }
@@ -587,16 +486,7 @@ namespace BCG {
ColumnInfo[] cis = new ColumnInfo[table.Columns.Count]; ColumnInfo[] cis = new ColumnInfo[table.Columns.Count];
foreach (DataColumn col in table.Columns) foreach (DataColumn col in table.Columns)
cis[col.Ordinal] = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType); cis[col.Ordinal] = new ColumnInfo(col.ColumnName, col.Ordinal, col.DataType);
IBlueTableSchema rv = new BlueTableSchema(cis, table.TableName); return new BlueTableSchema(cis, table.TableName);
rv = rv.add_id_col_if_missing();
return rv;
}
public static IBlueTableSchema add_id_col_if_missing(this IBlueTableSchema schema) {
IBlueTableSchema rv = schema;
if (!schema.columnNames.Contains("id")) {
rv = schema.add_columns(Tuple.Create(typeof(int), "id"));
}
return rv;
} }
/// <summary> /// <summary>
@@ -854,16 +744,10 @@ namespace BCG {
} }
public static IBlueRowCollection toIBlueRowCollection(DataTable dt) { public static IBlueRowCollection toIBlueRowCollection(DataTable dt) {
IBlueTableSchema schema = getSchema(dt); BlueTable retVal = new BlueTable(getSchema(dt));
BlueTable rv = new BlueTable(schema); foreach (DataRow dr in dt.Rows)
foreach (DataRow dr in dt.Rows) { retVal.addRow(dr);
MutableBlueRow nr = new MutableBlueRow(schema); return retVal;
foreach (DataColumn col in dr.Table.Columns)
nr.setValue(col.ColumnName, dr[col.ColumnName]);
nr.setValue("id", rv.Count);
rv.addRow(nr);
}
return rv;
} }
public static IBlueRowCollection toIBlueRowCollection(DataRow[] rows) { public static IBlueRowCollection toIBlueRowCollection(DataRow[] rows) {
@@ -935,20 +819,20 @@ namespace BCG {
/// </summary> /// </summary>
/// <param name="r1"></param> /// <param name="r1"></param>
/// <param name="r2"></param> /// <param name="r2"></param>
/// <param name="col_names"></param> /// <param name="colNames"></param>
/// <returns></returns> /// <returns></returns>
public static int compare_rows(IBlueRow r1, public static int compare_rows(IBlueRow r1,
IBlueRow r2, IBlueRow r2,
params string[] col_names) { params string[] colNames) {
int comp_res = 0; int retVal = 0;
foreach (string cn in col_names) { foreach (string colName in colNames) {
object val1 = r1[cn]; object val1 = r1[colName];
object val2 = r2[cn]; object val2 = r2[colName];
comp_res = _compare_row_cols(val1, val2); retVal = _compare_row_cols(val1, val2);
if (comp_res != 0) if (retVal != 0)
break; break;
} }
return comp_res; return retVal;
} }
private static int _compare_row_cols(object v1, object v2) { private static int _compare_row_cols(object v1, object v2) {
@@ -1724,10 +1608,8 @@ namespace BCG {
public static int[] get_column_indices(IBlueTableSchema schema, string[] col_names) { public static int[] get_column_indices(IBlueTableSchema schema, string[] col_names) {
List<int> rv = new List<int>(); List<int> rv = new List<int>();
foreach (string c in col_names) { foreach (string c in col_names)
Log.assert(schema.contains_column(c), $"Could not find col: {c} in table: {schema.tableName}");
rv.Add(schema[c].colNo); rv.Add(schema[c].colNo);
}
return rv.ToArray(); return rv.ToArray();
} }
@@ -1746,13 +1628,6 @@ namespace BCG {
return sb.ToString(); return sb.ToString();
} }
public static string id_string(IBlueRowCollection table) {
string rv = String.Empty;
foreach (IBlueRow r in table)
rv += $"{r["id"]} ";
return rv;
}
public static IDictionary<int, List<IBlueRow>> group_together(IBlueRowCollection rows, string[] group_by_col_names) { public static IDictionary<int, List<IBlueRow>> group_together(IBlueRowCollection rows, string[] group_by_col_names) {
BlueRowHasher hasher = new BlueRowHasher(group_by_col_names.Length); BlueRowHasher hasher = new BlueRowHasher(group_by_col_names.Length);
return group_together(rows, BlueDataUtility.get_column_indices(rows.schema, group_by_col_names), ref hasher); return group_together(rows, BlueDataUtility.get_column_indices(rows.schema, group_by_col_names), ref hasher);
@@ -1901,18 +1776,6 @@ namespace BCG {
#endregion #endregion
} }
public static IBlueRow find_max(IEnumerable<IBlueRow> rows, params string[] cols_to_compare) {
if (rows == null || rows.Count() == 0)
return null;
IBlueRow max_row = rows.ElementAt(0);
foreach (IBlueRow r in rows) {
int c = compare_rows(r, max_row, cols_to_compare);
if (c > 0)
max_row = r;
}
return max_row;
}
public static int compare_rows(IBlueRow r1, IBlueRow r2, Dictionary<string, bool> sort_map) { public static int compare_rows(IBlueRow r1, IBlueRow r2, Dictionary<string, bool> sort_map) {
foreach (KeyValuePair<string, bool> p in sort_map) { foreach (KeyValuePair<string, bool> p in sort_map) {
int comp = compare_send_dontcare_to_last(r1, p.Key, r2, p.Key, p.Value); int comp = compare_send_dontcare_to_last(r1, p.Key, r2, p.Key, p.Value);
@@ -2214,53 +2077,23 @@ namespace BCG {
try { try {
if (v == null || v == DBNull.Value) if (v == null || v == DBNull.Value)
return error_val; return error_val;
else {
if (v is DateTime) { if (v is DateTime)
return (DateTime)v; return (DateTime)v;
} else if (v is string) { else if (v is string) {
DateTime rv_dt; DateTime rv_dt = error_val;
DateTime.TryParse((string)v, out rv_dt);
// Try parsing the string as a DateTime
if (DateTime.TryParse((string)v,
System.Globalization.CultureInfo.InvariantCulture,
System.Globalization.DateTimeStyles.RoundtripKind,
out rv_dt)) {
return rv_dt; return rv_dt;
} else { } else {
return error_val; // Return the error value if parsing fails double oa_date = (double)Convert.ChangeType(v, typeof(Double));
return DateTime.FromOADate(oa_date);
} }
} else {
// Assume the input can be converted to a double and use FromOADate
double oa_date = (double)Convert.ChangeType(v, typeof(Double));
return DateTime.FromOADate(oa_date);
} }
} catch { } catch {
return error_val; return error_val;
} }
} }
// private static DateTime _get_datetime_value(object v, DateTime error_val) {
// try {
// if (v == null || v == DBNull.Value)
// return error_val;
// else {
// if (v is DateTime)
// return (DateTime)v;
// else if (v is string) {
// DateTime rv_dt = error_val;
// DateTime.TryParse((string)v, out rv_dt);
// return rv_dt;
// } else {
// double oa_date = (double)Convert.ChangeType(v, typeof(Double));
// return DateTime.FromOADate(oa_date);
// }
// }
// } catch {
// return error_val;
// }
// }
#endregion #endregion
#region CSV methods #region CSV methods
@@ -16,6 +16,12 @@ namespace bts {
_buffer_sz = buffer_sz; _buffer_sz = buffer_sz;
} }
// public OrderedBlueTable(List<IBlueRow> rows, int buffer_sz) {
// _buffer_sz = buffer_sz;
// foreach (IBlueRow r in rows)
// add_row(r);
// }
// returns the row that might need to be deleted to keep // returns the row that might need to be deleted to keep
// the circular buffer intact // the circular buffer intact
public IBlueRow add_row(IBlueRow row) { public IBlueRow add_row(IBlueRow row) {
@@ -8,9 +8,6 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="Google.Protobuf" Version="3.24.3" /> <PackageReference Include="Google.Protobuf" Version="3.24.3" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="OpenAI" Version="2.0.0" />
<PackageReference Include="SimMetrics.Net" Version="1.0.5" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
@@ -47,11 +47,6 @@ namespace BCG {
/// </summary> /// </summary>
/// <param name="assemblyName"></param> /// <param name="assemblyName"></param>
public void loadAssembly(string assemblyName) { public void loadAssembly(string assemblyName) {
Func<Type, bool> is_silverpine_only = (t) => {
if (!assemblyName.Contains("SharedStreams.dll"))
return true;
return t.Namespace == "bts.SilverPine";
};
bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assemblyName}"); bts.Log.Gen.info($"PrimitiveManager.loadAssembly assemblyName: {assemblyName}");
if (myAssemblyNames.Contains(assemblyName)) if (myAssemblyNames.Contains(assemblyName))
return; return;
@@ -61,7 +56,7 @@ namespace BCG {
Assembly.LoadFrom(assemblyName); Assembly.LoadFrom(assemblyName);
foreach (Type type in loadedAssembly.GetTypes()) foreach (Type type in loadedAssembly.GetTypes())
if (!type.IsInterface && is_silverpine_only(type) if (!type.IsInterface
&& !type.IsAbstract && !type.IsAbstract
&& type.IsSubclassOf(typeof(ViewStream)) && type.IsSubclassOf(typeof(ViewStream))
&& type.IsPublic) { && type.IsPublic) {
@@ -51,13 +51,13 @@ namespace BCG {
public virtual void make_batch_changes(IBlueRowCollection new_rows, public virtual void make_batch_changes(IBlueRowCollection new_rows,
IBlueRowCollection del_rows, IBlueRowCollection del_rows,
IBlueRowCollection added_rows) { IBlueRowCollection added_rows) {
// string log_msg = $"PubTableBroker.make_batch_changes table: {new_rows.schema.tableName} del: {del_rows.Count} added: {added_rows.Count}"; string log_msg = $"PubTableBroker.make_batch_changes table: {new_rows.schema.tableName} del: {del_rows.Count} added: {added_rows.Count}";
// foreach (IBlueRow r in del_rows) foreach (IBlueRow r in del_rows)
// log_msg += $"id: {r["id"]} "; log_msg += $"id: {r["id"]} ";
// log_msg += "addedRows: "; log_msg += "addedRows: ";
// foreach (IBlueRow r in added_rows) foreach (IBlueRow r in added_rows)
// log_msg += $"id: {r["id"]}"; log_msg += $"id: {r["id"]}";
// bts.Log.Gen.info(log_msg); bts.Log.Gen.info(log_msg);
SinglePubTableBroker table_publisher = null; SinglePubTableBroker table_publisher = null;
var name_key = new_rows.schema.tableName.ToLower(); var name_key = new_rows.schema.tableName.ToLower();
if (!_publisher_map.TryGetValue(name_key, out table_publisher)) if (!_publisher_map.TryGetValue(name_key, out table_publisher))
@@ -5,21 +5,18 @@ using System.Linq;
using bts; using bts;
namespace BCG namespace BCG {
{
// This class is responsible for mantaining the state fo a table. It gets the // This class is responsible for mantaining the state fo a table. It gets the
// bytes from the pubsubserver via the handle_updates call (called by VSD), deserializes // bytes from the pubsubserver via the handle_updates call (called by VSD), deserializes
// updates the state of the table, and fires the TableUpdateDelegate. // updates the state of the table, and fires the TableUpdateDelegate.
public class TableAccumulator2 : ITableAccumulator public class TableAccumulator2 : ITableAccumulator {
{
private readonly string _tag; private readonly string _tag;
private readonly ISubscriberCore _subscriber_core; private readonly ISubscriberCore _subscriber_core;
IStreamMsgDeserilizer _deser = new StreamIODeserializationController(); IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
private BlueTable _table; private BlueTable _table;
private TableUpdateDelegate _on_update_delegate; private TableUpdateDelegate _on_update_delegate;
public TableAccumulator2(string tag, ISubscriberCore sub_core) public TableAccumulator2(string tag, ISubscriberCore sub_core) {
{
_tag = tag; _tag = tag;
_subscriber_core = sub_core; _subscriber_core = sub_core;
} }
@@ -28,13 +25,10 @@ namespace BCG
//public bool is_active() { return _on_update_delegate != null; } //public bool is_active() { return _on_update_delegate != null; }
public event TableUpdateDelegate on_update public event TableUpdateDelegate on_update {
{ add {
add
{
Log.Gen.info("TableAccumulator2.on_update.add {_tag} subscribed"); Log.Gen.info("TableAccumulator2.on_update.add {_tag} subscribed");
if (_on_update_delegate == null) if (_on_update_delegate == null) {
{
//ie first subscription //ie first subscription
Log.Gen.info("TableAccumulator2.on_update {_tag} subscribed for the first time"); Log.Gen.info("TableAccumulator2.on_update {_tag} subscribed for the first time");
_subscriber_core.subscribe(_tag); _subscriber_core.subscribe(_tag);
@@ -44,17 +38,14 @@ namespace BCG
if (_table != null) if (_table != null)
value(_table, _table, new BlueTable(_table.schema)); value(_table, _table, new BlueTable(_table.schema));
} }
remove remove {
{
Log.Gen.info("TableAccumulator2.on_update.remove {_tag} unsubscribed"); Log.Gen.info("TableAccumulator2.on_update.remove {_tag} unsubscribed");
if (_on_update_delegate != null) if (_on_update_delegate != null) {
{
_on_update_delegate = _on_update_delegate =
Delegate.Remove(_on_update_delegate, value) as TableUpdateDelegate; Delegate.Remove(_on_update_delegate, value) as TableUpdateDelegate;
int n_subs = _on_update_delegate == null ? 0 : _on_update_delegate.GetInvocationList().Length; int n_subs = _on_update_delegate == null ? 0 : _on_update_delegate.GetInvocationList().Length;
Log.Gen.info("TableAccumulator2.on_update.remove {_tag} unsubscribed n_subs: {n_subs}"); Log.Gen.info("TableAccumulator2.on_update.remove {_tag} unsubscribed n_subs: {n_subs}");
if (_on_update_delegate == null) if (_on_update_delegate == null) {
{
_subscriber_core.unsubscribe(_tag); _subscriber_core.unsubscribe(_tag);
// TODO // TODO
//_table = null; //_table = null;
@@ -63,20 +54,17 @@ namespace BCG
} }
} }
private int _update_no = 0;
public void handle_updates(IEnumerable<byte[]> msgs) public void handle_updates(IEnumerable<byte[]> msgs) {
{
if (msgs == null || msgs.Count() == 0) if (msgs == null || msgs.Count() == 0)
return; return;
Log.Gen.info($"TableAccumulator2.handle_updates _update_no: {_update_no++} msgs: {msgs.Count()} msg_0: {msgs.ElementAt(0).Length}"); Log.Gen.info($"TableAccumulator2.handle_updates msgs: {msgs.Count()} msg_0: {msgs.ElementAt(0).Length}");
IBlueRowCollection table_0 = _table == null ? null : new BlueTable(_table); IBlueRowCollection table_0 = _table == null ? null : new BlueTable(_table);
foreach (var msg in msgs) foreach (var msg in msgs) {
{
bts.StreamIOMessage str_msg = _deser.deserialize(msg); bts.StreamIOMessage str_msg = _deser.deserialize(msg);
var state_txt = _table == null ? "NULL" : _table.Count.ToString();// BlueDataUtility.id_string(_table); var state_txt = _table == null ? "NULL" : _table.Count.ToString();
Log.Gen.info($"TableAccumulator2.handle_update tag: {_tag} sz: {msg.Length} str_msg: {str_msg} _state: {state_txt}"); Log.Gen.info($"TableAccumulator2.handle_update tag: {_tag} sz: {msg.Length} str_msg: {str_msg} _state: {state_txt}");
switch (str_msg.op) switch (str_msg.op) {
{
case bts.StreamIOMessage.StreamOp.CREATE: case bts.StreamIOMessage.StreamOp.CREATE:
_table = new BlueTable(str_msg.create_message.rows); _table = new BlueTable(str_msg.create_message.rows);
break; break;
@@ -89,13 +77,11 @@ namespace BCG
// this could be made faster if we just do this on the CREATE message, and pass-through // this could be made faster if we just do this on the CREATE message, and pass-through
// what we get on the MULTIPLE_REMOVE_ADD // what we get on the MULTIPLE_REMOVE_ADD
IBlueRowCollection d_rows, a_rows; IBlueRowCollection d_rows, a_rows;
//BlueDataUtility.compute_differences_id_based(_table, table_0, out d_rows, out a_rows); BlueDataUtility.compute_differences_id_based(_table, table_0, out d_rows, out a_rows);
BlueDataUtility.compute_differences_fast(_table, table_0, out d_rows, out a_rows);
_on_update_delegate(_table, a_rows, d_rows); _on_update_delegate(_table, a_rows, d_rows);
} }
public void update(Stream[] streams) public void update(Stream[] streams) {
{
} }
#endregion #endregion
@@ -95,8 +95,8 @@ namespace bts {
} }
public static IBlueTableSchema deserialize_schema(ref BinaryReader br) { public static IBlueTableSchema deserialize_schema(ref BinaryReader br) {
// var hex_txt = to_hex(br.BaseStream); var hex_txt = to_hex(br.BaseStream);
// bts.Log.Gen.info($"StreamSerializationUtils.deserialize_schema bytes: {hex_txt}"); bts.Log.Gen.info($"StreamSerializationUtils.deserialize_schema bytes: {hex_txt}");
string table_name = br.read_string_cpp(); string table_name = br.read_string_cpp();
int n_cols = br.ReadInt32(); // doing a br.Read was only advancing the data 1 byte!! int n_cols = br.ReadInt32(); // doing a br.Read was only advancing the data 1 byte!!
List<ColumnInfo> cis = new List<ColumnInfo>(); List<ColumnInfo> cis = new List<ColumnInfo>();
@@ -137,8 +137,8 @@ namespace bts {
public static ColumnInfo deserialize_column_info(ref BinaryReader br) { public static ColumnInfo deserialize_column_info(ref BinaryReader br) {
//col_no, name, type> //col_no, name, type>
// var hex_txt = to_hex(br.BaseStream); var hex_txt = to_hex(br.BaseStream);
// bts.Log.Gen.info($"StreamSerializationUtils.deserialize_column_info bytes: {hex_txt}"); bts.Log.Gen.info($"StreamSerializationUtils.deserialize_column_info bytes: {hex_txt}");
int col_no = br.ReadInt32(); int col_no = br.ReadInt32();
string col_name = br.read_string_cpp(); string col_name = br.read_string_cpp();
int tc_int = br.ReadInt32(); int tc_int = br.ReadInt32();
@@ -212,18 +212,5 @@ namespace bts {
return rv; return rv;
} }
public static string to_hex(byte[] byte_arr, int start_pos, int end_pos) {
var len = end_pos - start_pos;
StringBuilder hex_builder = new StringBuilder(len * 2);
int i = 0;
foreach (byte b in byte_arr) {
if (i >= start_pos && i < end_pos)
hex_builder.Append($"{b:X2} ");
i++;
}
return hex_builder.ToString().Trim();
}
} }
} }
@@ -7,23 +7,12 @@ namespace bts {
// for non stream tags // for non stream tags
public class PubsubServerGenericDataHandler : IPubSubServerTagHandler { public class PubsubServerGenericDataHandler : IPubSubServerTagHandler {
//byte[] _state = null; byte[] _state = null;
private MemoryStream _str = new MemoryStream();
private BinaryWriter _bw;
public PubsubServerGenericDataHandler() {
_bw = new BinaryWriter(_str);
}
public void update(string tag, byte[] msg) { public void update(string tag, byte[] msg) {
_str.Position = 0; _state = msg;
_bw.Write(msg);
Log.Gen.info($"PubsubServerGenericDataHandler.update tag: {tag} msg_len: {msg.Length}");
} }
public byte[] get_content(string tag) { public byte[] get_content(string tag) {
var rv = _str.ToArray(); return _state;
Log.Gen.info($"PubsubServerGenericDataHandler.get_content tag: {tag} rv: {rv.Length}");
return rv;
} }
public string[] supportedTagTypes { get; } public string[] supportedTagTypes { get; }
public void register(InternalPublishDelegate callback) { } public void register(InternalPublishDelegate callback) { }
@@ -44,7 +33,7 @@ namespace bts {
public void update(string tag, byte[] msg) { public void update(string tag, byte[] msg) {
StreamIOMessage str_msg = _deser.deserialize(msg); StreamIOMessage str_msg = _deser.deserialize(msg);
var state_txt = _state == null ? "NULL" : _state.Count.ToString(); var state_txt = _state == null ? "NULL" : _state.Count.ToString();
Log.Gen.info($"PubsubServerSingleStreamTagHandler.update (pre.1) tag: {tag} sz: {msg.Length} _state: {state_txt}"); Log.Gen.info($"PubsubServerSingleStreamTagHandler.update (pre.1) tag: {tag} sz: {msg.Length} str_msg: {str_msg} _state: {state_txt}");
switch (str_msg.op) { switch (str_msg.op) {
case StreamIOMessage.StreamOp.CREATE: case StreamIOMessage.StreamOp.CREATE:
//Log.assert(_state == null); //Log.assert(_state == null);
@@ -56,7 +45,7 @@ namespace bts {
_state.addRows(str_msg.multiple_remove_add_message.added_rows); _state.addRows(str_msg.multiple_remove_add_message.added_rows);
break; break;
} }
Log.Gen.info($"PubsubServerSingleStreamTagHandler.update (post) tag: {tag} sz: {msg.Length} state: {_state.Count}"); Log.Gen.info($"PubsubServerSingleStreamTagHandler.update (post) tag: {tag} sz: {msg.Length} str_msg: {str_msg} state: {_state.Count}");
} }
public byte[] get_content(string tag) { public byte[] get_content(string tag) {
@@ -71,7 +60,7 @@ namespace bts {
}; };
var rv = get_content_(); var rv = get_content_();
var rv_txt = rv == null ? "null" : rv.Length.ToString(); var rv_txt = rv == null ? "null" : rv.Length.ToString();
Log.Gen.info($"PubsubServerSingleStreamTagHandler.get_content rv: {rv_txt}"); Log.Gen.info($"PubsubServerStreamTagHandler.get_content rv: {rv_txt}");
return rv; return rv;
} }
public string[] supportedTagTypes { public string[] supportedTagTypes {
@@ -9,15 +9,7 @@
Letter [a-zA-Z] Letter [a-zA-Z]
BooleanConstant 'true'|'false' BooleanConstant 'true'|'false'
CSStringConstant \"([^\"\\]*|\\.)*\" CSStringConstant \"([^\"\\]*|\\.)*\"
YearInt [0-9]{4} DateTimeConstant ([1-9]|'10'|'11'|'12')\/([1-9]|([1-2][0-9])|'30'|'31')\/[0-9][0-9][0-9]*
MonthInt [0-9]{1,2}
DayInt [0-9]{1,2}
/**
* DateTimeConstant supports American (mm/dd/yyyy) and European (yyyy/mm/dd)
* formats. Month and day can be 1 or 2 digits. It does not validate that the
* numbers represent an actual date.
*/
DateTimeConstant ({MonthInt}\/{DayInt}\/{YearInt}|{YearInt}\/{MonthInt}\/{DayInt})
IntegerConstant -?[0-9]+ IntegerConstant -?[0-9]+
NullConstant 'null' NullConstant 'null'
RealConstant -?[0-9]+\.[0-9]+ RealConstant -?[0-9]+\.[0-9]+
@@ -1,6 +1,6 @@
{ {
"user": "bts", "user": "bts",
"sparkd": "http://192.168.1.80:8000/config", "sparkd": "http://localhost:8000/config",
"pubsub": { "pubsub": {
"milliseconds_between_sub_info_updates": 1000, "milliseconds_between_sub_info_updates": 1000,
"milliseconds_between_time_slice_updates": 5000, "milliseconds_between_time_slice_updates": 5000,
@@ -18,14 +18,14 @@
"publisher_routes": [ "publisher_routes": [
{ {
"name": "default", "name": "default",
"server_address": "bts-cloud://127.0.0.1:12345", "server_address": "bluto://localhost:12345",
"tag": ".*" "tag": ".*"
} }
], ],
"subscriber_routes": [ "subscriber_routes": [
{ {
"name": "default", "name": "default",
"server_address": "bts-cloud://127.0.0.1:12345", "server_address": "bluto://localhost:12345",
"tag": ".*" "tag": ".*"
} }
], ],
@@ -89,8 +89,8 @@
"name": "test_data", "name": "test_data",
"streams": [ "streams": [
{ {
"name": "milliseconds_1000_100", "name": "seconds_1_10",
"expression": "milliseconds(1000,100)", "expression": "second(1,10)",
"is_on_websocket": true "is_on_websocket": true
}, },
{ {
@@ -105,25 +105,5 @@
} }
] ]
} }
// {
// "name": "news_data",
// "streams": [
// {
// "name": "benzinga_news",
// "expression": "benzingaNews(\"mongodb://localhost:27017\",\"local\",\"news_headlines\", \"30s\", \"5d\")",
// "is_on_websocket": true
// },
// {
// "name": "benzinga_news_symbols",
// "expression": "benzingaNewsSymbols(benzinga_news)",
// "is_on_websocket": true
// },
// {
// "name": "benzinga_news_symbol_state",
// "expression": "benzingaNewsSymbolState(benzinga_news)",
// "is_on_websocket": true
// }
// ]
// }
] ]
} }
@@ -6,22 +6,7 @@
"streams": [ "streams": [
{ {
"name": "seconds_1_10", "name": "seconds_1_10",
"expression": "milliseconds(1000,10)", "expression": "second(1,10)",
"is_on_websocket": true
},
{
"name": "benzinga_news",
"expression": "benzingaNews(\"mongodb://localhost:27017\",\"local\",\"news_headlines\", \"30s\", \"5d\")",
"is_on_websocket": true
},
{
"name": "benzinga_news_symbols",
"expression": "benzingaNewsSymbols(benzinga_news)",
"is_on_websocket": true
},
{
"name": "benzinga_news_symbol_state",
"expression": "benzingaNewsSymbolState(benzinga_news)",
"is_on_websocket": true "is_on_websocket": true
}, },
{ {
@@ -37,4 +22,4 @@
] ]
} }
] ]
} }
@@ -10,8 +10,7 @@ using System.IO;
using System.Text.Json; using System.Text.Json;
using System.Text.Json.Serialization; using System.Text.Json.Serialization;
public class DataStreamPublisherConfig public class DataStreamPublisherConfig {
{
/// <summary> /// <summary>
/// The login user for sparkd /// The login user for sparkd
/// </summary> /// </summary>
@@ -39,17 +38,9 @@ public class DataStreamPublisherConfig
[JsonPropertyName("publishers")] [JsonPropertyName("publishers")]
public List<PublisherNode>? Publishers { get; set; } public List<PublisherNode>? Publishers { get; set; }
[JsonPropertyName("mongodb_url")] [JsonPropertyName("simple_echo_client")]
public string MONGO_DB_URL { get; set; } [Option("simple-echo-client", HelpText = "Run a simple echo client (won't publish tables or connect to sparkd)")]
public bool? SimpleEchoClient { get; set; }
[JsonPropertyName("mongodb_db_name")]
public string MONGO_DB_NAME { get; set; }
[JsonPropertyName("mongodb_db_collection")]
public string MONGO_DB_COLLECTION { get; set; }
[JsonPropertyName("benzinga_news_ttl")]
public string BENZINGA_NEWS_TTL { get; set; }
[JsonPropertyName("pubsub")] [JsonPropertyName("pubsub")]
public PubSubConfig? PubSub { get; set; } public PubSubConfig? PubSub { get; set; }
@@ -62,18 +53,18 @@ public class DataStreamPublisherConfig
public string? ConfigFile { get; set; } public string? ConfigFile { get; set; }
public static DataStreamPublisherConfig Default() => public static DataStreamPublisherConfig Default() =>
new() new() {
{ SimpleEchoClient = false,
PubSub = PubSubConfig.Default(), PubSub = PubSubConfig.Default(),
}; };
public void MergeWith(DataStreamPublisherConfig other) public void MergeWith(DataStreamPublisherConfig other) {
{
User ??= other.User; User ??= other.User;
Password ??= other.Password; Password ??= other.Password;
Sparkd ??= other.Sparkd; Sparkd ??= other.Sparkd;
Publishers ??= other.Publishers; Publishers ??= other.Publishers;
PubSub ??= other.PubSub; PubSub ??= other.PubSub;
SimpleEchoClient ??= other.SimpleEchoClient;
ConfigFile ??= other.ConfigFile; ConfigFile ??= other.ConfigFile;
} }
@@ -84,10 +75,8 @@ public class DataStreamPublisherConfig
/// <exception cref="InvalidOperationException"> /// <exception cref="InvalidOperationException">
/// JsonSerializer.Deserialize returns null /// JsonSerializer.Deserialize returns null
/// </exception> /// </exception>
public void MergeWithConfigFile() public void MergeWithConfigFile() {
{ if (ConfigFile is null) {
if (ConfigFile is null)
{
return; return;
} }
var cfgString = File.ReadAllText(ConfigFile); var cfgString = File.ReadAllText(ConfigFile);
@@ -23,10 +23,8 @@ using System.Threading.Tasks;
await Parser.Default.ParseArguments<DataStreamPublisherConfig>(args) await Parser.Default.ParseArguments<DataStreamPublisherConfig>(args)
.MapResult(RunAsync, errs => Task.FromResult(1)); .MapResult(RunAsync, errs => Task.FromResult(1));
async Task<int> RunAsync(DataStreamPublisherConfig cfg) async Task<int> RunAsync(DataStreamPublisherConfig cfg) {
{ try {
try
{
cfg.MergeWithConfigFile(); cfg.MergeWithConfigFile();
cfg.MergeWith(DataStreamPublisherConfig.Default()); cfg.MergeWith(DataStreamPublisherConfig.Default());
PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub)); PubSubConfig.SetGlobalConfig(Log.notnull(cfg.PubSub));
@@ -34,10 +32,12 @@ async Task<int> RunAsync(DataStreamPublisherConfig cfg)
Log.Gen.info("Starting DataStreamPublisher"); Log.Gen.info("Starting DataStreamPublisher");
var service_ctx = ServiceContext.create(cfg.Sparkd); var service_ctx = ServiceContext.create(cfg.Sparkd);
await service_ctx.login(cfg.User ?? "bts", cfg.Password ?? string.Empty); await service_ctx.login(cfg.User ?? "bts", cfg.Password ?? string.Empty);
_publish_streams(cfg.Publishers ?? new(0)); if (cfg.SimpleEchoClient ?? false) {
} start_echo_client();
catch (Exception ex) } else {
{ _publish_streams(cfg.Publishers ?? new(0));
}
} catch (Exception ex) {
// Write the error to console first, in case the logging system is compromised. // Write the error to console first, in case the logging system is compromised.
var error_msg = $"DataStreamPublisher threw exception:\n{ex}"; var error_msg = $"DataStreamPublisher threw exception:\n{ex}";
Console.Error.WriteLine(error_msg); Console.Error.WriteLine(error_msg);
@@ -48,39 +48,41 @@ async Task<int> RunAsync(DataStreamPublisherConfig cfg)
return 0; return 0;
} }
static ViewStream? build_stream(string? stream_expression) static ViewStream? build_stream(string? stream_expression) {
{ if (stream_expression is null) {
// Console.WriteLine("Hit key to continue");
// Console.ReadLine();
if (stream_expression is null)
{
return null; return null;
} }
try try {
{
bts.Log.Gen.info($"build_stream stream_expression: {stream_expression}"); bts.Log.Gen.info($"build_stream stream_expression: {stream_expression}");
var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression); var exp = DataStreamExpressionsInterpreter.Evaluate(stream_expression);
var rv = exp.getView(new Dictionary<string, StatementExpression>(0)); var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
return rv; return rv;
} } catch (Exception ex) {
catch (Exception ex)
{
bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}"); bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
return null; return null;
} }
} }
void _publish_streams(IEnumerable<PublisherNode> publishers) static void start_echo_client() {
{ Console.WriteLine("Starting DSPublisher in TCP client echo mode, Hit key to continue");
Console.ReadLine();
SimpleTCPClient client = new SimpleTCPClient("127.0.0.1", 12345, 1000, 1000);
client.start();
while (true) {
System.Threading.Thread.Sleep(1000);
}
// await SimpleTCPClient.start("127.0.0.1", 12345);
// Task.Delay(1000000000).Wait();
}
void _publish_streams(IEnumerable<PublisherNode> publishers) {
PubTableBroker pb_broker = new PubTableBroker(); PubTableBroker pb_broker = new PubTableBroker();
RemoteQueryManager csm = new RemoteQueryManager(); RemoteQueryManager csm = new RemoteQueryManager();
var csvPublishers = new List<CSVPublisher>(); var csvPublishers = new List<CSVPublisher>();
foreach (var publisher in publishers) foreach (var publisher in publishers) {
{ if (Directory.Exists(publisher.CsvDir)) {
if (Directory.Exists(publisher.CsvDir))
{
csvPublishers.Add(new(publisher.CsvDir, pb_broker)); csvPublishers.Add(new(publisher.CsvDir, pb_broker));
} }
} }
@@ -89,8 +91,7 @@ void _publish_streams(IEnumerable<PublisherNode> publishers)
.SelectMany(p => p.Streams.empty_if_null().Select(createViewPublisher)) .SelectMany(p => p.Streams.empty_if_null().Select(createViewPublisher))
.ToList(); .ToList();
while (true) while (true) {
{
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1)); System.Threading.Thread.Sleep(TimeSpan.FromSeconds(1));
ViewStreamDatabase.instance.doWork(); ViewStreamDatabase.instance.doWork();
csm.doWork(); csm.doWork();
@@ -98,19 +99,16 @@ void _publish_streams(IEnumerable<PublisherNode> publishers)
ManagedSubscriber.instance.doWork(); ManagedSubscriber.instance.doWork();
} }
ViewPublisher? createViewPublisher(PublishedStream ps) ViewPublisher? createViewPublisher(PublishedStream ps) {
{
Log.Gen.info( Log.Gen.info(
$"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}"); $"Instantiating ViewPublisher for: {JsonSerializer.Serialize(ps)}");
if (string.IsNullOrEmpty(ps.Name)) if (string.IsNullOrEmpty(ps.Name)) {
{
Log.Gen.warn( Log.Gen.warn(
$"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}"); $"Failed to instantiate ViewPublisher. Null/empty 'name': {JsonSerializer.Serialize(ps)}");
return null; return null;
} }
var vs = build_stream(ps.Expression); var vs = build_stream(ps.Expression);
if (vs is null) if (vs is null) {
{
Log.Gen.warn( Log.Gen.warn(
$"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}"); $"Failed to instantiate ViewPublisher. Invalid 'expression': {JsonSerializer.Serialize(ps)}");
return null; return null;
@@ -121,39 +119,31 @@ void _publish_streams(IEnumerable<PublisherNode> publishers)
} }
} }
void on_change(object obj) void on_change(object obj) {
{
bts.Log.Gen.info($"DSPublisher on_change dp: {obj}"); bts.Log.Gen.info($"DSPublisher on_change dp: {obj}");
} }
public class CSVPublisher : IWorker public class CSVPublisher : IWorker {
{
private IDictionary<string, ViewPublisher> _pubs = new Dictionary<string, ViewPublisher>(); private IDictionary<string, ViewPublisher> _pubs = new Dictionary<string, ViewPublisher>();
private readonly string _csv_directory; private readonly string _csv_directory;
private PubTableBroker _pub_table_broker; private PubTableBroker _pub_table_broker;
public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) public CSVPublisher(string csv_directory, PubTableBroker pub_table_broker) {
{
_csv_directory = csv_directory; _csv_directory = csv_directory;
_pub_table_broker = pub_table_broker; _pub_table_broker = pub_table_broker;
MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1)); MasterWorker.getInstance(false).addWorker(this, TimeSpan.FromSeconds(1));
} }
public void doWork() public void doWork() {
{
var file_names = Directory.GetFiles(_csv_directory); var file_names = Directory.GetFiles(_csv_directory);
foreach (var fn in file_names) foreach (var fn in file_names) {
{ if (!_pubs.ContainsKey(fn)) {
if (!_pubs.ContainsKey(fn))
{
_handle_new_file(fn); _handle_new_file(fn);
} }
} }
} }
private void _handle_new_file(string file_name) private void _handle_new_file(string file_name) {
{ if (file_name.EndsWith(".csv")) {
if (file_name.EndsWith(".csv"))
{
ViewStream vs = new CSVReaderStream(file_name); ViewStream vs = new CSVReaderStream(file_name);
vs = new SchemaGuesserStream(vs); vs = new SchemaGuesserStream(vs);
var toks = file_name.Split('\\'); var toks = file_name.Split('\\');
@@ -1,6 +1,5 @@
{ {
"user": "bts", "user": "bts",
"sparkd": "http://192.168.1.80:8000/config",
"pubsub": { "pubsub": {
"milliseconds_between_sub_info_updates": 1000, "milliseconds_between_sub_info_updates": 1000,
"milliseconds_between_time_slice_updates": 5000, "milliseconds_between_time_slice_updates": 5000,
@@ -18,14 +17,14 @@
"publisher_routes": [ "publisher_routes": [
{ {
"name": "default", "name": "default",
"server_address": "bts-cloud://192.168.1.80:12345", "server_address": "bluto://localhost:12345",
"tag": ".*" "tag": ".*"
} }
], ],
"subscriber_routes": [ "subscriber_routes": [
{ {
"name": "default", "name": "default",
"server_address": "bts-cloud://192.168.1.80:12345", "server_address": "bluto://localhost:12345",
"tag": ".*" "tag": ".*"
} }
], ],
@@ -59,4 +58,4 @@
"gc_activation_threshold": 0, "gc_activation_threshold": 0,
"wait_period_after_gc": 1000 "wait_period_after_gc": 1000
} }
} }
@@ -25,10 +25,8 @@ using System.Threading.Tasks;
namespace BCG namespace BCG {
{ public partial class MainForm : System.Windows.Forms.Form {
public partial class MainForm : System.Windows.Forms.Form
{
// static async Task _run() { // static async Task _run() {
// var login = "bts"; // var login = "bts";
// var passwd = String.Empty; // var passwd = String.Empty;
@@ -50,8 +48,7 @@ namespace BCG
private string myKQ; private string myKQ;
private TextWriter? myConsoleOutput; private TextWriter? myConsoleOutput;
public MainForm() public MainForm() {
{
//Task.Run(async () => await _run()); //Task.Run(async () => await _run());
InitializeComponent(); InitializeComponent();
// myKQ = ConfigurationRequester.getApplicationQualifier(); // myKQ = ConfigurationRequester.getApplicationQualifier();
@@ -64,40 +61,25 @@ namespace BCG
this.Load += new EventHandler(myLoad); this.Load += new EventHandler(myLoad);
this.Closing += new CancelEventHandler(myClosing); this.Closing += new CancelEventHandler(myClosing);
myQueryTextBox.TextChanged += (o, e) => myQueryTextBox.TextChanged += (o, e) => {
{
myQueryTextBox.ForeColor = Color.White; myQueryTextBox.ForeColor = Color.White;
}; };
} }
public string query_text public string query_text {
{
get { return myQueryTextBox.Text; } get { return myQueryTextBox.Text; }
set { myQueryTextBox.Text = value; } set { myQueryTextBox.Text = value; }
} }
private ViewStream myGetAllTagsStream() private ViewStream myGetAllTagsStream() {
{
// TODO: Port PublishedTagsStream from edge (then we can uncomment impl below) // TODO: Port PublishedTagsStream from edge (then we can uncomment impl below)
//ViewStream vs = new PubSubServerStateStream(); //ViewStream vs = new PubSubServerStateStream();
ViewStream vs = new PubSubServerStateStream(); ViewStream vs = new PubSubServerStateStream();
return new FilterStream(vs, "tag~='DS_'"); return new FilterStream(vs, "tag~='DS_'");
//return new LiteralStream("tag/String", "DS_table1;DS_table2"); //return new LiteralStream("tag/String", "DS_Comedians;DS_Baldies");
} }
private void mySetText() private void mySetText() {
{ this.Text = "Data Stream Query Analyzer | Site: " + mySite + (string.IsNullOrEmpty(myKQ) ? string.Empty : " | Key Qualifier: " + myKQ);
ViewStream routes = new PubsubRoutesStream();
routes.onChange += (o) => { };
var rows = routes.rows;
string route_txt = String.Empty;
foreach (IBlueRow r in rows)
{
var tag_regexp = r.get<string>("tag_regexp");
var server_url = r.get<string>("server_url");
route_txt += $"{tag_regexp} : {server_url},";
}
this.Text = $"Data Stream Query Analyzer | routes: {route_txt}";
myToolStripStatusLabelTime.Text = "Ready"; myToolStripStatusLabelTime.Text = "Ready";
myToolStripStatusLabelRows.Text = string.Empty; myToolStripStatusLabelRows.Text = string.Empty;
} }
@@ -106,12 +88,10 @@ namespace BCG
/// Gets the data in the form of a multilevel IDictionary /// Gets the data in the form of a multilevel IDictionary
/// </summary> /// </summary>
/// <returns></returns> /// <returns></returns>
private IDictionary myGetLibraryData() private IDictionary myGetLibraryData() {
{
SortedList retVal = new SortedList(); SortedList retVal = new SortedList();
string[] primitiveNames = PrimitiveManager.instance.getPrimitiveNames(); string[] primitiveNames = PrimitiveManager.instance.getPrimitiveNames();
foreach (string pName in primitiveNames) foreach (string pName in primitiveNames) {
{
string aName = PrimitiveManager.instance.getAssemblyName(pName); string aName = PrimitiveManager.instance.getAssemblyName(pName);
ICollection cNames = PrimitiveManager.instance.getConstructorSigns(pName); ICollection cNames = PrimitiveManager.instance.getConstructorSigns(pName);
if (!retVal.Contains(aName)) if (!retVal.Contains(aName))
@@ -126,15 +106,12 @@ namespace BCG
} }
private void myLoadAssemblies() private void myLoadAssemblies() {
{
myTreeView.Nodes.Clear(); myTreeView.Nodes.Clear();
IDictionary libraryData = myGetLibraryData(); IDictionary libraryData = myGetLibraryData();
foreach (DictionaryEntry de in libraryData) foreach (DictionaryEntry de in libraryData) {
{
TreeNode assemblyNode = new TreeNode(de.Key as String); TreeNode assemblyNode = new TreeNode(de.Key as String);
foreach (DictionaryEntry innerDe in (de.Value as IDictionary)) foreach (DictionaryEntry innerDe in (de.Value as IDictionary)) {
{
TreeNode primitiveNode = new TreeNode(innerDe.Key as string); TreeNode primitiveNode = new TreeNode(innerDe.Key as string);
foreach (string con in (innerDe.Value as ICollection)) foreach (string con in (innerDe.Value as ICollection))
primitiveNode.Nodes.Add(con); primitiveNode.Nodes.Add(con);
@@ -154,29 +131,23 @@ namespace BCG
}*/ }*/
} }
private void myHandleTableListChange(object o) private void myHandleTableListChange(object o) {
{
IBlueRowCollection rows = (o as DataPacket).newRows; IBlueRowCollection rows = (o as DataPacket).newRows;
myUpdateTableList(); myUpdateTableList();
} }
#region Persistence Keys #region Persistence Keys
public int textBoxHeight public int textBoxHeight {
{
get { return myQueryTextBox.Height; } get { return myQueryTextBox.Height; }
set { myQueryTextBox.Height = value; } set { myQueryTextBox.Height = value; }
} }
#endregion #endregion
private Stopwatch _stopwatch = new Stopwatch(); private void myTryExpression(string expStr) {
private void myTryExpression(string expStr)
{
_stopwatch.Restart();
myFreezeView(false); // force unfreeze the view myFreezeView(false); // force unfreeze the view
DataStreamExpression oldExp = myExp; DataStreamExpression oldExp = myExp;
try try {
{
var exp = DataStreamExpressionsInterpreter.Evaluate(expStr); var exp = DataStreamExpressionsInterpreter.Evaluate(expStr);
if (exp == null) if (exp == null)
@@ -202,9 +173,7 @@ namespace BCG
oldExp.getView(new Hashtable()).onChange -= new DataStream.ChangeDelegate(myCallback); oldExp.getView(new Hashtable()).onChange -= new DataStream.ChangeDelegate(myCallback);
*/ */
//myQueryTextBox.Text = expStr; //myQueryTextBox.Text = expStr;
} } catch (Exception exc) {
catch (Exception exc)
{
Log.Ui.warn(exc.ToString()); Log.Ui.warn(exc.ToString());
myDataGrid.DataSource = null; myDataGrid.DataSource = null;
@@ -218,8 +187,7 @@ namespace BCG
} }
} }
private string myExtractFromClass(TextReader r) private string myExtractFromClass(TextReader r) {
{
string line = r.ReadLine(); string line = r.ReadLine();
while (line != null && line.IndexOf("*Expression*") == -1) while (line != null && line.IndexOf("*Expression*") == -1)
line = r.ReadLine(); line = r.ReadLine();
@@ -232,8 +200,7 @@ namespace BCG
string result = ""; string result = "";
line = r.ReadLine(); line = r.ReadLine();
while (line != null && line.IndexOf("----------------------------------------------------------------------") == -1) while (line != null && line.IndexOf("----------------------------------------------------------------------") == -1) {
{
if (line != "") if (line != "")
result += line.Substring(" ".Length) + "\n"; result += line.Substring(" ".Length) + "\n";
line = r.ReadLine(); line = r.ReadLine();
@@ -244,24 +211,17 @@ namespace BCG
return result; return result;
} }
public void loadFile(string fileName) public void loadFile(string fileName) {
{ try {
try
{
TextReader r = new StreamReader(fileName); TextReader r = new StreamReader(fileName);
try try {
{
myQueryTextBox.Text = myExtractFromClass(r); myQueryTextBox.Text = myExtractFromClass(r);
} } finally {
finally
{
r.Close(); r.Close();
} }
myLastFileName = fileName; myLastFileName = fileName;
this.Text = fileName; this.Text = fileName;
} } catch (Exception exc) {
catch (Exception exc)
{
Log.Ui.warn(exc.ToString()); Log.Ui.warn(exc.ToString());
myDataGrid.DataSource = null; myDataGrid.DataSource = null;
@@ -272,8 +232,7 @@ namespace BCG
} }
private void myPrintCode(IBlueTableSchema schema, ICollection vars, string code) private void myPrintCode(IBlueTableSchema schema, ICollection vars, string code) {
{
myCSharpText.Text = "using System;\n\nnamespace BCG\n{"; myCSharpText.Text = "using System;\n\nnamespace BCG\n{";
@@ -293,8 +252,7 @@ namespace BCG
string lastVar = null; string lastVar = null;
string variableDeclarations = ""; string variableDeclarations = "";
foreach (string var in vars) foreach (string var in vars) {
{
variableDeclarations += " ViewStream " + var + ";\n"; variableDeclarations += " ViewStream " + var + ";\n";
lastVar = var; lastVar = var;
} }
@@ -324,16 +282,13 @@ namespace BCG
myCSharpText.Text += "}"; myCSharpText.Text += "}";
} }
static DataGridViewColumn _column_factory(ColumnInfo ci) static DataGridViewColumn _column_factory(ColumnInfo ci) {
{ return new DataGridViewTextBoxColumn {
return new DataGridViewTextBoxColumn
{
HeaderText = ci.colName, HeaderText = ci.colName,
Width = get_col_width(ci) Width = get_col_width(ci)
}; };
static int get_col_width(ColumnInfo ci) static int get_col_width(ColumnInfo ci) {
{
const int CHAR_WIDTH = 10; const int CHAR_WIDTH = 10;
const int MIN_WIDTH = 50; const int MIN_WIDTH = 50;
const int MIN_DATE_WIDTH = 200; const int MIN_DATE_WIDTH = 200;
@@ -344,15 +299,12 @@ namespace BCG
} }
// this is the stream callback // this is the stream callback
private void myCallback(object o) private void myCallback(object o) {
{
_stopwatch.Stop();
var timer = Stopwatch.StartNew(); var timer = Stopwatch.StartNew();
DataPacket dp = o as DataPacket; DataPacket dp = o as DataPacket;
myCrono.Stop(); myCrono.Stop();
Log.Ui.diag($"Received callback for {dp.schema.tableName}"); Log.Ui.diag($"Received callback for {dp.schema.tableName}");
if (o == null) if (o == null) {
{
System.Windows.Forms.MessageBox.Show(this, "Received a Null Data packet", System.Windows.Forms.MessageBox.Show(this, "Received a Null Data packet",
"Invalid Data", MessageBoxButtons.OK, MessageBoxIcon.Error); "Invalid Data", MessageBoxButtons.OK, MessageBoxIcon.Error);
return; return;
@@ -362,16 +314,14 @@ namespace BCG
myToolStripStatusLabelRows.BackColor = Color.Black; myToolStripStatusLabelRows.BackColor = Color.Black;
myToolStripStatusLabelTime.ForeColor = Color.White; myToolStripStatusLabelTime.ForeColor = Color.White;
myToolStripStatusLabelRows.ForeColor = Color.White; myToolStripStatusLabelRows.ForeColor = Color.White;
string time_text = "Last update: " + DateTime.Now.ToString("hh:mm:ss:fff tt") + $" time: {_stopwatch.ElapsedMilliseconds} msecs\t\t"; myToolStripStatusLabelTime.Text = "Last update: " + DateTime.Now.ToString("hh:mm:ss:fff tt");
myToolStripStatusLabelTime.Text = time_text;
myToolStripStatusLabelRows.Text = "New = " + dp.newRows.Count + " Added = " + dp.addedRows.Count + " Deleted = " + dp.deletedRows.Count; myToolStripStatusLabelRows.Text = "New = " + dp.newRows.Count + " Added = " + dp.addedRows.Count + " Deleted = " + dp.deletedRows.Count;
Log.Ui.diag(Text); Log.Ui.diag(Text);
int hscrollval = myChangeStyle ? 0 : myDataGrid.HorizScrollBarValue; int hscrollval = myChangeStyle ? 0 : myDataGrid.HorizScrollBarValue;
myDataGrid.DataSource = new SimpleBindableCollection(dp.newRows.schema, dp.newRows); myDataGrid.DataSource = new SimpleBindableCollection(dp.newRows.schema, dp.newRows);
var force = false; var force = false;
if (myChangeStyle && force) if (myChangeStyle && force) {
{
myChangeStyle = false; myChangeStyle = false;
myDataGrid.Columns.Clear(); myDataGrid.Columns.Clear();
myDataGrid.Columns.AddRange(Array.ConvertAll(dp.schema.columnInfo, _column_factory)); myDataGrid.Columns.AddRange(Array.ConvertAll(dp.schema.columnInfo, _column_factory));
@@ -398,8 +348,7 @@ namespace BCG
private DataStreamExpression? myExp; private DataStreamExpression? myExp;
private void myTimer_Tick(object sender, System.EventArgs e) private void myTimer_Tick(object sender, System.EventArgs e) {
{
// try // try
// { // {
ViewStreamDatabase.instance.doWork(); ViewStreamDatabase.instance.doWork();
@@ -416,27 +365,22 @@ namespace BCG
// } // }
} }
class StringWrapper class StringWrapper {
{ public StringWrapper(string s) {
public StringWrapper(string s)
{
myTableName = s; myTableName = s;
} }
public string tableName public string tableName {
{
get { return myTableName; } get { return myTableName; }
} }
private string myTableName; private string myTableName;
} }
private void MainForm_Closed(object sender, System.EventArgs e) private void MainForm_Closed(object sender, System.EventArgs e) {
{
if (myExp != null) if (myExp != null)
myExp.getView(new Dictionary<string, StatementExpression>(0)).onChange -= new DataStream.ChangeDelegate(myCallback); myExp.getView(new Dictionary<string, StatementExpression>(0)).onChange -= new DataStream.ChangeDelegate(myCallback);
} }
private void myMenuItemExecuteQueryClick(object sender, System.EventArgs e) private void myMenuItemExecuteQueryClick(object sender, System.EventArgs e) {
{
myQueryTextBox.SelectionStart = myQueryTextBox.Text.Length; myQueryTextBox.SelectionStart = myQueryTextBox.Text.Length;
myQueryTextBox.SelectionLength = myQueryTextBox.Text.Length; myQueryTextBox.SelectionLength = myQueryTextBox.Text.Length;
myQueryTextBox.SelectionColor = Color.White; myQueryTextBox.SelectionColor = Color.White;
@@ -444,8 +388,7 @@ namespace BCG
myTryExpression(myQueryTextBox.Text); myTryExpression(myQueryTextBox.Text);
} }
private void myToolStripMenuItemExecSel_Click(object sender, EventArgs e) private void myToolStripMenuItemExecSel_Click(object sender, EventArgs e) {
{
// int selstart = myQueryTextBox.SelectionStart; // int selstart = myQueryTextBox.SelectionStart;
// int sellength = myQueryTextBox.SelectionLength; // int sellength = myQueryTextBox.SelectionLength;
// myQueryTextBox.SelectionStart = 0; // myQueryTextBox.SelectionStart = 0;
@@ -460,8 +403,7 @@ namespace BCG
// myQueryTextBox.SelectionColor = Color.White; // myQueryTextBox.SelectionColor = Color.White;
} }
private void myMenuItemOpenClick(object sender, System.EventArgs e) private void myMenuItemOpenClick(object sender, System.EventArgs e) {
{
OpenFileDialog od = new OpenFileDialog(); OpenFileDialog od = new OpenFileDialog();
od.Filter = "C# class (*.cs)|*.cs"; od.Filter = "C# class (*.cs)|*.cs";
@@ -469,24 +411,21 @@ namespace BCG
loadFile(od.FileName); loadFile(od.FileName);
} }
private void mySave(string fileName) private void mySave(string fileName) {
{
using (TextWriter w = new StreamWriter(fileName)) using (TextWriter w = new StreamWriter(fileName))
w.Write(myCSharpText.Text); w.Write(myCSharpText.Text);
myLastFileName = fileName; myLastFileName = fileName;
this.Text = fileName; this.Text = fileName;
} }
private void myMenuItemSaveClick(object sender, System.EventArgs e) private void myMenuItemSaveClick(object sender, System.EventArgs e) {
{
if (myLastFileName == null) if (myLastFileName == null)
myMenuItemSaveAsClick(sender, e); myMenuItemSaveAsClick(sender, e);
else else
mySave(myLastFileName); mySave(myLastFileName);
} }
private void myMenuItemSaveAsClick(object sender, System.EventArgs e) private void myMenuItemSaveAsClick(object sender, System.EventArgs e) {
{
SaveFileDialog sd = new SaveFileDialog(); SaveFileDialog sd = new SaveFileDialog();
if (myLastFileName != null) if (myLastFileName != null)
sd.FileName = myLastFileName; sd.FileName = myLastFileName;
@@ -497,56 +436,47 @@ namespace BCG
mySave(sd.FileName); mySave(sd.FileName);
} }
private void myTableListBox_DoubleClick(object sender, System.EventArgs e) private void myTableListBox_DoubleClick(object sender, System.EventArgs e) {
{ if (myTableListBox.SelectedItem != null) {
if (myTableListBox.SelectedItem != null)
{
string tableName = myTableListBox.SelectedItem.ToString(); string tableName = myTableListBox.SelectedItem.ToString();
myQueryTextBox.Text = myQueryTextBox.Text == "" ? tableName : myQueryTextBox.Text + "\n" + tableName; myQueryTextBox.Text = myQueryTextBox.Text == "" ? tableName : myQueryTextBox.Text + "\n" + tableName;
myTryExpression(myQueryTextBox.Text); myTryExpression(myQueryTextBox.Text);
} }
} }
private void myDataGrid_MouseEnter(object sender, System.EventArgs e) private void myDataGrid_MouseEnter(object sender, System.EventArgs e) {
{
//myDataGrid.TableStyles[0].GridLineColor = Color.LightSteelBlue; //myDataGrid.TableStyles[0].GridLineColor = Color.LightSteelBlue;
// if (myDataGrid.TableStyles.Count > 0) // if (myDataGrid.TableStyles.Count > 0)
// myDataGrid.TableStyles[0].HeaderBackColor = Color.LightSteelBlue; // myDataGrid.TableStyles[0].HeaderBackColor = Color.LightSteelBlue;
} }
private void myDataGrid_MouseLeave(object sender, System.EventArgs e) private void myDataGrid_MouseLeave(object sender, System.EventArgs e) {
{
//myDataGrid.TableStyles[0].GridLineColor = Color.Gainsboro; //myDataGrid.TableStyles[0].GridLineColor = Color.Gainsboro;
// if (myDataGrid.TableStyles.Count > 0) // if (myDataGrid.TableStyles.Count > 0)
// myDataGrid.TableStyles[0].HeaderBackColor = Color.Gainsboro; // myDataGrid.TableStyles[0].HeaderBackColor = Color.Gainsboro;
} }
private void myCSharpText_MouseEnter(object sender, System.EventArgs e) private void myCSharpText_MouseEnter(object sender, System.EventArgs e) {
{
//myCSharpText.BackColor = Color.Gainsboro; //myCSharpText.BackColor = Color.Gainsboro;
myCSharpText.Font = new Font(myCSharpText.Font.Name, myCSharpText.Font.Size, FontStyle.Bold); myCSharpText.Font = new Font(myCSharpText.Font.Name, myCSharpText.Font.Size, FontStyle.Bold);
} }
private void myCSharpText_MouseLeave(object sender, System.EventArgs e) private void myCSharpText_MouseLeave(object sender, System.EventArgs e) {
{
//myCSharpText.BackColor = Color.White; //myCSharpText.BackColor = Color.White;
myCSharpText.Font = new Font(myCSharpText.Font.Name, myCSharpText.Font.Size, FontStyle.Regular); myCSharpText.Font = new Font(myCSharpText.Font.Name, myCSharpText.Font.Size, FontStyle.Regular);
} }
private void myTableListBox_MouseEnter(object sender, System.EventArgs e) private void myTableListBox_MouseEnter(object sender, System.EventArgs e) {
{
//myTableListBox.BackColor = Color.Gainsboro; //myTableListBox.BackColor = Color.Gainsboro;
myTableListBox.Font = new Font(myTableListBox.Font.Name, myTableListBox.Font.Size, FontStyle.Bold); myTableListBox.Font = new Font(myTableListBox.Font.Name, myTableListBox.Font.Size, FontStyle.Bold);
} }
private void myTableListBox_MouseLeave(object sender, System.EventArgs e) private void myTableListBox_MouseLeave(object sender, System.EventArgs e) {
{
//myTableListBox.BackColor = Color.White; //myTableListBox.BackColor = Color.White;
myTableListBox.Font = new Font(myTableListBox.Font.Name, myTableListBox.Font.Size, FontStyle.Regular); myTableListBox.Font = new Font(myTableListBox.Font.Name, myTableListBox.Font.Size, FontStyle.Regular);
} }
private void myLoad(object sender, EventArgs e) private void myLoad(object sender, EventArgs e) {
{
myFreezeView(false); myFreezeView(false);
myToolStripMenuItemSite.DropDownItems.Clear(); myToolStripMenuItemSite.DropDownItems.Clear();
@@ -570,9 +500,7 @@ namespace BCG
{ {
var ex = new Win32Exception(); // same as new Win32Exception(Marshal.GetLastWin32Error()) var ex = new Win32Exception(); // same as new Win32Exception(Marshal.GetLastWin32Error())
Console.WriteLine(ex.Message); Console.WriteLine(ex.Message);
} } else {
else
{
Console.WriteLine("Set stdout worked."); Console.WriteLine("Set stdout worked.");
} }
@@ -580,9 +508,7 @@ namespace BCG
{ {
var ex = new Win32Exception(); // same as new Win32Exception(Marshal.GetLastWin32Error()) var ex = new Win32Exception(); // same as new Win32Exception(Marshal.GetLastWin32Error())
Console.WriteLine(ex.Message); Console.WriteLine(ex.Message);
} } else {
else
{
Console.WriteLine("Set stderr worked."); Console.WriteLine("Set stderr worked.");
} }
Console.WriteLine("Redirecting console ouput to this window..."); Console.WriteLine("Redirecting console ouput to this window...");
@@ -598,12 +524,10 @@ namespace BCG
//_library = new LibraryForm(this); //_library = new LibraryForm(this);
} }
private void myMenuItemSiteClick(object sender, System.EventArgs e) private void myMenuItemSiteClick(object sender, System.EventArgs e) {
{
ToolStripMenuItem mi = (ToolStripMenuItem)sender; ToolStripMenuItem mi = (ToolStripMenuItem)sender;
if (mi.Text != mySite) if (mi.Text != mySite) {
{
mySite = mi.Text; mySite = mi.Text;
foreach (var m in mySiteItems) foreach (var m in mySiteItems)
m.Checked = false; m.Checked = false;
@@ -624,78 +548,63 @@ namespace BCG
} }
} }
private void myClosing(object sender, CancelEventArgs e) private void myClosing(object sender, CancelEventArgs e) {
{
} }
private void myHandleTreeNodeSelect(object sender, TreeViewEventArgs e) private void myHandleTreeNodeSelect(object sender, TreeViewEventArgs e) {
{ if (myTreeView.SelectedNode != null) {
if (myTreeView.SelectedNode != null) if (myTreeView.SelectedNode.LastNode == null) {
{
if (myTreeView.SelectedNode.LastNode == null)
{
string sn = myTreeView.SelectedNode.Parent.Text + "()"; string sn = myTreeView.SelectedNode.Parent.Text + "()";
myQueryTextBox.Text = myQueryTextBox.Text == "" ? sn : myQueryTextBox.Text + "\n" + sn; myQueryTextBox.Text = myQueryTextBox.Text == "" ? sn : myQueryTextBox.Text + "\n" + sn;
} }
} }
} }
private void myToolStripMenuItemExit_Click(object sender, EventArgs e) private void myToolStripMenuItemExit_Click(object sender, EventArgs e) {
{
Close(); Close();
} }
private void myToolStripMenuItemFreeze_Click(object sender, EventArgs e) private void myToolStripMenuItemFreeze_Click(object sender, EventArgs e) {
{
myFreezeView(myTimer.Enabled); myFreezeView(myTimer.Enabled);
} }
private void myFreezeView(bool freeze) private void myFreezeView(bool freeze) {
{
myTimer.Enabled = !freeze; myTimer.Enabled = !freeze;
myToolStripMenuItemFreeze.Checked = freeze; myToolStripMenuItemFreeze.Checked = freeze;
myToolStripMenuItemStep.Enabled = freeze; myToolStripMenuItemStep.Enabled = freeze;
} }
private void myToolStripMenuItemStep_Click(object sender, EventArgs e) private void myToolStripMenuItemStep_Click(object sender, EventArgs e) {
{
ViewStreamDatabase.instance.doWork(); ViewStreamDatabase.instance.doWork();
} }
private void myToolStripMenuItemLoadAss_Click(object sender, EventArgs e) private void myToolStripMenuItemLoadAss_Click(object sender, EventArgs e) {
{
OpenFileDialog ofd = new OpenFileDialog(); OpenFileDialog ofd = new OpenFileDialog();
ofd.ShowReadOnly = true; ofd.ShowReadOnly = true;
DialogResult dr = ofd.ShowDialog(this); DialogResult dr = ofd.ShowDialog(this);
if (ofd.CheckFileExists && ofd.CheckPathExists && dr == DialogResult.OK) if (ofd.CheckFileExists && ofd.CheckPathExists && dr == DialogResult.OK) {
{
PrimitiveManager.instance.loadAssembly(ofd.FileName); PrimitiveManager.instance.loadAssembly(ofd.FileName);
myLoadAssemblies(); myLoadAssemblies();
} }
} }
private void myToolStripMenuItemTest_Click(object sender, EventArgs e) private void myToolStripMenuItemTest_Click(object sender, EventArgs e) {
{ if (myCurrentStream != null) {
if (myCurrentStream != null)
{
myCurrentStream.onChange -= new DataStream.ChangeDelegate(myCallback); myCurrentStream.onChange -= new DataStream.ChangeDelegate(myCallback);
myCurrentStream.onChange += new DataStream.ChangeDelegate(myCallback); myCurrentStream.onChange += new DataStream.ChangeDelegate(myCallback);
} }
} }
private void myToolStripMenuItemClearConsole_Click(object sender, EventArgs e) private void myToolStripMenuItemClearConsole_Click(object sender, EventArgs e) {
{
myConsoleRichTextBox.Clear(); myConsoleRichTextBox.Clear();
Console.WriteLine("Redirecting console ouput to this window..."); Console.WriteLine("Redirecting console ouput to this window...");
} }
private void myTableFilterBox_TextChanged(object sender, EventArgs e) private void myTableFilterBox_TextChanged(object sender, EventArgs e) {
{
myUpdateTableList(); myUpdateTableList();
} }
private void myUpdateTableList() private void myUpdateTableList() {
{
IBlueRowCollection rows = myAllTagsStream.rows; IBlueRowCollection rows = myAllTagsStream.rows;
if (rows == null) if (rows == null)
return; return;
@@ -710,15 +619,12 @@ namespace BCG
myTableListBox.Items.Clear(); myTableListBox.Items.Clear();
SortedList<string, bool> sl = new SortedList<string, bool>(); SortedList<string, bool> sl = new SortedList<string, bool>();
if (string.IsNullOrEmpty(myTableFilterBox.Text)) if (string.IsNullOrEmpty(myTableFilterBox.Text))
foreach (IBlueRow r in rows) foreach (IBlueRow r in rows) {
{
sl[r["tag"].ToString().Substring(DSTagManager.get_stream_tag("").Length)] = true; sl[r["tag"].ToString().Substring(DSTagManager.get_stream_tag("").Length)] = true;
} }
else else {
{
Regex tableRegex = new Regex(myTableFilterBox.Text); Regex tableRegex = new Regex(myTableFilterBox.Text);
foreach (IBlueRow r in rows) foreach (IBlueRow r in rows) {
{
string thisTable = r["tag"].ToString().Substring(DSTagManager.get_stream_tag("").Length); string thisTable = r["tag"].ToString().Substring(DSTagManager.get_stream_tag("").Length);
if (tableRegex.IsMatch(thisTable)) if (tableRegex.IsMatch(thisTable))
sl[thisTable] = true; sl[thisTable] = true;
@@ -736,8 +642,7 @@ namespace BCG
myTableListBox.EndUpdate(); myTableListBox.EndUpdate();
} }
private void testGridToolStripMenuItem_Click(object sender, EventArgs e) private void testGridToolStripMenuItem_Click(object sender, EventArgs e) {
{
var timer = new Stopwatch(); var timer = new Stopwatch();
//while (true) //while (true)
//{ //{
@@ -759,11 +664,9 @@ namespace BCG
foreach (IBlueRow r in s.rows) foreach (IBlueRow r in s.rows)
rows.Add(r); rows.Add(r);
for (int i = 0; i < 100; i++) for (int i = 0; i < 100; i++) {
{
timer.Restart(); timer.Restart();
rows.Sort(delegate (IBlueRow r1, IBlueRow r2) rows.Sort(delegate (IBlueRow r1, IBlueRow r2) {
{
return _sort_dicts_index(r1, r2, 5, true); return _sort_dicts_index(r1, r2, 5, true);
}); });
timer.Stop(); timer.Stop();
@@ -771,22 +674,19 @@ namespace BCG
} }
} }
private static int mySortDictionaries(IBlueRow r1, IBlueRow r2, string column, bool low_to_high) private static int mySortDictionaries(IBlueRow r1, IBlueRow r2, string column, bool low_to_high) {
{
if (r1.Contains(column) && r2.Contains(column)) if (r1.Contains(column) && r2.Contains(column))
return compare_send_dontcare_to_last(r1[column], r2[column], low_to_high); return compare_send_dontcare_to_last(r1[column], r2[column], low_to_high);
else else
return 0; return 0;
} }
private static int _sort_dicts_index(IBlueRow r1, IBlueRow r2, int col_no, bool low_to_high) private static int _sort_dicts_index(IBlueRow r1, IBlueRow r2, int col_no, bool low_to_high) {
{
//return compare_send_dontcare_to_last(r1[col_no], r2[col_no], low_to_high); //return compare_send_dontcare_to_last(r1[col_no], r2[col_no], low_to_high);
return ((IComparable)r1[col_no]).CompareTo(r2[col_no]); return ((IComparable)r1[col_no]).CompareTo(r2[col_no]);
} }
internal static int compare_send_dontcare_to_last(object o1, object o2, bool low_to_high) internal static int compare_send_dontcare_to_last(object o1, object o2, bool low_to_high) {
{
if (myIsDontCare(o1)) if (myIsDontCare(o1))
return myIsDontCare(o2) ? 0 : 1; return myIsDontCare(o2) ? 0 : 1;
else else
@@ -794,16 +694,14 @@ namespace BCG
: ((IComparable)o2).CompareTo(o1)); : ((IComparable)o2).CompareTo(o1));
} }
private static bool myIsDontCare(object o) private static bool myIsDontCare(object o) {
{
return o == null || o == DBNull.Value || return o == null || o == DBNull.Value ||
o is string && String.IsNullOrEmpty((string)o) || o is string && String.IsNullOrEmpty((string)o) ||
o is double && (Double.IsInfinity((double)o) || Double.IsNaN((double)o)) || o is double && (Double.IsInfinity((double)o) || Double.IsNaN((double)o)) ||
o is float && (Single.IsInfinity((float)o) || Single.IsNaN((float)o)); o is float && (Single.IsInfinity((float)o) || Single.IsNaN((float)o));
} }
private void _new_grid_form(object args) private void _new_grid_form(object args) {
{
//System.Timers.Timer t = new System.Timers.Timer(); //System.Timers.Timer t = new System.Timers.Timer();
////t.SynchronizingObject ////t.SynchronizingObject
//BCG.Utilities.BlueEventLog.logEvent(String.Format("New BTSTabularGridTestForm created on thread {0}", System.Threading.Thread.CurrentThread.ManagedThreadId), //BCG.Utilities.BlueEventLog.logEvent(String.Format("New BTSTabularGridTestForm created on thread {0}", System.Threading.Thread.CurrentThread.ManagedThreadId),
@@ -815,13 +713,11 @@ namespace BCG
private void libraryToolStripMenuItem_Click(object sender, EventArgs e) { } private void libraryToolStripMenuItem_Click(object sender, EventArgs e) { }
private void myTableFilterLabel_Click(object sender, EventArgs e) private void myTableFilterLabel_Click(object sender, EventArgs e) {
{
} }
private void MainForm_Load(object sender, EventArgs e) private void MainForm_Load(object sender, EventArgs e) {
{
} }
} }
-6
View File
@@ -36,8 +36,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnExpressions", "Bt
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}" Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bts.ColumnSortExpressions", "Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj", "{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}"
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataStreamQueryAnalyzer", "DataStreamQueryAnalyzer\DataStreamQueryAnalyzer.csproj", "{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}"
EndProject
Global Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU Debug|Any CPU = Debug|Any CPU
@@ -115,9 +113,5 @@ Global
{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Debug|Any CPU.Build.0 = Debug|Any CPU {B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Debug|Any CPU.Build.0 = Debug|Any CPU
{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Release|Any CPU.ActiveCfg = Release|Any CPU {B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Release|Any CPU.ActiveCfg = Release|Any CPU
{B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Release|Any CPU.Build.0 = Release|Any CPU {B038CB72-4E81-4AF1-976F-ED8655C2C3D6}.Release|Any CPU.Build.0 = Release|Any CPU
{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}.Debug|Any CPU.Build.0 = Debug|Any CPU
{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}.Release|Any CPU.ActiveCfg = Release|Any CPU
{BFA5BA5C-927A-41A9-A696-1E2F4B5FFD15}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection EndGlobalSection
EndGlobal EndGlobal
@@ -1,77 +1,77 @@
// using System; using System;
// using System.Collections.Generic; using System.Collections.Generic;
// using System.Text; using System.Text;
// //using BCG.Utilities; //using BCG.Utilities;
// using System.Net; using System.Net;
// namespace BCG namespace BCG
// { {
// public class BlutoSubAddress : SubAddress public class BlutoSubAddress : SubAddress
// { {
// private string myIp; private string myIp;
// private int myPort; private int myPort;
// public static string prefix = "bluto"; public static string prefix = "bluto";
// public BlutoSubAddress(string ip, int port) public BlutoSubAddress(string ip, int port)
// { {
// myIp = ip; myIp = ip;
// myPort = port; myPort = port;
// } }
// public BlutoSubAddress(string serverAddress) public BlutoSubAddress(string serverAddress)
// { {
// string[] toks = serverAddress.Split(':'); string[] toks = serverAddress.Split(':');
// // TODO // TODO
// // BlueEventLog.logAssert(toks.Length == 3); // BlueEventLog.logAssert(toks.Length == 3);
// // BlueEventLog.logAssert(toks[0].ToLower().StartsWith(prefix)); // BlueEventLog.logAssert(toks[0].ToLower().StartsWith(prefix));
// string machineNameOrIP = toks[1].Remove(0, 2); string machineNameOrIP = toks[1].Remove(0, 2);
// if (machineNameOrIP.Split('.').Length == 4) if (machineNameOrIP.Split('.').Length == 4)
// myIp = toks[1].Remove(0, 2); myIp = toks[1].Remove(0, 2);
// else else
// { {
// //get the IP address from DNS //get the IP address from DNS
// IPAddress[] theAddressList = Dns.GetHostEntry(machineNameOrIP).AddressList; IPAddress[] theAddressList = Dns.GetHostEntry(machineNameOrIP).AddressList;
// //BlueEventLog.logAssert(theAddressList.Length == 1, "Error: machine has multiple ip addresses."); //BlueEventLog.logAssert(theAddressList.Length == 1, "Error: machine has multiple ip addresses.");
// //if we get multiple addresses, we want the one that is of the corret dotted notation //if we get multiple addresses, we want the one that is of the corret dotted notation
// bool oneWorked = false; bool oneWorked = false;
// foreach (IPAddress addr in theAddressList) foreach (IPAddress addr in theAddressList)
// { {
// if (addr.AddressFamily != System.Net.Sockets.AddressFamily.InterNetworkV6) if (addr.AddressFamily != System.Net.Sockets.AddressFamily.InterNetworkV6)
// { {
// myIp = addr.ToString(); myIp = addr.ToString();
// oneWorked = true; oneWorked = true;
}
}
// } if (!oneWorked)
// } {
throw new ArgumentException("Couldn't form server address from string " + serverAddress);
}
// if (!oneWorked) }
// { myPort = Convert.ToInt32(toks[2]);
// throw new ArgumentException("Couldn't form server address from string " + serverAddress); }
// }
// } public string ip
// myPort = Convert.ToInt32(toks[2]); {
// } get { return myIp; }
}
public int port
{
get { return myPort; }
}
public override string ToString()
// public string ip {
// { return prefix + "://" + ip + ":" + port;
// get { return myIp; } }
// } }
}
// public int port
// {
// get { return myPort; }
// }
// public override string ToString()
// {
// return prefix + "://" + ip + ":" + port;
// }
// }
// }
@@ -1,89 +1,88 @@
// using System; using System;
// using System.Collections.Generic; using System.Collections.Generic;
// using System.Text; using System.Text;
// using System.Threading; using System.Threading;
// using System.Runtime.InteropServices; using System.Runtime.InteropServices;
// TODO: remove namespace BCG
// namespace BCG {
// { public class SmartConnectionManagedWrapper
// public class SmartConnectionManagedWrapper {
// { public delegate void MessageDelegate(int msgType, byte[] msg);
// public delegate void MessageDelegate(int msgType, byte[] msg); private Dictionary<string, MessageDelegate> myServerToDel = new Dictionary<string, MessageDelegate>();
// private Dictionary<string, MessageDelegate> myServerToDel = new Dictionary<string, MessageDelegate>();
// public SmartConnectionManagedWrapper () public SmartConnectionManagedWrapper ()
// { {
// ClientToServerConnectionWrapper.init(); ClientToServerConnectionWrapper.init();
// Thread readThr = new Thread(new ThreadStart(myReceiveData)); Thread readThr = new Thread(new ThreadStart(myReceiveData));
// readThr.Start(); readThr.Start();
// } }
// public void initConnection(BlutoSubAddress subAddress) public void initConnection(BlutoSubAddress subAddress)
// { {
// ClientToServerConnectionWrapper.initServerConnection(ClientToServerConnectionWrapper.stringToBytes(subAddress.ip), ClientToServerConnectionWrapper.initServerConnection(ClientToServerConnectionWrapper.stringToBytes(subAddress.ip),
// subAddress.port); subAddress.port);
// } }
// public void send(BlutoSubAddress address, byte[] data) public void send(BlutoSubAddress address, byte[] data)
// { {
// byte[] ip = ClientToServerConnectionWrapper.stringToBytes (address.ip); byte[] ip = ClientToServerConnectionWrapper.stringToBytes (address.ip);
// ClientToServerConnectionWrapper.sendDataToServer(ip, address.port, data, data.Length); ClientToServerConnectionWrapper.sendDataToServer(ip, address.port, data, data.Length);
// } }
public void resgisterDataCallback(BlutoSubAddress address, MessageDelegate del)
{
lock (this)
{
myServerToDel.Add(address.ToString(), del);
}
}
// public void resgisterDataCallback(BlutoSubAddress address, MessageDelegate del) public void unregisterDataCallback(BlutoSubAddress address, MessageDelegate del)
// { {
// lock (this) lock (this)
// { {
// myServerToDel.Add(address.ToString(), del); myServerToDel.Remove(address.ip);
// } }
// } }
// public void unregisterDataCallback(BlutoSubAddress address, MessageDelegate del) public string getClientAsSeenByServer(string serverURL)
// { {
// lock (this) return "";
// { }
// myServerToDel.Remove(address.ip);
// }
// }
// public string getClientAsSeenByServer(string serverURL) private unsafe void myReceiveData()
// { {
// return ""; while (true)
// } {
BlutoTCPClientMessage** psClientPackets = null;
int npts = 0;
///returns as soon as it gets the first element
ClientToServerConnectionWrapper.getServerData(1000, 1, &psClientPackets, out npts);
for (int i = 0; i < npts; i++)
{
string ip = Marshal.PtrToStringAnsi(psClientPackets[i]->ip);
int port = psClientPackets[i]->port;
string serverURL = new BlutoSubAddress(ip, port).ToString();
// private unsafe void myReceiveData() byte[] msg = null;
// { if (psClientPackets[i]->msgType == 1)
// while (true) {
// { msg = new byte[psClientPackets[i]->size];
// BlutoTCPClientMessage** psClientPackets = null; Marshal.Copy(psClientPackets[i]->msg, msg, 0, msg.Length);
// int npts = 0; }
// ///returns as soon as it gets the first element
// ClientToServerConnectionWrapper.getServerData(1000, 1, &psClientPackets, out npts);
// for (int i = 0; i < npts; i++)
// {
// string ip = Marshal.PtrToStringAnsi(psClientPackets[i]->ip);
// int port = psClientPackets[i]->port;
// string serverURL = new BlutoSubAddress(ip, port).ToString();
// byte[] msg = null; MessageDelegate cb = null;
// if (psClientPackets[i]->msgType == 1) lock (this)
// { {
// msg = new byte[psClientPackets[i]->size]; myServerToDel.TryGetValue(serverURL, out cb);
// Marshal.Copy(psClientPackets[i]->msg, msg, 0, msg.Length); }
// } if (cb != null)
cb(psClientPackets[i]->msgType, msg);
}
}
}
// MessageDelegate cb = null; }
// lock (this) }
// {
// myServerToDel.TryGetValue(serverURL, out cb);
// }
// if (cb != null)
// cb(psClientPackets[i]->msgType, msg);
// }
// }
// }
// }
// }
@@ -72,6 +72,6 @@ namespace BCG {
/// <param name="del"></param> /// <param name="del"></param>
void register_data_callback(PubSubServerDataDelegate del); void register_data_callback(PubSubServerDataDelegate del);
//void register_send_confirmation_callback(SendConfirmationDelegate del); void register_send_confirmation_callback(SendConfirmationDelegate del);
} }
} }
@@ -1,52 +1,52 @@
// using System; using System;
// //using System.Messaging; //using System.Messaging;
// using BCG; using BCG;
// //using BCG.Utilities; //using BCG.Utilities;
// namespace BCG namespace BCG
// { {
// public class MSMQSubAddress : SubAddress public class MSMQSubAddress : SubAddress
// { {
// private string myIP, myQ; private string myIP, myQ;
// public static readonly string prefix = "msmq"; public static readonly string prefix = "msmq";
// public MSMQSubAddress (string adr) public MSMQSubAddress (string adr)
// { {
// try try
// { {
// // throw away prefix // throw away prefix
// string s = adr; string s = adr;
// s = s.Substring(prefix.Length + "://".Length); s = s.Substring(prefix.Length + "://".Length);
// // grab IP // grab IP
// myIP = s.Substring(0,s.IndexOf("/")); myIP = s.Substring(0,s.IndexOf("/"));
// // throw away ip // throw away ip
// s = s.Substring(myIP.Length + 1); s = s.Substring(myIP.Length + 1);
// myQ = s; myQ = s;
}
catch (Exception)
{
throw (new InvalidSubAddress(adr + " is not a valid queue address"));
}
}
// } public MSMQSubAddress (string ip, string simpleQName)
// catch (Exception) {
// { myIP = ip;
// throw (new InvalidSubAddress(adr + " is not a valid queue address")); myQ = simpleQName;
// } }
// }
// public MSMQSubAddress (string ip, string simpleQName) public string ip {get {return myIP;}}
// {
// myIP = ip;
// myQ = simpleQName;
// }
// public string ip {get {return myIP;}} public string simpleQueueName {get {return myQ;}}
// public string simpleQueueName {get {return myQ;}} public override string ToString() { return prefix + "://" + myIP + "/" + myQ; }
// public override string ToString() { return prefix + "://" + myIP + "/" + myQ; } public string formatName
{
// public string formatName get{return "FormatName:DIRECT=TCP:" + myIP + "\\private$\\" + myQ;}
// { }
// get{return "FormatName:DIRECT=TCP:" + myIP + "\\private$\\" + myQ;}
// } }
}
// }
// }
@@ -1,262 +1,296 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Text; using System.Text;
using System.Net; using System.Net;
using System.Net.Sockets; using System.Net.Sockets;
using System.IO; using System.IO;
//using BCG.Utilities; //using BCG.Utilities;
//using BCG.Utilities.General; //using BCG.Utilities.General;
using System.Threading; using System.Threading;
using mbts; using mbts;
using bts; using bts;
using System.Linq; using System.Linq;
using System.Reflection.Metadata.Ecma335; using System.Reflection.Metadata.Ecma335;
using Bts.PubSub; using Bts.PubSub;
using Microsoft.Win32;
using System.ComponentModel; namespace BCG {
/// <summary>
namespace BCG { /// Keeps an ITCPConnection for talking to the server. It also monitors heartbeats
/// <summary> /// from the server, and in case of missing hbs, informs its clients, and tries to
/// Keeps an ITCPConnection for talking to the server. It also monitors heartbeats /// restablish the connection.
/// from the server, and in case of missing hbs, informs its clients, and tries to /// Threads :
/// restablish the connection. /// its assumed that IPubSubServerConnection functions (including doWork) will
/// Threads : /// be called from the same thread. ITCPMessageHandler.onData and
/// its assumed that IPubSubServerConnection functions (including doWork) will /// ITCPConnectionHandler.onConnection will be called on any arbitrary thread
/// be called from the same thread. ITCPMessageHandler.onData and ///
/// ITCPConnectionHandler.onConnection are called on an arbitrary thread. /// </summary>
/// _conn_callback is called from within doWork, and _data_callback is called grom public class PubSubServerManagedTCPConnection : IPubSubServerConnection,// external facing interface to communicate msgs to Pubsubserver
// an arbitrary data thread ITCPMessageHandler, // to get messages from ITCPCOnnection
/// </summary> ITCPConnectionHandler { // to get connections from AsyncTCPConnector_APM
public class PubSubServerManagedTCPConnection : IPubSubServerConnection,// external facing interface to communicate msgs to Pubsubserver private ITCPConnection _connection, _new_connection;
ITCPMessageHandler, // to get messages from ITCPCOnnection private readonly BlutoSubAddress _server_address;
ITCPConnectionHandler { // to get connections from AsyncTCPConnector_APM // if a message(with confirm) can not be sent , its id is queued here,
private ITCPConnection _connection; // on reconnection, _on_send_onfirm_callback is fired with a negative on these ids, so the user
private readonly RemoteConnectionInfo _server_rc_info; //can re-send it. The re-send mechanism is not in implemented in this class, but by the user of this class
private readonly bool _is_sub; private ThreadSafeQueue<string> _negative_unsent_confs = new ThreadSafeQueue<string>();
// if _data_callback is not subscribed to , the user of this class can poll updates, // in onData, data is queued here, for polling
// in which case this queue will be used private SyncQueue<byte[]> _in_data_queue = new SyncQueue<byte[]>();
private SyncQueue<byte[]> _in_data_queue = new SyncQueue<byte[]>(); private DateTime _last_hb = DateTime.Now;
private DateTime _last_hb = DateTime.Now; private PubSubServerConnectionStatusChange _conn_callback;
private PubSubServerConnectionStatusChange _conn_callback; private PubSubServerDataDelegate _data_callback;
private PubSubServerDataDelegate _data_callback; private SendConfirmationDelegate _on_send_confirm_callback;
private readonly TimeSpan _connection_drop_threshold = PubSubConfig.Instance.ConnectionDropThreshold(); private readonly TimeSpan _connection_drop_threshold = PubSubConfig.Instance.ConnectionDropThreshold();
private ThreadSafeQueue<ConnectionStatusChange> _conn_cb_queue = new ThreadSafeQueue<ConnectionStatusChange>();
public PubSubServerManagedTCPConnection(string server_url)
// the callbacks for connection, comes from the ITCPConnection, when a connection is broken, : this(new BlutoSubAddress(server_url)) { }
// and from AsyncTCPConnector whne a connection is established. ALl that info is assembled here and
// enqueued, deuqued oin the doWork and acted on public PubSubServerManagedTCPConnection(BlutoSubAddress server_address) {
private class ConnectionStatusChange { _server_address = server_address;
public enum Status { _connect_to_ps_server();
CONNECTED, }
DISCONNECTED
} private void _connect_to_ps_server() {
public readonly Status connection_satus; bts.Log.Gen.info($"PubSubServerManagedTCPConnection._connect_to_ps_server server: {_server_address}");
public readonly ITCPConnection new_connection = null; AsyncTCPConnector_APM.connect_to(_server_address, this);
public ConnectionStatusChange(Status connection_satus_, ITCPConnection new_connection_) { }
connection_satus = connection_satus_;
new_connection = new_connection_; #region IPubSubServerConnection Members
}
public override string ToString() { public void send_with_confirmation(byte[] data, ref string conf_id) {
return connection_satus.ToString(); bts.Log.Gen.info($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} data: {data.Length} _connection : {_connection != null}");
} if (_connection != null) {
} _connection.sendWithConfirmation(data, ref conf_id, (msg_id, confirmed) => {
if (_on_send_confirm_callback != null)
public PubSubServerManagedTCPConnection(RemoteConnectionInfo server_address, bool is_sub) { _on_send_confirm_callback(msg_id, confirmed);
Log.Gen.info("PubSubServerManagedTCPConnection.cons server_address: {server_address} is_sub: {is_sub}"); });
_server_rc_info = server_address; } else {
_is_sub = is_sub; bts.Log.Gen.warn($"PubSubServerManagedTCPConnection.sendWithConfirmation server: {_server_address} no connection, buffering send");
// this is where it tries to connect to the pubsub server, and gets a callback on onConnection conf_id = Guid.NewGuid().ToString();
_connect_to_ps_server(); _negative_unsent_confs.enqueue(conf_id);
} }
}
private void _connect_to_ps_server() {
bts.Log.Gen.info($"PubSubServerManagedTCPConnection._connect_to_ps_server server _server_rc_info: {_server_rc_info} is_sub: {_is_sub}"); private void _send(byte[] data) {
// onConnection will be called on an arbitrary thread bts.Log.Gen.info($"PubSubServerManagedTCPConnection._send server: {_server_address} data: {data.Length} _connection : {_connection != null}");
AsyncTCPConnector_AsyncWrapper.connect_to(_server_rc_info, this); if (_connection != null) {
} bts.Log.Gen.info("PubSubServerManagedTCPConnection.send server: " + _server_address + "data: " + data.Length);
_connection.send(data);
#region IPubSubServerConnection Members }
}
private void _send(byte[] data) {
bts.Log.Gen.info($"PubSubServerManagedTCPConnection._send client: {client_url} server: {_server_rc_info} data: {data.Length} is_sub: {_is_sub} _connection : {_connection != null}"); public void send(bts.PubSubClientMessage ps_msg) {
if (_connection != null) { bts.Log.Gen.info($"PubSubServerManagedTCPConnection.send server: {_server_address} ps_msg: {ps_msg}");
_connection.send(data); var msg_bytes = ps_msg.serialize_into_bytes();
bts.Log.Gen.info("PubSubServerManagedTCPConnection.send, data is sent, server: " + _server_rc_info + "data: " + data.Length); PubSubClientMessage msg_retrv = new PubSubClientMessage();
} else { msg_retrv.consruct_deserialize(msg_bytes);
bts.Log.Gen.warn($"PubSubServerManagedTCPConnection._send, could not send data, server: {_server_rc_info} data: {data.Length} is_sub: {_is_sub} _connection : {_connection != null} "); bts.Log.Gen.info($"PubSubServerManagedTCPConnection.send server: {_server_address} msg_retrv: {msg_retrv}");
} _send(msg_bytes);
} }
public void send(bts.PubSubClientMessage ps_msg) { // private void send(Stream str) {
bts.Log.Gen.info($"PubSubServerManagedTCPConnection.send server: {_server_rc_info} ps_msg: {ps_msg}"); // BinaryReader br = new BinaryReader(str);
// whys it constructed again? // _send(br.ReadBytes(Convert.ToInt32(str.Length - str.Position)));
var msg_bytes = ps_msg.serialize_into_bytes(); // }
//bts.Log.Gen.info($"PubSubServerManagedTCPConnection.send server: {_server_rc_info} msg_retrv: {msg_retrv}");
_send(msg_bytes); // private void send(PubSubMessage msg) {
} // send(msg.to_stream());
// }
public IEnumerable<bts.PubSubServerMessage> get_updates(TimeSpan timeout, int maxElements) {
List<byte[]> rv_raw = _in_data_queue.dequeueMultipleElements(Convert.ToInt32(timeout.TotalMilliseconds), maxElements); // public void send_with_confirmation(PubSubMessage msg, ref string conf_id) {
List<bts.PubSubServerMessage> rv = new List<PubSubServerMessage>(); // var bytes = msg.to_bytes();
var i = 0; // send_with_confirmation(bytes, ref conf_id);
foreach (var b in rv_raw) { // }
PubSubServerMessage ps_msg = new PubSubServerMessage();
ps_msg.consruct_deserialize(b); public void send_with_confirmation(bts.PubSubClientMessage msg, ref string conf_id, SendConfirmationDelegate del) {
bts.Log.Gen.info($"PubSubServerManagedTCPConnection.get_updates downloaded: {ps_msg} msg_no: {i} / {rv_raw.Count}"); if (_connection == null) {
rv.Add(ps_msg); // quq
i++; }
} MemoryStream str = new MemoryStream();
return rv; msg.serialize_into_str(str);
} var bytes = StreamUtilityMethods.getBytesFromStream(str);
_connection.sendWithConfirmation(bytes, ref conf_id, del);
public string server_url { }
get { return _server_rc_info.ToString(); }
} public IEnumerable<bts.PubSubServerMessage> get_updates(TimeSpan timeout, int maxElements) {
List<byte[]> rv_raw = _in_data_queue.dequeueMultipleElements(Convert.ToInt32(timeout.TotalMilliseconds), maxElements);
public string client_url { List<bts.PubSubServerMessage> rv = new List<PubSubServerMessage>();
get { var i = 0;
if (_connection == null) foreach (var b in rv_raw) {
return RemoteConnectionInfo.null_instance.ToString(); PubSubServerMessage ps_msg = new PubSubServerMessage();
return _connection.get_self_address().ToString(); ps_msg.consruct_deserialize(b);
} bts.Log.Gen.info($"PubSubServerManagedTCPConnection.get_updates downloaded: {ps_msg} msg_no: {i} / {rv_raw.Count}");
} rv.Add(ps_msg);
i++;
public void register_connection_callback(PubSubServerConnectionStatusChange del) { }
_conn_callback = del; return rv;
} }
public void register_data_callback(PubSubServerDataDelegate del) { public string server_url {
_data_callback = del; get { return _server_address.ToString(); }
} }
#endregion public string client_url {
get {
#region IDisposable Members string rv = "";
lock (this) {
private bool _is_disposed = false; if (_connection == null) {
public void Dispose() { rv = "";
if (isDisposed) } else {
return; if (PubSubConfig.Instance.IpTranslation.is_null_or_empty()) {
bts.Log.Gen.info("PubSubServerManagedTCPConnection.Dispose"); string ip;
if (_connection != null) { int port;
_connection.Dispose(); _connection.getSelfAddress(out ip, out port);
} rv = new BlutoSubAddress(ip, port).ToString();
_is_disposed = true; } else {
} rv = "";
public bool isDisposed { }
get { }
return _is_disposed; }
} return rv;
} }
}
#endregion
public void register_connection_callback(PubSubServerConnectionStatusChange del) {
#region ITCPMessageHandler Members _conn_callback = del;
}
/// <summary>
/// This method is executed on an arbitrary CLR thread. public void register_data_callback(PubSubServerDataDelegate del) {
/// </summary> _data_callback = del;
/// <param name="message"></param> }
public void onData(RemoteMessage message) {
bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onData connection: {_connection} message: {message}"); public void register_send_confirmation_callback(SendConfirmationDelegate del) {
// TODO check for disconenction _on_send_confirm_callback = del;
switch (message.messageType) { }
case RemoteMessageType.Data:
if (_data_callback != null) #endregion
_data_callback(message.remoteConnectionInfo.ToString(), message.message);
else #region IDisposable Members
_in_data_queue.enqueue(message.message);
break; private bool _is_disposed = false;
case RemoteMessageType.Heartbeat: public void Dispose() {
_last_hb = DateTime.Now; bts.Log.Gen.info("PubSubServerManagedTCPConnection.Dispose");
break; //throw new Exception("The method or operation is not implemented.");
case RemoteMessageType.Connection: if (_connection != null) {
// since this is called from the connection, this condition will never hit, ssince _connection.Dispose();
//new connections are detected only by AsyncTCPConnector
Log.assert(false); }
break; _is_disposed = true;
case RemoteMessageType.Disconnection: }
bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onData case RemoteMessageType.Disconnection: message: {message}"); public bool isDisposed {
_conn_cb_queue.enqueue(new ConnectionStatusChange(ConnectionStatusChange.Status.DISCONNECTED, null)); get {
break; return _is_disposed;
}
}
} }
#endregion #endregion
#region ITCPConnectionHandler Members #region ITCPMessageHandler Members
/// will be called on on an arbitrary .Net thread /// <summary>
public void onConnection(ITCPConnection connection) { /// This method is executed on an arbitrary CLR thread. To avoid blocking it
var server_rci = connection.get_peer_address(); /// data is written to the queue and method returns. myLastHB being a primitive
bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onConnection connection: {_connection_txt(connection)}"); /// (long) is safe for multi threaded operations
if (_server_rc_info.Equals(server_rci)) { /// </summary>
_conn_cb_queue.enqueue(new ConnectionStatusChange(ConnectionStatusChange.Status.CONNECTED, connection)); /// <param name="message"></param>
// _last_hb = DateTime.Now; public void onData(RemoteMessage message) {
// _new_connection = connection; bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onData message: {message}");
} // TODO check for disconenction
} switch (message.messageType) {
case RemoteMessageType.Data:
#endregion if (_data_callback != null)
_data_callback(message.remoteConnectionInfo.ToString(), message.message);
void _check_missing_hb() { else
if (!_connection.isDisposed) { _in_data_queue.enqueue(message.message);
TimeSpan hb_interval = DateTime.Now - _last_hb; break;
bool hb_missed = hb_interval > _connection_drop_threshold; case RemoteMessageType.Heartbeat:
if (hb_missed) { _last_hb = DateTime.Now;
bts.Log.Gen.warn("PubSubServerManagedTCPConnection._dispose_and_reconnect_if_missing_hb disposing connecion: {myServerAddress}"); break;
ConnectionStatusChange ci = new ConnectionStatusChange(ConnectionStatusChange.Status.DISCONNECTED, null); case RemoteMessageType.Connection:
_conn_cb_queue.enqueue(ci); _conn_callback(_server_address.ToString(), true);
} break;
} else { case RemoteMessageType.Disconnection:
// already disposed, nothing to do if (_conn_callback != null)
} _conn_callback(_server_address.ToString(), false);
} _connection.Dispose();
_connect_to_ps_server();
private static string _connection_txt(ITCPConnection conn) { break;
if (conn != null) {
return $"conn_type: {conn.GetType()} server: {conn.get_peer_address()} self: {conn.get_self_address()}"; }
} }
return "NULL";
} #endregion
#region IWorker Members #region ITCPConnectionHandler Members
/// <summary> /// will be called on on an rbitrary .Net thread
/// should be called on the same thread as the other user functions. public void onConnection(ITCPConnection connection) {
/// the design only allows for onConnection to be on a different string ip;
/// thread. int port;
/// </summary> connection.getPeerAddress(out ip, out port);
public void doWork() { BlutoSubAddress add = new BlutoSubAddress(ip, port);
//_dispose_and_reconnect_if_missing_hb(); bts.Log.Gen.info($"PubSubServerManagedTCPConnection.onConnection add: {add.ToString()} connection type: {connection.GetType()}");
if (_conn_cb_queue.getCount() > 0) { if (add.ToString() == _server_address.ToString()) {
var css = _conn_cb_queue.dequeueAll(); _last_hb = DateTime.Now;
foreach (var c in css) { _new_connection = connection;
Log.Gen.info($"PubSubServerManagedTCPConnection.doWork c: {c}"); }
switch (c.connection_satus) { }
case ConnectionStatusChange.Status.CONNECTED: {
Log.assert(c.new_connection != null); #endregion
c.new_connection.register(this);
_connection = c.new_connection; void _dispose_and_reconnect_if_missing_hb() {
_last_hb = DateTime.Now; if (!_connection.isDisposed) {
if (_conn_callback != null) TimeSpan hb_interval = DateTime.Now - _last_hb;
_conn_callback(_server_rc_info.ToString(), true); bool hb_missed = hb_interval > _connection_drop_threshold;
break; if (hb_missed) {
} if (_conn_callback != null)
case ConnectionStatusChange.Status.DISCONNECTED: { _conn_callback(_server_address.ToString(), false);
_connection.Dispose(); bts.Log.Gen.info("PubSubServerManagedTCPConnection._dispose_and_reconnect_if_missing_hb disposing connecion: {myServerAddress}");
_connect_to_ps_server(); _connection.Dispose();
if (_conn_callback != null) _connect_to_ps_server();
_conn_callback(_server_rc_info.ToString(), false); }
break; } else {
} // already disposed, nothing to do
} }
} }
}
} void _check_for_new_connection() {
if (_new_connection != null) {
#endregion _connection = _new_connection;
_new_connection = null; // since we are using its presence as a marker of nw connections
} bts.Log.Gen.info("PubSubServerManagedTCPConnection._check_for_new_connection assigning new connection");
} _connection.register(this);
if (_conn_callback != null)
_conn_callback(_server_address.ToString(), true);
if (_on_send_confirm_callback != null) {
var msg_ids = _negative_unsent_confs.dequeueAll();
foreach (var id in msg_ids)
_on_send_confirm_callback(id, false);
}
}
}
#region IWorker Members
/// <summary>
/// should be called on the same thread as the other user functions.
/// the design only allows for onConnection to be on a different
/// thread.
/// </summary>
public void doWork() {
//_dispose_and_reconnect_if_missing_hb();
//try {
_check_for_new_connection();
// } catch (Exception ex) {
// bts.Log.Gen.warn($"PubsSubServerManagedTCPConnection.doWork ex: {ex}");
// }
}
#endregion
// private void myOnSendConfirm(string messageId, bool confirmed) {
// _negative_confs.enqueue(new Pair<string, bool>(messageId, confirmed));
// }
}
}
@@ -3,38 +3,55 @@ using System.Collections.Generic;
using System.Text; using System.Text;
//using BCG.Utilities; //using BCG.Utilities;
namespace BCG { namespace BCG
{
/// <summary> /// <summary>
/// Every instance of PublisherCore and SubscriberCore will have its /// Every instance of PublisherCore and SubscriberCore will have its
/// factory. Rather than have once singelton factory, this approach /// factory. Rather than have once singelton factory, this approach
/// is preffered since multiple pub/sub core can connect to the same /// is preffered since multiple pub/sub core can connect to the same
/// connection and make it difficultt o manage connections. /// connection and make it difficultt o manage connections.
/// </summary> /// </summary>
public class PubSubServerConnectionFactory : IDisposable { public class PubSubServerConnectionFactory : IDisposable
private Dictionary<RemoteConnectionInfo, IPubSubServerConnection> _conn_map = {
new Dictionary<RemoteConnectionInfo, IPubSubServerConnection>(); private bool myIsSub;
private readonly bool _is_sub; private Dictionary<string, IPubSubServerConnection> myConnections = new Dictionary<string, IPubSubServerConnection>();
internal PubSubServerConnectionFactory(bool is_sub) { internal PubSubServerConnectionFactory (bool isSub)
_is_sub = is_sub; {
myIsSub = isSub;
} }
public IPubSubServerConnection get_connection(RemoteConnectionInfo server_rci) { public IPubSubServerConnection getConnection(string url)
IPubSubServerConnection rv = null; {
if (!_conn_map.TryGetValue(server_rci, out rv)) { if (!myConnections.ContainsKey(url))
rv = _build_connection(server_rci); myConnections.Add(url, myBuildConnection(url));
_conn_map.Add(server_rci, rv); return myConnections[url];
}
private IPubSubServerConnection myBuildConnection(string url)
{
if (url.StartsWith(MSMQSubAddress.prefix))
{
// TODO, no more MSMQ
//Diagnostics.assert(false);
return null;
// MSMQSubAddress serverAdd = new MSMQSubAddress(url);
// return new PubSubServerMSMQConnection(serverAdd.ToString(), myIsSub);
}
else if (url.StartsWith(BlutoSubAddress.prefix))
{
return new PubSubServerManagedTCPConnection(url);
}
else
{
// BlueEventLog.logAssert(false, "Unknown transport, only msmq and blutoTcp allowed");
return null;
} }
return rv;
}
private IPubSubServerConnection _build_connection(RemoteConnectionInfo server_rci) {
return new PubSubServerManagedTCPConnection(server_rci, _is_sub);
} }
#region IDisposable Members #region IDisposable Members
public void Dispose() { } public void Dispose() {}
#endregion #endregion
} }
@@ -124,14 +124,7 @@ namespace bts {
public long publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); public long publish_time_epoch_ms = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
public byte[] data_to_publish; public byte[] data_to_publish;
public void sanity_check() {
Log.assert(!String.IsNullOrEmpty(tag_type));
Log.assert(!String.IsNullOrEmpty(tag));
Log.assert(!String.IsNullOrEmpty(connection));
}
public void serialize_into_str(Stream str) { public void serialize_into_str(Stream str) {
sanity_check();
BinaryWriter bw = new BinaryWriter(str); BinaryWriter bw = new BinaryWriter(str);
bw.write_string_cpp(connection); bw.write_string_cpp(connection);
bw.write_string_cpp(publishing_app); bw.write_string_cpp(publishing_app);
@@ -24,7 +24,6 @@ namespace bts {
} }
private static Dictionary<string, TagHandlerEntry> _config_map = new Dictionary<string, TagHandlerEntry>() private static Dictionary<string, TagHandlerEntry> _config_map = new Dictionary<string, TagHandlerEntry>()
{ {
{String.Empty, new TagHandlerEntry(DSTagManager.DataStreamTagType , "DataStreamCore", "bts.PubsubServerStreamTagHandler")},
{DSTagManager.DataStreamTagType, new TagHandlerEntry(DSTagManager.DataStreamTagType , "DataStreamCore", "bts.PubsubServerStreamTagHandler")}, {DSTagManager.DataStreamTagType, new TagHandlerEntry(DSTagManager.DataStreamTagType , "DataStreamCore", "bts.PubsubServerStreamTagHandler")},
{"System_Sub", new TagHandlerEntry("System_Sub", "PubSub", "BCG.SystemSubscriptionMapPublisher")}, {"System_Sub", new TagHandlerEntry("System_Sub", "PubSub", "BCG.SystemSubscriptionMapPublisher")},
{"System_Stats", new TagHandlerEntry("System_Stats", "PubSub", "BCG.SystemStatsPublisher")} {"System_Stats", new TagHandlerEntry("System_Stats", "PubSub", "BCG.SystemStatsPublisher")}
@@ -8,11 +8,11 @@ namespace BCG {
public interface IPubSubRouter { public interface IPubSubRouter {
// event PubSubTagURLChangeDelegate onSubURLTagRouteChange; event PubSubTagURLChangeDelegate onSubURLTagRouteChange;
// event PubSubTagURLChangeDelegate onPubURLTagRouteChange; event PubSubTagURLChangeDelegate onPubURLTagRouteChange;
IEnumerable<PubSubRoute> sub_routes(); List<string> get_all_pubsub_sub_urls();
List<string> get_all_pubsub_pub_urlss();
RemoteConnectionInfo get_sub_route_for_tag(string tag); string get_sub_url_for_tag(string tag);
RemoteConnectionInfo get_pub_route_for_tag(string tag); string get_pub_url_for_tag(string tag);
} }
} }
@@ -5,38 +5,43 @@ using System.Text.RegularExpressions;
using System.Text.Json.Serialization; using System.Text.Json.Serialization;
using System.Text.Json; using System.Text.Json;
using System.Text.Json.Nodes; using System.Text.Json.Nodes;
using System.Runtime.InteropServices;
namespace BCG { namespace BCG {
[JsonConverter(typeof(PubSubRouteJsonConverter))] [JsonConverter(typeof(PubSubRouteJsonConverter))]
public class PubSubRoute : IComparable<PubSubRoute> { public class PubSubRoute : IComparable<PubSubRoute> {
public string server_address; public string serverAddress;
public string tag_reg_exp; public string tagRegExp;
//route names are used in intermediary code (i.e. while substations are being deployed) //route names are used in intermediary code (i.e. while substations are being deployed)
//so that MC Hammer can advertise them. //so that MC Hammer can advertise them.
public string name; public string name;
private Regex _regex = null; private Regex myRegex = null;
public PubSubRoute() { } public PubSubRoute() { }
public PubSubRoute(string the_server_address, string the_tag_reg_exp) { public PubSubRoute(string theServerAddress, string theTagRegExp) {
server_address = the_server_address; serverAddress = theServerAddress;
tag_reg_exp = the_tag_reg_exp; tagRegExp = theTagRegExp;
} }
// public PubSubRoute()
// {
// // serverAddress = string.Format("bluto://{0}:{1}", cfg.serverAddress, cfg.serverPort);
// // tagRegExp = cfg.tagsRegex;
// }
public string[] getAllPubSubs() { public string[] getAllPubSubs() {
return server_address.Split(';'); return serverAddress.Split(';');
} }
public Regex get_regex() { public Regex getRegex() {
return _regex ?? new Regex(tag_reg_exp); return myRegex ?? new Regex(tagRegExp);
} }
#region IComparable<PubSubRoute> Members #region IComparable<PubSubRoute> Members
public int CompareTo(PubSubRoute other) { public int CompareTo(PubSubRoute other) {
return server_address == other.server_address ? tag_reg_exp.CompareTo(other.tag_reg_exp) return serverAddress == other.serverAddress ? tagRegExp.CompareTo(other.tagRegExp)
: server_address.CompareTo(other.server_address); : serverAddress.CompareTo(other.serverAddress);
} }
#endregion #endregion
@@ -46,8 +51,8 @@ namespace BCG {
return false; return false;
else { else {
PubSubRoute other = obj as PubSubRoute; PubSubRoute other = obj as PubSubRoute;
return server_address.Equals(other.server_address) && return serverAddress.Equals(other.serverAddress) &&
tag_reg_exp.Equals(other.tag_reg_exp); tagRegExp.Equals(other.tagRegExp);
} }
} }
@@ -56,33 +61,29 @@ namespace BCG {
} }
public static PubSubRoute get_default_route() { public static PubSubRoute get_default_route() {
return new PubSubRoute("bluto://127.0.0.1:12345", ".*"); return new PubSubRoute("bluto://127.0.0.0.1:12345", ".*");
}
public override string ToString() {
return $"tag_reg_exp: {tag_reg_exp} server_address: {server_address}";
} }
} }
public static class PubSubRouteExtensions { public static class PubSubRouteExtensions {
public static bool IsMatch(this PubSubRoute route, string tag) => public static bool IsMatch(this PubSubRoute route, string tag) =>
route.get_regex().IsMatch(tag); route.getRegex().IsMatch(tag);
} }
public class PubSubRouteJsonConverter : JsonConverter<PubSubRoute> { public class PubSubRouteJsonConverter : JsonConverter<PubSubRoute> {
public override PubSubRoute? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { public override PubSubRoute? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
var json_obj = JsonSerializer.Deserialize<JsonObject>(ref reader); var jsonObj = JsonSerializer.Deserialize<JsonObject>(ref reader);
if (json_obj is null) { if (jsonObj is null) {
return null; return null;
} }
var rv = new PubSubRoute(); var rv = new PubSubRoute();
ReadString("name", ref rv.name); ReadString("name", ref rv.name);
ReadString("server_address", ref rv.server_address); ReadString("server_address", ref rv.serverAddress);
ReadString("tag", ref rv.tag_reg_exp); ReadString("tag", ref rv.tagRegExp);
return rv; return rv;
void ReadString(string key, ref string valueOut) { void ReadString(string key, ref string valueOut) {
if (json_obj.TryGetPropertyValue(key, out var valueJson)) { if (jsonObj.TryGetPropertyValue(key, out var valueJson)) {
var valueStr = valueJson?.GetValue<string>(); var valueStr = valueJson?.GetValue<string>();
// Only (over)write valueOut if non-null // Only (over)write valueOut if non-null
if (valueStr is not null) { if (valueStr is not null) {
@@ -95,8 +96,8 @@ namespace BCG {
public override void Write(Utf8JsonWriter writer, PubSubRoute value, JsonSerializerOptions options) { public override void Write(Utf8JsonWriter writer, PubSubRoute value, JsonSerializerOptions options) {
var jsonObj = new JsonObject { var jsonObj = new JsonObject {
["name"] = value.name, ["name"] = value.name,
["server_address"] = value.server_address, ["server_address"] = value.serverAddress,
["tag"] = value.tag_reg_exp, ["tag"] = value.tagRegExp,
}; };
JsonSerializer.Serialize(writer, jsonObj); JsonSerializer.Serialize(writer, jsonObj);
} }
@@ -1,83 +1,97 @@
using System; using System;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using System.Collections; using System.Collections;
using System.Collections.Generic; using System.Collections.Generic;
using BCG; using BCG;
using Bts.PubSub; using Bts.PubSub;
using bts; using bts;
using System.Linq; using System.Linq;
//using BCG.Utilities; //using BCG.Utilities;
//using bts; //using bts;
namespace BCG { namespace BCG {
public class PubSubRouter : IPubSubRouter { public class PubSubRouter : IPubSubRouter {
// private Regex[] mySubRouteRegExps; private Regex[] myPubRouteRegExps;
private PubSubRoute[] _pub_routes; private Regex[] mySubRouteRegExps;
private PubSubRoute[] _sub_routes; private PubSubRoute[] myPubRoutes;
private PubSubRoute[] mySubRoutes;
public PubSubRouter() : this(
PubSubConfig.Instance.SubscriberRoutes?.ToArray() public PubSubRouter() : this(
?? throw new InvalidOperationException("Subscriber routes are not configured"), PubSubConfig.Instance.SubscriberRoutes?.ToArray()
PubSubConfig.Instance.PublisherRoutes?.ToArray() ?? throw new InvalidOperationException("Subscriber routes are not configured"),
?? throw new InvalidOperationException("Publisher routes are not configured")) { } PubSubConfig.Instance.PublisherRoutes?.ToArray()
?? throw new InvalidOperationException("Publisher routes are not configured")) { }
public PubSubRouter(params PubSubRoute[] routes) : this(routes, routes) { }
public PubSubRouter(params PubSubRoute[] routes) : this(routes, routes) { }
public PubSubRouter(PubSubRoute[] sub_routes, PubSubRoute[] pub_routes) {
string log_msg = String.Empty; public PubSubRouter(PubSubRoute[] subRoutes, PubSubRoute[] pubRoutes) {
foreach (var route in sub_routes) { mySubRouteRegExps = new Regex[subRoutes.Length];
log_msg += $" sub_route: {route}"; myPubRouteRegExps = new Regex[pubRoutes.Length];
}
foreach (var route in pub_routes) { mySubRoutes = subRoutes;
log_msg += $" pub_route: {route}"; myPubRoutes = pubRoutes;
}
Log.Gen.info($"PubSubRouter routes: {log_msg}"); for (int i = 0; i < subRoutes.Length; ++i)
_sub_routes = sub_routes; mySubRouteRegExps[i] = new Regex(subRoutes[i].tagRegExp);
_pub_routes = pub_routes; for (int i = 0; i < pubRoutes.Length; ++i)
} myPubRouteRegExps[i] = new Regex(pubRoutes[i].tagRegExp);
}
#region IPubSubRouter methods
private int myGetRouteIndex(string forTag, PubSubRoute[] routes, Regex[] regexes) {
public IEnumerable<PubSubRoute> sub_routes() { for (int i = 0; i < routes.Length; i++)
return _sub_routes; if (regexes[i].IsMatch(forTag))
} return i;
public RemoteConnectionInfo get_pub_route_for_tag(string tag) { throw new PubSubRouteNotFound(String.Format("No route found for {0}", forTag));
if (!_pub_routes.TryGetFirst(r => r?.IsMatch(tag) ?? false, out var route)) { }
throw new ArgumentException($"Failed to find route for tag '{tag}'", nameof(tag));
} #region IPubSubRouter methods
Log.Gen.info($"PubSubRouter.get_sub_url_for_tag tag: {tag} route: {route}");
return RemoteConnectionInfo.parse_url(route.server_address); private string myPickAServer(string tag, string[] urls) {
} if (urls.Length == 1)
return urls[0];
public RemoteConnectionInfo get_sub_route_for_tag(string tag) { else {
if (!_sub_routes.TryGetFirst(r => r?.IsMatch(tag) ?? false, out var route)) { uint hc = (uint)tag.GetHashCode();
throw new ArgumentException($"Failed to find route for tag '{tag}'", nameof(tag)); return urls[hc % urls.Length];
} }
Log.Gen.info($"PubSubRouter.get_sub_url_for_tag tag: {tag} route: {route}"); }
return RemoteConnectionInfo.parse_url(route.server_address);
} public string get_pub_url_for_tag(string tag) {
if (!myPubRoutes.TryGetFirst(r => r?.IsMatch(tag) ?? false, out var route)) {
public List<string> get_all_pubsub_sub_urls() => throw new ArgumentException($"Failed to find route for tag '{tag}'", nameof(tag));
_sub_routes.Select(r => r.server_address).ToList(); }
Log.Gen.info($"PubSubRouter.get_sub_url_for_tag. tag: {tag} route: {route}");
public List<string> get_all_pubsub_pub_urlss() => return route.serverAddress;
_pub_routes.Select(r => r.server_address).ToList(); }
public event PubSubTagURLChangeDelegate onSubURLTagRouteChange; public string get_sub_url_for_tag(string tag) {
public event PubSubTagURLChangeDelegate onPubURLTagRouteChange; if (!mySubRoutes.TryGetFirst(r => r?.IsMatch(tag) ?? false, out var route)) {
throw new ArgumentException($"Failed to find route for tag '{tag}'", nameof(tag));
#endregion }
Log.Gen.info($"PubSubRouter.get_sub_url_for_tag. tag: {tag} route: {route}");
//this function is put here so that the events are used, in order to prevent build errors. return route.serverAddress;
private void myNeverCallThisFunction() { }
if (onPubURLTagRouteChange == null || onSubURLTagRouteChange == null) {
int aVar = 2; public List<string> get_all_pubsub_sub_urls() =>
aVar = aVar + 2; mySubRoutes.Select(r => r.serverAddress).ToList();
}
} public List<string> get_all_pubsub_pub_urlss() =>
myPubRoutes.Select(r => r.serverAddress).ToList();
} public event PubSubTagURLChangeDelegate onSubURLTagRouteChange;
} public event PubSubTagURLChangeDelegate onPubURLTagRouteChange;
#endregion
//this function is put here so that the events are used, in order to prevent build errors.
private void myNeverCallThisFunction() {
if (onPubURLTagRouteChange == null || onSubURLTagRouteChange == null) {
int aVar = 2;
aVar = aVar + 2;
}
}
}
}
+29 -29
View File
@@ -1,31 +1,31 @@
// using System; using System;
// using BCG; using BCG;
// //using BCG.Utilities; //using BCG.Utilities;
// namespace BCG namespace BCG
// { {
// public class InvalidSubAddress : Exception public class InvalidSubAddress : Exception
// { {
// public InvalidSubAddress(string msg) : base (msg){} public InvalidSubAddress(string msg) : base (msg){}
// } }
// public abstract class SubAddress public abstract class SubAddress
// { {
// public static SubAddress decode (string s) public static SubAddress decode (string s)
// { {
// //if (s.StartsWith(TcpSubAddress.prefix)) //if (s.StartsWith(TcpSubAddress.prefix))
// // return new TcpSubAddress(s); // return new TcpSubAddress(s);
// //else if (s.StartsWith(SuperSocketAddress.prefix)) //else if (s.StartsWith(SuperSocketAddress.prefix))
// // return new SuperSocketAddress(s); // return new SuperSocketAddress(s);
// if (s.StartsWith(MSMQSubAddress.prefix)) if (s.StartsWith(MSMQSubAddress.prefix))
// return new MSMQSubAddress(s); return new MSMQSubAddress(s);
// else if (s.StartsWith(BlutoSubAddress.prefix)) else if (s.StartsWith(BlutoSubAddress.prefix))
// return new BlutoSubAddress(s); return new BlutoSubAddress(s);
// else else
// { {
// //BlueEventLog.logAssert(false, "Unknown pubsub transport for : " + s); //BlueEventLog.logAssert(false, "Unknown pubsub transport for : " + s);
// return null; return null;
// } }
// } }
// } }
// } }
@@ -27,39 +27,35 @@ public class PubSubConfig {
[JsonPropertyName("simple_echo_server")] [JsonPropertyName("simple_echo_server")]
public bool? SimpleEchoServer { get; set; } public bool? SimpleEchoServer { get; set; }
// unused
[JsonPropertyName("milliseconds_between_sub_info_updates")] [JsonPropertyName("milliseconds_between_sub_info_updates")]
public int? MillisecondsBetweenSubInfoUpdates { get; set; } = 1000; public int? MillisecondsBetweenSubInfoUpdates { get; set; } = 1000;
// unused
[JsonPropertyName("milliseconds_between_time_slice_updates")] [JsonPropertyName("milliseconds_between_time_slice_updates")]
public int? MillisecondsBetweenTimeSliceUpdates { get; set; } = 5000; public int? MillisecondsBetweenTimeSliceUpdates { get; set; } = 5000;
// unused
[JsonPropertyName("milliseconds_between_no_buffer_space_retries")] [JsonPropertyName("milliseconds_between_no_buffer_space_retries")]
public int? MillisecondsBetweenNoBufferRetries { get; set; } = 100; public int? MillisecondsBetweenNoBufferRetries { get; set; } = 100;
// unused
[JsonPropertyName("retries_on_no_buffer_space")] [JsonPropertyName("retries_on_no_buffer_space")]
public int? RetriesOnNoBufferSpace { get; set; } = 1; public int? RetriesOnNoBufferSpace { get; set; } = 1;
// unused
[JsonPropertyName("do_it_all_seconds_between_log")] [JsonPropertyName("do_it_all_seconds_between_log")]
public int? DoItAllSecondsBetweenLog { get; set; } = 0; public int? DoItAllSecondsBetweenLog { get; set; } = 0;
// Used by the PubSubMessageReactor to pass to the ManagedTCPMessageHb (but unsed within it) // Milliseconds
[JsonPropertyName("heartbeat_interval")] [JsonPropertyName("heartbeat_interval")]
public int? HeartbeatInterval { get; set; } = 15000; public int? HeartbeatInterval { get; set; } = 15000;
[JsonPropertyName("missing_heartbeats_before_gc")] [JsonPropertyName("missing_heartbeats_before_gc")]
public int? MissingHeartbeatsBeforeGc { get; set; } = 50; public int? MissingHeartbeatsBeforeGc { get; set; } = 50;
// unused [JsonPropertyName("use_managed_sockets")]
public bool? UseManagedSockets { get; set; } = true;
// Milliseconds // Milliseconds
[JsonPropertyName("state_update_period")] [JsonPropertyName("state_update_period")]
public int? StateUpdatePeriod { get; set; } = 5000; public int? StateUpdatePeriod { get; set; } = 5000;
// unused
// Milliseconds // Milliseconds
[JsonPropertyName("statistics_update_period")] [JsonPropertyName("statistics_update_period")]
public int? StatisticsUpdatePeriod { get; set; } = 5000; public int? StatisticsUpdatePeriod { get; set; } = 5000;
@@ -68,20 +64,67 @@ public class PubSubConfig {
[JsonPropertyName("disposed_connection_check_period")] [JsonPropertyName("disposed_connection_check_period")]
public int? DisposedConnectionCheckPeriod { get; set; } = 1000; public int? DisposedConnectionCheckPeriod { get; set; } = 1000;
// Milliseconds
[JsonPropertyName("message_processing_warning_threshold")]
public int? MessageProcessingWarningThreshold { get; set; } = 1000;
[JsonPropertyName("max_packet_size")]
public int? MaxPacketSize { get; set; } = 500000000;
[JsonPropertyName("blue_comm_must_die")]
public bool? BlueCommMustDie { get; set; } = true;
[JsonPropertyName("crushable_tags")]
public List<string>? CrushableTags { get; set; }
[JsonPropertyName("tag_type_to_assembly_map")] [JsonPropertyName("tag_type_to_assembly_map")]
public TagTypeToAssembly? TagTypeToAssemblyMap { get; set; } public TagTypeToAssembly? TagTypeToAssemblyMap { get; set; }
// unused [JsonPropertyName("server_queue")]
public string? ServerQueue { get; set; } = "PubSubQueue";
[JsonPropertyName("port_range")]
public PubSubPortRange? PortRange { get; set; }
[JsonPropertyName("bind_to_random_port")]
public bool? BindToRandomPort { get; set; } = false;
[JsonPropertyName("debug_tags")]
public string? DebugTags { get; set; }
[JsonPropertyName("latency_report_threshold_ms")] [JsonPropertyName("latency_report_threshold_ms")]
public int? LatencyReportThresholdMs { get; set; } public int? LatencyReportThresholdMs { get; set; }
// unused [JsonPropertyName("use_substations")]
public bool? UseSubstations { get; set; }
[JsonPropertyName("use_dynamic_clients")]
public bool? UseDynamicClients { get; set; }
[JsonPropertyName("is_substation")]
public bool? IsSubstation { get; set; }
[JsonPropertyName("internally_publish_sub_info")] [JsonPropertyName("internally_publish_sub_info")]
public bool? InternallyPublishSubInfo { get; set; } public bool? InternallyPublishSubInfo { get; set; }
[JsonPropertyName("use_managed_clients_in_native_code")]
public bool? UseManagedClientsInNativeCode { get; set; }
[JsonPropertyName("compressed_tags")] [JsonPropertyName("compressed_tags")]
public string? CompressedTags { get; set; } public string? CompressedTags { get; set; }
[JsonPropertyName("tags_to_report_compression_ratio")]
public string? TagsToReportCompressionRatio { get; set; }
[JsonPropertyName("disable_msmq")]
public bool? DisableMsmq { get; set; }
[JsonPropertyName("use_prepublished_server_state")]
public bool? UsePrepublishedServerState { get; set; }
[JsonPropertyName("ip_translation")]
public string? IpTranslation { get; set; }
[JsonPropertyName("gc_activation_threshold")] [JsonPropertyName("gc_activation_threshold")]
public int? GcActivationThreshold { get; set; } public int? GcActivationThreshold { get; set; }
@@ -105,22 +148,38 @@ public class PubSubConfig {
DoItAllSecondsBetweenLog = 0, DoItAllSecondsBetweenLog = 0,
HeartbeatInterval = 15000, HeartbeatInterval = 15000,
MissingHeartbeatsBeforeGc = 50, MissingHeartbeatsBeforeGc = 50,
UseManagedSockets = true,
StateUpdatePeriod = 5000, StateUpdatePeriod = 5000,
StatisticsUpdatePeriod = 5000, StatisticsUpdatePeriod = 5000,
DisposedConnectionCheckPeriod = 1000, DisposedConnectionCheckPeriod = 1000,
MessageProcessingWarningThreshold = 1000,
ServerPort = 12345, ServerPort = 12345,
MaxPacketSize = 500000000,
BlueCommMustDie = true,
CrushableTags = new(0),
TagTypeToAssemblyMap = new(), TagTypeToAssemblyMap = new(),
ServerQueue = "PubSubQueue",
PortRange = new(30400, 31700),
BindToRandomPort = false,
DebugTags = NOTAGS,
LatencyReportThresholdMs = 1500, LatencyReportThresholdMs = 1500,
UseSubstations = false,
IsSubstation = false,
InternallyPublishSubInfo = true, InternallyPublishSubInfo = true,
UseManagedClientsInNativeCode = false,
CompressedTags = NOTAGS, CompressedTags = NOTAGS,
TagsToReportCompressionRatio = NOTAGS,
DisableMsmq = false,
UsePrepublishedServerState = false,
IpTranslation = null,
GcActivationThreshold = 0, GcActivationThreshold = 0,
WaitPeriodAfterGc = 1000, WaitPeriodAfterGc = 1000,
SimpleEchoServer = false, SimpleEchoServer = false,
PublisherRoutes = new() { PublisherRoutes = new() {
new("bluto://127.0.0.1:12345", ".*") new("bluto://127.0.0.0.1:12345", ".*")
}, },
SubscriberRoutes = new() { SubscriberRoutes = new() {
new("bluto://127.0.0.1:12345", ".*") new("bluto://127.0.0.0.1:12345", ".*")
} }
}; };
return rv; return rv;
@@ -134,14 +193,30 @@ public class PubSubConfig {
DoItAllSecondsBetweenLog ??= other.DoItAllSecondsBetweenLog; DoItAllSecondsBetweenLog ??= other.DoItAllSecondsBetweenLog;
HeartbeatInterval ??= other.HeartbeatInterval; HeartbeatInterval ??= other.HeartbeatInterval;
MissingHeartbeatsBeforeGc ??= other.MissingHeartbeatsBeforeGc; MissingHeartbeatsBeforeGc ??= other.MissingHeartbeatsBeforeGc;
UseManagedSockets ??= other.UseManagedSockets;
StateUpdatePeriod ??= other.StateUpdatePeriod; StateUpdatePeriod ??= other.StateUpdatePeriod;
StatisticsUpdatePeriod ??= other.StatisticsUpdatePeriod; StatisticsUpdatePeriod ??= other.StatisticsUpdatePeriod;
DisposedConnectionCheckPeriod ??= other.DisposedConnectionCheckPeriod; DisposedConnectionCheckPeriod ??= other.DisposedConnectionCheckPeriod;
MessageProcessingWarningThreshold ??= other.MessageProcessingWarningThreshold;
ServerPort ??= other.ServerPort; ServerPort ??= other.ServerPort;
MaxPacketSize ??= other.MaxPacketSize;
BlueCommMustDie ??= other.BlueCommMustDie;
CrushableTags ??= other.CrushableTags;
TagTypeToAssemblyMap ??= other.TagTypeToAssemblyMap; TagTypeToAssemblyMap ??= other.TagTypeToAssemblyMap;
ServerQueue ??= other.ServerQueue;
PortRange ??= other.PortRange;
BindToRandomPort ??= other.BindToRandomPort;
DebugTags ??= other.DebugTags;
LatencyReportThresholdMs ??= other.LatencyReportThresholdMs; LatencyReportThresholdMs ??= other.LatencyReportThresholdMs;
UseSubstations ??= other.UseSubstations;
IsSubstation ??= other.IsSubstation;
InternallyPublishSubInfo ??= other.InternallyPublishSubInfo; InternallyPublishSubInfo ??= other.InternallyPublishSubInfo;
UseManagedClientsInNativeCode ??= other.UseManagedClientsInNativeCode;
CompressedTags ??= other.CompressedTags; CompressedTags ??= other.CompressedTags;
TagsToReportCompressionRatio ??= other.TagsToReportCompressionRatio;
DisableMsmq ??= other.DisableMsmq;
UsePrepublishedServerState ??= other.UsePrepublishedServerState;
IpTranslation ??= other.IpTranslation;
GcActivationThreshold ??= other.GcActivationThreshold; GcActivationThreshold ??= other.GcActivationThreshold;
WaitPeriodAfterGc ??= other.WaitPeriodAfterGc; WaitPeriodAfterGc ??= other.WaitPeriodAfterGc;
SimpleEchoServer ??= other.SimpleEchoServer; SimpleEchoServer ??= other.SimpleEchoServer;
@@ -1,542 +0,0 @@
// using System;
// using System.Collections.Generic;
// using System.Text;
// using System.Net;
// using System.Net.Sockets;
// using System.IO;
// //using BCG.Utilities;
// using BCG.Utilities.General;
// using System.Runtime.InteropServices;
// namespace BCG
// {
// /// <summary>
// /// This class, mantains a connection to a peer socket, using
// /// the .Net async TCP library. It sends heartbeats at regular
// /// intervals (config). It is in active states when messages
// /// are sent and received without an error state. It assumed
// /// dead state when either of the following happens
// /// BeginReceive returns a SocketError
// /// BeginSend returns a SocketError
// /// EndReceive returns a SocketError
// /// EndSend returns a SocketError
// /// As soon as it assumes dead state it stops sending or
// /// receiving any messages (including heartbeats).
// /// The user of this class can either detect that by
// /// missing heartbeats or checking the isConnected property.
// /// Missing should be the more general way, since if the
// /// peer dies in a 'silent' way, where none of the above
// /// methods indicate an error, missing heartbeats is guaranteed
// /// to detect it.
// /// </summary>
// public class AsyncTCPConnection : ITCPConnection, IWorker, IDisposable
// {
// /// <summary>
// /// This is used as a state for the Async send. Reason being
// /// that if onSend indicates that it could not send the full
// /// data (ie EndSend does not return the full size), the offset
// /// is used to resend the data
// /// </summary>
// public class AsyncMessage
// {
// public byte[] buffer;
// public int offset;
// public string messageId;
// public SendConfirmationDelegate onSendDel;
// public AsyncMessage(byte[] theBuffer, int theOffset)
// : this(theBuffer, theOffset, String.Empty, null) { }
// public AsyncMessage(byte[] theBuffer, int theOffset, string theMessageId, SendConfirmationDelegate theOnSendDel)
// {
// buffer = theBuffer;
// offset = theOffset;
// messageId = theMessageId;
// onSendDel = theOnSendDel;
// }
// public int length
// {
// get
// {
// return buffer.Length - offset;
// }
// }
// }
// enum ReadState
// {
// readingSize,
// readingData
// };
// #region work around for .Net socket's LocalEndPoint bug
// [StructLayout(LayoutKind.Sequential)]
// private struct in_addr
// {
// [MarshalAs(UnmanagedType.ByValArray, SizeConst = 4)]
// public byte[] sin_addr;
// }
// [StructLayout(LayoutKind.Sequential)]
// private struct sockaddr
// {
// public short sin_family;
// public ushort sin_port;
// public in_addr sin_addr;
// [MarshalAs(UnmanagedType.ByValArray, SizeConst = 8)]
// public byte[] sin_zero;
// }
// [DllImport("wsock32.dll")]
// private static extern int getsockname(IntPtr s, ref sockaddr name, ref int namelen);
// [DllImport("wsock32.dll")]
// private static extern int getpeername(IntPtr s, ref sockaddr name, ref int namelen);
// #endregion
// private readonly int myMaxMessageSize = PubSub3Config.instance.PS3_MaxPacketSize;
// private RemoteConnectionInfo myPeer, mySelf;
// private Socket mySock;
// private ITCPMessageHandler myMessageHandler;
// private byte[] myCurrentReadBuffer;
// private int myCurrentReadBufferOffset;
// private ReadState myCurrentReadState;
// private bool myAlive = true;
// private byte[] myHBMessage;
// private TimeSpan myHBTimeSpan;
// private MasterWorker myMasterWorker = MasterWorker.getInstance(true);
// private bool myHBReceived = false, myDataReceived = false, myHBSent = false;
// private DateTime myLastHBReceivedTime = new DateTime(1900, 1, 1);
// public AsyncTCPConnection(Socket sock, TimeSpan heartBeatTimeSpan)
// {
// myHBTimeSpan = heartBeatTimeSpan;
// mySock = sock;
// mySock.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.DontRoute, true);
// string ip;
// int port;
// getPeerAddress(out ip, out port);
// myPeer = new RemoteConnectionInfo(ip, port);
// getSelfAddress(out ip, out port);
// mySelf = new RemoteConnectionInfo(ip, port);
// // BlueEventLog.logEvent(String.Format("AsyncTCPConnection created : self = {0} , peer = {1}",
// // mySelf.ToString(), myPeer.ToString()), PS3LogLevels.connection);
// }
// public override string ToString()
// {
// return String.Format("peer = {0}, self = {1}", myPeer.ToString(), mySelf.ToString());
// }
// #region ITCPConnection Members
// public void sendWithConfirmation(byte[] data, ref string confId, SendConfirmationDelegate del)
// {
// confId = Guid.NewGuid().ToString();
// byte[] msg = myBuildDataMessage(data);
// myBeginOrResumeSending(new AsyncMessage(msg, 0, confId, del));
// }
// public void send(byte[] data)
// {
// byte[] msg = myBuildDataMessage(data);
// myBeginOrResumeSending(new AsyncMessage(msg, 0));
// }
// public void register(ITCPMessageHandler messageHandler)
// {
// // BlueEventLog.logAssert(myMessageHandler == null, "Message Handler is already assigned");
// // BlueEventLog.logEvent(String.Format("AsyncTCPConnection for peer = {0} being registered", myPeer.ToString()),
// // PS3LogLevels.connection);
// myMessageHandler = messageHandler;
// myMasterWorker.addWorker(this, myHBTimeSpan);
// myReadSize();
// }
// public void getPeerAddress(out string ip, out int port)
// {
// ip = "";
// port = 0;
// if (myAlive)
// {
// IPEndPoint ipEp = mySock.RemoteEndPoint as IPEndPoint;
// ip = ipEp.Address.ToString();
// port = ipEp.Port;
// }
// ///.Net has some problems getting the port, so using
// ///native code for that
// //sockaddr addr = new sockaddr();
// //int namelen = 16;
// //getsockname(mySock.Handle, ref addr, ref namelen);
// //port = addr.sin_port;
// }
// public void getSelfAddress(out string ip, out int port)
// {
// ip = "";
// port = 0;
// if (myAlive)
// {
// IPEndPoint ipEp = mySock.LocalEndPoint as IPEndPoint;
// ip = ipEp.Address.ToString();
// port = ipEp.Port;
// }
// ///due to this bug in .Net socket, we have to resort to
// ///native code
// //sockaddr addr = new sockaddr();
// //int namelen = 16;
// //getsockname(mySock.Handle, ref addr, ref namelen);
// //port = addr.sin_port;
// }
// public bool isDisposed
// {
// get { return !myAlive; }
// }
// #endregion
// #region IWorker Members
// bool _last_connected = false;
// public void doWork()
// {
// if (mySock != null)
// //if (mySock.Connected != _last_connected)
// {
// Console.WriteLine($"AsyncTCPConnection.doWork connected: {mySock.Connected}");
// _last_connected = mySock.Connected;
// }
// if (!myHBSent)
// {
// // BlueEventLog.logEvent(String.Format("Sending heartbeat to peer {0} for the first time", myPeer.ToString()),
// // PS3LogLevels.connection);
// myHBSent = true;
// }
// myBeginOrResumeSending(new AsyncMessage(myBuildHeartbeatMessage(), 0));
// }
// #endregion
// #region sending data
// private void myBeginOrResumeSending(AsyncMessage message)
// {
// if (myAlive)
// {
// try
// {
// //Console.WriteLine($"AsyncTCPConnection.myBeginOrResumeSending len: {message.length} connected: {mySock.Connected}");
// SocketError beginSendErr;
// mySock.BeginSend(message.buffer, message.offset, message.length, SocketFlags.None,
// out beginSendErr, new AsyncCallback(myOnSend), message);
// Console.WriteLine($"AsyncTCPConnection.myBeginOrResumeSending len: {message.length} socker_error: {beginSendErr} connected: {mySock.Connected}");
// if (beginSendErr != SocketError.Success)
// myHandleError(beginSendErr);
// }
// catch (Exception ex)
// {
// myHandleError(ex);
// }
// }
// }
// private void myOnSend(IAsyncResult res)
// {
// if (myAlive)
// {
// try
// {
// SocketError writeErr;
// int bytesSent = mySock.EndSend(res, out writeErr);
// Console.WriteLine($"AsyncTCPConnection.myOnSend bytes: {bytesSent} writeErr: {writeErr} connected: {mySock.Connected}");
// AsyncMessage message = res.AsyncState as AsyncMessage;
// message.offset += bytesSent;
// if (writeErr == SocketError.Success)
// {
// if (message.offset == message.buffer.Length)
// {
// // if confirmation needs to be sent
// if (String.Empty != message.messageId)
// {
// message.onSendDel(message.messageId, true);
// }
// }
// else
// {
// ///resume writing rest of the message
// // BlueEventLog.logEvent("could not send the whole message all at once , messageSize = " +
// // message.buffer.Length + " offset = " + message.offset, PS3LogLevels.info);
// myBeginOrResumeSending(message);
// }
// }
// else
// {
// myHandleError(writeErr);
// }
// }
// catch (Exception ex)
// {
// myHandleError(ex);
// }
// }
// }
// /// <summary>
// /// Assembles the heartbeat message, and uses it for
// /// future references
// /// </summary>
// /// <returns></returns>
// private byte[] myBuildHeartbeatMessage()
// {
// if (myHBMessage == null)
// {
// BinaryWriter bw = new BinaryWriter(new MemoryStream());
// int sz = 4;
// int msgType = 1;
// bw.Write(sz);
// bw.Write(msgType);
// myHBMessage = (bw.BaseStream as MemoryStream).ToArray();
// }
// return myHBMessage;
// }
// /// <summary>
// /// Assembles the data message. This is necessary
// /// for the other side to understand the message. This can also
// /// be achieved by not forming the message and use the
// /// BeginSend (List<>) function rather than BeginSend (byte[])
// /// </summary>
// /// <param name="data"></param>
// /// <returns></returns>
// private byte[] myBuildDataMessage(byte[] data)
// {
// BinaryWriter bw = new BinaryWriter(new MemoryStream());
// bw.Write(data.Length + 8);
// bw.Write(0);
// bw.Write(0); ///mid for backward compatibility
// bw.Write(data);
// return (bw.BaseStream as MemoryStream).ToArray();
// }
// #endregion
// #region receiving data
// private void myReadSize()
// {
// myCurrentReadBuffer = new byte[4];
// myCurrentReadBufferOffset = 0;
// myCurrentReadState = ReadState.readingSize;
// myBeginOrResumeReading();
// }
// private void myReadData(int dataSize)
// {
// myCurrentReadBuffer = new byte[dataSize];
// myCurrentReadBufferOffset = 0;
// myCurrentReadState = ReadState.readingData;
// myBeginOrResumeReading();
// }
// private void myBeginOrResumeReading()
// {
// if (myAlive)
// {
// try
// {
// int beginReceiveAttempts = 0;
// while (true)
// {
// beginReceiveAttempts++;
// SocketError readSockErr;
// mySock.BeginReceive(myCurrentReadBuffer, myCurrentReadBufferOffset,
// myCurrentReadBuffer.Length - myCurrentReadBufferOffset,
// SocketFlags.None, out readSockErr,
// new AsyncCallback(myOnRead),
// myCurrentReadBuffer);
// Console.WriteLine($"AsyncTCPConnection.myBeginOrResumeReading socket_error: {readSockErr} connected: {mySock.Connected}");
// if (readSockErr == SocketError.NoBufferSpaceAvailable)
// {
// string msg = String.Format("Not enough buffer space to receive {0} bytes. Attempt: {1} of {2}.",
// myCurrentReadBuffer.Length - myCurrentReadBufferOffset,
// beginReceiveAttempts, PubSub3Config.instance.PS3_retriesOnNoBufferSpace + 1);
// //BlueEventLog.logEvent(msg, PS3LogLevels.disconnection);
// if (beginReceiveAttempts < (PubSub3Config.instance.PS3_retriesOnNoBufferSpace + 1))
// System.Threading.Thread.Sleep(PubSub3Config.instance.PS3_millisecondsBetweenNoBufferSpaceRetries);
// else
// {
// //BlueEventLog.logEvent("Could not recover from NoBufferSpace condition. Breaking connection.", PS3LogLevels.disconnection);
// myHandleError(readSockErr);
// }
// }
// else if (readSockErr == SocketError.Success)
// {
// if (beginReceiveAttempts > 1)
// // BlueEventLog.logEvent(String.Format("Recovered from NoBufferSpace after {0} tries",beginReceiveAttempts-1) ,
// // PS3LogLevels.disconnection);
// break;
// }
// else
// myHandleError(readSockErr);
// }
// }
// catch (Exception ex)
// {
// myHandleError(ex);
// }
// }
// }
// private void myOnRead(IAsyncResult res)
// {
// if (myAlive)
// {
// try
// {
// SocketError readErr;
// Console.WriteLine($"AsyncTCPConnection.myOnRead pre connected: {mySock.Connected}");
// int bytesRead = mySock.EndReceive(res, out readErr);
// Console.WriteLine($"AsyncTCPConnection.myOnRead bytes: {bytesRead} readErr: {readErr} connected: {mySock.Connected}");
// myCurrentReadBufferOffset += bytesRead;
// if (readErr == SocketError.Success)
// {
// if (myCurrentReadBufferOffset == myCurrentReadBuffer.Length)
// {
// ///read the whole of the buffer
// if (myCurrentReadState == ReadState.readingSize)
// {
// ///the size has already been read, now convert
// ///it into an int, and start reading the data
// int dataSize = new BinaryReader(new MemoryStream(myCurrentReadBuffer)).ReadInt32();
// if (dataSize > myMaxMessageSize)
// {
// string errMessage = "Encountered malformed message, size indicated = " + dataSize;
// //BlueEventLog.logEvent(errMessage, "Error");
// myHandleError(new ApplicationException(errMessage));
// }
// else
// myReadData(dataSize);
// }
// else
// {
// ///the whole data has been finally read, time to
// ///tell the subscriber about it
// BinaryReader br = new BinaryReader(new MemoryStream(myCurrentReadBuffer));
// int msgType = br.ReadInt32();
// if (msgType == 0)
// {
// ///skips the message ID
// br.ReadInt32();
// byte[] data = br.ReadBytes(myCurrentReadBuffer.Length - 8);
// ///callbacks can be made through thread pool to avoid this
// ///thread from being blocked, by this callback
// if (!myDataReceived)
// {
// myDataReceived = true;
// Console.WriteLine($"AsyncTCPConnection Data received for the first time from peer {myPeer.ToString()}");
// }
// myMessageHandler.onData(new RemoteMessage(data, RemoteMessageType.Data, myPeer));
// }
// else if (msgType == 1)
// {
// if (!myHBReceived)
// {
// myHBReceived = true;
// Console.WriteLine($"AsyncTCPConnection HB received for the first time from peer {myPeer.ToString()}");
// }
// myLastHBReceivedTime = DateTime.Now;
// myMessageHandler.onData(new RemoteMessage(null, RemoteMessageType.Heartbeat, myPeer));
// }
// myReadSize();
// }
// }
// else
// {
// ///still part of the buffer left to read
// ///continue reading
// myBeginOrResumeReading();
// }
// }
// else
// {
// myHandleError(readErr);
// }
// }
// catch (Exception ex)
// {
// myHandleError(ex);
// }
// }
// }
// #endregion
// #region IDisposable Members
// public void Dispose()
// {
// Console.WriteLine("AsyncTCPConnection.Dispose()");
// lock (this)
// {
// if (myAlive)
// {
// myMasterWorker.removeWorker(this);
// myAlive = false;
// // BlueEventLog.logEvent(String.Format("AsycnTCPConnection with peer {0} being disposed, last HB was received at {1}" ,
// // myPeer.ToString(), myLastHBReceivedTime.ToLongTimeString()),
// // PS3LogLevels.disconnection);
// try
// {
// mySock.Close();
// }
// catch (Exception ex)
// {
// String.Format("Exception of type {0} thrown while trying to close socket connected to {1} , with" +
// " the following exception msg {2}", ex.GetType().ToString(), myPeer.ToString(), ex.Message);
// //BlueEventLog.logEvent(ex);
// if (ex is OutOfMemoryException)
// throw ex;
// }
// }
// }
// }
// #endregion
// private void myHandleError(object err)
// {
// lock (this)
// {
// string errMsg = "";
// if (err is Exception)
// {
// errMsg = String.Format("AsyncTCPConnection.Dispose Encountered an exception of type '{0}' on the socket connected to ip = {1} , port = {2} ; " +
// "Exception message = {3} , if the socket has not been disposed, it will be disposed now", err.GetType(), myPeer.ip, myPeer.port, (err as Exception).Message);
// if (err is OutOfMemoryException)
// throw (err as OutOfMemoryException);
// }
// else if (err is SocketError)
// {
// errMsg = String.Format("AsyncTCPConnection.Dispose Encountered a SocketError of type '{0}' on the socket connected to ip = {1} , port = {2} ; " +
// "if the socket has not been disposed, it will be disposed now", err.ToString(), myPeer.ip, myPeer.port);
// Console.WriteLine($"AsyncTCPConnection.myHandleError errMsg: {errMsg}");
// }
// //BlueEventLog.logEvent(errMsg, PS3LogLevels.disconnection);
// if (myAlive)
// {
// Dispose();
// }
// }
// }
// }
// }
@@ -1,248 +0,0 @@
// using BCG;
// using Google.Protobuf;
// using System;
// using System.IO;
// using System.Linq;
// using System.Net;
// using System.Net.Sockets;
// using System.Runtime.CompilerServices;
// using System.Text;
// using System.Threading.Tasks;
// using BCG;
// using System.Xml.Serialization;
// using System.Reflection;
// using BCG.Utilities.General;
// using System.Net.WebSockets;
// namespace BCG {
// public class AsyncTCPConnection2 : ITCPConnection, IWorker, IDisposable {
// private Socket _socket;
// // tied to the socket above
// private NetworkStream _nw_stream;
// private readonly ConnectionSide _side;
// ITCPMessageHandler _message_handler;
// private int _msg_id = 0;
// RemoteConnectionInfo _peer_address;
// RemoteConnectionInfo _self_address;
// private TimeSpan _hb_period = TimeSpan.FromSeconds(1);
// private DateTime _last_hb_sent = DateTime.MinValue;
// private DateTime _last_hb_received = DateTime.MaxValue;
// private static byte _msg_type_data = 1;
// private static byte _msg_type_hb = 0;
// private bool _is_disposed = false;
// SyncQueue<QueuedMessage> _data_send_queue = new SyncQueue<QueuedMessage>();
// public class QueuedMessage {
// public string id;
// public SendConfirmationDelegate del;
// public byte[] data;
// public QueuedMessage(string id, SendConfirmationDelegate del, byte[] data) {
// id = id;
// del = del;
// data = data;
// }
// public QueuedMessage(byte[] data)
// : this(String.Empty, null, data) { }
// }
// public enum ConnectionSide {
// Client,
// Server
// }
// public AsyncTCPConnection2(Socket socket, ConnectionSide side) {
// _socket = socket;
// _nw_stream = new NetworkStream(socket);
// _side = side;
// IPEndPoint r_ep = socket.RemoteEndPoint as IPEndPoint;
// _peer_address = new RemoteConnectionInfo(r_ep.Address.ToString(), r_ep.Port);
// var loc_ep = socket.LocalEndPoint as IPEndPoint;
// _self_address = new RemoteConnectionInfo(loc_ep.Address.ToString(), loc_ep.Port);
// }
// public void register(ITCPMessageHandler message_handler) {
// _message_handler = message_handler;
// Task.Run(async () => _keep_reading());
// Task.Run(async () => await _keep_sending());
// //_keep_reading();
// // if add a _keep_sending threadis added then we can maek send a non async
// // methid making it much easier to integrate with the rest
// }
// // this thread pulls
// private async Task _keep_sending() {
// bts.Log.Gen.info($"TCPConnection._keep_sending starting");
// while (!_is_disposed) {
// try {
// QueuedMessage m = _data_send_queue.dequeue(100);
// if (m == null) {
// continue;
// }
// await send_async(m.data);
// if (String.IsNullOrEmpty(m.id)) {
// m.del(m.id, true);
// }
// } catch (Exception ex) {
// bts.Log.Gen.error($"TCPConnection._keep_sending ex: {ex.Message}");
// await Task.Delay(100);
// _handle_error(ex);
// }
// }
// }
// private async Task _keep_reading() {
// bts.Log.Gen.info($"TCPConnection._keep_reading starting");
// while (!_is_disposed) {
// try {
// byte[] msg = await TCPSocketUtils.read_bytes_async(_nw_stream);
// bts.Log.Gen.info($"AsyncTCPConnection2._keep_reading side: {_side} self: {get_self_address()} peer: {get_peer_address()} msg len: {msg.Length}");
// await _handle_msg(msg);
// } catch (Exception ex) {
// bts.Log.Gen.error($"TCPConnection._keep_reading ex: {ex.Message}");
// await Task.Delay(100);
// _handle_error(ex);
// }
// }
// }
// private async Task _handle_msg(byte[] msg) {
// // read the first byte to read the msg type and deal with it accordingly
// BinaryReader br = new BinaryReader(new MemoryStream(msg));
// byte msg_type = br.ReadByte();
// if (msg_type == _msg_type_data) {
// byte[] data = br.ReadBytes(msg.Length - 1);
// RemoteMessage m = new RemoteMessage(data, RemoteMessageType.Data, _peer_address);
// _message_handler.onData(m);
// } else {
// RemoteMessage m = new RemoteMessage(null, RemoteMessageType.Heartbeat, _peer_address);
// _message_handler.onData(m);
// }
// }
// // not very efficient
// private static byte[] _prefix_msg_type(byte[] data) {
// MemoryStream ms = new MemoryStream();
// BinaryWriter bw = new BinaryWriter(ms);
// bw.Write(_msg_type_data);
// bw.Write(data);
// ms.Position = 0;
// return ms.ToArray();
// }
// private async void _handle_error(Exception ex) {
// bts.Log.Gen.error($"TCPConnection._handle_error ex: {ex}");
// try {
// var socket = _nw_stream.Socket;
// socket.Close();
// socket.Dispose();
// } catch (Exception ex2) {
// bts.Log.Gen.error($"TCPConnection._handle_error got exception when closing Socket ex: {ex}");
// }
// _is_disposed = true;
// RemoteMessage rm = new RemoteMessage(new byte[] { }, RemoteMessageType.Disconnection, _peer_address);
// _message_handler.onData(rm);
// }
// public void send_with_confirmation(byte[] data, string confId, SendConfirmationDelegate del) { }
// public async Task send_with_confirmation_async(byte[] data, string conf_id, SendConfirmationDelegate del) {
// try {
// conf_id = (++_msg_id).ToString();
// await TCPSocketUtils.send_bytes_async(_nw_stream, data);
// del(conf_id, true);
// } catch (Exception ex) {
// _handle_error(ex);
// }
// }
// public void sendWithConfirmation(byte[] data, ref string conf_id, SendConfirmationDelegate del) {
// conf_id = (++_msg_id).ToString();
// send(data);
// del(conf_id, true);
// }
// public void send(byte[] data) {
// Task.Run(async () => await send_async(data));
// bts.Log.Gen.info($"AsyncTCPConnection2.send self: {get_self_address()} peer: {get_peer_address()} msg len: {data.Length} sent successfully");
// //_data_send_queue.enqueue(new QueuedMessage(data));
// }
// public async Task send_async(byte[] data) {
// // prefix 4 bytes for type (data vs hb)
// MemoryStream str = new MemoryStream();
// BinaryWriter bw = new BinaryWriter(str);
// bw.Write(_msg_type_data);
// bw.Write(data);
// try {
// await TCPSocketUtils.send_bytes_async(_nw_stream, str.ToArray());
// } catch (Exception ex) {
// bts.Log.Gen.error($"AsyncTCPConnection2.send_async ex: {ex}");
// _handle_error(ex);
// }
// }
// public void getPeerAddress(out string ip, out int port) {
// ip = _peer_address.ip;
// port = _peer_address.port;
// ///.Net has some problems getting the port, so using
// ///native code for that
// //sockaddr addr = new sockaddr();
// //int namelen = 16;
// //getsockname(mySock.Handle, ref addr, ref namelen);
// //port = addr.sin_port;
// }
// public RemoteConnectionInfo get_peer_address() {
// string ip; int port;
// getPeerAddress(out ip, out port);
// return new RemoteConnectionInfo(ip, port);
// }
// public RemoteConnectionInfo get_self_address() {
// string ip; int port;
// getSelfAddress(out ip, out port);
// return new RemoteConnectionInfo(ip, port);
// }
// public void getSelfAddress(out string ip, out int port) {
// ip = _self_address.ip;
// port = _self_address.port;
// }
// private async void _send_hb() {
// if (_last_hb_sent - DateTime.Now > _hb_period) {
// MemoryStream ms = new MemoryStream();
// BinaryWriter bw = new BinaryWriter(ms);
// bw.Write(_msg_type_hb);
// ms.Position = 0;
// byte[] hb_msg = ms.ToArray();
// await TCPSocketUtils.send_bytes_async(_nw_stream, hb_msg);
// }
// }
// public void doWork() {
// _send_hb();
// }
// public void Dispose() {
// _nw_stream.Close();
// _nw_stream.Dispose();
// _is_disposed = true;
// }
// public bool isDisposed {
// get {
// return _is_disposed;
// }
// }
// }
// }
@@ -15,7 +15,6 @@ namespace bts {
private ITCPConnectionHandler _connection_handler; private ITCPConnectionHandler _connection_handler;
private HashSet<string> _key_map = new HashSet<string>(); private HashSet<string> _key_map = new HashSet<string>();
public AsyncTCPConnectionAcceptor_APM(int port, ITCPConnectionHandler connection_handler) { public AsyncTCPConnectionAcceptor_APM(int port, ITCPConnectionHandler connection_handler) {
Log.Gen.info($"AsyncTCPConnectionAcceptor_APM. cons started on port: {port}");
_port = port; _port = port;
_connection_handler = connection_handler; _connection_handler = connection_handler;
} }
@@ -26,9 +25,7 @@ namespace bts {
} }
private void _init() { private void _init() {
Log.Gen.info($"AsyncTCPConnectionAcceptor_APM._init started on port: {_port}");
_accept_socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); _accept_socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
TCPSocketUtils.set_server_socket_options(ref _accept_socket);
IPEndPoint endpoint = new IPEndPoint(IPAddress.Any, _port); IPEndPoint endpoint = new IPEndPoint(IPAddress.Any, _port);
_accept_socket.Bind(endpoint); _accept_socket.Bind(endpoint);
_accept_socket.Listen(250); _accept_socket.Listen(250);
@@ -36,7 +33,6 @@ namespace bts {
private void _begin_accepting() { private void _begin_accepting() {
var accept_key = Guid.NewGuid().ToString(); var accept_key = Guid.NewGuid().ToString();
Log.Gen.info($"AsyncTCPConnectionAcceptor_APM._begin_accepting port: {_port} accept_key: {accept_key}");
var res = _accept_socket.BeginAccept(new AsyncCallback(_on_accept), accept_key); var res = _accept_socket.BeginAccept(new AsyncCallback(_on_accept), accept_key);
if (res != null && res.IsCompleted && res.CompletedSynchronously) if (res != null && res.IsCompleted && res.CompletedSynchronously)
_on_accept(res); _on_accept(res);
@@ -66,8 +62,11 @@ namespace bts {
_connection_handler.onConnection(newConn); _connection_handler.onConnection(newConn);
// go back to accepting more conns // go back to accepting more conns
_begin_accepting(); _begin_accepting();
var peer_rci = newConn.get_peer_address(); string ip; int port;
Log.Gen.info($"AsyncTCPConnectionAcceptor_APM._on_accept peer_rci: {peer_rci}"); newConn.getPeerAddress(out ip, out port);
string connMsg = String.Format("Connection to port {2} made from {0}:{1}", ip, port, _port);
// BlueEventLog.logEvent(connMsg, PS3LogLevels.connection);
//Console.WriteLine(connMsg);
} }
@@ -1,58 +0,0 @@
using System.Net.Sockets;
using System.Net;
using BCG;
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
namespace bts {
// opens a TCP port and starts listening to connections. Unlike AsyncTCPConnectionAcceptor_APM,
// it uses TcpListener, instead of raw sockets.
public class AsyncTCPConnectionAcceptor_AsyncWrapper {
private readonly int _accept_port;
private Socket _accept_socket;
private ITCPConnectionHandler _connection_handler;
private TcpListener _server;
private Action<ITCPConnection> _cbs = (c) => { };
public AsyncTCPConnectionAcceptor_AsyncWrapper(int accept_port, ITCPConnectionHandler connection_handler) {
Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper.cons started on port: {accept_port}");
_accept_port = accept_port;
_connection_handler = connection_handler;
}
public void resgiter_callback(Action<ITCPConnection> cb) {
_cbs += cb;
}
public void start() {
Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper._init started on port: {_accept_port}");
_server = new TcpListener(IPAddress.Any, _accept_port);
_server.Start();
Task.Run(() => _accept_connections_forever());
}
private void _accept_connections_forever() {
Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper._begin_accepting port: {_accept_port}");
while (true) {
// this is a blocking call
Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper._begin_accepting port: {_accept_port} starting blocking call AcceptTcpClient");
TcpClient tcp_client = _server.AcceptTcpClient();
ITCPConnection conn = _create_connection(tcp_client);
Log.Gen.info($"AsyncTCPConnectionAcceptor_AsyncWrapper._begin_accepting port: {_accept_port}, new connection, self: {conn.get_self_address()} peer: {conn.get_peer_address()} calling onConnection now");
_connection_handler.onConnection(conn);
}
//_cbs(conn);
}
private static ITCPConnection _create_connection(TcpClient tcp_client) {
ITCPConnection conn = new AsyncTCPConnection_AsyncWrapper(tcp_client);
// ITCPConnection conn = new AsyncTCPConnection_APM(tcp_client.GetStream().Socket, TimeSpan.FromSeconds(1), ITCPConnection.ConnectionSide.Client);
return conn;
}
}
}
Loaded 100 of 1013 files, more files were not shown because too many files have changed in this diff. Show more