Compare commits

..
Author SHA1 Message Date
John Colvin 8676d4e270 activ md check-pt
Started to resolve circular deps between libspark <-> sparkd
2024-01-09 13:55:11 -06:00
John Colvin 08706cb75d refactor: Don't log parse errors in JsonReader::read_or
This function is designed for cases where the key may not exist, or the
value may be invalid
2023-12-20 11:41:06 -05:00
John Colvin f64e5fe1bf scratch: activ md check point 2023-12-15 13:17:51 -05:00
901 changed files with 27277 additions and 49241 deletions

No files matched your search

-125
View File
@@ -1,125 +0,0 @@
FROM ubuntu:22.04
ARG DEBIAN_FRONTEND=noninteractive
# Note installations that take a while (more than 10s)
RUN apt-get update
RUN apt-get install -y ca-certificates
RUN apt-get install -y ccache
RUN apt-get install -y curl
RUN apt-get install -y fish
RUN apt-get install -y gdb
# git-all: ~250s
RUN apt-get install -y git-all
RUN apt-get install -y gnupg
# liblttng-ust-dev: ~30s
RUN apt-get install -y liblttng-ust-dev
RUN apt-get install -y libssh2-1-dev
RUN apt-get install -y locales
RUN apt-get install -y ninja-build
# openssh-server: ~40s
RUN apt-get install -y openssh-server
RUN apt-get install -y pkg-config
RUN apt-get install -y software-properties-common
RUN apt-get install -y sudo
RUN apt-get install -y tzdata
RUN apt-get install -y vim
RUN apt-get install -y zip
# Python: ~30s
RUN apt-get install -y \
python3-pip \
python3-venv \
&& python3 -m pip install --upgrade pip
# GCC
ARG gcc_major=13
RUN \
add-apt-repository ppa:ubuntu-toolchain-r/test \
&& apt-get update \
&& apt-get install -y g++-$gcc_major
# Clang
ARG clang_major=17
RUN \
curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key 2> /dev/null \
| gpg --dearmor - \
| tee /usr/share/keyrings/llvm-snapshot.gpg > /dev/null \
&& echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-$clang_major main" \
| tee -a "/etc/apt/sources.list.d/llvm.list" \
&& apt-get update \
&& apt-get install -y \
clang-$clang_major \
clang-format-$clang_major \
clangd-$clang_major
# CMake
RUN \
curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
| gpg --dearmor - \
| tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \
&& echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
| tee /etc/apt/sources.list.d/kitware.list > /dev/null \
&& apt-get update \
&& apt-get install -y cmake
# Dotnet
RUN apt-get update && apt-get install -y dotnet7
# Node
ARG node_major=18
RUN mkdir -p /etc/apt/keyrings \
&& curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \
| gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
&& echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$node_major.x nodistro main" \
| tee /etc/apt/sources.list.d/nodesource.list \
&& apt-get update \
&& apt-get install -y nodejs
# Emumba (deps)
RUN apt-get update && apt-get -y install rapidjson-dev
# Emumba
COPY emumba /emumba
RUN dpkg -i /emumba/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb \
&& rm -r /emumba
# Pcap
RUN apt-get update && apt-get install -y libpcap0.8-dev
# Databento (deps)
RUN apt-get update && apt-get -y install libzstd-dev
# Set up default executables (update-alternatives)
RUN \
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-$clang_major 100; \
update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-$clang_major 100; \
update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-$clang_major 100; \
update-alternatives --install /usr/bin/clangd clangd /usr/bin/clangd-$clang_major 100; \
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$gcc_major 100; \
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-$gcc_major 100
# Add compiler symlinks for ccache
RUN \
ln -s /usr/bin/ccache /usr/local/bin/clang; \
ln -s /usr/bin/ccache /usr/local/bin/clang++; \
ln -s /usr/bin/ccache /usr/local/bin/clang-$clang_major; \
ln -s /usr/bin/ccache /usr/local/bin/clang++-$clang_major; \
ln -s /usr/bin/ccache /usr/local/bin/gcc; \
ln -s /usr/bin/ccache /usr/local/bin/g++
RUN locale-gen en_US.UTF-8
ARG docker_user="bts"
ARG docker_uid="1000"
ARG docker_pass="bts"
RUN \
groupadd -g "$docker_uid" "$docker_user" \
&& useradd \
-m \
-u "$docker_uid" \
-g "$docker_uid" \
-G sudo \
--password "$(openssl passwd -1 "$docker_pass")" \
"$docker_user"
-50
View File
@@ -1,50 +0,0 @@
{
"dockerFile": "Dockerfile",
// Creates local dir to mount to HOME in docker
"initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
"hostRequirements": {
"memory": "4gb",
"storage": "6gb"
},
"postCreateCommand": "/home/bts/spark/docker/sparkd/dev/devcontainer_post_create.sh",
"containerUser": "bts",
"workspaceMount": "source=${localWorkspaceFolder},target=/home/bts/spark,type=bind",
"workspaceFolder": "/home/bts/spark",
"updateRemoteUserUID": true,
"shutdownAction": "none",
// Note: If running multiple containers on the same host, each must use a
// unique set of ports. Usually we just add 1000 to each port as needed. For
// example, if 8000-8003 is taken, go with 9000-9003, then 10000-10003, etc.
"runArgs": [
"--shm-size",
"2gb",
"-p",
"8000:8000",
"-p",
"8001:8001",
"-p",
"8002:8002",
"-p",
"8003:8003",
"-p",
"12345:12345"
],
"mounts": [
"source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind"
],
"customizations": {
"vscode": {
"settings": {
"clangd.arguments": [
"--compile-commands-dir=${workspaceFolder}/build/clang/debug"
]
},
"extensions": [
"eamodio.gitlens",
"llvm-vs-code-extensions.vscode-clangd",
"ms-vscode.cmake-tools",
"ms-vscode.cpptools"
]
}
}
}
@@ -1,4 +0,0 @@
#!/usr/bin/env bash
this_dir="$(realpath "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")")"
sudo dpkg -i "$this_dir/emumba-deribit-connector-3.3.1-Linux.LICENSE_DISABLED.deb"
-3
View File
@@ -1,3 +0,0 @@
*.cmd text=auto eol=crlf
*.cs text=auto eol=crlf
*.sh text=auto eol=lf
+4 -6
View File
@@ -8,9 +8,7 @@ cmake-build-debug/
.idea/ .idea/
.dir-locals.el .dir-locals.el
*.pyc *.pyc
**.vscode/* **.vscode
!.vscode/launch.json
!.vscode/tasks.json
.vs .vs
obj/ obj/
*.csproj.user *.csproj.user
@@ -49,13 +47,13 @@ trace_data/
/src/.clang-format /src/.clang-format
/src/server/sparkd/cmds/spartan/analysis/.ipynb_checkpoints /src/server/sparkd/cmds/spartan/analysis/.ipynb_checkpoints
/src/server/sparkd/cmds/spartan/analysis/data /src/server/sparkd/cmds/spartan/analysis/data
.tmp/ *.tmp/*
.venv*/ /.venv*/*
.cache/ .cache/
/src/Spark.Admin.WebApp/tsconfig.*.g.json /src/Spark.Admin.WebApp/tsconfig.*.g.json
/src/Spark.Admin.WebApp/src/bin/ /src/Spark.Admin.WebApp/src/bin/
**/CMakeDeps.cmake **/CMakeDeps.cmake
**/CMakeDeps_win.cmake **/CMakeDeps_win.cmake
.devcontainer/ **/devcontainer.json
*.Generated.cs *.Generated.cs
*.lst *.lst
-158
View File
@@ -1,158 +0,0 @@
{
"version": "0.2.0",
"configurations": [
{
"name": ".NET Core Attach",
"type": "coreclr",
"request": "attach"
},
{
"name": "DataStreamPublisher",
"type": "coreclr",
"request": "launch",
"preLaunchTask": "Build DataStream (Debug)",
"program": "",
"linux": {
"program": "DataStreamPublisher",
},
"windows": {
"program": "DataStreamPublisher.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json"
],
"cwd": "build/Debug/bin/net7.0",
"stopAtEntry": false,
"console": "integratedTerminal"
},
{
"name": "(gdb) Launch",
"type": "cppdbg",
"request": "launch",
"program": "/home/bts/spark/build/sparkd-debug-clang/server/sparkd",
"args": ["run"],
"stopAtEntry": false,
"cwd": "/home/bts/spark/build/sparkd-debug-clang/server",
"environment": [],
"externalConsole": false,
"MIMode": "gdb",
},
{
"name": "DataStreamQueryAnalyzer",
"type": "coreclr",
"request": "launch",
//"preLaunchTask": "Build DataStreamQueryAnalyzer (Debug)",
"program": "",
"windows": {
"program": "DataStreamQueryAnalyzer.exe",
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json"
],
"cwd": "build/Debug/bin/net7.0-windows",
"env": {
"PYTHON_EXE": "${workspaceFolder}/.venv/Scripts/python.exe"
},
},
"stopAtEntry": false,
"console": "integratedTerminal",
},
{
"name": "PubSubServer",
"type": "coreclr",
"request": "launch",
"preLaunchTask": "Build DataStream (Debug)",
"program": "",
"linux": {
"program": "PubSubServer",
},
"windows": {
"program": "PubSubServer.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json"
],
"cwd": "build/Debug/bin/net7.0",
"stopAtEntry": false,
"console": "integratedTerminal"
},
{
"name": "Spark",
"type": "coreclr",
"request": "launch",
"program": "",
"windows": {
"preLaunchTask": "Build Spark (Debug)",
"program": "Spark.exe",
"cwd": "build/Debug/bin/net7.0-windows",
},
"stopAtEntry": false,
"console": "integratedTerminal"
},
{
"name": "Spark (Parameterized)",
"type": "coreclr",
"request": "launch",
"preLaunchTask": "Build Spark (Debug)",
"program": "",
"windows": {
"program": "Spark.exe",
"args": [
"--sparkd",
"${input:sparkConfigServer}"
],
"cwd": "build/Debug/bin/net7.0-windows",
},
"stopAtEntry": false,
"console": "integratedTerminal"
},
{
"name": "Sparkd",
"type": "coreclr",
"request": "launch",
"preLaunchTask": "Build Sparkd (Debug | Clang)",
"program": "",
"linux": {
"program": "sparkd",
},
"args": [
"run"
],
"cwd": "build/sparkd-debug-clang/server",
"stopAtEntry": false,
"console": "integratedTerminal"
}
],
"inputs": [
{
"id": "dataStreamPublisherConfig",
"type": "pickString",
"description": "Publisher Configuration",
"options": [
"MarketDataPublisher",
"SimpleEchoClient",
"SimpleTestDataPublisher",
"TestDataPublisher",
],
"default": "MarketDataPublisher",
},
{
"id": "pubSubServerConfig",
"type": "pickString",
"description": "PubSub Configuration",
"options": [
"PubSubServer",
"SimpleEchoClient",
],
"default": "PubSubServer",
},
{
"id": "sparkConfigServer",
"type": "promptString",
"description": "Config Server URL",
"default": "http://localhost:8000/config",
},
]
}
-828
View File
@@ -1,828 +0,0 @@
{
"version": "2.0.0",
"tasks": [
{
"label": "Build Api Server",
"group": "build",
"type": "process",
"command": "",
"linux": {
"command": "tools/build_api_server.sh",
},
"windows": {
"command": "tools/build_api_server.cmd",
},
"args": [
"--configuration",
"${input:buildConfiguration}",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Api Server (Debug)",
"group": "build",
"type": "process",
"command": "",
"linux": {
"command": "tools/build_api_server.sh",
},
"windows": {
"command": "tools/build_api_server.cmd",
},
"args": [
"--configuration",
"Debug",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Api Server (Release)",
"group": "build",
"type": "process",
"command": "",
"windows": {
"command": "tools/build_api_server.cmd",
},
"linux": {
"command": "tools/build_api_server.sh",
},
"args": [
"--configuration",
"Release",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build DataStream",
"group": "build",
"type": "process",
"command": "",
"linux": {
"command": "tools/build_data_stream.sh",
},
"windows": {
"command": "tools/build_data_stream.cmd",
},
"args": [
"--configuration",
"${input:buildConfiguration}",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build DataStream (Debug)",
"group": "build",
"type": "process",
"command": "",
"linux": {
"command": "tools/build_data_stream.sh",
},
"windows": {
"command": "tools/build_data_stream.cmd",
},
"args": [
"--configuration",
"Debug",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build DataStream (Release)",
"group": "build",
"type": "process",
"command": "",
"linux": {
"command": "tools/build_data_stream.sh",
},
"windows": {
"command": "tools/build_data_stream.cmd",
},
"args": [
"--configuration",
"Release",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Spark",
"group": "build",
"type": "process",
"command": "",
"windows": {
"command": "tools/build_spark.cmd",
},
"args": [
"--configuration",
"${input:buildConfiguration}",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Spark (Debug)",
"group": "build",
"type": "process",
"command": "",
"windows": {
"command": "tools/build_spark.cmd",
},
"args": [
"--configuration",
"Debug",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Spark (Release)",
"group": "build",
"type": "process",
"command": "",
"windows": {
"command": "tools/build_spark.cmd",
},
"args": [
"--configuration",
"Release",
],
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Sparkd",
"group": "build",
"type": "shell",
"command": "",
"linux": {
"command": "tools/build_sparkd.sh",
"args": [
"--preset",
"${input:sparkdCmakePreset}",
],
},
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Sparkd (Debug)",
"group": "build",
"type": "shell",
"command": "",
"linux": {
"command": "tools/build_sparkd.sh",
"args": [
"--build-type",
"Debug",
"--compiler",
"${input:sparkdCompiler}"
],
},
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Sparkd (Debug | Gcc)",
"group": "build",
"type": "shell",
"command": "",
"linux": {
"command": "tools/build_sparkd.sh",
"args": [
"--build-type",
"Debug",
"--compiler",
"gcc"
],
},
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Sparkd (Debug | Clang)",
"group": "build",
"type": "shell",
"command": "",
"linux": {
"command": "tools/build_sparkd.sh",
"args": [
"--build-type",
"Debug",
"--compiler",
"clang"
],
},
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Sparkd (Release)",
"group": "build",
"type": "shell",
"command": "",
"linux": {
"command": "tools/build_sparkd.sh",
"args": [
"--build-type",
"Release",
"--compiler",
"${input:sparkdCompiler}"
],
},
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Sparkd (Release | Gcc)",
"group": "build",
"type": "shell",
"command": "",
"linux": {
"command": "tools/build_sparkd.sh",
"args": [
"--build-type",
"Release",
"--compiler",
"gcc"
],
},
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Build Sparkd (Release | Clang)",
"group": "build",
"type": "shell",
"command": "",
"linux": {
"command": "tools/build_sparkd.sh",
"args": [
"--build-type",
"Release",
"--compiler",
"clang"
],
},
"icon": {
"id": "terminal",
"color": "terminal.ansiCyan"
},
},
{
"label": "Edit DataStream Config",
"type": "process",
"command": "",
"linux": {
"command": "editor",
"args": [
"${env:HOME}/.local/share/DataStream/conf/spark.conf"
],
},
"windows": {
"command": "vim",
"args": [
"${env:APPDATA}/bts/DataStream/conf/spark.conf"
],
},
"presentation": {
"reveal": "always",
"focus": true,
"panel": "shared",
"showReuseMessage": false,
"close": true,
},
"icon": {
"id": "wrench"
},
},
{
"label": "Edit Spark Config",
"type": "process",
"command": "",
"windows": {
"command": "vim",
"args": [
"${env:APPDATA}/bts/conf/spark.conf"
],
},
"presentation": {
"reveal": "always",
"focus": true,
"panel": "dedicated",
"showReuseMessage": false,
"close": true,
},
"icon": {
"id": "wrench"
},
},
{
"label": "Run DataStreamQueryAnalyzer",
"dependsOn": "Build DataStream",
"type": "process",
"command": "",
"windows": {
"command": "DataStreamQueryAnalyzer.exe",
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
],
"options": {
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
},
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run DataStreamQueryAnalyzer (Debug)",
"dependsOn": "Build DataStream (Debug)",
"type": "process",
"command": "",
"windows": {
"command": "DataStreamQueryAnalyzer.exe",
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
],
"options": {
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
},
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run DataStreamQueryAnalyzer (Release)",
"dependsOn": "Build DataStream (Release)",
"type": "process",
"command": "",
"windows": {
"command": "DataStreamQueryAnalyzer.exe",
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/DataStreamQueryAnalyzer/Configurations/DataStreamQueryAnalyzer.json",
],
"options": {
"cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
},
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run DataStreamPublisher",
"dependsOn": "Build DataStream",
"type": "process",
"command": "",
"linux": {
"command": "DataStreamPublisher",
},
"windows": {
"command": "DataStreamPublisher.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
],
"options": {
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run DataStreamPublisher (Debug)",
"dependsOn": "Build DataStream (Debug)",
"type": "process",
"command": "",
"linux": {
"command": "DataStreamPublisher",
},
"windows": {
"command": "DataStreamPublisher.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
],
"options": {
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run DataStreamPublisher (Release)",
"dependsOn": "Build DataStream (Release)",
"type": "process",
"command": "",
"linux": {
"command": "DataStreamPublisher",
},
"windows": {
"command": "DataStreamPublisher.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/DataStreamPublisher/Configurations/${input:dataStreamPublisherConfig}.json",
],
"options": {
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run PubSubServer",
"dependsOn": "Build DataStream",
"type": "process",
"command": "",
"linux": {
"command": "PubSubServer",
},
"windows": {
"command": "PubSubServer.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
],
"options": {
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run PubSubServer (Debug)",
"dependsOn": "Build DataStream (Debug)",
"type": "process",
"command": "",
"linux": {
"command": "PubSubServer",
},
"windows": {
"command": "PubSubServer.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
],
"options": {
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run PubSubServer (Debug | Skip Build)",
"type": "process",
"command": "",
"linux": {
"command": "PubSubServer",
},
"windows": {
"command": "PubSubServer.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
],
"options": {
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run PubSubServer (Release)",
"dependsOn": "Build DataStream (Release)",
"type": "process",
"command": "",
"linux": {
"command": "PubSubServer",
},
"windows": {
"command": "PubSubServer.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
],
"options": {
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run PubSubServer (Release | Skip Build)",
"type": "process",
"command": "",
"linux": {
"command": "PubSubServer",
},
"windows": {
"command": "PubSubServer.exe",
},
"args": [
"--config",
"${workspaceFolder}/src/DataStreamSuite/PubSubServer/Configurations/${input:pubSubServerConfig}.json",
],
"options": {
"cwd": "${workspaceFolder}/build/Release/bin/net7.0"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run Spark",
"dependsOn": "Build Spark",
"type": "process",
"command": "",
"windows": {
"command": "Spark.exe",
"options": {
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0-windows"
},
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
}
},
{
"label": "Run Spark (Debug)",
"dependsOn": "Build Spark (Debug)",
"type": "process",
"command": "",
"windows": {
"command": "Spark.exe",
"options": {
"cwd": "${workspaceFolder}/build/Debug/bin/net7.0-windows"
},
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
}
},
{
"label": "Run Spark (Release)",
"dependsOn": "Build Spark (Release)",
"type": "process",
"command": "",
"windows": {
"command": "Spark.exe",
"options": {
"cwd": "${workspaceFolder}/build/Release/bin/net7.0-windows"
},
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
}
},
{
"label": "Run Spark Config Query",
"dependsOn": "Build Spark",
"type": "process",
"command": "",
"linux": {
"command": "Spark.Console",
},
"windows": {
"command": "Spark.Console.exe",
},
"args": [
"config",
"--query",
"${input:sparkConfigQuery}",
],
"options": {
"cwd": "${workspaceFolder}/build/${input:buildConfiguration}/bin/net7.0"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run Sparkd",
"dependsOn": "Build Sparkd",
"type": "process",
"command": "",
"linux": {
"command": "sparkd",
},
"args": [
"run"
],
"options": {
"cwd": "${workspaceFolder}/build/${input:sparkdCmakePreset}/server"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "Run Sparkd Config Query",
"dependsOn": "Build Sparkd",
"type": "process",
"command": "",
"linux": {
"command": "sparkd",
},
"args": [
"config",
"--query",
"${input:sparkdConfigQuery}",
],
"options": {
"cwd": "${workspaceFolder}/build/${input:sparkdCompiler}/${input:sparkdBuildType}/server"
},
"icon": {
"id": "run",
"color": "terminal.ansiGreen",
},
},
{
"label": "View DataStreamQueryAnalyzer Logs (Last Run)",
"type": "process",
"command": "",
"windows": {
"command": "tools/logging/show_last_run.cmd",
"args": [
"DataStream",
"DataStreamQueryAnalyzer.exe",
"'Starting DataStreamQueryAnalyzer'"
]
},
"presentation": {
"reveal": "always",
"focus": true,
"panel": "shared",
"showReuseMessage": false,
"close": true,
},
"icon": {
"id": "wrench"
},
},
{
"label": "View PubSubServer Logs (Last Run)",
"type": "process",
"command": "",
"linux": {
"command": "tools/logging/show_last_run.sh",
"args": [
"${env:HOME}/.local/share/DataStream/data/logging.PubSubServer",
"Starting PubSubServer"
]
},
"windows": {
"command": "tools/logging/show_last_run.cmd",
"args": [
"DataStream",
"PubSubServer.exe",
"Starting PubSubServer"
]
},
"presentation": {
"reveal": "always",
"focus": true,
"panel": "shared",
"showReuseMessage": false,
"close": true,
},
"icon": {
"id": "wrench"
},
},
],
"inputs": [
{
"id": "buildConfiguration",
"type": "pickString",
"description": "Configuration",
"options": [
"Debug",
"Release",
],
"default": "Debug",
},
{
"id": "sparkdCmakePreset",
"type": "pickString",
"description": "Configuration",
"options": [
"sparkd-debug-clang",
"sparkd-debug-gcc",
"sparkd-release-clang",
"sparkd-release-gcc",
],
"default": "sparkd-debug-clang",
},
{
"id": "sparkdCompiler",
"type": "pickString",
"description": "Configuration",
"options": [
"clang",
"gcc",
],
"default": "sparkd-debug-clang",
},
{
"id": "dataStreamPublisherConfig",
"type": "pickString",
"description": "Publisher Configuration",
"options": [
"MarketDataPublisher",
"SimpleEchoClient",
"SimpleTestDataPublisher",
"TestDataPublisher",
],
"default": "MarketDataPublisher",
},
{
"id": "pubSubServerConfig",
"type": "pickString",
"description": "PubSub Configuration",
"options": [
"PubSubServer",
"SimpleEchoClient",
],
"default": "PubSubServer",
},
{
"id": "sparkConfigQuery",
"type": "promptString",
"description": "Config Type(s)"
},
{
"id": "sparkdConfigQuery",
"type": "promptString",
"description": "Config Type(s)"
}
]
}
+51 -14
View File
@@ -14,17 +14,12 @@ set(
CACHE CACHE
STRING STRING
"External dependencies root folder (should contain folders: lib, bin, include") "External dependencies root folder (should contain folders: lib, bin, include")
# SHARED_MEMORY_PREFIX serves as a kind of namespace so multiple applications
# can run simultaneously without fighting over the same shm files (e.g., sparkd,
# api server, etc.).
set(SHARED_MEMORY_PREFIX "bts" CACHE STRING "Prefix to use when creating shared memory files")
set(SPARK_BIN "${EXTERNALS_ROOT}/bin") set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include") set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>") set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
set(BUILD_API_SERVER OFF CACHE BOOL "Build the api server")
set(BUILD_DOCS OFF CACHE BOOL "Build documentation") set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
set(BUILD_SDK OFF CACHE BOOL "Build the sdk") set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
set(EXPORT_MD_CONSUMER OFF CACHE BOOL "Export (extern) C functions for MD consumer interop")
if (WIN32) if (WIN32)
set(BUILD_SPARKD OFF CACHE BOOL "Build sparkd") set(BUILD_SPARKD OFF CACHE BOOL "Build sparkd")
set(BUILD_ADMIN_WEB_APP OFF CACHE BOOL "Build the Admin WebApp") set(BUILD_ADMIN_WEB_APP OFF CACHE BOOL "Build the Admin WebApp")
@@ -36,13 +31,13 @@ endif()
include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib) include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
link_directories(${SPARK_LIB}) link_directories(${SPARK_LIB})
add_compile_definitions(SHARED_MEMORY_PREFIX="${SHARED_MEMORY_PREFIX}") if(BUILD_API_SERVER)
add_compile_definitions(APP_DATA_DIR="${APP_DATA_DIR}") add_compile_definitions(SPARK_ENABLE_API_SERVER)
if(DEFINED APP_DATA_DIR_OVERRIDE)
add_compile_definitions(APP_DATA_DIR_OVERRIDE="${APP_DATA_DIR_OVERRIDE}")
endif() endif()
add_compile_definitions(EXPORT_MD_CONSUMER="${EXPORT_MD_CONSUMER}")
if(WIN32) if(WIN32)
add_compile_definitions(SHARED_MEMORY_PREFIX="spark")
add_compile_definitions(APP_DATA_DIR="\$\{APPDATA\}/bts/")
# subdirs # subdirs
add_subdirectory(src/lib) add_subdirectory(src/lib)
else() else()
@@ -52,7 +47,13 @@ else()
set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install") set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>") set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")
if(BUILD_SPARKD) if(BUILD_API_SERVER)
add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/spark_api/")
add_compile_definitions(SHARED_MEMORY_PREFIX="spark_api")
elseif(BUILD_SPARKD)
add_compile_definitions(SHARED_MEMORY_PREFIX="sparkd")
add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/sparkdata/")
# Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or # Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
# read by their code). We override some below. # read by their code). We override some below.
add_compile_definitions(SPARK_ENABLE_CCAPI) add_compile_definitions(SPARK_ENABLE_CCAPI)
@@ -114,6 +115,9 @@ else()
add_subdirectory(src/server/sparkd) add_subdirectory(src/server/sparkd)
add_subdirectory(src/data_files) add_subdirectory(src/data_files)
endif() endif()
if (BUILD_ACTIV_MD)
add_subdirectory(src/app_activ-tester)
endif()
add_subdirectory(src/lib) add_subdirectory(src/lib)
if (BUILD_ADMIN_WEB_APP) if (BUILD_ADMIN_WEB_APP)
add_subdirectory(src/Spark.Admin.WebApp) add_subdirectory(src/Spark.Admin.WebApp)
@@ -126,14 +130,42 @@ else()
endif() endif()
if(BUILD_SPARKD) if(BUILD_SPARKD)
function(copy_shared_lib lib_file_name)
add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/src/server/sparkd/${lib_file_name}
COMMAND ${CMAKE_COMMAND} -E copy
${EXTERNALS_ROOT}/lib/Debug/${lib_file_name}
${CMAKE_BINARY_DIR}/src/server/sparkd/${lib_file_name}
DEPENDS ${EXTERNALS_ROOT}/lib/Debug/${lib_file_name})
endfunction()
copy_shared_lib(libActivAgentApplication_x86-64_ubuntu22-04_gcc123_mdid.so)
copy_shared_lib(libActivBase_x86-64_ubuntu22-04_gcc123_mdid.so)
copy_shared_lib(libActivFramework_x86-64_ubuntu22-04_gcc123_mdid.so)
copy_shared_lib(libActivOneApi_x86-64_ubuntu22-04_gcc123_mdid.so)
copy_shared_lib(libActivDatabase_x86-64_ubuntu22-04_gcc123_mdid.so)
copy_shared_lib(libActivServices_x86-64_ubuntu22-04_gcc123_mdid.so)
copy_shared_lib(libBoostAtomic_x86-64_ubuntu22-04_gcc123_mdid.so)
copy_shared_lib(libBoostChrono_x86-64_ubuntu22-04_gcc123_mdid.so)
copy_shared_lib(libBoostFilesystem_x86-64_ubuntu22-04_gcc123_mdid.so)
copy_shared_lib(libBoostProgramOptions_x86-64_ubuntu22-04_gcc123_mdid.so)
add_custom_target(activ_shared_libs ALL DEPENDS
${CMAKE_BINARY_DIR}/src/server/sparkd/libActivAgentApplication_x86-64_ubuntu22-04_gcc123_mdid.so
${CMAKE_BINARY_DIR}/src/server/sparkd/libActivBase_x86-64_ubuntu22-04_gcc123_mdid.so
${CMAKE_BINARY_DIR}/src/server/sparkd/libActivFramework_x86-64_ubuntu22-04_gcc123_mdid.so
${CMAKE_BINARY_DIR}/src/server/sparkd/libActivOneApi_x86-64_ubuntu22-04_gcc123_mdid.so
${CMAKE_BINARY_DIR}/src/server/sparkd/libActivDatabase_x86-64_ubuntu22-04_gcc123_mdid.so
${CMAKE_BINARY_DIR}/src/server/sparkd/libActivServices_x86-64_ubuntu22-04_gcc123_mdid.so
${CMAKE_BINARY_DIR}/src/server/sparkd/libBoostAtomic_x86-64_ubuntu22-04_gcc123_mdid.so
${CMAKE_BINARY_DIR}/src/server/sparkd/libBoostChrono_x86-64_ubuntu22-04_gcc123_mdid.so
${CMAKE_BINARY_DIR}/src/server/sparkd/libBoostFilesystem_x86-64_ubuntu22-04_gcc123_mdid.so
${CMAKE_BINARY_DIR}/src/server/sparkd/libBoostProgramOptions_x86-64_ubuntu22-04_gcc123_mdid.so)
add_custom_command( add_custom_command(
OUTPUT ${CMAKE_BINARY_DIR}/server OUTPUT ${CMAKE_BINARY_DIR}/server
COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/${CMAKE_BUILD_TYPE}/libonixs-fix-engine.so.4 ${CMAKE_BINARY_DIR}/server
COMMAND COMMAND
ln ARGS -sf ln ARGS -sf
${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0 ${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
@@ -152,6 +184,11 @@ else()
FILES FILES
"${SPARK_LIB}/libACE-5.7.5.so" "${SPARK_LIB}/libACE-5.7.5.so"
"${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0" "${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
"${SPARK_LIB}/libActivFramework_x86-64_ubuntu22-04_gcc123_mdid.so"
"${SPARK_LIB}/libActivOneApi_x86-64_ubuntu22-04_gcc123_mdid.so"
"${SPARK_LIB}/libActivServices_x86-64_ubuntu22-04_gcc123_mdid.so"
"${SPARK_LIB}/libBoostAtomic_x86-64_ubuntu22-04_gcc123_mdid.so"
"${SPARK_LIB}/libBoostFilesystem_x86-64_ubuntu22-04_gcc123_mdid.so"
DESTINATION "${SPARK_INSTALL_PREFIX}/bin") DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
endif() endif()
endif() endif()
+87 -115
View File
@@ -15,13 +15,11 @@
"rhs": "Windows" "rhs": "Windows"
}, },
"generator": "Visual Studio 17 2022", "generator": "Visual Studio 17 2022",
"binaryDir": "${sourceDir}/build/${presetName}", "binaryDir": "${sourceDir}/build",
"environment": { "environment": {
"SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600" "SPARK_CMAKE_CXX_FLAGS": "/MP /WX /wd4250 /wd4251 /wd4244 /wd4267 /wd4355 /wd4275 /wd4345 /bigobj /EHsc /DCURL_STATICLIB /D_WINSOCK_DEPRECATED_NO_WARNINGS /D__TBB_NO_IMPLICIT_LINKAGE /DBOOST_ALL_NO_LIB /std:c++20 /D_SCL_SECURE_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0600"
}, },
"cacheVariables": { "cacheVariables": {
"APP_DATA_DIR": "${dollar}{APPDATA}/bts",
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_DATA_DIR}",
"CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}", "CMAKE_CXX_FLAGS": "$env{SPARK_CMAKE_CXX_FLAGS}",
"CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd", "CMAKE_CXX_FLAGS_DEBUG": "$env{SPARK_CMAKE_CXX_FLAGS} /MTd",
"CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT", "CMAKE_CXX_FLAGS_RELEASE": "$env{SPARK_CMAKE_CXX_FLAGS} /MT",
@@ -32,12 +30,7 @@
"EXTERNALS_ROOT": "${sourceDir}/externals/windows", "EXTERNALS_ROOT": "${sourceDir}/externals/windows",
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf", "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0", "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
"DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows", "DOTNET_WINDOWS_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0-windows"
"EXPORT_MD_CONSUMER": {
"type": "BOOL",
"value": "ON"
},
"SHARED_MEMORY_PREFIX": "spark"
} }
}, },
@@ -53,8 +46,6 @@
"binaryDir": "${sourceDir}/build/${presetName}", "binaryDir": "${sourceDir}/build/${presetName}",
"cacheVariables": { "cacheVariables": {
"platform": "", "platform": "",
"APP_DATA_DIR": "${dollar}{HOME}/sparkdata",
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARKD_DATA_DIR}",
"CMAKE_BUILD_WITH_INSTALL_RPATH": { "CMAKE_BUILD_WITH_INSTALL_RPATH": {
"type": "BOOL", "type": "BOOL",
"value": "ON" "value": "ON"
@@ -71,8 +62,7 @@
"BUILD_ADMIN_WEB_APP": { "BUILD_ADMIN_WEB_APP": {
"type": "BOOL", "type": "BOOL",
"value": "ON" "value": "ON"
}, }
"SHARED_MEMORY_PREFIX": "sparkd"
} }
}, },
@@ -105,72 +95,114 @@
{ {
"name": "sparkd-debug-clang", "name": "sparkd-debug-clang",
"inherits": "sparkd-debug", "inherits": "sparkd-debug",
"displayName": "Sparkd (Debug | Clang)", "displayName": "Debug (clang)",
"description": "Debug build compiled w/ clang", "description": "Debug build compiled w/ clang",
"binaryDir": "${sourceDir}/build/clang/debug",
"cacheVariables": { "cacheVariables": {
"CMAKE_C_COMPILER": "clang", "CMAKE_C_COMPILER": "clang",
"CMAKE_CXX_COMPILER": "clang++", "CMAKE_CXX_COMPILER": "clang++",
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG" "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/debug/sparkd"
} }
}, },
{ {
"name": "sparkd-debug-gcc", "name": "sparkd-debug-gcc",
"inherits": "sparkd-debug", "inherits": "sparkd-debug",
"displayName": "Sparkd (Debug | Gcc)", "displayName": "Debug (gcc)",
"description": "Debug build compiled w/ gcc", "description": "Debug build compiled w/ gcc",
"binaryDir": "${sourceDir}/build/gcc/debug",
"cacheVariables": { "cacheVariables": {
"CMAKE_C_COMPILER": "gcc", "CMAKE_C_COMPILER": "gcc-12",
"CMAKE_CXX_COMPILER": "g++", "CMAKE_CXX_COMPILER": "g++-12",
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG" "CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -DDEBUG",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/debug/sparkd"
}
},
{
"name": "activ-tester",
"inherits": "sparkd-debug-gcc",
"displayName": "Activ Tester Debug (gcc)",
"description": "Activ Tester Debug build compiled w/ gcc",
"binaryDir": "${sourceDir}/build/${presetName}",
"cacheVariables": {
"BUILD_ACTIV_MD": {
"type": "BOOL",
"value": "ON"
},
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}/sparkd",
"CMAKE_TOOLCHAIN_FILE": "${sourceDir}/vcpkg/scripts/buildsystems/vcpkg.cmake"
} }
}, },
{ {
"name": "sparkd-release-clang", "name": "sparkd-release-clang",
"inherits": "sparkd-release", "inherits": "sparkd-release",
"displayName": "Sparkd (Release | Clang)", "displayName": "Release (clang)",
"description": "Release build compiled w/ clang", "description": "Release build compiled w/ clang",
"binaryDir": "${sourceDir}/build/clang/release",
"cacheVariables": { "cacheVariables": {
"CMAKE_C_COMPILER": "clang", "CMAKE_C_COMPILER": "clang",
"CMAKE_CXX_COMPILER": "clang++", "CMAKE_CXX_COMPILER": "clang++",
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3" "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/clang/release/sparkd"
} }
}, },
{ {
"name": "sparkd-release-gcc", "name": "sparkd-release-gcc",
"inherits": "sparkd-release", "inherits": "sparkd-release",
"displayName": "Sparkd (Release | Gcc)", "displayName": "Release (gcc)",
"description": "Release build compiled w/ gcc", "description": "Release build compiled w/ gcc",
"binaryDir": "${sourceDir}/build/gcc/release",
"cacheVariables": { "cacheVariables": {
"CMAKE_C_COMPILER": "gcc", "CMAKE_C_COMPILER": "gcc",
"CMAKE_CXX_COMPILER": "g++", "CMAKE_CXX_COMPILER": "g++",
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers -Wno-stringop-overflow --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3" "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -Wno-stringop-overread -Wno-maybe-uninitialized -fno-omit-frame-pointer -DNDEBUG -O3",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/gcc/release/sparkd"
}
},
{
"name": "spark-api-server-base",
"hidden": true,
"binaryDir": "${sourceDir}/build/${presetName}",
"cacheVariables": {
"BUILD_API_SERVER": {
"type": "bool",
"value": "ON"
},
"CONFIG_DEFAULTS_PATH": "config/profiles/md_dev/*.conf;config/profiles/default/*.conf"
} }
}, },
{ {
"name": "spark-api-server-windows", "name": "spark-api-server-windows",
"inherits": "spark", "inherits": "spark",
"displayName": "Spark Api Server (Windows)", "displayName": "Spark Api Server",
"cacheVariables": { "cacheVariables": {
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}", "BUILD_API_SERVER": {
"SHARED_MEMORY_PREFIX": "spark_api_server" "type": "bool",
"value": "ON"
}
} }
}, },
{ {
"name": "spark-api-server-linux", "name": "spark-api-server-linux",
"inherits": "sparkd", "inherits": "sparkd",
"displayName": "Spark Api Server (Linux)", "displayName": "Spark Api Server",
"generator": "Ninja Multi-Config", "generator": "Ninja Multi-Config",
"cacheVariables": { "cacheVariables": {
"APP_DATA_DIR_OVERRIDE": "${dollar}{SPARK_API_SERVER_DATA_DIR}",
"BUILD_ADMIN_WEB_APP": { "BUILD_ADMIN_WEB_APP": {
"type": "BOOL", "type": "BOOL",
"value": "OFF" "value": "OFF"
}, },
"BUILD_API_SERVER": {
"type": "bool",
"value": "ON"
},
"BUILD_SPARKD": { "BUILD_SPARKD": {
"type": "BOOL", "type": "BOOL",
"value": "OFF" "value": "OFF"
@@ -181,11 +213,7 @@
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3", "CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}", "CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf", "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf",
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0", "DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0"
"EXPORT_MD_CONSUMER": {
"type": "BOOL",
"value": "ON"
}
} }
}, },
@@ -208,62 +236,13 @@
}, },
{ {
"name": "data-stream-suite-windows", "name": "data-stream-suite",
"condition": {
"type": "equals",
"lhs": "${hostSystemName}",
"rhs": "Windows"
},
"inherits": "spark", "inherits": "spark",
"displayName": "Data Stream Suite (Windows)", "displayName": "Data Stream Suite",
"description": "Data Stream utils ported over from Edge", "description": "Data Stream utils ported over from Edge",
"binaryDir": "${sourceDir}/build/${presetName}", "binaryDir": "${sourceDir}/build/${presetName}",
"cacheVariables": { "cacheVariables": {
"APP_DATA_DIR": "${dollar}{APPDATA}/bts/DataStream", "CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf"
"APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
"SHARED_MEMORY_PREFIX": "data_stream"
}
},
{
"name": "data-stream-suite-linux",
"displayName": "Data Stream Suite (Linux)",
"description": "Data Stream utils ported over from Edge",
"condition": {
"type": "equals",
"lhs": "${hostSystemName}",
"rhs": "Linux"
},
"inherits": "sparkd",
"generator": "Ninja Multi-Config",
"cacheVariables": {
"APP_DATA_DIR": "${dollar}{HOME}/.local/share/DataStream",
"APP_DATA_DIR_OVERRIDE": "${dollar}{DATA_STREAM_DATA_DIR}",
"BUILD_ADMIN_WEB_APP": {
"type": "BOOL",
"value": "OFF"
},
"BUILD_SDK": {
"type": "BOOL",
"value": "OFF"
},
"BUILD_SPARKD": {
"type": "BOOL",
"value": "OFF"
},
"CMAKE_C_COMPILER": "clang",
"CMAKE_CXX_COMPILER": "clang++",
"CMAKE_CXX_FLAGS_DEBUG": "-g -fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -DDEBUG",
"CMAKE_CXX_FLAGS_RELEASE": "-fPIC -Wall -pedantic -Wextra -Werror -Wno-missing-braces -Wno-ignored-qualifiers --std=c++20 -fdiagnostics-color -fno-omit-frame-pointer -DNDEBUG -O3",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/install/${presetName}",
"CONFIG_DEFAULTS_PATH": "config/profiles/default/*.conf;config/profiles/data_stream/*.conf",
"DOTNET_BINARY_DIR": "${sourceDir}/build/$<CONFIG>/bin/net7.0",
"EXPORT_MD_CONSUMER": {
"type": "BOOL",
"value": "ON"
},
"SHARED_MEMORY_PREFIX": "data_stream"
} }
} }
], ],
@@ -281,7 +260,7 @@
{ {
"name": "spark-debug", "name": "spark-debug",
"displayName": "Spark (Debug)", "displayName": "Debug",
"inherits": "spark", "inherits": "spark",
"configurePreset": "spark", "configurePreset": "spark",
"configuration": "Debug" "configuration": "Debug"
@@ -289,7 +268,7 @@
{ {
"name": "spark-release", "name": "spark-release",
"displayName": "Spark (Release)", "displayName": "Release",
"inherits": "spark", "inherits": "spark",
"configurePreset": "spark", "configurePreset": "spark",
"configuration": "Release" "configuration": "Release"
@@ -317,28 +296,35 @@
{ {
"name": "sparkd-debug-clang", "name": "sparkd-debug-clang",
"configurePreset": "sparkd-debug-clang", "configurePreset": "sparkd-debug-clang",
"displayName": "Sparkd (Debug | Clang)", "displayName": "Debug (clang)",
"inherits": "sparkd" "inherits": "sparkd"
}, },
{ {
"name": "sparkd-debug-gcc", "name": "sparkd-debug-gcc",
"configurePreset": "sparkd-debug-gcc", "configurePreset": "sparkd-debug-gcc",
"displayName": "Sparkd (Debug | Gcc)", "displayName": "Debug (gcc)",
"inherits": "sparkd"
},
{
"name": "activ-tester",
"configurePreset": "activ-tester",
"displayName": "Activ Tester Debug (gcc)",
"inherits": "sparkd" "inherits": "sparkd"
}, },
{ {
"name": "sparkd-release-clang", "name": "sparkd-release-clang",
"configurePreset": "sparkd-release-clang", "configurePreset": "sparkd-release-clang",
"displayName": "Sparkd (Release | Clang)", "displayName": "Release (clang)",
"inherits": "sparkd" "inherits": "sparkd"
}, },
{ {
"name": "sparkd-release-gcc", "name": "sparkd-release-gcc",
"configurePreset": "sparkd-release-gcc", "configurePreset": "sparkd-release-gcc",
"displayName": "Sparkd (Release | Gcc)", "displayName": "Release (gcc)",
"inherits": "sparkd" "inherits": "sparkd"
}, },
@@ -355,14 +341,14 @@
{ {
"name": "spark-api-server-windows-debug", "name": "spark-api-server-windows-debug",
"displayName": "Spark Api Server (Windows | Debug)", "displayName": "Spark Api Server (Debug)",
"inherits": "spark-api-server-windows", "inherits": "spark-api-server-windows",
"configuration": "Debug" "configuration": "Debug"
}, },
{ {
"name": "spark-api-server-windows-release", "name": "spark-api-server-windows-release",
"displayName": "Spark Api Server (Windows | Release)", "displayName": "Spark Api Server (Debug)",
"inherits": "spark-api-server-windows", "inherits": "spark-api-server-windows",
"configuration": "Release" "configuration": "Release"
}, },
@@ -381,14 +367,14 @@
{ {
"name": "spark-api-server-linux-debug", "name": "spark-api-server-linux-debug",
"inherits": "spark-api-server-linux", "inherits": "spark-api-server-linux",
"displayName": "Spark Api Server (Linux | Debug)", "displayName": "Spark Api Server (Debug)",
"configuration": "Debug" "configuration": "Debug"
}, },
{ {
"name": "spark-api-server-linux-release", "name": "spark-api-server-linux-release",
"inherits": "spark-api-server-linux", "inherits": "spark-api-server-linux",
"displayName": "Spark Api Server (Linux | Release)", "displayName": "Spark Api Server (Release)",
"configuration": "Release" "configuration": "Release"
}, },
@@ -408,30 +394,16 @@
}, },
{ {
"name": "data-stream-suite-windows-debug", "name": "data-stream-suite-debug",
"displayName": "Data Stream Suite (Windows | Debug)", "displayName": "DataStreamSuite (Debug)",
"configurePreset": "data-stream-suite-windows", "configurePreset": "data-stream-suite",
"configuration": "Debug" "configuration": "Debug"
}, },
{ {
"name": "data-stream-suite-windows-release", "name": "data-stream-suite-release",
"displayName": "Data Stream Suite (Windows | Release)", "displayName": "DataStreamSuite (Release)",
"configurePreset": "data-stream-suite-windows", "configurePreset": "data-stream-suite",
"configuration": "Release"
},
{
"name": "data-stream-suite-linux-debug",
"displayName": "Data Stream Suite (Linux | Debug)",
"configurePreset": "data-stream-suite-linux",
"configuration": "Debug"
},
{
"name": "data-stream-suite-linux-release",
"displayName": "Data Stream Suite (Linux | Release)",
"configurePreset": "data-stream-suite-linux",
"configuration": "Release" "configuration": "Release"
} }
] ]
+124
View File
@@ -0,0 +1,124 @@
# Symbols {
# value {
# venues {
# name: "VENUE_BINANCE_US"
# spot: "BTC"
# spot: "ETH"
# }
# }
# }
#
# MarketDataConfig {
# value {
# binance_us {
# num_depth_levels: 5
# security_criteria {
# clauses {
# security_types: SECURITY_TYPE_SPOT
# symbols: "BTC"
# }
# }
# }
# }
# }
MarketDataConfig {
value {
# binance {
# num_depth_levels: 5
# security_criteria {
# clauses {
# security_types: SECURITY_TYPE_SPOT
# symbols: "BTC"
# symbols: "ETH"
# }
# }
# url_rest_base: "https://testnet.binance.vision"
# url_ws_base: "wss://testnet.binance.vision"
# }
# binance_usds_futures {
# num_depth_levels: 5
# security_criteria {
# clauses {
# security_types: SECURITY_TYPE_FUTURE
# symbols: "BTC"
# symbols: "ETH"
# }
# }
# security_desc_formattings {
# expression: "^(.+)(USD)(?=_)(.*)"
# format: "$1USDT$3"
# }
# url_rest_base: "https://testnet.binancefuture.com"
# url_ws_base: "wss://fstream.binance.com"
# }
binance_us {
num_depth_levels: 5
security_criteria {
clauses {
security_types: SECURITY_TYPE_SPOT
symbols: "BTC"
symbols: "ETH"
}
}
url_rest_base: "https://api.binance.us"
url_ws_base: "wss://stream.binance.us:9443"
}
# deribit {
# num_depth_levels: 10
# security_criteria {
# clauses {
# security_types: SECURITY_TYPE_SPOT
# security_types: SECURITY_TYPE_FUTURE
# security_types: SECURITY_TYPE_OPTION
# security_types: SECURITY_TYPE_SPREAD
# symbols: "BTC"
# symbols: "ETH"
# }
# }
# currencies: "BTC"
# currencies: "USDC"
# url_rest_base: "https://test.deribit.com"
# url_ws_base: "wss://test.deribit.com"
# }
# okx {
# num_depth_levels: 5
# security_criteria {
# clauses {
# security_types: SECURITY_TYPE_SPOT
# security_types: SECURITY_TYPE_FUTURE
# security_types: SECURITY_TYPE_OPTION
# symbols: "BTC"
# symbols: "ETH"
# }
# }
# url_rest_base: "https://aws.okx.com"
# url_ws_base: "wss://wsaws.okx.com:8443"
# is_protocol_ping_pong_enabled: true
# protocol_ping_interval_ms: 60000
# protocol_pong_timeout_ms: 30000
# is_application_ping_pong_enabled: true
# application_ping_interval_ms: 20000
# application_pong_timeout_ms: 10000
# }
# recording {
# num_of_seconds_to_run: 300
# venue: VENUE_DERIBIT
# }
# replay {
# venue: VENUE_DERIBIT
# }
}
}
Symbols {
value {
venues {
name: "VENUE_BINANCE_US"
spot: "BTC"
spot: "ETH"
}
}
}
+210
View File
@@ -0,0 +1,210 @@
cmake_minimum_required(VERSION 3.12)
project(spark)
set(CMAKE_CXX_STANDARD 20)
if(WIN32)
set(platform "windows")
else()
set(platform "linux")
endif()
set(
EXTERNALS_ROOT
"${CMAKE_SOURCE_DIR}/externals/${platform}"
CACHE
STRING
"External dependencies root folder (should contain folders: lib, bin, include")
set(SPARK_BIN "${EXTERNALS_ROOT}/bin")
set(SPARK_INCLUDE "${EXTERNALS_ROOT}/include")
set(SPARK_LIB "${EXTERNALS_ROOT}/lib/$<CONFIG>")
set(BUILD_API_SERVER OFF CACHE BOOL "Build the api server")
set(BUILD_DOCS OFF CACHE BOOL "Build documentation")
set(BUILD_SDK OFF CACHE BOOL "Build the sdk")
if (WIN32)
set(BUILD_SPARKD OFF CACHE BOOL "Build sparkd")
set(BUILD_ADMIN_WEB_APP OFF CACHE BOOL "Build the Admin WebApp")
else()
set(BUILD_SPARKD ON CACHE BOOL "Build sparkd")
set(BUILD_ADMIN_WEB_APP ON CACHE BOOL "Build the Admin WebApp")
endif()
include_directories(BEFORE SYSTEM ${SPARK_INCLUDE} ${EXTERNALS_ROOT}/include/zlib)
link_directories(${SPARK_LIB})
if(BUILD_API_SERVER)
add_compile_definitions(SPARK_ENABLE_API_SERVER)
endif()
if(WIN32)
add_compile_definitions(SHARED_MEMORY_PREFIX="spark")
add_compile_definitions(APP_DATA_DIR="\$\{APPDATA\}/bts/")
# subdirs
add_subdirectory(src/lib)
else()
# make sparkd look for libspark.so in its directory
set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
set(CMAKE_INSTALL_RPATH ".")
set(CMAKE_INSTALL_PREFIX "${CMAKE_SOURCE_DIR}/install")
set(SPARK_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/$<CONFIG>")
if(BUILD_API_SERVER)
add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/spark_api/")
add_compile_definitions(SHARED_MEMORY_PREFIX="spark_api")
elseif(BUILD_SPARKD)
add_compile_definitions(SHARED_MEMORY_PREFIX="sparkd")
add_compile_definitions(APP_DATA_DIR="\$\{HOME\}/sparkdata/")
# Note: All macros that begin w/ 'CCAPI' are ccapi's macros (defined and/or
# read by their code). We override some below.
add_compile_definitions(SPARK_ENABLE_CCAPI)
# This allows us to override hardcoded exchange-urls with ones read from
# config (i.e., resolved at runtime rather than compile-time)
add_compile_definitions(CCAPI_EXPOSE_INTERNAL)
# Enable ccapi services
add_compile_definitions(CCAPI_ENABLE_SERVICE_MARKET_DATA)
add_compile_definitions(CCAPI_ENABLE_SERVICE_EXECUTION_MANAGEMENT)
# Enable ccapi exchanges
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE)
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_COIN_FUTURES)
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_US)
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_BINANCE_USDS_FUTURES)
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_COINBASE)
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_DERIBIT)
add_compile_definitions(CCAPI_ENABLE_EXCHANGE_OKX)
set(CCAPI_LOG_LEVEL
"Warn"
CACHE STRING "<Fatal|Error|Warn|Info|Debug|Trace> Log level for ccapi")
string(TOUPPER "${CCAPI_LOG_LEVEL}" _ccapi_log_level_upper)
add_compile_definitions("CCAPI_ENABLE_LOG_${_ccapi_log_level_upper}")
# ** Binance **
add_compile_definitions(CCAPI_BINANCE_URL_REST_BASE="https://api.binance.com")
add_compile_definitions(BINANCE_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
add_compile_definitions(BINANCE_PRICE_STATS_PATH="/api/v3/ticker/24hr")
# ** Binance US **
add_compile_definitions(BINANCE_US_INSTRUMENTS_PATH="/api/v3/exchangeInfo")
add_compile_definitions(BINANCE_US_PRICE_STATS_PATH="/api/v3/ticker/24hr")
# ** Binance USDS Futures **
add_compile_definitions(CCAPI_BINANCE_USDS_FUTURES_URL_REST_BASE="https://dapi.binance.com")
add_compile_definitions(BINANCE_USDS_FUTURES_INSTRUMENTS_PATH="/dapi/v1/exchangeInfo")
add_compile_definitions(BINANCE_USDS_FUTURES_PRICE_STATS_PATH="/dapi/v1/ticker/24hr")
# ** Deribit **
add_compile_definitions(DERIBIT_INSTRUMENT_PATH="/api/v2/public/get_instrument")
add_compile_definitions(DERIBIT_INSTRUMENTS_PATH="/api/v2/public/get_instruments")
add_compile_definitions(DERIBIT_COMBOS_PATH="/api/v2/public/get_combos")
add_compile_definitions(DERIBIT_TICKER_PATH="/api/v2/public/ticker")
add_compile_definitions(DERIBIT_CURRENCIES_METHOD="public/get_currencies")
add_compile_definitions(DERIBIT_INSTRUMENT_METHOD="public/get_instrument")
add_compile_definitions(DERIBIT_INSTRUMENTS_METHOD="public/get_instruments")
add_compile_definitions(DERIBIT_COMBO_DETAILS_METHOD="public/get_combo_details")
add_compile_definitions(DERIBIT_COMBOS_METHOD="public/get_combos")
# ** Okx **
add_compile_definitions(OKX_INSTRUMENTS_PATH="/api/v5/public/instruments")
add_compile_definitions(OKX_UNDERLYING_PATH="/api/v5/public/underlying")
endif()
# subdirs
if (BUILD_SPARKD)
add_subdirectory(src/server/sparkd)
add_subdirectory(src/data_files)
endif()
add_subdirectory(src/lib)
if (BUILD_ADMIN_WEB_APP)
add_subdirectory(src/Spark.Admin.WebApp)
endif()
if(BUILD_DOCS)
add_subdirectory(doc/user)
endif()
if(BUILD_SDK)
add_subdirectory(src/sdk)
endif()
if(BUILD_SPARKD)
function(copy_shared_lib lib_file_name)
add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/src/server/sparkd/${lib_file_name}
COMMAND ${CMAKE_COMMAND} -E copy
${EXTERNALS_ROOT}/lib/Debug/${lib_file_name}
${CMAKE_BINARY_DIR}/src/server/sparkd/${lib_file_name}
DEPENDS ${EXTERNALS_ROOT}/lib/Debug/${lib_file_name})
add_custom_target(copy_shared_lib_${lib_file_name} ALL DEPENDS ${CMAKE_BINARY_DIR}/src/server/sparkd/${lib_file_name})
endfunction()
add_custom_command(
OUTPUT ${CMAKE_BINARY_DIR}/server
COMMAND mkdir -p ${CMAKE_BINARY_DIR}/server
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/server/sparkd/sparkd ${CMAKE_BINARY_DIR}/server
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/data_files ${CMAKE_BINARY_DIR}/server
COMMAND ln ARGS -sf ${CMAKE_BINARY_DIR}/src/lib/libspark.so ${CMAKE_BINARY_DIR}/server
COMMAND
ln ARGS -sf
${EXTERNALS_ROOT}/lib/Release/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0
${CMAKE_BINARY_DIR}/server
COMMAND
${CMAKE_COMMAND} -E copy
${EXTERNALS_ROOT}/lib/Debug/libActivAgentApplication_x86-64_ubuntu22-04_gcc114_mdid.so
${CMAKE_BINARY_DIR}/server
COMMAND
${CMAKE_COMMAND} -E copy
${EXTERNALS_ROOT}/lib/Debug/libActivBase_x86-64_ubuntu22-04_gcc114_mdid.so
${CMAKE_BINARY_DIR}/server
COMMAND
${CMAKE_COMMAND} -E copy
${EXTERNALS_ROOT}/lib/Debug/libActivFramework_x86-64_ubuntu22-04_gcc114_mdid.so
${CMAKE_BINARY_DIR}/server
COMMAND
${CMAKE_COMMAND} -E copy
${EXTERNALS_ROOT}/lib/Debug/libActivOneApi_x86-64_ubuntu22-04_gcc114_mdid.so
${CMAKE_BINARY_DIR}/server
COMMAND
${CMAKE_COMMAND} -E copy
${EXTERNALS_ROOT}/lib/Debug/libActivDatabase_x86-64_ubuntu22-04_gcc114_mdid.so
${CMAKE_BINARY_DIR}/server
COMMAND
${CMAKE_COMMAND} -E copy
${EXTERNALS_ROOT}/lib/Debug/libActivServices_x86-64_ubuntu22-04_gcc114_mdid.so
${CMAKE_BINARY_DIR}/server
COMMAND
${CMAKE_COMMAND} -E copy
${EXTERNALS_ROOT}/lib/Debug/libBoostAtomic_x86-64_ubuntu22-04_gcc114_mdid.so
${CMAKE_BINARY_DIR}/server
COMMAND
${CMAKE_COMMAND} -E copy
${EXTERNALS_ROOT}/lib/Debug/libBoostChrono_x86-64_ubuntu22-04_gcc114_mdid.so
${CMAKE_BINARY_DIR}/server
COMMAND
${CMAKE_COMMAND} -E copy
${EXTERNALS_ROOT}/lib/Debug/libBoostFilesystem_x86-64_ubuntu22-04_gcc114_mdid.so
${CMAKE_BINARY_DIR}/server
COMMAND
${CMAKE_COMMAND} -E copy
${EXTERNALS_ROOT}/lib/Debug/libBoostProgramOptions_x86-64_ubuntu22-04_gcc114_mdid.so
${CMAKE_BINARY_DIR}/server
COMMAND ln ARGS -sf ${EXTERNALS_ROOT}/lib/Release/libACE-5.7.5.so ${CMAKE_BINARY_DIR}/server
COMMAND mkdir ARGS -p ${CMAKE_BINARY_DIR}/server/plugins
COMMENT "Creating server runtime layout"
DEPENDS sparkd data_files_dir)
add_custom_target(server_layout ALL DEPENDS ${CMAKE_BINARY_DIR}/server)
install(
DIRECTORY "${CMAKE_BINARY_DIR}/server/plugins"
DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
install(
FILES
"${SPARK_LIB}/libACE-5.7.5.so"
"${SPARK_LIB}/libOnixS.IceImpactMulticastPriceFeedHandlerCpp.so.3.16.0.0"
"${SPARK_LIB}/libActivFramework_x86-64_ubuntu22-04_gcc114_mdid.so"
"${SPARK_LIB}/libActivOneApi_x86-64_ubuntu22-04_gcc114_mdid.so"
"${SPARK_LIB}/libActivServices_x86-64_ubuntu22-04_gcc114_mdid.so"
"${SPARK_LIB}/libBoostAtomic_x86-64_ubuntu22-04_gcc114_mdid.so"
"${SPARK_LIB}/libBoostFilesystem_x86-64_ubuntu22-04_gcc114_mdid.so"
DESTINATION "${SPARK_INSTALL_PREFIX}/bin")
endif()
endif()
Executable
+33
View File
@@ -0,0 +1,33 @@
diff.astextplain.textconv=astextplain
filter.lfs.clean=git-lfs clean -- %f
filter.lfs.smudge=git-lfs smudge -- %f
filter.lfs.process=git-lfs filter-process
filter.lfs.required=true
http.sslbackend=openssl
http.sslcainfo=C:/Program Files/Git/mingw64/etc/ssl/certs/ca-bundle.crt
core.autocrlf=true
core.fscache=true
core.symlinks=false
pull.rebase=false
credential.helper=manager
credential.https://dev.azure.com.usehttppath=true
init.defaultbranch=master
status.short=true
core.repositoryformatversion=0
core.filemode=false
core.bare=false
core.logallrefupdates=true
core.symlinks=false
core.ignorecase=true
submodule.active=.
remote.origin.url=git@bitbucket.org:bluetradesys/spark.git
remote.origin.fetch=+refs/heads/*:refs/remotes/origin/*
branch.spark_crypto.dev.remote=origin
branch.spark_crypto.dev.merge=refs/heads/spark_crypto.dev
submodule.docker/linux.url=git@bitbucket.org:bluetradesys/linux_docker.git
submodule.docker/windows.url=git@bitbucket.org:bluetradesys/windows_docker.git
submodule.externals.url=git@bitbucket.org:bluetradesys/externals.git
branch.development.remote=origin
branch.development.merge=refs/heads/development
user.name=John Colvin
user.email=john.colvin@bluetradesys.com
+12
View File
@@ -0,0 +1,12 @@
core.repositoryformatversion=0
core.filemode=true
core.bare=false
core.logallrefupdates=true
remote.origin.url=git@bitbucket.org:bluetradesys/spark.git
remote.origin.fetch=+refs/heads/*:refs/remotes/origin/*
branch.spark_crypto.dev.remote=origin
branch.spark_crypto.dev.merge=refs/heads/spark_crypto.dev
branch.development.remote=origin
branch.development.merge=refs/heads/development
submodule.externals.active=true
submodule.externals.url=git@bitbucket.org:bluetradesys/externals.git
+8
View File
@@ -0,0 +1,8 @@
function(init_executable source_dir)
file(
GLOB_RECURSE ${PROJECT_NAME}_Sources
RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
${source_dir}/*.cpp)
add_executable(${PROJECT_NAME} ${${PROJECT_NAME}_Sources})
target_include_directories(${PROJECT_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
endfunction(init_executable)
Submodule docker/linux deleted from 56c61231f3.
+2 -8
View File
@@ -48,15 +48,9 @@ RUN apt-get update && apt-get install -y \
# cmake # cmake
RUN \ RUN \
curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \ curl -fsSL https://apt.kitware.com/keys/kitware-archive-latest.asc 2> /dev/null \
| gpg --dearmor - \ | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg > /dev/null \
| tee /usr/share/keyrings/kitware-archive-keyring.gpg > /dev/null \ && apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
&& echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" \
| tee /etc/apt/sources.list.d/kitware.list > /dev/null \
&& apt-get update \ && apt-get update \
&& rm /usr/share/keyrings/kitware-archive-keyring.gpg \
&& apt-get install -y kitware-archive-keyring \
&& echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs)-rc main" \
| tee -a /etc/apt/sources.list.d/kitware.list >/dev/null \
&& apt-get install -y cmake && apt-get install -y cmake
# node # node
+2 -3
View File
@@ -1,7 +1,6 @@
{ {
"image": "bluetradingsystems/sparkd-dev:latest", "image": "bluetradingsystems/sparkd-dev:latest",
// Creates local dir to mount to HOME in docker "initializeCommand": "${localWorkspaceFolder}/docker/sparkd/dev/devcontainer_initialize.sh",
"initializeCommand": "mkdir -p ${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId}",
"hostRequirements": { "hostRequirements": {
"memory": "4gb", "memory": "4gb",
"storage": "6gb" "storage": "6gb"
@@ -28,6 +27,6 @@
"8003:8003" "8003:8003"
], ],
"mounts": [ "mounts": [
"source=${localEnv:HOME}/.bts/devcontainer-mounts/${devcontainerId},target=/home/bts,type=bind" "source=${localEnv:HOME}/.sparkd-dev/mounts/sparkd-dev,target=/home/bts,type=bind"
] ]
} }
+3
View File
@@ -0,0 +1,3 @@
#!/usr/bin/env bash
mkdir -p "$HOME/.sparkd-dev/mounts/sparkd-dev"
Submodule docker/windows deleted from de7d40c09d.
+3
View File
@@ -24,3 +24,6 @@
those re-entered explicitly those re-entered explicitly
- With this release we add per user location and smp id tags. At least - With this release we add per user location and smp id tags. At least
the location will have to be set through the admin UI for every user the location will have to be set through the admin UI for every user
libBoostFilesystem_x86-64_ubuntu22-04_gcc123_mdid.so: ELF 64-bit LSB shared object, x86-64, version 1 (GNU/Linux), dynamically linked, BuildID[sha1]=75427020663511112b3aadeef5023e4f368aafd4, stripped
/usr/lib/x86_64-linux-gnu/libboost_filesystem.so.1.74.0: ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, BuildID[sha1]=f0cb6f85ac07ed93816bae06c584d4a1fabb0e70, stripped
@@ -8,7 +8,7 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="Google.Protobuf" Version="3.24.3" /> <PackageReference Include="Google.Protobuf" Version="3.24.3" />
<PackageReference Include="YaccLexTools" Version="1.2.2" /> <PackageReference Include="YaccLexTools" Version="1.2.0" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
+34 -15
View File
@@ -1,22 +1,41 @@
using System.Diagnostics; using mbts.spark;
namespace BCG.Utilities.General; using System;
using System.Runtime;
/// <summary> namespace BCG.Utilities.General
/// Summary description for Crono. {
/// </summary> /// <summary>
public class Crono { /// Summary description for Crono.
readonly Stopwatch myTimer = new(); /// </summary>
public class Crono
{
public Crono()
{
myTimer = new HighPerformanceTimer();
}
public void start() => public void start()
myTimer.Start(); {myTimer.start();}
public void stop() => public void stop()
myTimer.Stop(); {myTimer.stop();}
public TimeSpan time => public TimeSpan time
myTimer.Elapsed; {
get
{
return new TimeSpan((long) (myTimer.totalMilliseconds * TimeSpan.TicksPerMillisecond));
}
}
public double miliseconds
{
get
{return myTimer.totalMilliseconds;}
}
private HighPerformanceTimer myTimer;
}
public double miliseconds =>
myTimer.Elapsed.TotalMilliseconds;
} }
+192 -192
View File
@@ -1,192 +1,192 @@
using System; using System;
using System.Collections; using System.Collections;
using System.Collections.Generic; using System.Collections.Generic;
using System.Collections.Concurrent; using System.Collections.Concurrent;
using System.Reflection; using System.Reflection;
using Microsoft.Win32; using Microsoft.Win32;
namespace BCG namespace BCG
{ {
public enum RegistryRoot { HKCU, HKLM } public enum RegistryRoot { HKCU, HKLM }
[Serializable] [Serializable]
public class BlueRegistry public class BlueRegistry
{ {
public const string BlueSubKey = @"Software\Blue Capital Group\"; public const string BlueSubKey = @"Software\Blue Capital Group\";
public static readonly string ApplicationPrefix; public static readonly string ApplicationPrefix;
protected string myFullPath; protected string myFullPath;
public BlueRegistry(string subPath) public BlueRegistry(string subPath)
{ {
myFullPath = string.Concat(BlueRegistry.ApplicationPrefix, @"\", subPath); myFullPath = string.Concat(BlueRegistry.ApplicationPrefix, @"\", subPath);
} }
public BlueRegistry(BlueRegistry parent, string subPath) public BlueRegistry(BlueRegistry parent, string subPath)
{ {
myFullPath = string.Concat(parent.myFullPath, @"\", subPath); myFullPath = string.Concat(parent.myFullPath, @"\", subPath);
} }
static BlueRegistry() static BlueRegistry()
{ {
string app = "Applications"; string app = "Applications";
Assembly ass = Assembly.GetEntryAssembly(); Assembly ass = Assembly.GetEntryAssembly();
if (ass != null) if (ass != null)
{ {
app = ass.GetName().Name; app = ass.GetName().Name;
object[] attrs = ass.GetCustomAttributes( object[] attrs = ass.GetCustomAttributes(
Type.GetType("System.Reflection.AssemblyTitleAttribute"), true); Type.GetType("System.Reflection.AssemblyTitleAttribute"), true);
if (attrs.Length > 0 && ((AssemblyTitleAttribute)attrs[0]).Title != string.Empty) if (attrs.Length > 0 && ((AssemblyTitleAttribute)attrs[0]).Title != string.Empty)
{ {
app = ((AssemblyTitleAttribute)attrs[0]).Title; app = ((AssemblyTitleAttribute)attrs[0]).Title;
} }
} }
ApplicationPrefix = string.Concat(BlueRegistry.BlueSubKey, app); ApplicationPrefix = string.Concat(BlueRegistry.BlueSubKey, app);
} }
public object getValue(string name) public object getValue(string name)
{ {
return getValue(RegistryRoot.HKCU, name); return getValue(RegistryRoot.HKCU, name);
} }
public object getValue(string name, object defaultValue) public object getValue(string name, object defaultValue)
{ {
return getValue(RegistryRoot.HKCU, name, defaultValue); return getValue(RegistryRoot.HKCU, name, defaultValue);
} }
public void setValue(string name, object value) public void setValue(string name, object value)
{ {
setValue(RegistryRoot.HKCU, name, value); setValue(RegistryRoot.HKCU, name, value);
} }
public void deleteValue(string name) public void deleteValue(string name)
{ {
deleteValue(RegistryRoot.HKCU, name); deleteValue(RegistryRoot.HKCU, name);
} }
private RegistryKey myGetRegistryKey(RegistryRoot root) private RegistryKey myGetRegistryKey(RegistryRoot root)
{ {
if (root == RegistryRoot.HKCU) if (root == RegistryRoot.HKCU)
return Registry.CurrentUser.CreateSubKey(myFullPath); return Registry.CurrentUser.CreateSubKey(myFullPath);
else else
return Registry.LocalMachine.CreateSubKey(myFullPath); return Registry.LocalMachine.CreateSubKey(myFullPath);
} }
#region Generic Methods #region Generic Methods
public virtual void setValue(RegistryRoot root, string name, object value) internal virtual void setValue(RegistryRoot root, string name, object value)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
rk.SetValue(name, value); rk.SetValue(name, value);
} }
} }
internal virtual object getValue(RegistryRoot root, string name) internal virtual object getValue(RegistryRoot root, string name)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
return rk.GetValue(name); return rk.GetValue(name);
} }
} }
internal virtual void deleteValue(RegistryRoot root, string name) internal virtual void deleteValue(RegistryRoot root, string name)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
rk.DeleteValue(name); rk.DeleteValue(name);
} }
} }
public virtual object getValue(RegistryRoot root, string name, object defaultValue) internal virtual object getValue(RegistryRoot root, string name, object defaultValue)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
return rk.GetValue(name, defaultValue); return rk.GetValue(name, defaultValue);
} }
} }
#endregion #endregion
#region Double #region Double
internal virtual void setDouble(RegistryRoot root, string name, double value) internal virtual void setDouble(RegistryRoot root, string name, double value)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
DoubleRegistryHandler.setValue(rk, name, value); DoubleRegistryHandler.setValue(rk, name, value);
} }
} }
internal virtual double getDouble(RegistryRoot root, string name) internal virtual double getDouble(RegistryRoot root, string name)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
return DoubleRegistryHandler.getValue(rk, name); return DoubleRegistryHandler.getValue(rk, name);
} }
} }
#endregion #endregion
#region Enum #region Enum
internal virtual void setEnum(RegistryRoot root, string name, Enum value) internal virtual void setEnum(RegistryRoot root, string name, Enum value)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
EnumRegistryHandler.setValue(rk, name, value); EnumRegistryHandler.setValue(rk, name, value);
} }
} }
internal virtual Enum getEnum(RegistryRoot root, string name, Type enumType) internal virtual Enum getEnum(RegistryRoot root, string name, Type enumType)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
return EnumRegistryHandler.getValue(rk, name, enumType); return EnumRegistryHandler.getValue(rk, name, enumType);
} }
} }
#endregion #endregion
#region Array #region Array
internal virtual void setArray(RegistryRoot root, string name, Array array) internal virtual void setArray(RegistryRoot root, string name, Array array)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
ArrayRegistryHandler.setValue(rk, name, array); ArrayRegistryHandler.setValue(rk, name, array);
} }
} }
internal virtual Array getArray(RegistryRoot root, string name, Type elementType) internal virtual Array getArray(RegistryRoot root, string name, Type elementType)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
return ArrayRegistryHandler.getValue(rk, name, elementType); return ArrayRegistryHandler.getValue(rk, name, elementType);
} }
} }
#endregion #endregion
#region SortedList #region SortedList
internal virtual void setSortedList(RegistryRoot root, string name, SortedList slist) internal virtual void setSortedList(RegistryRoot root, string name, SortedList slist)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
SortedListRegistryHandler.setValue(rk, name, slist); SortedListRegistryHandler.setValue(rk, name, slist);
} }
} }
internal virtual SortedList getSortedList(RegistryRoot root, string name) internal virtual SortedList getSortedList(RegistryRoot root, string name)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
return SortedListRegistryHandler.getValue(rk, name); return SortedListRegistryHandler.getValue(rk, name);
} }
} }
internal virtual void deleteSortedList(RegistryRoot root, string name) internal virtual void deleteSortedList(RegistryRoot root, string name)
{ {
using (RegistryKey rk = myGetRegistryKey(root)) using (RegistryKey rk = myGetRegistryKey(root))
{ {
SortedListRegistryHandler.deleteValue(rk, name); SortedListRegistryHandler.deleteValue(rk, name);
} }
} }
#endregion #endregion
} }
} }
@@ -1,9 +1,10 @@
<Project Sdk="Microsoft.NET.Sdk"> <Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup> <PropertyGroup>
<TargetFramework>net7.0</TargetFramework> <TargetFramework>net7.0-windows</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings> <ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable> <Nullable>enable</Nullable>
<UseWindowsForms>true</UseWindowsForms>
</PropertyGroup> </PropertyGroup>
</Project> </Project>
@@ -1,166 +1,166 @@
using System; using System;
using System.Reflection; using System.Reflection;
using System.Windows.Forms; using System.Windows.Forms;
using System.Drawing; using System.Drawing;
using System.Diagnostics; using System.Diagnostics;
using Microsoft.Win32; using Microsoft.Win32;
namespace BCG namespace BCG
{ {
public class PersistWindowState public class PersistWindowState
{ {
// event info that allows form to persist extra window state data // event info that allows form to persist extra window state data
public delegate void WindowStateHandler (object sender, BlueRegistry bluereg); public delegate void WindowStateHandler (object sender, BlueRegistry bluereg);
public event WindowStateHandler LoadState; public event WindowStateHandler LoadState;
public event WindowStateHandler SaveState; public event WindowStateHandler SaveState;
private Form myParent; private Form myParent;
private BlueRegistry myRegistry; private BlueRegistry myRegistry;
private int myNormalLeft; private int myNormalLeft;
private int myNormalTop; private int myNormalTop;
private int myNormalWidth; private int myNormalWidth;
private int myNormalHeight; private int myNormalHeight;
private bool myResizable = true; private bool myResizable = true;
private FormWindowState myWindowState; private FormWindowState myWindowState;
private bool myAllowSaveMinimized = false; private bool myAllowSaveMinimized = false;
private bool mySavePosition = true; private bool mySavePosition = true;
private bool mySaveState = true; private bool mySaveState = true;
public PersistWindowState (bool savePosition) public PersistWindowState (bool savePosition)
{ {
mySavePosition = savePosition; mySavePosition = savePosition;
} }
public PersistWindowState () public PersistWindowState ()
{ {
} }
public bool saveState public bool saveState
{ {
[DebuggerStepThrough] [DebuggerStepThrough]
set { mySaveState = value; } set { mySaveState = value; }
} }
public Form parent public Form parent
{ {
[DebuggerStepThrough] [DebuggerStepThrough]
get { return myParent; } get { return myParent; }
set set
{ {
myParent = value; myParent = value;
myRegistry = new BlueRegistry (myParent.Text); myRegistry = new BlueRegistry (myParent.Text);
// subscribe to parent form's events // subscribe to parent form's events
myParent.Closing += new System.ComponentModel.CancelEventHandler(myOnClosing); myParent.Closing += new System.ComponentModel.CancelEventHandler(myOnClosing);
myParent.Resize += new System.EventHandler(myOnResize); myParent.Resize += new System.EventHandler(myOnResize);
myParent.Move += new System.EventHandler(myOnMove); myParent.Move += new System.EventHandler(myOnMove);
myParent.Load += new System.EventHandler(myOnLoad); myParent.Load += new System.EventHandler(myOnLoad);
// get initial width and height in case form is never resized // get initial width and height in case form is never resized
myNormalWidth = myParent.Width; myNormalWidth = myParent.Width;
myNormalHeight = myParent.Height; myNormalHeight = myParent.Height;
} }
} }
public BlueRegistry blueRegistry public BlueRegistry blueRegistry
{ {
[DebuggerStepThrough] [DebuggerStepThrough]
get { return myRegistry; } get { return myRegistry; }
[DebuggerStepThrough] [DebuggerStepThrough]
set { myRegistry = value; } set { myRegistry = value; }
} }
public bool resizable public bool resizable
{ {
[DebuggerStepThrough] [DebuggerStepThrough]
set { myResizable = value; } set { myResizable = value; }
} }
public bool allowSaveMinimized public bool allowSaveMinimized
{ {
[DebuggerStepThrough] [DebuggerStepThrough]
set { myAllowSaveMinimized = value; } set { myAllowSaveMinimized = value; }
} }
private void myOnResize(object sender, System.EventArgs e) private void myOnResize(object sender, System.EventArgs e)
{ {
// save width and height // save width and height
if (myParent.WindowState == FormWindowState.Normal && myResizable) if (myParent.WindowState == FormWindowState.Normal && myResizable)
{ {
myNormalWidth = myParent.Width; myNormalWidth = myParent.Width;
myNormalHeight = myParent.Height; myNormalHeight = myParent.Height;
} }
} }
private void myOnMove(object sender, System.EventArgs e) private void myOnMove(object sender, System.EventArgs e)
{ {
// save position // save position
if(myParent.WindowState == FormWindowState.Normal) if(myParent.WindowState == FormWindowState.Normal)
{ {
myNormalLeft = myParent.Left; myNormalLeft = myParent.Left;
myNormalTop = myParent.Top; myNormalTop = myParent.Top;
} }
// save state // save state
myWindowState = myParent.WindowState; myWindowState = myParent.WindowState;
} }
private void myOnClosing(object sender, System.ComponentModel.CancelEventArgs e) private void myOnClosing(object sender, System.ComponentModel.CancelEventArgs e)
{ {
// save position, size and state // save position, size and state
if (mySavePosition) if (mySavePosition)
{ {
myRegistry.setValue (RegistryRoot.HKCU, "Left", myNormalLeft); myRegistry.setValue (RegistryRoot.HKCU, "Left", myNormalLeft);
myRegistry.setValue (RegistryRoot.HKCU, "Top", myNormalTop); myRegistry.setValue (RegistryRoot.HKCU, "Top", myNormalTop);
} }
if (myResizable) if (myResizable)
{ {
myRegistry.setValue (RegistryRoot.HKCU, "Width", myNormalWidth); myRegistry.setValue (RegistryRoot.HKCU, "Width", myNormalWidth);
myRegistry.setValue (RegistryRoot.HKCU, "Height", myNormalHeight); myRegistry.setValue (RegistryRoot.HKCU, "Height", myNormalHeight);
} }
// check if we are allowed to save the state as minimized (not normally) // check if we are allowed to save the state as minimized (not normally)
if (!myAllowSaveMinimized) if (!myAllowSaveMinimized)
{ {
if(myWindowState == FormWindowState.Minimized) if(myWindowState == FormWindowState.Minimized)
myWindowState = FormWindowState.Normal; myWindowState = FormWindowState.Normal;
} }
if (mySaveState) if (mySaveState)
myRegistry.setValue (RegistryRoot.HKCU, "WindowState", (int) myWindowState); myRegistry.setValue (RegistryRoot.HKCU, "WindowState", (int) myWindowState);
// fire SaveState event // fire SaveState event
if (SaveState != null) if (SaveState != null)
SaveState (this, myRegistry); SaveState (this, myRegistry);
} }
private void myOnLoad (object sender, System.EventArgs e) private void myOnLoad (object sender, System.EventArgs e)
{ {
if (mySavePosition) if (mySavePosition)
{ {
int left = (int) myRegistry.getValue (RegistryRoot.HKCU, "Left", myParent.Left); int left = (int) myRegistry.getValue (RegistryRoot.HKCU, "Left", myParent.Left);
int top = (int) myRegistry.getValue (RegistryRoot.HKCU, "Top", myParent.Top); int top = (int) myRegistry.getValue (RegistryRoot.HKCU, "Top", myParent.Top);
myParent.Location = new Point (left, top); myParent.Location = new Point (left, top);
} }
if (myResizable) if (myResizable)
{ {
int width = (int) myRegistry.getValue (RegistryRoot.HKCU, "Width", myParent.Width); int width = (int) myRegistry.getValue (RegistryRoot.HKCU, "Width", myParent.Width);
int height = (int) myRegistry.getValue (RegistryRoot.HKCU, "Height", myParent.Height); int height = (int) myRegistry.getValue (RegistryRoot.HKCU, "Height", myParent.Height);
myParent.Size = new Size (width, height); myParent.Size = new Size (width, height);
} }
if (mySaveState) if (mySaveState)
{ {
FormWindowState windowState = (FormWindowState) myRegistry.getValue (RegistryRoot.HKCU, "WindowState", FormWindowState windowState = (FormWindowState) myRegistry.getValue (RegistryRoot.HKCU, "WindowState",
(int) myParent.WindowState); (int) myParent.WindowState);
myParent.WindowState = windowState; myParent.WindowState = windowState;
} }
// fire LoadState event // fire LoadState event
if(LoadState != null) if(LoadState != null)
LoadState (this, myRegistry); LoadState (this, myRegistry);
} }
} }
} }
@@ -1,24 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="YaccLexTools" Version="1.2.2" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
<ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
<ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
</ItemGroup>
<ItemGroup Label="ColumnExpression">
<Scanner Include="ColumnExpressionTokens.lex" />
<Parser Include="ColumnExpressionGrammar.y" />
</ItemGroup>
</Project>
Whitespace-only changes.
@@ -1,62 +0,0 @@
namespace Bts.ColumnExpressions;
public class ColumnExpression {
public ColumnExpression(string name, Type? type = null, object? value = null) {
Name = name;
Type = type
?? value?.GetType()
?? throw new ArgumentException(
"Type must be specified explicitly when value is null", nameof(type));
Value = value
?? DBNull.Value;
}
public string Name { get; }
public Type Type { get; }
public object Value { get; }
public static ColumnExpression Create<T>(string name, T value, Type? type = null) {
// Try to convert value to 'type' to verify they are compatible
// Note: Can't use `System.Type.IsAssignableFrom` because it doesn't
// account for implicit conversions (e.g., int -> long), just inheritance.
if (type is not null && !ConvertUtils.try_convert(type, value, out _)) {
throw new ArgumentException(
$"Type '{type}' is not assignable from value type '{value?.GetType()}'");
}
return new(name, type ?? typeof(T), value);
}
internal static Type ParseType(string typeId) {
return Candidates().ExceptNull().FirstOrDefault() ??
throw new ArgumentException($"Failed to parse Type from '{typeId}'", nameof(typeId));
IEnumerable<Type?> Candidates() {
if (typeId.Length == 0) {
yield break;
}
yield return Type.GetType(typeId);
yield return Type.GetType($"System.{typeId}");
if (typeId.Length > 1) {
yield return Type.GetType($"System.{char.ToUpper(typeId[0]) + typeId[1..]}");
}
yield return typeId.ToLower() switch {
"float64" or "double" or "real64" or "number" => typeof(double),
"float32" or "single" or "real32" or "real" or "float" => typeof(float),
"int16" or "short" or "integer16" or "signed16" => typeof(short),
"int32" or "int" or "integer" or "integer32" or "signed" => typeof(int),
"int64" or "long" or "integer64" or "signed64" => typeof(long),
"uint16" or "ushort" or "unsigned16" => typeof(ushort),
"uint32" or "uint" or "unsigned" => typeof(uint),
"uint64" or "ulong" or "unsigned64" => typeof(ulong),
"char" => typeof(char),
"byte" or "uint8" => typeof(byte),
"bool" or "boolean" => typeof(bool),
"string" or "text" => typeof(string),
"date" or "datetime" => typeof(DateTime),
_ => null
};
}
}
}
@@ -1,49 +0,0 @@
%namespace Bts.ColumnExpressions
%partial
%visibility internal
%tokentype ColumnExpressionToken
%parsertype ColumnExpressionParser
%union {
public int Integer;
public double Real;
public bool Boolean;
public string String;
public string Identifier;
public string TypeId;
public DateTime DateTime;
public Tuple<string, Type> ColumnDefinition;
public ColumnExpression ColumnExpression;
public ImmutableList<ColumnExpression> ColumnExpressions;
}
%token BOOLEAN_CONSTANT, INTEGER_CONSTANT, REAL_CONSTANT, DATE_TIME_CONSTANT, STRING_CONSTANT
%token IDENTIFIER, EQUALS, TYPE_ID, OF_TYPE
%nonassoc EQUALS
%type <ColumnDefinition> ColumnDefinition
%type <ColumnExpression> ColumnExpression
%type <ColumnExpressions> ColumnExpressions
%%
ColumnExpressions :
ColumnExpression { $$ = ImmutableList.CreateRange(new[] { $1 }); }
| ColumnExpressions ColumnExpression { $$ = $1.Add($2); }
;
ColumnExpression :
ColumnDefinition EQUALS INTEGER_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Integer, $1.Item2); }
| ColumnDefinition EQUALS REAL_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Real, $1.Item2); }
| ColumnDefinition EQUALS DATE_TIME_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.DateTime, $1.Item2); }
| ColumnDefinition EQUALS BOOLEAN_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.Boolean, $1.Item2); }
| ColumnDefinition EQUALS STRING_CONSTANT { $$ = ColumnExpression.Create($1.Item1, $3.String, $1.Item2); }
| ColumnDefinition { $$ = new($1.Item1, $1.Item2); }
;
ColumnDefinition :
IDENTIFIER { $$ = new($1.Identifier, null); }
| IDENTIFIER OF_TYPE IDENTIFIER { $$ = new($1.Identifier, ColumnExpression.ParseType($3.Identifier)); }
| IDENTIFIER OF_TYPE TYPE_ID { $$ = new($1.Identifier, ColumnExpression.ParseType($3.TypeId)); }
;
@@ -1,18 +0,0 @@
namespace Bts.ColumnExpressions;
using System.IO;
using System.Text;
public class ColumnExpressionInterpreter {
public static IList<ColumnExpression> Evaluate(Stream stream) {
var parser = new ColumnExpressionParser();
var res = parser.Parse(stream);
return res?.ColumnExpressions ?? ImmutableList<ColumnExpression>.Empty.Add(
res?.ColumnExpression ?? throw new Exception("Failed to parse column(s) expression"));
}
public static IList<ColumnExpression> Evaluate(string str) {
using var ms = new MemoryStream(Encoding.Default.GetBytes(str));
return Evaluate(ms);
}
}
@@ -1,18 +0,0 @@
namespace Bts.ColumnExpressions;
using System.Text;
partial class ColumnExpressionParser {
public ColumnExpressionParser() : base(null) { }
public ValueType? Parse(string s) {
using var ms = new MemoryStream(Encoding.Default.GetBytes(s));
return Parse(ms);
}
public ValueType? Parse(Stream stream) {
Scanner = new ColumnExpressionScanner(stream);
Parse();
return CurrentSemanticValue;
}
}
@@ -1,9 +0,0 @@
namespace Bts.ColumnExpressions;
partial class ColumnExpressionScanner {
public override void yyerror(string format, params object[] args) {
base.yyerror(format, args);
Console.WriteLine(format, args);
Console.WriteLine();
}
}
@@ -1,85 +0,0 @@
%namespace Bts.ColumnExpressions
%scannertype ColumnExpressionScanner
%visibility internal
%tokentype ColumnExpressionToken
%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers
/* Literal Constants */
Letter [a-zA-Z]
BooleanConstant true|false
StringConstant '([^\'\\]*|\\.)*'
DateTimeConstant ([1-9]|'10'|'11'|'12')\/([1-9]|([1-2][0-9])|'30'|'31')\/[0-9][0-9][0-9]*
IntegerConstant -?[0-9]+
NullConstant 'null'
RealConstant -?[0-9]+\.[0-9]+
/* Relational Operators */
Equals =
OfType /
Identifier {Letter}({Letter}*[0-9]*_*)*
NamespacePrefix {Identifier}\.
TypeId {NamespacePrefix}*{Identifier}
/* Utility Tokens */
OpenParenthesis \(
CloseParenthesis \)
Comma ,
QuoteOpen \[
QuoteClose \]
/* Comments */
LineComment \/\/[^\n\r]*(\n|\r|\r\n)
%%
{Equals} {
return (int)ColumnExpressionToken.EQUALS;
}
{OfType} {
return (int)ColumnExpressionToken.OF_TYPE;
}
{StringConstant} {
yylval.String = yytext.Substring(1, yytext.Length - 2).Replace(@"\'","'");
return (int)ColumnExpressionToken.STRING_CONSTANT;
}
{BooleanConstant} {
yylval.Boolean = yytext.ToLower() switch {
"true" => true,
"false" => false,
_ => throw new Exception(
string.Format("Expected yytext to be 'true' or 'false' (actually '{0}')", yytext))
};
return (int)ColumnExpressionToken.BOOLEAN_CONSTANT;
}
{DateTimeConstant} {
yylval.DateTime = DateTime.Parse(yytext);
return (int)ColumnExpressionToken.DATE_TIME_CONSTANT;
}
{IntegerConstant} {
yylval.Integer = int.Parse(yytext);
return (int)ColumnExpressionToken.INTEGER_CONSTANT;
}
{RealConstant} {
yylval.Real = double.Parse(yytext);
return (int)ColumnExpressionToken.REAL_CONSTANT;
}
{Identifier} {
yylval.Identifier = yytext;
return (int)ColumnExpressionToken.IDENTIFIER;
}
{TypeId} {
yylval.TypeId = yytext;
return (int)ColumnExpressionToken.TYPE_ID;
}
%%
@@ -1,720 +0,0 @@
// ==============================================================
// <auto-generated>
// This code automatically produced from an embedded resource.
// Do not edit this file, or it will become incompatible with
// the specification from which it was generated.
// </auto-generated>
// ==============================================================
using System;
using System.IO;
using System.Text;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Runtime.Serialization;
using System.Globalization;
namespace QUT.GplexBuffers
{
// Code copied from GPLEX embedded resource
[Serializable]
internal class BufferException : Exception
{
public BufferException() { }
public BufferException(string message) : base(message) { }
public BufferException(string message, Exception innerException)
: base(message, innerException) { }
protected BufferException(SerializationInfo info, StreamingContext context)
: base(info, context) { }
}
internal abstract class ScanBuff
{
private string fileNm;
public const int EndOfFile = -1;
public const int UnicodeReplacementChar = 0xFFFD;
public bool IsFile { get { return (fileNm != null); } }
public string FileName { get { return fileNm; } set { fileNm = value; } }
public abstract int Pos { get; set; }
public abstract int Read();
public virtual void Mark() { }
public abstract string GetString(int begin, int limit);
public static ScanBuff GetBuffer(string source)
{
return new StringBuffer(source);
}
public static ScanBuff GetBuffer(IList<string> source)
{
return new LineBuffer(source);
}
#if (!NOFILES)
public static ScanBuff GetBuffer(Stream source)
{
return new BuildBuffer(source);
}
#if (!BYTEMODE)
public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
{
return new BuildBuffer(source, fallbackCodePage);
}
#endif // !BYTEMODE
#endif // !NOFILES
}
#region Buffer classes
// ==============================================================
// ===== Definitions for various ScanBuff derived classes ====
// ==============================================================
// =============== String input ================
// ==============================================================
/// <summary>
/// This class reads characters from a single string as
/// required, for example, by Visual Studio language services
/// </summary>
sealed class StringBuffer : ScanBuff
{
string str; // input buffer
int bPos; // current position in buffer
int sLen;
public StringBuffer(string source)
{
this.str = source;
this.sLen = source.Length;
this.FileName = null;
}
public override int Read()
{
if (bPos < sLen) return str[bPos++];
else if (bPos == sLen) { bPos++; return '\n'; } // one strike, see new line
else { bPos++; return EndOfFile; } // two strikes and you're out!
}
public override string GetString(int begin, int limit)
{
// "limit" can be greater than sLen with the BABEL
// option set. Read returns a "virtual" EOL if
// an attempt is made to read past the end of the
// string buffer. Without the guard any attempt
// to fetch yytext for a token that includes the
// EOL will throw an index exception.
if (limit > sLen) limit = sLen;
if (limit <= begin) return "";
else return str.Substring(begin, limit - begin);
}
public override int Pos
{
get { return bPos; }
set { bPos = value; }
}
public override string ToString() { return "StringBuffer"; }
}
// ==============================================================
// The LineBuff class contributed by Nigel Horspool,
// nigelh@cs.uvic.cs
// ==============================================================
sealed class LineBuffer : ScanBuff
{
IList<string> line; // list of source lines from a file
int numLines; // number of strings in line list
string curLine; // current line in that list
int cLine; // index of current line in the list
int curLen; // length of current line
int curLineStart; // position of line start in whole file
int curLineEnd; // position of line end in whole file
int maxPos; // max position ever visited in whole file
int cPos; // ordinal number of code in source
// Constructed from a list of strings, one per source line.
// The lines have had trailing '\n' characters removed.
public LineBuffer(IList<string> lineList)
{
line = lineList;
numLines = line.Count;
cPos = curLineStart = 0;
curLine = (numLines > 0 ? line[0] : "");
maxPos = curLineEnd = curLen = curLine.Length;
cLine = 1;
FileName = null;
}
public override int Read()
{
if (cPos < curLineEnd)
return curLine[cPos++ - curLineStart];
if (cPos++ == curLineEnd)
return '\n';
if (cLine >= numLines)
return EndOfFile;
curLine = line[cLine];
curLen = curLine.Length;
curLineStart = curLineEnd + 1;
curLineEnd = curLineStart + curLen;
if (curLineEnd > maxPos)
maxPos = curLineEnd;
cLine++;
return curLen > 0 ? curLine[0] : '\n';
}
// To speed up searches for the line containing a position
private int cachedPosition;
private int cachedIxdex;
private int cachedLineStart;
// Given a position pos within the entire source, the results are
// ix -- the index of the containing line
// lstart -- the position of the first character on that line
private void findIndex(int pos, out int ix, out int lstart)
{
if (pos >= cachedPosition)
{
ix = cachedIxdex; lstart = cachedLineStart;
}
else
{
ix = lstart = 0;
}
while (ix < numLines)
{
int len = line[ix].Length + 1;
if (pos < lstart + len) break;
lstart += len;
ix++;
}
cachedPosition = pos;
cachedIxdex = ix;
cachedLineStart = lstart;
}
public override string GetString(int begin, int limit)
{
if (begin >= maxPos || limit <= begin) return "";
int endIx, begIx, endLineStart, begLineStart;
findIndex(begin, out begIx, out begLineStart);
int begCol = begin - begLineStart;
findIndex(limit, out endIx, out endLineStart);
int endCol = limit - endLineStart;
string s = line[begIx];
if (begIx == endIx)
{
// the usual case, substring all on one line
return (endCol <= s.Length) ?
s.Substring(begCol, endCol - begCol)
: s.Substring(begCol) + "\n";
}
// the string spans multiple lines, yuk!
StringBuilder sb = new StringBuilder();
if (begCol < s.Length)
sb.Append(s.Substring(begCol));
for (; ; )
{
sb.Append("\n");
s = line[++begIx];
if (begIx >= endIx) break;
sb.Append(s);
}
if (endCol <= s.Length)
{
sb.Append(s.Substring(0, endCol));
}
else
{
sb.Append(s);
sb.Append("\n");
}
return sb.ToString();
}
public override int Pos
{
get { return cPos; }
set
{
cPos = value;
findIndex(cPos, out cLine, out curLineStart);
// cLine should be the *next* line after curLine.
curLine = (cLine < numLines ? line[cLine++] : "");
curLineEnd = curLineStart + curLine.Length;
}
}
public override string ToString() { return "LineBuffer"; }
}
#if (!NOFILES)
// ==============================================================
// ===== class BuildBuff : for unicode text files ========
// ==============================================================
class BuildBuffer : ScanBuff
{
// Double buffer for char stream.
class BufferElement
{
StringBuilder bldr = new StringBuilder();
StringBuilder next = new StringBuilder();
int minIx;
int maxIx;
int brkIx;
bool appendToNext;
internal BufferElement() { }
internal int MaxIndex { get { return maxIx; } }
// internal int MinIndex { get { return minIx; } }
internal char this[int index]
{
get
{
if (index < minIx || index >= maxIx)
throw new BufferException("Index was outside data buffer");
else if (index < brkIx)
return bldr[index - minIx];
else
return next[index - brkIx];
}
}
internal void Append(char[] block, int count)
{
maxIx += count;
if (appendToNext)
this.next.Append(block, 0, count);
else
{
this.bldr.Append(block, 0, count);
brkIx = maxIx;
appendToNext = true;
}
}
internal string GetString(int start, int limit)
{
if (limit <= start)
return "";
if (start >= minIx && limit <= maxIx)
if (limit < brkIx) // String entirely in bldr builder
return bldr.ToString(start - minIx, limit - start);
else if (start >= brkIx) // String entirely in next builder
return next.ToString(start - brkIx, limit - start);
else // Must do a string-concatenation
return
bldr.ToString(start - minIx, brkIx - start) +
next.ToString(0, limit - brkIx);
else
throw new BufferException("String was outside data buffer");
}
internal void Mark(int limit)
{
if (limit > brkIx + 16) // Rotate blocks
{
StringBuilder temp = bldr;
bldr = next;
next = temp;
next.Length = 0;
minIx = brkIx;
brkIx = maxIx;
}
}
}
BufferElement data = new BufferElement();
int bPos; // Postion index in the StringBuilder
BlockReader NextBlk; // Delegate that serves char-arrays;
private string EncodingName
{
get
{
StreamReader rdr = NextBlk.Target as StreamReader;
return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
}
}
public BuildBuffer(Stream stream)
{
FileStream fStrm = (stream as FileStream);
if (fStrm != null) FileName = fStrm.Name;
NextBlk = BlockReaderFactory.Raw(stream);
}
#if (!BYTEMODE)
public BuildBuffer(Stream stream, int fallbackCodePage)
{
FileStream fStrm = (stream as FileStream);
if (fStrm != null) FileName = fStrm.Name;
NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
}
#endif
/// <summary>
/// Marks a conservative lower bound for the buffer,
/// allowing space to be reclaimed. If an application
/// needs to call GetString at arbitrary past locations
/// in the input stream, Mark() is not called.
/// </summary>
public override void Mark() { data.Mark(bPos - 2); }
public override int Pos
{
get { return bPos; }
set { bPos = value; }
}
/// <summary>
/// Read returns the ordinal number of the next char, or
/// EOF (-1) for an end of stream. Note that the next
/// code point may require *two* calls of Read().
/// </summary>
/// <returns></returns>
public override int Read()
{
//
// Characters at positions
// [data.offset, data.offset + data.bldr.Length)
// are available in data.bldr.
//
if (bPos < data.MaxIndex)
{
// ch0 cannot be EOF
return (int)data[bPos++];
}
else // Read from underlying stream
{
// Experimental code, blocks of page size
char[] chrs = new char[4096];
int count = NextBlk(chrs, 0, 4096);
if (count == 0)
return EndOfFile;
else
{
data.Append(chrs, count);
return (int)data[bPos++];
}
}
}
public override string GetString(int begin, int limit)
{
return data.GetString(begin, limit);
}
public override string ToString()
{
return "StringBuilder buffer, encoding: " + this.EncodingName;
}
}
// =============== End ScanBuff-derived classes ==================
internal delegate int BlockReader(char[] block, int index, int number);
// A delegate factory, serving up a delegate that
// reads a block of characters from the underlying
// encoded stream, via a StreamReader object.
//
internal static class BlockReaderFactory
{
public static BlockReader Raw(Stream stream)
{
return delegate(char[] block, int index, int number)
{
byte[] b = new byte[number];
int count = stream.Read(b, 0, number);
int i = 0;
int j = index;
for (; i < count; i++, j++)
block[j] = (char)b[i];
return count;
};
}
#if (!BYTEMODE)
public static BlockReader Get(Stream stream, int fallbackCodePage)
{
Encoding encoding;
int preamble = Preamble(stream);
if (preamble != 0) // There is a valid BOM here!
encoding = Encoding.GetEncoding(preamble);
else if (fallbackCodePage == -1) // Fallback is "raw" bytes
return Raw(stream);
else if (fallbackCodePage != -2) // Anything but "guess"
encoding = Encoding.GetEncoding(fallbackCodePage);
else // This is the "guess" option
{
int guess = new Guesser(stream).GuessCodePage();
stream.Seek(0, SeekOrigin.Begin);
if (guess == -1) // ==> this is a 7-bit file
encoding = Encoding.ASCII;
else if (guess == 65001)
encoding = Encoding.UTF8;
else // ==> use the machine default
encoding = Encoding.Default;
}
StreamReader reader = new StreamReader(stream, encoding);
return reader.Read;
}
static int Preamble(Stream stream)
{
int b0 = stream.ReadByte();
int b1 = stream.ReadByte();
if (b0 == 0xfe && b1 == 0xff)
return 1201; // UTF16BE
if (b0 == 0xff && b1 == 0xfe)
return 1200; // UTF16LE
int b2 = stream.ReadByte();
if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
return 65001; // UTF8
//
// There is no unicode preamble, so we
// return denoter for the machine default.
//
stream.Seek(0, SeekOrigin.Begin);
return 0;
}
#endif // !BYTEMODE
}
#endif // !NOFILES
#endregion Buffer classes
// ==============================================================
// ============ class CodePageHandling =============
// ==============================================================
#if (!NOFILES)
public static class CodePageHandling
{
public static int GetCodePage(string option)
{
string command = option.ToUpperInvariant();
if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
command = command.Substring(9);
try
{
if (command.Equals("RAW"))
return -1;
else if (command.Equals("GUESS"))
return -2;
else if (command.Equals("DEFAULT"))
return 0;
else if (char.IsDigit(command[0]))
return int.Parse(command, CultureInfo.InvariantCulture);
else
{
Encoding enc = Encoding.GetEncoding(command);
return enc.CodePage;
}
}
catch (FormatException)
{
Console.Error.WriteLine(
"Invalid format \"{0}\", using machine default", option);
}
catch (ArgumentException)
{
Console.Error.WriteLine(
"Unknown code page \"{0}\", using machine default", option);
}
return 0;
}
}
#region guesser
#if (!BYTEMODE)
// ==============================================================
// ============ Encoding Guesser =============
// ==============================================================
/// <summary>
/// This class provides a simple finite state automaton that
/// scans the file looking for (1) valid UTF-8 byte patterns,
/// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
/// The method then guesses whether it is UTF-8 or maybe some
/// local machine default encoding. This works well for the
/// various Latin encodings.
/// </summary>
internal class Guesser
{
ScanBuff buffer;
public int GuessCodePage() { return Scan(); }
const int maxAccept = 10;
const int initial = 0;
const int eofNum = 0;
const int goStart = -1;
const int INITIAL = 0;
const int EndToken = 0;
#region user code
/*
* Reads the bytes of a file to determine if it is
* UTF-8 or a single-byte code page file.
*/
public long utfX;
public long uppr;
#endregion user code
int state;
int currentStart = startState[0];
int code;
#region ScannerTables
static int[] startState = new int[] { 11, 0 };
#region CharacterMap
static sbyte[] map = new sbyte[256] {
/* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
/* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
/* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
/* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
/* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
/* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
/* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
/* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
#endregion
static sbyte[][] nextState = new sbyte[][] {
new sbyte[] {0, 0, 0, 0, 0, 0},
new sbyte[] {-1, -1, 10, -1, -1, -1},
new sbyte[] {-1, -1, -1, -1, -1, -1},
new sbyte[] {-1, -1, 8, -1, -1, -1},
new sbyte[] {-1, -1, 5, -1, -1, -1},
new sbyte[] {-1, -1, 6, -1, -1, -1},
new sbyte[] {-1, -1, 7, -1, -1, -1},
null,
new sbyte[] {-1, -1, 9, -1, -1, -1},
null,
null,
new sbyte[] {-1, 1, 2, 3, 4, 2}
};
[SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
// Reason for suppression: cannot have self-reference in array initializer.
static Guesser()
{
nextState[7] = nextState[2];
nextState[9] = nextState[2];
nextState[10] = nextState[2];
}
int NextState()
{
if (code == ScanBuff.EndOfFile)
return eofNum;
else
return nextState[state][map[code]];
}
#endregion
public Guesser(System.IO.Stream file) { SetSource(file); }
public void SetSource(System.IO.Stream source)
{
this.buffer = new BuildBuffer(source);
code = buffer.Read();
}
int Scan()
{
for (; ; )
{
int next;
state = currentStart;
while ((next = NextState()) == goStart)
code = buffer.Read();
state = next;
code = buffer.Read();
while ((next = NextState()) > eofNum)
{
state = next;
code = buffer.Read();
}
if (state <= maxAccept)
{
#region ActionSwitch
#pragma warning disable 162
switch (state)
{
case eofNum:
switch (currentStart)
{
case 11:
if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
else if (uppr * 10 > utfX) return 0; /* default code page */
else return 65001; /* UTF-8 encoding */
break;
}
return EndToken;
case 1: // Recognized '{Upper128}', Shortest string "\xC0"
case 2: // Recognized '{Upper128}', Shortest string "\x80"
case 3: // Recognized '{Upper128}', Shortest string "\xE0"
case 4: // Recognized '{Upper128}', Shortest string "\xF0"
uppr++;
break;
case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80"
uppr += 2;
break;
case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80"
uppr += 3;
break;
case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80"
utfX += 3;
break;
case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80"
uppr += 2;
break;
case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80"
utfX += 2;
break;
case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80"
utfX++;
break;
default:
break;
}
#pragma warning restore 162
#endregion
}
}
}
} // end class Guesser
#endif // !BYTEMODE
#endregion
#endif // !NOFILES
// End of code copied from embedded resource
}
@@ -1,884 +0,0 @@
// ==============================================================
// <auto-generated>
// This code automatically produced from an embedded resource.
// Do not edit this file, or it will become incompatible with
// the specification from which it was generated.
// </auto-generated>
// ==============================================================
// Gardens Point Parser Generator
// Copyright (c) Wayne Kelly, QUT 2005-2010
// (see accompanying GPPGcopyright.rtf)
using System;
using System.Text;
using System.Globalization;
using System.Collections.Generic;
using System.Runtime.Serialization;
using System.Diagnostics.CodeAnalysis;
namespace QUT.Gppg {
/// <summary>
/// Abstract class for GPPG shift-reduce parsers.
/// Parsers generated by GPPG derive from this base
/// class, overriding the abstract Initialize() and
/// DoAction() methods.
/// </summary>
/// <typeparam name="TValue">Semantic value type</typeparam>
/// <typeparam name="TSpan">Location type</typeparam>
#if EXPORT_GPPG
public abstract class ShiftReduceParser<TValue, TSpan>
#else
internal abstract class ShiftReduceParser<TValue, TSpan>
#endif
where TSpan : IMerge<TSpan>, new() {
private AbstractScanner<TValue, TSpan> scanner;
/// <summary>
/// The abstract scanner for this parser.
/// </summary>
protected AbstractScanner<TValue, TSpan> Scanner {
get { return scanner; }
set { scanner = value; }
}
/// <summary>
/// Constructor for base class
/// </summary>
/// <param name="scanner">Scanner instance for this parser</param>
protected ShiftReduceParser( AbstractScanner<TValue, TSpan> scanner ) {
this.scanner = scanner;
}
// ==============================================================
// TECHNICAL EXPLANATION.
// Why the next two fields are not exposed via properties.
// ==============================================================
// These fields are of the generic parameter types, and are
// frequently instantiated as struct types in derived classes.
// Semantic actions are defined in the derived classes and refer
// to instance fields of these structs. In such cases the code
// "get_CurrentSemanticValue().myField = blah;" will fail since
// the getter pushes the value of the field, not the reference.
// So, in the presence of properties, gppg would need to encode
// such field accesses as ...
// "tmp = get_CurrentSemanticValue(); // Fetch value
// tmp.myField = blah; // update
// set_CurrentSemanticValue(tmp); " // Write update back.
// There is no issue if TValue is restricted to be a ref type.
// The same explanation applies to scanner.yylval.
// ==============================================================
/// <summary>
/// The current value of the "$$" symbolic variable in the parser
/// </summary>
[SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
protected TValue CurrentSemanticValue;
/// <summary>
/// The current value of the "@$" symbolic variable in the parser
/// </summary>
[SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
protected TSpan CurrentLocationSpan;
protected int NextToken;
private TSpan LastSpan;
private State FsaState;
private bool recovering;
private int tokensSinceLastError;
private PushdownPrefixState<State> StateStack = new PushdownPrefixState<State>();
private PushdownPrefixState<TValue> valueStack = new PushdownPrefixState<TValue>();
private PushdownPrefixState<TSpan> locationStack = new PushdownPrefixState<TSpan>();
/// <summary>
/// The stack of semantic value (YYSTYPE) values.
/// </summary>
protected PushdownPrefixState<TValue> ValueStack { get { return valueStack; } }
/// <summary>
/// The stack of location value (YYLTYPE) varlues.
/// </summary>
protected PushdownPrefixState<TSpan> LocationStack { get { return locationStack; } }
private int errorToken;
private int endOfFileToken;
private string[] nonTerminals;
private State[] states;
private Rule[] rules;
/// <summary>
/// Initialization method to allow derived classes
/// to insert the rule list into this base class.
/// </summary>
/// <param name="rules">The array of Rule objects</param>
protected void InitRules( Rule[] rules ) { this.rules = rules; }
/// <summary>
/// Initialization method to allow derived classes to
/// insert the states table into this base class.
/// </summary>
/// <param name="states">The pre-initialized states table</param>
protected void InitStates( State[] states ) { this.states = states; }
/// <summary>
/// OBSOLETE FOR VERSION 1.4.0
/// </summary>
/// <param name="size"></param>
protected void InitStateTable( int size ) { states = new State[size]; }
/// <summary>
/// Initialization method to allow derived classes
/// to insert the special value for the error and EOF tokens.
/// </summary>
/// <param name="err">The error state ordinal</param>
/// <param name="end">The EOF stat ordinal</param>
protected void InitSpecialTokens( int err, int end ) {
errorToken = err;
endOfFileToken = end;
}
/// <summary>
/// Initialization method to allow derived classes to
/// insert the non-terminal symbol names into this base class.
/// </summary>
/// <param name="names">Non-terminal symbol names</param>
protected void InitNonTerminals( string[] names ) { nonTerminals = names; }
#region YYAbort, YYAccept etcetera.
[Serializable]
[SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
// Reason for FxCop message suppression -
// This exception cannot escape from the local context
private class AcceptException : Exception {
internal AcceptException() { }
protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
}
[Serializable]
[SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
// Reason for FxCop message suppression -
// This exception cannot escape from the local context
private class AbortException : Exception {
internal AbortException() { }
protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
}
[Serializable]
[SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
// Reason for FxCop message suppression -
// This exception cannot escape from the local context
private class ErrorException : Exception {
internal ErrorException() { }
protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
}
// The following methods are only called from within
// a semantic action. The thrown exceptions can never
// propagate outside the ShiftReduceParser class in
// which they are nested.
/// <summary>
/// Force parser to terminate, returning "true"
/// </summary>
protected static void YYAccept() { throw new AcceptException(); }
/// <summary>
/// Force parser to terminate, returning "false"
/// </summary>
protected static void YYAbort() { throw new AbortException(); }
/// <summary>
/// Force parser to terminate, returning
/// "false" if error recovery fails.
/// </summary>
protected static void YYError() { throw new ErrorException(); }
/// <summary>
/// Check if parser in error recovery state.
/// </summary>
protected bool YYRecovering { get { return recovering; } }
#endregion
/// <summary>
/// Abstract base method. ShiftReduceParser calls this
/// to initialize the base class data structures. Concrete
/// parser classes must override this method.
/// </summary>
protected abstract void Initialize();
/// <summary>
/// Main entry point of the Shift-Reduce Parser.
/// </summary>
/// <returns>True if parse succeeds, else false for
/// unrecoverable errors</returns>
public bool Parse() {
Initialize(); // allow derived classes to instantiate rules, states and nonTerminals
NextToken = 0;
FsaState = states[0];
StateStack.Push( FsaState );
valueStack.Push( CurrentSemanticValue );
LocationStack.Push( CurrentLocationSpan );
while (true) {
#if TRACE_ACTIONS
Console.Error.WriteLine("Entering state {0} ", FsaState.number);
DisplayStack();
#endif
int action = FsaState.defaultAction;
if (FsaState.ParserTable != null) {
if (NextToken == 0) {
// We save the last token span, so that the location span
// of production right hand sides that begin or end with a
// nullable production will be correct.
LastSpan = scanner.yylloc;
NextToken = scanner.yylex();
#if TRACE_ACTIONS
Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) );
#endif
}
#if TRACE_ACTIONS
else
Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) );
#endif
if (FsaState.ParserTable.ContainsKey( NextToken ))
action = FsaState.ParserTable[NextToken];
}
if (action > 0) // shift
{
Shift( action );
}
else if (action < 0) // reduce
{
try {
Reduce( -action );
if (action == -1) // accept
return true;
}
catch (AbortException) {
return false;
}
catch (AcceptException) {
return true;
}
catch (ErrorException) {
if (!ErrorRecovery())
return false;
else
throw; // Rethrow x, preserving information.
}
}
else if (action == 0 && !ErrorRecovery()) // error
return false;
}
}
private void Shift( int stateIndex ) {
#if TRACE_ACTIONS
Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken));
#endif
FsaState = states[stateIndex];
valueStack.Push( scanner.yylval );
StateStack.Push( FsaState );
LocationStack.Push( scanner.yylloc );
if (recovering) {
if (NextToken != errorToken)
tokensSinceLastError++;
if (tokensSinceLastError > 5)
recovering = false;
}
if (NextToken != endOfFileToken)
NextToken = 0;
}
private void Reduce( int ruleNumber ) {
#if TRACE_ACTIONS
DisplayRule(ruleNumber);
#endif
Rule rule = rules[ruleNumber];
int rhLen = rule.RightHandSide.Length;
//
// Default actions for unit productions.
//
if (rhLen == 1) {
CurrentSemanticValue = valueStack.TopElement(); // Default action: $$ = $1;
CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1;
}
else if (rhLen == 0) {
// Create a new blank value.
// Explicit semantic action may mutate this value
CurrentSemanticValue = default( TValue );
// The location span for an empty production will start with the
// beginning of the next lexeme, and end with the finish of the
// previous lexeme. This gives the correct behaviour when this
// nonsense value is used in later Merge operations.
CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ?
scanner.yylloc.Merge( LastSpan ) :
default( TSpan ));
}
else {
// Default action: $$ = $1;
CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen];
// Default action "@$ = @1.Merge(@N)" for location info.
TSpan at1 = LocationStack[LocationStack.Depth - rhLen];
TSpan atN = LocationStack[LocationStack.Depth - 1];
CurrentLocationSpan =
((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan ));
}
DoAction( ruleNumber );
for (int i = 0; i < rule.RightHandSide.Length; i++) {
StateStack.Pop();
valueStack.Pop();
LocationStack.Pop();
}
FsaState = StateStack.TopElement();
if (FsaState.Goto.ContainsKey( rule.LeftHandSide ))
FsaState = states[FsaState.Goto[rule.LeftHandSide]];
StateStack.Push( FsaState );
valueStack.Push( CurrentSemanticValue );
LocationStack.Push( CurrentLocationSpan );
}
/// <summary>
/// Execute the selected action from array.
/// Must be overriden in derived classes.
/// </summary>
/// <param name="actionNumber">Index of the action to perform</param>
protected abstract void DoAction( int actionNumber );
private bool ErrorRecovery() {
bool discard;
if (!recovering) // if not recovering from previous error
ReportError();
if (!FindErrorRecoveryState())
return false;
//
// The interim fix for the "looping in error recovery"
// artifact involved moving the setting of the recovering
// bool until after invalid tokens have been discarded.
//
ShiftErrorToken();
discard = DiscardInvalidTokens();
recovering = true;
tokensSinceLastError = 0;
return discard;
}
private void ReportError() {
StringBuilder errorMsg = new StringBuilder();
errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) );
if (FsaState.ParserTable.Count < 7) {
bool first = true;
foreach (int terminal in FsaState.ParserTable.Keys) {
if (first)
errorMsg.Append( ", expecting " );
else
errorMsg.Append( ", or " );
errorMsg.Append( TerminalToString( terminal ) );
first = false;
}
}
scanner.yyerror( errorMsg.ToString() );
}
private void ShiftErrorToken() {
int old_next = NextToken;
NextToken = errorToken;
Shift( FsaState.ParserTable[NextToken] );
#if TRACE_ACTIONS
Console.Error.WriteLine("Entering state {0} ", FsaState.number);
#endif
NextToken = old_next;
}
private bool FindErrorRecoveryState() {
while (true) // pop states until one found that accepts error token
{
if (FsaState.ParserTable != null &&
FsaState.ParserTable.ContainsKey( errorToken ) &&
FsaState.ParserTable[errorToken] > 0) // shift
return true;
#if TRACE_ACTIONS
Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number);
#endif
StateStack.Pop();
valueStack.Pop();
LocationStack.Pop();
#if TRACE_ACTIONS
DisplayStack();
#endif
if (StateStack.IsEmpty()) {
#if TRACE_ACTIONS
Console.Error.WriteLine("Aborting: didn't find a state that accepts error token");
#endif
return false;
}
else
FsaState = StateStack.TopElement();
}
}
private bool DiscardInvalidTokens() {
int action = FsaState.defaultAction;
if (FsaState.ParserTable != null) {
// Discard tokens until find one that works ...
while (true) {
if (NextToken == 0) {
#if TRACE_ACTIONS
Console.Error.Write("Reading a token: ");
#endif
NextToken = scanner.yylex();
}
#if TRACE_ACTIONS
Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken));
#endif
if (NextToken == endOfFileToken)
return false;
if (FsaState.ParserTable.ContainsKey( NextToken ))
action = FsaState.ParserTable[NextToken];
if (action != 0)
return true;
else {
#if TRACE_ACTIONS
Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken));
#endif
NextToken = 0;
}
}
}
else if (recovering && tokensSinceLastError == 0) {
//
// Boolean recovering is not set until after the first
// error token has been shifted. Thus if we get back
// here with recovering set and no tokens read we are
// looping on the same error recovery action. This
// happens if current_state.ParserTable is null because
// the state has an LR(0) reduction, but not all
// lookahead tokens are valid. This only occurs for
// error productions that *end* on "error".
//
// This action discards tokens one at a time until
// the looping stops. Another attack would be to always
// use the LALR(1) table if a production ends on "error"
//
#if TRACE_ACTIONS
Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken));
#endif
if (NextToken == endOfFileToken)
return false;
NextToken = 0;
return true;
}
else
return true;
}
/// <summary>
/// Traditional YACC method. Discards the next input token.
/// </summary>
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )]
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )]
// Reason for FxCop message suppression -
// This is a traditional name for YACC-like functionality
protected void yyclearin() { NextToken = 0; }
/// <summary>
/// Tradional YACC method. Clear the "recovering" flag.
/// </summary>
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )]
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )]
// Reason for FxCop message suppression -
// This is a traditional name for YACC-like functionality
protected void yyerrok() { recovering = false; }
/// <summary>
/// OBSOLETE FOR VERSION 1.4.0
/// Method used by derived types to insert new
/// state instances in the "states" array.
/// </summary>
/// <param name="stateNumber">index of the state</param>
/// <param name="state">data for the state</param>
protected void AddState( int stateNumber, State state ) {
states[stateNumber] = state;
state.number = stateNumber;
}
private void DisplayStack() {
Console.Error.Write( "State stack is now:" );
for (int i = 0; i < StateStack.Depth; i++)
Console.Error.Write( " {0}", StateStack[i].number );
Console.Error.WriteLine();
}
private void DisplayRule( int ruleNumber ) {
Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber );
DisplayProduction( rules[ruleNumber] );
}
private void DisplayProduction( Rule rule ) {
if (rule.RightHandSide.Length == 0)
Console.Error.Write( "/* empty */ " );
else
foreach (int symbol in rule.RightHandSide)
Console.Error.Write( "{0} ", SymbolToString( symbol ) );
Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) );
}
/// <summary>
/// Abstract state class naming terminal symbols.
/// This is overridden by derived classes with the
/// name (or alias) to be used in error messages.
/// </summary>
/// <param name="terminal">The terminal ordinal</param>
/// <returns></returns>
protected abstract string TerminalToString( int terminal );
private string SymbolToString( int symbol ) {
if (symbol < 0)
return nonTerminals[-symbol - 1];
else
return TerminalToString( symbol );
}
/// <summary>
/// Return text representation of argument character
/// </summary>
/// <param name="input">The character to convert</param>
/// <returns>String representation of the character</returns>
protected static string CharToString( char input ) {
switch (input) {
case '\a': return @"'\a'";
case '\b': return @"'\b'";
case '\f': return @"'\f'";
case '\n': return @"'\n'";
case '\r': return @"'\r'";
case '\t': return @"'\t'";
case '\v': return @"'\v'";
case '\0': return @"'\0'";
default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input );
}
}
}
/// <summary>
/// Classes implementing this interface must supply a
/// method that merges two location objects to return
/// a new object of the same type.
/// GPPG-generated parsers have the default location
/// action equivalent to "@$ = @1.Merge(@N);" where N
/// is the right-hand-side length of the production.
/// </summary>
/// <typeparam name="TSpan">The Location type</typeparam>
#if EXPORT_GPPG
public interface IMerge<TSpan>
#else
internal interface IMerge<TSpan>
#endif
{
/// <summary>
/// Interface method that creates a location object from
/// the current and last object. Typically used to create
/// a location object extending from the start of the @1
/// object to the end of the @N object.
/// </summary>
/// <param name="last">The lexically last object to merge</param>
/// <returns>The merged location object</returns>
TSpan Merge( TSpan last );
}
/// <summary>
/// This is the default class that carries location
/// information from the scanner to the parser.
/// If you don't declare "%YYLTYPE Foo" the parser
/// will expect to deal with this type.
/// </summary>
#if EXPORT_GPPG
public class LexLocation : IMerge<LexLocation>
#else
[SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )]
internal class LexLocation : IMerge<LexLocation>
#endif
{
private int startLine; // start line
private int startColumn; // start column
private int endLine; // end line
private int endColumn; // end column
/// <summary>
/// The line at which the text span starts.
/// </summary>
public int StartLine { get { return startLine; } }
/// <summary>
/// The column at which the text span starts.
/// </summary>
public int StartColumn { get { return startColumn; } }
/// <summary>
/// The line on which the text span ends.
/// </summary>
public int EndLine { get { return endLine; } }
/// <summary>
/// The column of the first character
/// beyond the end of the text span.
/// </summary>
public int EndColumn { get { return endColumn; } }
/// <summary>
/// Default no-arg constructor.
/// </summary>
public LexLocation() { }
/// <summary>
/// Constructor for text-span with given start and end.
/// </summary>
/// <param name="sl">start line</param>
/// <param name="sc">start column</param>
/// <param name="el">end line </param>
/// <param name="ec">end column</param>
public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; }
/// <summary>
/// Create a text location which spans from the
/// start of "this" to the end of the argument "last"
/// </summary>
/// <param name="last">The last location in the result span</param>
/// <returns>The merged span</returns>
public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); }
}
/// <summary>
/// Abstract scanner class that GPPG expects its scanners to
/// extend.
/// </summary>
/// <typeparam name="TValue">Semantic value type YYSTYPE</typeparam>
/// <typeparam name="TSpan">Source location type YYLTYPE</typeparam>
#if EXPORT_GPPG
public abstract class AbstractScanner<TValue, TSpan>
#else
internal abstract class AbstractScanner<TValue, TSpan>
#endif
where TSpan : IMerge<TSpan> {
/// <summary>
/// Lexical value optionally set by the scanner. The value
/// is of the %YYSTYPE type declared in the parser spec.
/// </summary>
[SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )]
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )]
// Reason for FxCop message suppression -
// This is a traditional name for YACC-like functionality
// A field must be declared for this value of parametric type,
// since it may be instantiated by a value struct. If it were
// implemented as a property, machine generated code in derived
// types would not be able to select on the returned value.
#pragma warning disable 649
public TValue yylval; // Lexical value: set by scanner
#pragma warning restore 649
/// <summary>
/// Current scanner location property. The value is of the
/// type declared by %YYLTYPE in the parser specification.
/// </summary>
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )]
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )]
// Reason for FxCop message suppression -
// This is a traditional name for YACC-like functionality
public virtual TSpan yylloc {
get { return default( TSpan ); } // Empty implementation allowing
set { /* skip */ } // yylloc to be ignored entirely.
}
/// <summary>
/// Main call point for LEX-like scanners. Returns an int
/// corresponding to the token recognized by the scanner.
/// </summary>
/// <returns>An int corresponding to the token</returns>
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )]
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )]
// Reason for FxCop message suppression -
// This is a traditional name for YACC-like functionality
public abstract int yylex();
/// <summary>
/// Traditional error reporting provided by LEX-like scanners
/// to their YACC-like clients.
/// </summary>
/// <param name="format">Message format string</param>
/// <param name="args">Optional array of args</param>
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )]
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )]
// Reason for FxCop message suppression -
// This is a traditional name for YACC-like functionality
public virtual void yyerror( string format, params object[] args ) { }
}
/// <summary>
/// Encapsulated state for the parser.
/// Opaque to users, visible to the tool-generated code.
/// </summary>
#if EXPORT_GPPG
public class State
{
/// <summary>
/// The number of states in the automaton.
/// </summary>
public int number;
#else
internal class State {
/// <summary>
/// The index of this state in the states array.
/// </summary>
internal int number;
#endif
internal Dictionary<int, int> ParserTable; // Terminal -> ParseAction
internal Dictionary<int, int> Goto; // NonTerminal -> State;
internal int defaultAction; // = 0; // ParseAction
/// <summary>
/// State transition data for this state. Pairs of elements of the
/// goto array associate symbol ordinals with next state indices.
/// The actions array is passed to another constructor.
/// </summary>
/// <param name="actions">The action list</param>c
/// <param name="goToList">Next state data</param>
public State( int[] actions, int[] goToList )
: this( actions ) {
Goto = new Dictionary<int, int>();
for (int i = 0; i < goToList.Length; i += 2)
Goto.Add( goToList[i], goToList[i + 1] );
}
/// <summary>
/// Action data for this state. Pairs of elements of the
/// action array associate action ordinals with each of
/// those symbols that have actions in the current state.
/// </summary>
/// <param name="actions">The action array</param>
public State( int[] actions ) {
ParserTable = new Dictionary<int, int>();
for (int i = 0; i < actions.Length; i += 2)
ParserTable.Add( actions[i], actions[i + 1] );
}
/// <summary>
/// Set the default action for this state.
/// </summary>
/// <param name="defaultAction">Ordinal of the default action</param>
public State( int defaultAction ) {
this.defaultAction = defaultAction;
}
/// <summary>
/// Set the default action and the state transition table.
/// </summary>
/// <param name="defaultAction">The default action</param>
/// <param name="goToList">Transitions from this state</param>
public State( int defaultAction, int[] goToList )
: this( defaultAction ) {
Goto = new Dictionary<int, int>();
for (int i = 0; i < goToList.Length; i += 2)
Goto.Add( goToList[i], goToList[i + 1] );
}
}
/// <summary>
/// Rule representation at runtime.
/// </summary>
#if EXPORT_GPPG
public class Rule
#else
internal class Rule
#endif
{
internal int LeftHandSide; // symbol
internal int[] RightHandSide; // symbols
/// <summary>
/// Rule constructor. This holds the ordinal of
/// the left hand side symbol, and the list of
/// right hand side symbols, in lexical order.
/// </summary>
/// <param name="left">The LHS non-terminal</param>
/// <param name="right">The RHS symbols, in lexical order</param>
public Rule( int left, int[] right ) {
this.LeftHandSide = left;
this.RightHandSide = right;
}
}
/// <summary>
/// Stack utility for the shift-reduce parser.
/// GPPG parsers have three instances:
/// (1) The parser state stack, T = QUT.Gppg.State,
/// (2) The semantic value stack, T = TValue,
/// (3) The location stack, T = TSpan.
/// </summary>
/// <typeparam name="T"></typeparam>
#if EXPORT_GPPG
public class PushdownPrefixState<T>
#else
internal class PushdownPrefixState<T>
#endif
{
// Note that we cannot use the BCL Stack<T> class
// here as derived types need to index into stacks.
//
private T[] array = new T[8];
private int tos = 0;
/// <summary>
/// Indexer for values of the stack below the top.
/// </summary>
/// <param name="index">index of the element, starting from the bottom</param>
/// <returns>the selected element</returns>
public T this[int index] { get { return array[index]; } }
/// <summary>
/// The current depth of the stack.
/// </summary>
public int Depth { get { return tos; } }
internal void Push( T value ) {
if (tos >= array.Length) {
T[] newarray = new T[array.Length * 2];
System.Array.Copy( array, newarray, tos );
array = newarray;
}
array[tos++] = value;
}
internal T Pop() {
T rslt = array[--tos];
array[tos] = default( T );
return rslt;
}
internal T TopElement() { return array[tos - 1]; }
internal bool IsEmpty() { return tos == 0; }
}
}
@@ -1,4 +0,0 @@
global using System;
global using System.Linq;
global using System.Collections.Generic;
global using System.Collections.Immutable;
@@ -1,24 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="YaccLexTools" Version="1.2.2" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
<ProjectReference Include="..\ExpressionCore\ExpressionCore.csproj" />
<ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
</ItemGroup>
<ItemGroup Label="ColumnSortExpression">
<Scanner Include="ColumnSortExpressionTokens.lex" />
<Parser Include="ColumnSortExpressionGrammar.y" />
</ItemGroup>
</Project>
@@ -1,11 +0,0 @@
namespace Bts.ColumnSortExpressions;
public class ColumnSortExpression {
public ColumnSortExpression(string name, bool isAscending = true) {
Name = name;
IsAscending = isAscending;
}
public string Name { get; }
public bool IsAscending { get; }
}
@@ -1,29 +0,0 @@
%namespace Bts.ColumnSortExpressions
%partial
%visibility internal
%tokentype ColumnSortExpressionToken
%parsertype ColumnSortExpressionParser
%union {
public string Identifier;
public ColumnSortExpression Column;
public ImmutableList<ColumnSortExpression> Columns;
}
%token ASCENDING, DESCENDING, IDENTIFIER
%type <Column> Column
%type <Columns> Columns
%%
Columns :
Column { $$ = ImmutableList.CreateRange(new[] { $1 }); }
| Columns Column { $$ = $1.Add($2); }
;
Column :
IDENTIFIER { $$ = new($1.Identifier, true); }
| ASCENDING IDENTIFIER { $$ = new($2.Identifier, true); }
| DESCENDING IDENTIFIER { $$ = new($2.Identifier, false); }
;
@@ -1,18 +0,0 @@
namespace Bts.ColumnSortExpressions;
using System.IO;
using System.Text;
public class ColumnSortExpressionInterpreter {
public static IList<ColumnSortExpression> Evaluate(Stream stream) {
var parser = new ColumnSortExpressionParser();
var res = parser.Parse(stream);
return res?.Columns ?? ImmutableList<ColumnSortExpression>.Empty.Add(
res?.Column ?? throw new Exception("Failed to parse column(s) expression"));
}
public static IList<ColumnSortExpression> Evaluate(string str) {
using var ms = new MemoryStream(Encoding.Default.GetBytes(str));
return Evaluate(ms);
}
}
@@ -1,18 +0,0 @@
namespace Bts.ColumnSortExpressions;
using System.Text;
partial class ColumnSortExpressionParser {
public ColumnSortExpressionParser() : base(null) { }
public ValueType? Parse(string s) {
using var ms = new MemoryStream(Encoding.Default.GetBytes(s));
return Parse(ms);
}
public ValueType? Parse(Stream stream) {
Scanner = new ColumnSortExpressionScanner(stream);
Parse();
return CurrentSemanticValue;
}
}
@@ -1,9 +0,0 @@
namespace Bts.ColumnSortExpressions;
partial class ColumnSortExpressionScanner {
public override void yyerror(string format, params object[] args) {
base.yyerror(format, args);
Console.WriteLine(format, args);
Console.WriteLine();
}
}
@@ -1,30 +0,0 @@
%namespace Bts.ColumnSortExpressions
%scannertype ColumnSortExpressionScanner
%visibility internal
%tokentype ColumnSortExpressionToken
%option stack, minimize, parser, verbose, persistbuffer, noembedbuffers
/* Literal Constants */
Letter [a-zA-Z]
Identifier {Letter}({Letter}*[0-9]*_*)*
Ascending +
Descending -
%%
{Ascending} {
return (int)ColumnSortExpressionToken.ASCENDING;
}
{Descending} {
return (int)ColumnSortExpressionToken.DESCENDING;
}
{Identifier} {
yylval.Identifier = yytext;
return (int)ColumnSortExpressionToken.IDENTIFIER;
}
%%
@@ -1,720 +0,0 @@
// ==============================================================
// <auto-generated>
// This code automatically produced from an embedded resource.
// Do not edit this file, or it will become incompatible with
// the specification from which it was generated.
// </auto-generated>
// ==============================================================
using System;
using System.IO;
using System.Text;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Runtime.Serialization;
using System.Globalization;
namespace QUT.GplexBuffers
{
// Code copied from GPLEX embedded resource
[Serializable]
internal class BufferException : Exception
{
public BufferException() { }
public BufferException(string message) : base(message) { }
public BufferException(string message, Exception innerException)
: base(message, innerException) { }
protected BufferException(SerializationInfo info, StreamingContext context)
: base(info, context) { }
}
internal abstract class ScanBuff
{
private string fileNm;
public const int EndOfFile = -1;
public const int UnicodeReplacementChar = 0xFFFD;
public bool IsFile { get { return (fileNm != null); } }
public string FileName { get { return fileNm; } set { fileNm = value; } }
public abstract int Pos { get; set; }
public abstract int Read();
public virtual void Mark() { }
public abstract string GetString(int begin, int limit);
public static ScanBuff GetBuffer(string source)
{
return new StringBuffer(source);
}
public static ScanBuff GetBuffer(IList<string> source)
{
return new LineBuffer(source);
}
#if (!NOFILES)
public static ScanBuff GetBuffer(Stream source)
{
return new BuildBuffer(source);
}
#if (!BYTEMODE)
public static ScanBuff GetBuffer(Stream source, int fallbackCodePage)
{
return new BuildBuffer(source, fallbackCodePage);
}
#endif // !BYTEMODE
#endif // !NOFILES
}
#region Buffer classes
// ==============================================================
// ===== Definitions for various ScanBuff derived classes ====
// ==============================================================
// =============== String input ================
// ==============================================================
/// <summary>
/// This class reads characters from a single string as
/// required, for example, by Visual Studio language services
/// </summary>
sealed class StringBuffer : ScanBuff
{
string str; // input buffer
int bPos; // current position in buffer
int sLen;
public StringBuffer(string source)
{
this.str = source;
this.sLen = source.Length;
this.FileName = null;
}
public override int Read()
{
if (bPos < sLen) return str[bPos++];
else if (bPos == sLen) { bPos++; return '\n'; } // one strike, see new line
else { bPos++; return EndOfFile; } // two strikes and you're out!
}
public override string GetString(int begin, int limit)
{
// "limit" can be greater than sLen with the BABEL
// option set. Read returns a "virtual" EOL if
// an attempt is made to read past the end of the
// string buffer. Without the guard any attempt
// to fetch yytext for a token that includes the
// EOL will throw an index exception.
if (limit > sLen) limit = sLen;
if (limit <= begin) return "";
else return str.Substring(begin, limit - begin);
}
public override int Pos
{
get { return bPos; }
set { bPos = value; }
}
public override string ToString() { return "StringBuffer"; }
}
// ==============================================================
// The LineBuff class contributed by Nigel Horspool,
// nigelh@cs.uvic.cs
// ==============================================================
sealed class LineBuffer : ScanBuff
{
IList<string> line; // list of source lines from a file
int numLines; // number of strings in line list
string curLine; // current line in that list
int cLine; // index of current line in the list
int curLen; // length of current line
int curLineStart; // position of line start in whole file
int curLineEnd; // position of line end in whole file
int maxPos; // max position ever visited in whole file
int cPos; // ordinal number of code in source
// Constructed from a list of strings, one per source line.
// The lines have had trailing '\n' characters removed.
public LineBuffer(IList<string> lineList)
{
line = lineList;
numLines = line.Count;
cPos = curLineStart = 0;
curLine = (numLines > 0 ? line[0] : "");
maxPos = curLineEnd = curLen = curLine.Length;
cLine = 1;
FileName = null;
}
public override int Read()
{
if (cPos < curLineEnd)
return curLine[cPos++ - curLineStart];
if (cPos++ == curLineEnd)
return '\n';
if (cLine >= numLines)
return EndOfFile;
curLine = line[cLine];
curLen = curLine.Length;
curLineStart = curLineEnd + 1;
curLineEnd = curLineStart + curLen;
if (curLineEnd > maxPos)
maxPos = curLineEnd;
cLine++;
return curLen > 0 ? curLine[0] : '\n';
}
// To speed up searches for the line containing a position
private int cachedPosition;
private int cachedIxdex;
private int cachedLineStart;
// Given a position pos within the entire source, the results are
// ix -- the index of the containing line
// lstart -- the position of the first character on that line
private void findIndex(int pos, out int ix, out int lstart)
{
if (pos >= cachedPosition)
{
ix = cachedIxdex; lstart = cachedLineStart;
}
else
{
ix = lstart = 0;
}
while (ix < numLines)
{
int len = line[ix].Length + 1;
if (pos < lstart + len) break;
lstart += len;
ix++;
}
cachedPosition = pos;
cachedIxdex = ix;
cachedLineStart = lstart;
}
public override string GetString(int begin, int limit)
{
if (begin >= maxPos || limit <= begin) return "";
int endIx, begIx, endLineStart, begLineStart;
findIndex(begin, out begIx, out begLineStart);
int begCol = begin - begLineStart;
findIndex(limit, out endIx, out endLineStart);
int endCol = limit - endLineStart;
string s = line[begIx];
if (begIx == endIx)
{
// the usual case, substring all on one line
return (endCol <= s.Length) ?
s.Substring(begCol, endCol - begCol)
: s.Substring(begCol) + "\n";
}
// the string spans multiple lines, yuk!
StringBuilder sb = new StringBuilder();
if (begCol < s.Length)
sb.Append(s.Substring(begCol));
for (; ; )
{
sb.Append("\n");
s = line[++begIx];
if (begIx >= endIx) break;
sb.Append(s);
}
if (endCol <= s.Length)
{
sb.Append(s.Substring(0, endCol));
}
else
{
sb.Append(s);
sb.Append("\n");
}
return sb.ToString();
}
public override int Pos
{
get { return cPos; }
set
{
cPos = value;
findIndex(cPos, out cLine, out curLineStart);
// cLine should be the *next* line after curLine.
curLine = (cLine < numLines ? line[cLine++] : "");
curLineEnd = curLineStart + curLine.Length;
}
}
public override string ToString() { return "LineBuffer"; }
}
#if (!NOFILES)
// ==============================================================
// ===== class BuildBuff : for unicode text files ========
// ==============================================================
class BuildBuffer : ScanBuff
{
// Double buffer for char stream.
class BufferElement
{
StringBuilder bldr = new StringBuilder();
StringBuilder next = new StringBuilder();
int minIx;
int maxIx;
int brkIx;
bool appendToNext;
internal BufferElement() { }
internal int MaxIndex { get { return maxIx; } }
// internal int MinIndex { get { return minIx; } }
internal char this[int index]
{
get
{
if (index < minIx || index >= maxIx)
throw new BufferException("Index was outside data buffer");
else if (index < brkIx)
return bldr[index - minIx];
else
return next[index - brkIx];
}
}
internal void Append(char[] block, int count)
{
maxIx += count;
if (appendToNext)
this.next.Append(block, 0, count);
else
{
this.bldr.Append(block, 0, count);
brkIx = maxIx;
appendToNext = true;
}
}
internal string GetString(int start, int limit)
{
if (limit <= start)
return "";
if (start >= minIx && limit <= maxIx)
if (limit < brkIx) // String entirely in bldr builder
return bldr.ToString(start - minIx, limit - start);
else if (start >= brkIx) // String entirely in next builder
return next.ToString(start - brkIx, limit - start);
else // Must do a string-concatenation
return
bldr.ToString(start - minIx, brkIx - start) +
next.ToString(0, limit - brkIx);
else
throw new BufferException("String was outside data buffer");
}
internal void Mark(int limit)
{
if (limit > brkIx + 16) // Rotate blocks
{
StringBuilder temp = bldr;
bldr = next;
next = temp;
next.Length = 0;
minIx = brkIx;
brkIx = maxIx;
}
}
}
BufferElement data = new BufferElement();
int bPos; // Postion index in the StringBuilder
BlockReader NextBlk; // Delegate that serves char-arrays;
private string EncodingName
{
get
{
StreamReader rdr = NextBlk.Target as StreamReader;
return (rdr == null ? "raw-bytes" : rdr.CurrentEncoding.BodyName);
}
}
public BuildBuffer(Stream stream)
{
FileStream fStrm = (stream as FileStream);
if (fStrm != null) FileName = fStrm.Name;
NextBlk = BlockReaderFactory.Raw(stream);
}
#if (!BYTEMODE)
public BuildBuffer(Stream stream, int fallbackCodePage)
{
FileStream fStrm = (stream as FileStream);
if (fStrm != null) FileName = fStrm.Name;
NextBlk = BlockReaderFactory.Get(stream, fallbackCodePage);
}
#endif
/// <summary>
/// Marks a conservative lower bound for the buffer,
/// allowing space to be reclaimed. If an application
/// needs to call GetString at arbitrary past locations
/// in the input stream, Mark() is not called.
/// </summary>
public override void Mark() { data.Mark(bPos - 2); }
public override int Pos
{
get { return bPos; }
set { bPos = value; }
}
/// <summary>
/// Read returns the ordinal number of the next char, or
/// EOF (-1) for an end of stream. Note that the next
/// code point may require *two* calls of Read().
/// </summary>
/// <returns></returns>
public override int Read()
{
//
// Characters at positions
// [data.offset, data.offset + data.bldr.Length)
// are available in data.bldr.
//
if (bPos < data.MaxIndex)
{
// ch0 cannot be EOF
return (int)data[bPos++];
}
else // Read from underlying stream
{
// Experimental code, blocks of page size
char[] chrs = new char[4096];
int count = NextBlk(chrs, 0, 4096);
if (count == 0)
return EndOfFile;
else
{
data.Append(chrs, count);
return (int)data[bPos++];
}
}
}
public override string GetString(int begin, int limit)
{
return data.GetString(begin, limit);
}
public override string ToString()
{
return "StringBuilder buffer, encoding: " + this.EncodingName;
}
}
// =============== End ScanBuff-derived classes ==================
internal delegate int BlockReader(char[] block, int index, int number);
// A delegate factory, serving up a delegate that
// reads a block of characters from the underlying
// encoded stream, via a StreamReader object.
//
internal static class BlockReaderFactory
{
public static BlockReader Raw(Stream stream)
{
return delegate(char[] block, int index, int number)
{
byte[] b = new byte[number];
int count = stream.Read(b, 0, number);
int i = 0;
int j = index;
for (; i < count; i++, j++)
block[j] = (char)b[i];
return count;
};
}
#if (!BYTEMODE)
public static BlockReader Get(Stream stream, int fallbackCodePage)
{
Encoding encoding;
int preamble = Preamble(stream);
if (preamble != 0) // There is a valid BOM here!
encoding = Encoding.GetEncoding(preamble);
else if (fallbackCodePage == -1) // Fallback is "raw" bytes
return Raw(stream);
else if (fallbackCodePage != -2) // Anything but "guess"
encoding = Encoding.GetEncoding(fallbackCodePage);
else // This is the "guess" option
{
int guess = new Guesser(stream).GuessCodePage();
stream.Seek(0, SeekOrigin.Begin);
if (guess == -1) // ==> this is a 7-bit file
encoding = Encoding.ASCII;
else if (guess == 65001)
encoding = Encoding.UTF8;
else // ==> use the machine default
encoding = Encoding.Default;
}
StreamReader reader = new StreamReader(stream, encoding);
return reader.Read;
}
static int Preamble(Stream stream)
{
int b0 = stream.ReadByte();
int b1 = stream.ReadByte();
if (b0 == 0xfe && b1 == 0xff)
return 1201; // UTF16BE
if (b0 == 0xff && b1 == 0xfe)
return 1200; // UTF16LE
int b2 = stream.ReadByte();
if (b0 == 0xef && b1 == 0xbb && b2 == 0xbf)
return 65001; // UTF8
//
// There is no unicode preamble, so we
// return denoter for the machine default.
//
stream.Seek(0, SeekOrigin.Begin);
return 0;
}
#endif // !BYTEMODE
}
#endif // !NOFILES
#endregion Buffer classes
// ==============================================================
// ============ class CodePageHandling =============
// ==============================================================
#if (!NOFILES)
public static class CodePageHandling
{
public static int GetCodePage(string option)
{
string command = option.ToUpperInvariant();
if (command.StartsWith("CodePage:", StringComparison.OrdinalIgnoreCase))
command = command.Substring(9);
try
{
if (command.Equals("RAW"))
return -1;
else if (command.Equals("GUESS"))
return -2;
else if (command.Equals("DEFAULT"))
return 0;
else if (char.IsDigit(command[0]))
return int.Parse(command, CultureInfo.InvariantCulture);
else
{
Encoding enc = Encoding.GetEncoding(command);
return enc.CodePage;
}
}
catch (FormatException)
{
Console.Error.WriteLine(
"Invalid format \"{0}\", using machine default", option);
}
catch (ArgumentException)
{
Console.Error.WriteLine(
"Unknown code page \"{0}\", using machine default", option);
}
return 0;
}
}
#region guesser
#if (!BYTEMODE)
// ==============================================================
// ============ Encoding Guesser =============
// ==============================================================
/// <summary>
/// This class provides a simple finite state automaton that
/// scans the file looking for (1) valid UTF-8 byte patterns,
/// (2) bytes >= 0x80 which are not part of a UTF-8 sequence.
/// The method then guesses whether it is UTF-8 or maybe some
/// local machine default encoding. This works well for the
/// various Latin encodings.
/// </summary>
internal class Guesser
{
ScanBuff buffer;
public int GuessCodePage() { return Scan(); }
const int maxAccept = 10;
const int initial = 0;
const int eofNum = 0;
const int goStart = -1;
const int INITIAL = 0;
const int EndToken = 0;
#region user code
/*
* Reads the bytes of a file to determine if it is
* UTF-8 or a single-byte code page file.
*/
public long utfX;
public long uppr;
#endregion user code
int state;
int currentStart = startState[0];
int code;
#region ScannerTables
static int[] startState = new int[] { 11, 0 };
#region CharacterMap
static sbyte[] map = new sbyte[256] {
/* '\0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* '\x10' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* '\x20' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* '0' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* '@' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* 'P' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* '`' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* 'p' */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
/* '\x80' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
/* '\x90' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
/* '\xA0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
/* '\xB0' */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
/* '\xC0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
/* '\xD0' */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
/* '\xE0' */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
/* '\xF0' */ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5 };
#endregion
static sbyte[][] nextState = new sbyte[][] {
new sbyte[] {0, 0, 0, 0, 0, 0},
new sbyte[] {-1, -1, 10, -1, -1, -1},
new sbyte[] {-1, -1, -1, -1, -1, -1},
new sbyte[] {-1, -1, 8, -1, -1, -1},
new sbyte[] {-1, -1, 5, -1, -1, -1},
new sbyte[] {-1, -1, 6, -1, -1, -1},
new sbyte[] {-1, -1, 7, -1, -1, -1},
null,
new sbyte[] {-1, -1, 9, -1, -1, -1},
null,
null,
new sbyte[] {-1, 1, 2, 3, 4, 2}
};
[SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline")]
// Reason for suppression: cannot have self-reference in array initializer.
static Guesser()
{
nextState[7] = nextState[2];
nextState[9] = nextState[2];
nextState[10] = nextState[2];
}
int NextState()
{
if (code == ScanBuff.EndOfFile)
return eofNum;
else
return nextState[state][map[code]];
}
#endregion
public Guesser(System.IO.Stream file) { SetSource(file); }
public void SetSource(System.IO.Stream source)
{
this.buffer = new BuildBuffer(source);
code = buffer.Read();
}
int Scan()
{
for (; ; )
{
int next;
state = currentStart;
while ((next = NextState()) == goStart)
code = buffer.Read();
state = next;
code = buffer.Read();
while ((next = NextState()) > eofNum)
{
state = next;
code = buffer.Read();
}
if (state <= maxAccept)
{
#region ActionSwitch
#pragma warning disable 162
switch (state)
{
case eofNum:
switch (currentStart)
{
case 11:
if (utfX == 0 && uppr == 0) return -1; /* raw ascii */
else if (uppr * 10 > utfX) return 0; /* default code page */
else return 65001; /* UTF-8 encoding */
break;
}
return EndToken;
case 1: // Recognized '{Upper128}', Shortest string "\xC0"
case 2: // Recognized '{Upper128}', Shortest string "\x80"
case 3: // Recognized '{Upper128}', Shortest string "\xE0"
case 4: // Recognized '{Upper128}', Shortest string "\xF0"
uppr++;
break;
case 5: // Recognized '{Utf8pfx4}{Utf8cont}', Shortest string "\xF0\x80"
uppr += 2;
break;
case 6: // Recognized '{Utf8pfx4}{Utf8cont}{2}', Shortest string "\xF0\x80\x80"
uppr += 3;
break;
case 7: // Recognized '{Utf8pfx4}{Utf8cont}{3}', Shortest string "\xF0\x80\x80\x80"
utfX += 3;
break;
case 8: // Recognized '{Utf8pfx3}{Utf8cont}', Shortest string "\xE0\x80"
uppr += 2;
break;
case 9: // Recognized '{Utf8pfx3}{Utf8cont}{2}', Shortest string "\xE0\x80\x80"
utfX += 2;
break;
case 10: // Recognized '{Utf8pfx2}{Utf8cont}', Shortest string "\xC0\x80"
utfX++;
break;
default:
break;
}
#pragma warning restore 162
#endregion
}
}
}
} // end class Guesser
#endif // !BYTEMODE
#endregion
#endif // !NOFILES
// End of code copied from embedded resource
}
@@ -1,884 +0,0 @@
// ==============================================================
// <auto-generated>
// This code automatically produced from an embedded resource.
// Do not edit this file, or it will become incompatible with
// the specification from which it was generated.
// </auto-generated>
// ==============================================================
// Gardens Point Parser Generator
// Copyright (c) Wayne Kelly, QUT 2005-2010
// (see accompanying GPPGcopyright.rtf)
using System;
using System.Text;
using System.Globalization;
using System.Collections.Generic;
using System.Runtime.Serialization;
using System.Diagnostics.CodeAnalysis;
namespace QUT.Gppg {
/// <summary>
/// Abstract class for GPPG shift-reduce parsers.
/// Parsers generated by GPPG derive from this base
/// class, overriding the abstract Initialize() and
/// DoAction() methods.
/// </summary>
/// <typeparam name="TValue">Semantic value type</typeparam>
/// <typeparam name="TSpan">Location type</typeparam>
#if EXPORT_GPPG
public abstract class ShiftReduceParser<TValue, TSpan>
#else
internal abstract class ShiftReduceParser<TValue, TSpan>
#endif
where TSpan : IMerge<TSpan>, new() {
private AbstractScanner<TValue, TSpan> scanner;
/// <summary>
/// The abstract scanner for this parser.
/// </summary>
protected AbstractScanner<TValue, TSpan> Scanner {
get { return scanner; }
set { scanner = value; }
}
/// <summary>
/// Constructor for base class
/// </summary>
/// <param name="scanner">Scanner instance for this parser</param>
protected ShiftReduceParser( AbstractScanner<TValue, TSpan> scanner ) {
this.scanner = scanner;
}
// ==============================================================
// TECHNICAL EXPLANATION.
// Why the next two fields are not exposed via properties.
// ==============================================================
// These fields are of the generic parameter types, and are
// frequently instantiated as struct types in derived classes.
// Semantic actions are defined in the derived classes and refer
// to instance fields of these structs. In such cases the code
// "get_CurrentSemanticValue().myField = blah;" will fail since
// the getter pushes the value of the field, not the reference.
// So, in the presence of properties, gppg would need to encode
// such field accesses as ...
// "tmp = get_CurrentSemanticValue(); // Fetch value
// tmp.myField = blah; // update
// set_CurrentSemanticValue(tmp); " // Write update back.
// There is no issue if TValue is restricted to be a ref type.
// The same explanation applies to scanner.yylval.
// ==============================================================
/// <summary>
/// The current value of the "$$" symbolic variable in the parser
/// </summary>
[SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
protected TValue CurrentSemanticValue;
/// <summary>
/// The current value of the "@$" symbolic variable in the parser
/// </summary>
[SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
protected TSpan CurrentLocationSpan;
protected int NextToken;
private TSpan LastSpan;
private State FsaState;
private bool recovering;
private int tokensSinceLastError;
private PushdownPrefixState<State> StateStack = new PushdownPrefixState<State>();
private PushdownPrefixState<TValue> valueStack = new PushdownPrefixState<TValue>();
private PushdownPrefixState<TSpan> locationStack = new PushdownPrefixState<TSpan>();
/// <summary>
/// The stack of semantic value (YYSTYPE) values.
/// </summary>
protected PushdownPrefixState<TValue> ValueStack { get { return valueStack; } }
/// <summary>
/// The stack of location value (YYLTYPE) varlues.
/// </summary>
protected PushdownPrefixState<TSpan> LocationStack { get { return locationStack; } }
private int errorToken;
private int endOfFileToken;
private string[] nonTerminals;
private State[] states;
private Rule[] rules;
/// <summary>
/// Initialization method to allow derived classes
/// to insert the rule list into this base class.
/// </summary>
/// <param name="rules">The array of Rule objects</param>
protected void InitRules( Rule[] rules ) { this.rules = rules; }
/// <summary>
/// Initialization method to allow derived classes to
/// insert the states table into this base class.
/// </summary>
/// <param name="states">The pre-initialized states table</param>
protected void InitStates( State[] states ) { this.states = states; }
/// <summary>
/// OBSOLETE FOR VERSION 1.4.0
/// </summary>
/// <param name="size"></param>
protected void InitStateTable( int size ) { states = new State[size]; }
/// <summary>
/// Initialization method to allow derived classes
/// to insert the special value for the error and EOF tokens.
/// </summary>
/// <param name="err">The error state ordinal</param>
/// <param name="end">The EOF stat ordinal</param>
protected void InitSpecialTokens( int err, int end ) {
errorToken = err;
endOfFileToken = end;
}
/// <summary>
/// Initialization method to allow derived classes to
/// insert the non-terminal symbol names into this base class.
/// </summary>
/// <param name="names">Non-terminal symbol names</param>
protected void InitNonTerminals( string[] names ) { nonTerminals = names; }
#region YYAbort, YYAccept etcetera.
[Serializable]
[SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
// Reason for FxCop message suppression -
// This exception cannot escape from the local context
private class AcceptException : Exception {
internal AcceptException() { }
protected AcceptException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
}
[Serializable]
[SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
// Reason for FxCop message suppression -
// This exception cannot escape from the local context
private class AbortException : Exception {
internal AbortException() { }
protected AbortException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
}
[Serializable]
[SuppressMessage( "Microsoft.Design", "CA1064:ExceptionsShouldBePublic" )]
// Reason for FxCop message suppression -
// This exception cannot escape from the local context
private class ErrorException : Exception {
internal ErrorException() { }
protected ErrorException( SerializationInfo i, StreamingContext c ) : base( i, c ) { }
}
// The following methods are only called from within
// a semantic action. The thrown exceptions can never
// propagate outside the ShiftReduceParser class in
// which they are nested.
/// <summary>
/// Force parser to terminate, returning "true"
/// </summary>
protected static void YYAccept() { throw new AcceptException(); }
/// <summary>
/// Force parser to terminate, returning "false"
/// </summary>
protected static void YYAbort() { throw new AbortException(); }
/// <summary>
/// Force parser to terminate, returning
/// "false" if error recovery fails.
/// </summary>
protected static void YYError() { throw new ErrorException(); }
/// <summary>
/// Check if parser in error recovery state.
/// </summary>
protected bool YYRecovering { get { return recovering; } }
#endregion
/// <summary>
/// Abstract base method. ShiftReduceParser calls this
/// to initialize the base class data structures. Concrete
/// parser classes must override this method.
/// </summary>
protected abstract void Initialize();
/// <summary>
/// Main entry point of the Shift-Reduce Parser.
/// </summary>
/// <returns>True if parse succeeds, else false for
/// unrecoverable errors</returns>
public bool Parse() {
Initialize(); // allow derived classes to instantiate rules, states and nonTerminals
NextToken = 0;
FsaState = states[0];
StateStack.Push( FsaState );
valueStack.Push( CurrentSemanticValue );
LocationStack.Push( CurrentLocationSpan );
while (true) {
#if TRACE_ACTIONS
Console.Error.WriteLine("Entering state {0} ", FsaState.number);
DisplayStack();
#endif
int action = FsaState.defaultAction;
if (FsaState.ParserTable != null) {
if (NextToken == 0) {
// We save the last token span, so that the location span
// of production right hand sides that begin or end with a
// nullable production will be correct.
LastSpan = scanner.yylloc;
NextToken = scanner.yylex();
#if TRACE_ACTIONS
Console.Error.WriteLine( "Reading: Next token is {0}", TerminalToString( NextToken ) );
#endif
}
#if TRACE_ACTIONS
else
Console.Error.WriteLine( "Next token is still {0}", TerminalToString( NextToken ) );
#endif
if (FsaState.ParserTable.ContainsKey( NextToken ))
action = FsaState.ParserTable[NextToken];
}
if (action > 0) // shift
{
Shift( action );
}
else if (action < 0) // reduce
{
try {
Reduce( -action );
if (action == -1) // accept
return true;
}
catch (AbortException) {
return false;
}
catch (AcceptException) {
return true;
}
catch (ErrorException) {
if (!ErrorRecovery())
return false;
else
throw; // Rethrow x, preserving information.
}
}
else if (action == 0 && !ErrorRecovery()) // error
return false;
}
}
private void Shift( int stateIndex ) {
#if TRACE_ACTIONS
Console.Error.Write("Shifting token {0}, ", TerminalToString(NextToken));
#endif
FsaState = states[stateIndex];
valueStack.Push( scanner.yylval );
StateStack.Push( FsaState );
LocationStack.Push( scanner.yylloc );
if (recovering) {
if (NextToken != errorToken)
tokensSinceLastError++;
if (tokensSinceLastError > 5)
recovering = false;
}
if (NextToken != endOfFileToken)
NextToken = 0;
}
private void Reduce( int ruleNumber ) {
#if TRACE_ACTIONS
DisplayRule(ruleNumber);
#endif
Rule rule = rules[ruleNumber];
int rhLen = rule.RightHandSide.Length;
//
// Default actions for unit productions.
//
if (rhLen == 1) {
CurrentSemanticValue = valueStack.TopElement(); // Default action: $$ = $1;
CurrentLocationSpan = LocationStack.TopElement(); // Default action "@$ = @1;
}
else if (rhLen == 0) {
// Create a new blank value.
// Explicit semantic action may mutate this value
CurrentSemanticValue = default( TValue );
// The location span for an empty production will start with the
// beginning of the next lexeme, and end with the finish of the
// previous lexeme. This gives the correct behaviour when this
// nonsense value is used in later Merge operations.
CurrentLocationSpan = (scanner.yylloc != null && LastSpan != null ?
scanner.yylloc.Merge( LastSpan ) :
default( TSpan ));
}
else {
// Default action: $$ = $1;
CurrentSemanticValue = valueStack[LocationStack.Depth - rhLen];
// Default action "@$ = @1.Merge(@N)" for location info.
TSpan at1 = LocationStack[LocationStack.Depth - rhLen];
TSpan atN = LocationStack[LocationStack.Depth - 1];
CurrentLocationSpan =
((at1 != null && atN != null) ? at1.Merge( atN ) : default( TSpan ));
}
DoAction( ruleNumber );
for (int i = 0; i < rule.RightHandSide.Length; i++) {
StateStack.Pop();
valueStack.Pop();
LocationStack.Pop();
}
FsaState = StateStack.TopElement();
if (FsaState.Goto.ContainsKey( rule.LeftHandSide ))
FsaState = states[FsaState.Goto[rule.LeftHandSide]];
StateStack.Push( FsaState );
valueStack.Push( CurrentSemanticValue );
LocationStack.Push( CurrentLocationSpan );
}
/// <summary>
/// Execute the selected action from array.
/// Must be overriden in derived classes.
/// </summary>
/// <param name="actionNumber">Index of the action to perform</param>
protected abstract void DoAction( int actionNumber );
private bool ErrorRecovery() {
bool discard;
if (!recovering) // if not recovering from previous error
ReportError();
if (!FindErrorRecoveryState())
return false;
//
// The interim fix for the "looping in error recovery"
// artifact involved moving the setting of the recovering
// bool until after invalid tokens have been discarded.
//
ShiftErrorToken();
discard = DiscardInvalidTokens();
recovering = true;
tokensSinceLastError = 0;
return discard;
}
private void ReportError() {
StringBuilder errorMsg = new StringBuilder();
errorMsg.AppendFormat( "Syntax error, unexpected {0}", TerminalToString( NextToken ) );
if (FsaState.ParserTable.Count < 7) {
bool first = true;
foreach (int terminal in FsaState.ParserTable.Keys) {
if (first)
errorMsg.Append( ", expecting " );
else
errorMsg.Append( ", or " );
errorMsg.Append( TerminalToString( terminal ) );
first = false;
}
}
scanner.yyerror( errorMsg.ToString() );
}
private void ShiftErrorToken() {
int old_next = NextToken;
NextToken = errorToken;
Shift( FsaState.ParserTable[NextToken] );
#if TRACE_ACTIONS
Console.Error.WriteLine("Entering state {0} ", FsaState.number);
#endif
NextToken = old_next;
}
private bool FindErrorRecoveryState() {
while (true) // pop states until one found that accepts error token
{
if (FsaState.ParserTable != null &&
FsaState.ParserTable.ContainsKey( errorToken ) &&
FsaState.ParserTable[errorToken] > 0) // shift
return true;
#if TRACE_ACTIONS
Console.Error.WriteLine("Error: popping state {0}", StateStack.TopElement().number);
#endif
StateStack.Pop();
valueStack.Pop();
LocationStack.Pop();
#if TRACE_ACTIONS
DisplayStack();
#endif
if (StateStack.IsEmpty()) {
#if TRACE_ACTIONS
Console.Error.WriteLine("Aborting: didn't find a state that accepts error token");
#endif
return false;
}
else
FsaState = StateStack.TopElement();
}
}
private bool DiscardInvalidTokens() {
int action = FsaState.defaultAction;
if (FsaState.ParserTable != null) {
// Discard tokens until find one that works ...
while (true) {
if (NextToken == 0) {
#if TRACE_ACTIONS
Console.Error.Write("Reading a token: ");
#endif
NextToken = scanner.yylex();
}
#if TRACE_ACTIONS
Console.Error.WriteLine("Next token is {0}", TerminalToString(NextToken));
#endif
if (NextToken == endOfFileToken)
return false;
if (FsaState.ParserTable.ContainsKey( NextToken ))
action = FsaState.ParserTable[NextToken];
if (action != 0)
return true;
else {
#if TRACE_ACTIONS
Console.Error.WriteLine("Error: Discarding {0}", TerminalToString(NextToken));
#endif
NextToken = 0;
}
}
}
else if (recovering && tokensSinceLastError == 0) {
//
// Boolean recovering is not set until after the first
// error token has been shifted. Thus if we get back
// here with recovering set and no tokens read we are
// looping on the same error recovery action. This
// happens if current_state.ParserTable is null because
// the state has an LR(0) reduction, but not all
// lookahead tokens are valid. This only occurs for
// error productions that *end* on "error".
//
// This action discards tokens one at a time until
// the looping stops. Another attack would be to always
// use the LALR(1) table if a production ends on "error"
//
#if TRACE_ACTIONS
Console.Error.WriteLine("Error: panic discard of {0}", TerminalToString(NextToken));
#endif
if (NextToken == endOfFileToken)
return false;
NextToken = 0;
return true;
}
else
return true;
}
/// <summary>
/// Traditional YACC method. Discards the next input token.
/// </summary>
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyclearin" )]
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyclearin" )]
// Reason for FxCop message suppression -
// This is a traditional name for YACC-like functionality
protected void yyclearin() { NextToken = 0; }
/// <summary>
/// Tradional YACC method. Clear the "recovering" flag.
/// </summary>
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerrok" )]
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerrok" )]
// Reason for FxCop message suppression -
// This is a traditional name for YACC-like functionality
protected void yyerrok() { recovering = false; }
/// <summary>
/// OBSOLETE FOR VERSION 1.4.0
/// Method used by derived types to insert new
/// state instances in the "states" array.
/// </summary>
/// <param name="stateNumber">index of the state</param>
/// <param name="state">data for the state</param>
protected void AddState( int stateNumber, State state ) {
states[stateNumber] = state;
state.number = stateNumber;
}
private void DisplayStack() {
Console.Error.Write( "State stack is now:" );
for (int i = 0; i < StateStack.Depth; i++)
Console.Error.Write( " {0}", StateStack[i].number );
Console.Error.WriteLine();
}
private void DisplayRule( int ruleNumber ) {
Console.Error.Write( "Reducing stack by rule {0}, ", ruleNumber );
DisplayProduction( rules[ruleNumber] );
}
private void DisplayProduction( Rule rule ) {
if (rule.RightHandSide.Length == 0)
Console.Error.Write( "/* empty */ " );
else
foreach (int symbol in rule.RightHandSide)
Console.Error.Write( "{0} ", SymbolToString( symbol ) );
Console.Error.WriteLine( "-> {0}", SymbolToString( rule.LeftHandSide ) );
}
/// <summary>
/// Abstract state class naming terminal symbols.
/// This is overridden by derived classes with the
/// name (or alias) to be used in error messages.
/// </summary>
/// <param name="terminal">The terminal ordinal</param>
/// <returns></returns>
protected abstract string TerminalToString( int terminal );
private string SymbolToString( int symbol ) {
if (symbol < 0)
return nonTerminals[-symbol - 1];
else
return TerminalToString( symbol );
}
/// <summary>
/// Return text representation of argument character
/// </summary>
/// <param name="input">The character to convert</param>
/// <returns>String representation of the character</returns>
protected static string CharToString( char input ) {
switch (input) {
case '\a': return @"'\a'";
case '\b': return @"'\b'";
case '\f': return @"'\f'";
case '\n': return @"'\n'";
case '\r': return @"'\r'";
case '\t': return @"'\t'";
case '\v': return @"'\v'";
case '\0': return @"'\0'";
default: return string.Format( CultureInfo.InvariantCulture, "'{0}'", input );
}
}
}
/// <summary>
/// Classes implementing this interface must supply a
/// method that merges two location objects to return
/// a new object of the same type.
/// GPPG-generated parsers have the default location
/// action equivalent to "@$ = @1.Merge(@N);" where N
/// is the right-hand-side length of the production.
/// </summary>
/// <typeparam name="TSpan">The Location type</typeparam>
#if EXPORT_GPPG
public interface IMerge<TSpan>
#else
internal interface IMerge<TSpan>
#endif
{
/// <summary>
/// Interface method that creates a location object from
/// the current and last object. Typically used to create
/// a location object extending from the start of the @1
/// object to the end of the @N object.
/// </summary>
/// <param name="last">The lexically last object to merge</param>
/// <returns>The merged location object</returns>
TSpan Merge( TSpan last );
}
/// <summary>
/// This is the default class that carries location
/// information from the scanner to the parser.
/// If you don't declare "%YYLTYPE Foo" the parser
/// will expect to deal with this type.
/// </summary>
#if EXPORT_GPPG
public class LexLocation : IMerge<LexLocation>
#else
[SuppressMessage( "Microsoft.Performance", "CA1812:AvoidUninstantiatedInternalClasses" )]
internal class LexLocation : IMerge<LexLocation>
#endif
{
private int startLine; // start line
private int startColumn; // start column
private int endLine; // end line
private int endColumn; // end column
/// <summary>
/// The line at which the text span starts.
/// </summary>
public int StartLine { get { return startLine; } }
/// <summary>
/// The column at which the text span starts.
/// </summary>
public int StartColumn { get { return startColumn; } }
/// <summary>
/// The line on which the text span ends.
/// </summary>
public int EndLine { get { return endLine; } }
/// <summary>
/// The column of the first character
/// beyond the end of the text span.
/// </summary>
public int EndColumn { get { return endColumn; } }
/// <summary>
/// Default no-arg constructor.
/// </summary>
public LexLocation() { }
/// <summary>
/// Constructor for text-span with given start and end.
/// </summary>
/// <param name="sl">start line</param>
/// <param name="sc">start column</param>
/// <param name="el">end line </param>
/// <param name="ec">end column</param>
public LexLocation( int sl, int sc, int el, int ec ) { startLine = sl; startColumn = sc; endLine = el; endColumn = ec; }
/// <summary>
/// Create a text location which spans from the
/// start of "this" to the end of the argument "last"
/// </summary>
/// <param name="last">The last location in the result span</param>
/// <returns>The merged span</returns>
public LexLocation Merge( LexLocation last ) { return new LexLocation( this.startLine, this.startColumn, last.endLine, last.endColumn ); }
}
/// <summary>
/// Abstract scanner class that GPPG expects its scanners to
/// extend.
/// </summary>
/// <typeparam name="TValue">Semantic value type YYSTYPE</typeparam>
/// <typeparam name="TSpan">Source location type YYLTYPE</typeparam>
#if EXPORT_GPPG
public abstract class AbstractScanner<TValue, TSpan>
#else
internal abstract class AbstractScanner<TValue, TSpan>
#endif
where TSpan : IMerge<TSpan> {
/// <summary>
/// Lexical value optionally set by the scanner. The value
/// is of the %YYSTYPE type declared in the parser spec.
/// </summary>
[SuppressMessage( "Microsoft.Design", "CA1051:DoNotDeclareVisibleInstanceFields" )]
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylval" )]
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylval" )]
// Reason for FxCop message suppression -
// This is a traditional name for YACC-like functionality
// A field must be declared for this value of parametric type,
// since it may be instantiated by a value struct. If it were
// implemented as a property, machine generated code in derived
// types would not be able to select on the returned value.
#pragma warning disable 649
public TValue yylval; // Lexical value: set by scanner
#pragma warning restore 649
/// <summary>
/// Current scanner location property. The value is of the
/// type declared by %YYLTYPE in the parser specification.
/// </summary>
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylloc" )]
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylloc" )]
// Reason for FxCop message suppression -
// This is a traditional name for YACC-like functionality
public virtual TSpan yylloc {
get { return default( TSpan ); } // Empty implementation allowing
set { /* skip */ } // yylloc to be ignored entirely.
}
/// <summary>
/// Main call point for LEX-like scanners. Returns an int
/// corresponding to the token recognized by the scanner.
/// </summary>
/// <returns>An int corresponding to the token</returns>
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yylex" )]
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yylex" )]
// Reason for FxCop message suppression -
// This is a traditional name for YACC-like functionality
public abstract int yylex();
/// <summary>
/// Traditional error reporting provided by LEX-like scanners
/// to their YACC-like clients.
/// </summary>
/// <param name="format">Message format string</param>
/// <param name="args">Optional array of args</param>
[SuppressMessage( "Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyerror" )]
[SuppressMessage( "Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyerror" )]
// Reason for FxCop message suppression -
// This is a traditional name for YACC-like functionality
public virtual void yyerror( string format, params object[] args ) { }
}
/// <summary>
/// Encapsulated state for the parser.
/// Opaque to users, visible to the tool-generated code.
/// </summary>
#if EXPORT_GPPG
public class State
{
/// <summary>
/// The number of states in the automaton.
/// </summary>
public int number;
#else
internal class State {
/// <summary>
/// The index of this state in the states array.
/// </summary>
internal int number;
#endif
internal Dictionary<int, int> ParserTable; // Terminal -> ParseAction
internal Dictionary<int, int> Goto; // NonTerminal -> State;
internal int defaultAction; // = 0; // ParseAction
/// <summary>
/// State transition data for this state. Pairs of elements of the
/// goto array associate symbol ordinals with next state indices.
/// The actions array is passed to another constructor.
/// </summary>
/// <param name="actions">The action list</param>c
/// <param name="goToList">Next state data</param>
public State( int[] actions, int[] goToList )
: this( actions ) {
Goto = new Dictionary<int, int>();
for (int i = 0; i < goToList.Length; i += 2)
Goto.Add( goToList[i], goToList[i + 1] );
}
/// <summary>
/// Action data for this state. Pairs of elements of the
/// action array associate action ordinals with each of
/// those symbols that have actions in the current state.
/// </summary>
/// <param name="actions">The action array</param>
public State( int[] actions ) {
ParserTable = new Dictionary<int, int>();
for (int i = 0; i < actions.Length; i += 2)
ParserTable.Add( actions[i], actions[i + 1] );
}
/// <summary>
/// Set the default action for this state.
/// </summary>
/// <param name="defaultAction">Ordinal of the default action</param>
public State( int defaultAction ) {
this.defaultAction = defaultAction;
}
/// <summary>
/// Set the default action and the state transition table.
/// </summary>
/// <param name="defaultAction">The default action</param>
/// <param name="goToList">Transitions from this state</param>
public State( int defaultAction, int[] goToList )
: this( defaultAction ) {
Goto = new Dictionary<int, int>();
for (int i = 0; i < goToList.Length; i += 2)
Goto.Add( goToList[i], goToList[i + 1] );
}
}
/// <summary>
/// Rule representation at runtime.
/// </summary>
#if EXPORT_GPPG
public class Rule
#else
internal class Rule
#endif
{
internal int LeftHandSide; // symbol
internal int[] RightHandSide; // symbols
/// <summary>
/// Rule constructor. This holds the ordinal of
/// the left hand side symbol, and the list of
/// right hand side symbols, in lexical order.
/// </summary>
/// <param name="left">The LHS non-terminal</param>
/// <param name="right">The RHS symbols, in lexical order</param>
public Rule( int left, int[] right ) {
this.LeftHandSide = left;
this.RightHandSide = right;
}
}
/// <summary>
/// Stack utility for the shift-reduce parser.
/// GPPG parsers have three instances:
/// (1) The parser state stack, T = QUT.Gppg.State,
/// (2) The semantic value stack, T = TValue,
/// (3) The location stack, T = TSpan.
/// </summary>
/// <typeparam name="T"></typeparam>
#if EXPORT_GPPG
public class PushdownPrefixState<T>
#else
internal class PushdownPrefixState<T>
#endif
{
// Note that we cannot use the BCL Stack<T> class
// here as derived types need to index into stacks.
//
private T[] array = new T[8];
private int tos = 0;
/// <summary>
/// Indexer for values of the stack below the top.
/// </summary>
/// <param name="index">index of the element, starting from the bottom</param>
/// <returns>the selected element</returns>
public T this[int index] { get { return array[index]; } }
/// <summary>
/// The current depth of the stack.
/// </summary>
public int Depth { get { return tos; } }
internal void Push( T value ) {
if (tos >= array.Length) {
T[] newarray = new T[array.Length * 2];
System.Array.Copy( array, newarray, tos );
array = newarray;
}
array[tos++] = value;
}
internal T Pop() {
T rslt = array[--tos];
array[tos] = default( T );
return rslt;
}
internal T TopElement() { return array[tos - 1]; }
internal bool IsEmpty() { return tos == 0; }
}
}
@@ -1,4 +0,0 @@
global using System;
global using System.Linq;
global using System.Collections.Generic;
global using System.Collections.Immutable;
@@ -8,7 +8,7 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="Google.Protobuf" Version="3.24.3" /> <PackageReference Include="Google.Protobuf" Version="3.24.3" />
<PackageReference Include="YaccLexTools" Version="1.2.2" /> <PackageReference Include="YaccLexTools" Version="1.2.0" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
@@ -9,17 +9,9 @@
<ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" /> <ProjectReference Include="..\ArithmeticExpressions\ArithmeticExpressions.csproj" />
<ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" /> <ProjectReference Include="..\BaseUtilities\BaseUtilities.csproj" />
<ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" /> <ProjectReference Include="..\BooleanExpressions\BooleanExpressions.csproj" />
<ProjectReference Include="..\Bts.ColumnExpressions\Bts.ColumnExpressions.csproj" />
<ProjectReference Include="..\Bts.ColumnSortExpressions\Bts.ColumnSortExpressions.csproj" />
<ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" /> <ProjectReference Include="..\DataStreamCore\DataStreamCore.csproj" />
<ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" /> <ProjectReference Include="..\DataStreamExpressions\DataStreamExpressions.csproj" />
<ProjectReference Include="..\..\client\spark_core\spark_core.csproj" /> <ProjectReference Include="..\..\client\spark_core\spark_core.csproj" />
</ItemGroup> </ItemGroup>
<ItemGroup>
<PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
<PackageReference Include="MongoDB.Bson" Version="2.23.1" />
<PackageReference Include="MongoDB.Driver" Version="2.23.1" />
</ItemGroup>
</Project> </Project>
@@ -1,26 +0,0 @@
namespace Bts.DataStream;
using BCG;
using Spark.Core.Disposables;
using System;
using System.Reactive.Linq;
public static class DataStreamExtensions {
public static ViewStream AsViewStream(this IObservable<DataPacket> source) =>
source as ViewStream ?? new ObservableViewStream(source);
public static IObservable<DataPacket> AsObservable(this ViewStream source) {
return source as IObservable<DataPacket> ?? Observable.Create<DataPacket>(Subscribe);
IDisposable Subscribe(IObserver<DataPacket> observer) {
source.onChange += OnChange;
return Disposable.Create(() => source.onChange -= OnChange);
void OnChange(object? newValue) =>
observer.OnNext(newValue as DataPacket ?? throw new ArgumentException(
$"Expected DataPacket, got {newValue?.GetType()}", nameof(newValue)));
}
}
}
@@ -1,29 +0,0 @@
namespace Bts.DataStream;
using BCG;
using bts;
using System;
using System.IO;
public class PandasScriptStream : ViewStream {
public PandasScriptStream(ViewStream vs, string file)
: this(vs, file, null) { }
public PandasScriptStream(ViewStream vs, string file, string? pythonExe)
: this(vs, file, pythonExe, ViewStreamDatabase.instance) { }
public PandasScriptStream(ViewStream vs, string file, string? pythonExe, ViewStreamDatabase vsd) {
mySetRealSource(new PandasStream(vs, readFile(), pythonExe, vsd));
string readFile() {
try {
return File.ReadAllText(file);
} catch (Exception ex) {
Log.Gen.error($"PandasScriptStream error reading file: {ex}");
throw;
}
}
}
}
@@ -1,130 +0,0 @@
namespace Bts.DataStream;
using BCG;
using BCG.Extensions;
using BCG.Utilities.General;
using bts;
using bts.collections;
using Bts.DataStreamCore;
using Spark.Core.Disposables;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
public class PandasStream : ViewStream, IWorker {
readonly string inputFile = $"PandasStream_{Guid.NewGuid()}.csv";
readonly string outputFile = $"PandasStream_{Guid.NewGuid()}.csv";
readonly string pythonExe;
readonly ViewStreamDatabase vsd;
readonly CSVSaverStream inputCsvStream;
readonly Lazy<string> pythonCode;
IDictionary<int, IBlueRow>? lastRowMap = null;
int inputVersion = 0;
int outputVersion = 0;
DateTime lastFireTime = DateTime.MinValue;
TimeSpan fireThrottle = TimeSpan.FromSeconds(3);
IDisposable? dispose;
public PandasStream(ViewStream vs, string dataFrameExpression)
: this(vs, dataFrameExpression, null, ViewStreamDatabase.instance) { }
public PandasStream(ViewStream vs, string dataFrameExpression, string? pythonExe, ViewStreamDatabase vsd) {
pythonExe ??= Environment.GetEnvironmentVariable("PYTHON_EXE") ?? @"C:\Users\johnc\GoogleDrive\Repos\bts\spark\.venv\Scripts\python.exe";
this.pythonExe = pythonExe;
this.vsd = vsd;
// This should fire an update when it writes to the output file
inputCsvStream = new CSVSaverStream(vs, inputFile, true, int.MaxValue, true);
pythonCode = new(getPythonCode);
string getPythonCode() {
var imports =
@"import numpy as np
import pandas as pd
import sys
";
var bd = new StringBuilder(imports);
bd.AppendLine($"df=pd.read_csv('{inputFile}')");
bd.AppendLine(dataFrameExpression);
bd.AppendLine($"df.to_csv('{outputFile}', index=False)");
return bd.ToString();
}
}
protected override void myOnFirstSubscriber() {
vsd.addWorker(this);
inputCsvStream.onChange += onInputChanged;
this.dispose = Disposable.Create(dispose);
void onInputChanged(object newValue) =>
inputVersion++;
void dispose() {
inputCsvStream.onChange -= onInputChanged;
vsd.removeWorker(this);
deleteFile(inputFile);
deleteFile(outputFile);
}
void deleteFile(string file, [CallerArgumentExpression(nameof(file))] string name = "") {
if (!File.Exists(file)) {
return;
}
try {
File.Delete(file);
} catch (Exception ex) {
Log.Gen.error($"PandasStream. Error deleting {name} file '{file}': {ex}");
}
}
}
protected override void myOnLastSubscriber() =>
Disposable.dispose(ref dispose);
static void executePythonCode(string pythonCode, string? pythonExe = null) {
pythonExe ??= "python";
var start = new ProcessStartInfo {
FileName = pythonExe,
Arguments = "-c \"" + pythonCode.Replace("\"", "\\\"") + "\"",
UseShellExecute = false,
RedirectStandardOutput = true
};
using var process = Process.Start(start)
?? throw new InvalidOperationException("Failed to launch python");
process.WaitForExit();
}
void IWorker.doWork() {
if (outputVersion == inputVersion) {
return;
}
var now = DateTime.Now;
var elapsedSinceLastFire = now - lastFireTime;
if (elapsedSinceLastFire < fireThrottle) {
return;
}
outputVersion = inputVersion;
executePythonCode(pythonCode.Value, pythonExe);
var rows = BlueDataUtility.createRowsFromCSVFileWithoutTypes(outputFile, out var schema);
var rowMap = rows as IDictionary<int, IBlueRow> ?? rows.Cast<IBlueRow>().ToDictionary(r => r.Id());
var comp = BlueRowEqualityComparer.Instance;
var deletedRows = new BlueTable(schema);
var addedRows = new BlueTable(schema);
if (lastRowMap is not null) {
deletedRows.SetItems(lastRowMap.Where(oldRow =>
!(rowMap.TryGetValue(oldRow.Key, out var newRow) && comp.Equals(newRow, oldRow.Value))));
addedRows.SetItems(rowMap.Where(newRow =>
!(lastRowMap.TryGetValue(newRow.Key, out var oldRow) && comp.Equals(oldRow, newRow.Value))));
}
lastRowMap = rowMap;
lastFireTime = DateTime.Now;
myFireEvent(new DataPacket(rows, addedRows, deletedRows));
}
}
@@ -1,79 +0,0 @@
using BCG.Utilities.General;
using System;
namespace BCG {
public class TCPConnectionTestStream : ViewStream, IWorker, ITCPConnectionHandler, ITCPMessageHandler {
class State {
public int n_sends = 0;
public int n_recvs = 0;
}
private readonly string _ip;
private readonly int _port;
private readonly ViewStreamDatabase _vsd;
State _state = new State();
private readonly int _msg_size;
private IBlueTableSchema _schema;
private TimeSpan _send_period;
private ITCPConnection _connection = null;
private byte[] _buff;
public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds) :
this(ip, port, msg_size, send_period_seconds, ViewStreamDatabase.instance) { }
public TCPConnectionTestStream(string ip, int port, int msg_size, int send_period_seconds, ViewStreamDatabase vsd) {
_ip = ip;
_port = port;
_msg_size = msg_size;
_vsd = vsd;
_schema = BlueDataUtility.form_schema("TCPClient",
Tuple.Create("id", TypeCode.Int32),
Tuple.Create("n_sends", TypeCode.UInt32),
Tuple.Create("n_recvs", TypeCode.UInt32));
_send_period = TimeSpan.FromSeconds(send_period_seconds);
_buff = new byte[msg_size];
for (int i = 0; i < _buff.Length; i++) {
_buff[i] = (byte)i;
}
}
protected override void myOnFirstSubscriber() {
MasterWorker.getInstance(false).addWorker(this, _send_period);
var server_rci = new RemoteConnectionInfo(_ip, _port);
bts.AsyncTCPConnector_APM.connect_to(server_rci, this);
}
protected override void myOnLastSubscriber() {
MasterWorker.getInstance(false).removeWorker(this);
}
public void doWork() {
if (_connection != null) {
_connection.send(_buff);
_state.n_sends++;
}
IBlueRow r = new BlueRow(_schema, 0, _state.n_sends, _state.n_recvs);
BlueTable rows = new BlueTable(r.schema);
rows.addRow(r);
var dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket, true);
myFireEvent(dp);
}
public void onConnection(ITCPConnection connection) {
_connection = connection;
_connection.register(this);
}
public bool isDisposed { get; }
public void onData(RemoteMessage msg) {
bts.Log.Gen.info($"TCPConnectionTestStream received msg: {msg}");
_state.n_recvs++;
}
public void Dispose() {
}
}
}
@@ -1,3 +0,0 @@
file_path = r"C:\Temp\test2.txt" # r prefix for raw string to avoid escape characters
with open(file_path, "w") as f:
f.write("Hello, world!\n")
@@ -1,37 +0,0 @@
namespace Bts.DataStream;
using BCG;
using bts;
using System;
public class ObservableViewStream : ViewStream, IObservable<DataPacket>, IDisposable {
readonly IObservable<DataPacket> source;
IDisposable? subscription;
public ObservableViewStream(IObservable<DataPacket> source) =>
this.source = source;
public void Dispose() {
GC.SuppressFinalize(this);
if (subscription is not null) {
subscription.Dispose();
subscription = null;
}
}
protected override void myOnFirstSubscriber() {
Log.isnull(subscription);
subscription = source.Subscribe(myFireEvent);
}
protected override void myOnLastSubscriber() {
Log.notnull(subscription);
subscription.Dispose();
subscription = null;
}
IDisposable IObservable<DataPacket>.Subscribe(IObserver<DataPacket> observer) =>
source.Subscribe(observer);
}
@@ -0,0 +1,32 @@
using BCG;
namespace BCG
{
// this is a stream so we can run it from the DSQA (since dsqa can only load streams)
[DataStreamUserRespresentation("add_to_cloud")]
public class AddToCloudStream : ViewStream
{
private readonly string _name, _stream_exp;
private StreamOffloadRequestor _requestor = new StreamOffloadRequestor();
public AddToCloudStream(string name, params string[] lines)
: this(name, 0, lines)
{
}
public AddToCloudStream(string name, int persist, params string[] lines)
{
_name = name;
_stream_exp = getExpression(lines);
_requestor.add_to_cloud(_name, _stream_exp, persist == 1);
mySetRealSource(new TableStream("cloud_streams"));
}
static string getExpression(params string[] lines) {
for (int i = 0; i < lines.Length; i++) {
// unescape quotes
lines[i] = lines[i].Replace("\\\"", "\"");
}
return string.Join("\n", lines);
}
}
}
@@ -1,22 +1,31 @@
using BCG;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using mbts.config;
using bts.utils;
using System.Threading.Tasks;
using Bts.DataStreamExpressions; using Bts.DataStreamExpressions;
using System.Collections.Generic;
using BCG.Utilities.General; using BCG.Utilities.General;
using bts;
namespace BCG { namespace BCG
{
// listens to the requests posted on "cloud_streams" request table, and // listens to the requests posted on "cloud_streams" request table, and
// if unfulfilled starts a ViewPubsliher // if unfulfilled starts a ViewPubsliher
public class RemoteQueryManager : IWorker { public class CloudStreamManager : IWorker
public class StreamPubUnit { {
public class StreamPubUnit
{
public string name; public string name;
public string stream_expression; public string stream_expression;
public ViewStream stream; public ViewStream stream;
public bool persist; public bool persist;
public ViewPublisher view_publisher; public ViewPublisher view_publisher;
public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_) { public StreamPubUnit(string name_, string stream_expression_, ViewStream stream_, bool persist_, ViewPublisher view_publisher_)
{
name = name_; name = name_;
stream_expression = stream_expression_; stream_expression = stream_expression_;
stream = stream_; stream = stream_;
@@ -24,7 +33,7 @@ namespace BCG {
view_publisher = view_publisher_; view_publisher = view_publisher_;
} }
} }
static ViewStream? build_stream(string stream_expression) static ViewStream build_stream(string stream_expression)
{ {
try try
{ {
@@ -32,7 +41,9 @@ namespace BCG {
var rv = exp.getView(new Dictionary<string, StatementExpression>(0)); var rv = exp.getView(new Dictionary<string, StatementExpression>(0));
return rv; return rv;
} catch (Exception ex) { }
catch (Exception ex)
{
bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}"); bts.Log.assert(false, $"Could not build stream: {stream_expression} ex: {ex}");
return null; return null;
} }
@@ -41,37 +52,34 @@ namespace BCG {
_stream_pub_map = new Dictionary<string, StreamPubUnit>(); _stream_pub_map = new Dictionary<string, StreamPubUnit>();
private ViewStream _stream = new TableStream("cloud_streams"); private ViewStream _stream = new TableStream("cloud_streams");
private PubTableBroker _broker = new PubTableBroker(); private PubTableBroker _broker = new PubTableBroker();
private ManagedSubscriber _sub = ManagedSubscriber.instance;
public CloudStreamManager()
public RemoteQueryManager() { {
_stream.onChange += _on_str; _stream.onChange += _on_str;
_sub.subscribe("RemoteRequest", (string tag, byte[] data) => {
RemoteRequest req = RemoteRequest.construct_from_bytes(data);
_spawn_remote_stream(req.published_table_name, req.stream_expression, false);
});
} }
private void _on_str(object obj) private void _on_str(object obj)
{ {
if (obj is not DataPacket dp) { DataPacket dp = obj as DataPacket;
return;
}
foreach (IBlueRow r in dp.newRows) foreach (IBlueRow r in dp.newRows)
{ {
string name = r.get<string>("name"); string name = r.get<string>("name");
bool persist = r.get<bool>("persist"); bool persist = r.get<bool>("persist");
string stream_exp = r.get<string>("stream_exp"); string stream_exp = r.get<string>("stream_exp");
if (!_stream_pub_map.ContainsKey(name)) { if (!_stream_pub_map.ContainsKey(name))
_spawn_remote_stream(name, stream_exp, persist); {
_assign(name, stream_exp, persist);
} }
} }
} }
private void _spawn_remote_stream(string name, string stream_expression, bool persist) { private void _assign(string name, string stream_expression, bool persist)
{
bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}"); bts.Log.Gen.info($"CloudStreamManager.assign name: {name} stream_expression: {stream_expression}");
ViewStream stream = build_stream(stream_expression); ViewStream stream = build_stream(stream_expression);
_expose_to_ws(name, stream); _expose_to_ws(name, stream);
if (persist) { if (persist)
{
stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000); // 10MBs stream = new PersistorStream(stream, name, "C:\\Temp\\", 1000000); // 10MBs
} }
ViewPublisher view_pub = new ViewPublisher(name, _broker, stream); ViewPublisher view_pub = new ViewPublisher(name, _broker, stream);
@@ -79,12 +87,14 @@ namespace BCG {
_stream_pub_map.Add(name, u); _stream_pub_map.Add(name, u);
} }
private void _expose_to_ws(string name, ViewStream stream) { private void _expose_to_ws(string name, ViewStream stream)
{
// TODO: john // TODO: john
} }
public void doWork() { public void doWork()
{
} }
} }
} }
@@ -1,48 +0,0 @@
using BCG;
using bts;
using System;
namespace BCG {
// this is a stream so we can run it from the DSQA (since dsqa can only load streams)
[DataStreamUserRespresentation("remoteQuery")]
public class RemoteQueryStream : ViewStream {
private readonly string _name, _stream_exp;
private PublisherCore _publisher_core = new PublisherCore();
private ViewStream _source_str;
public RemoteQueryStream(params string[] lines) {
_stream_exp = get_expression(lines);
_name = Guid.NewGuid().ToString();
_name = $"remotereq_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds().ToString()}";
//_name.Replace("-", "_");
}
protected override void myOnFirstSubscriber() {
RemoteRequest rem_req = new RemoteRequest(_stream_exp, _name);
byte[] rem_req_bytes = rem_req.serialize();
_publisher_core.publish("System", "RemoteRequest", rem_req_bytes);
_source_str = new TableStream(_name);
_source_str.onChange += new ChangeDelegate(_handle_data);
}
protected override void myOnLastSubscriber() {
//maybe there should be a way to turn remote requests off ?
_source_str.onChange -= new ChangeDelegate(_handle_data);
}
static string get_expression(params string[] lines) {
for (int i = 0; i < lines.Length; i++) {
// unescape quotes
lines[i] = lines[i].Replace("\\\"", "\"");
}
return string.Join("\n", lines);
}
private void _handle_data(object o) {
myFireEvent(o);
}
}
}
@@ -1,31 +0,0 @@
namespace bts {
using System.IO;
using BCG;
using System.IO;
using System.Threading.Tasks.Dataflow;
public class RemoteRequest {
public readonly string stream_expression;
public readonly string published_table_name;
public RemoteRequest(string stream_expression_, string published_table_name_) {
stream_expression = stream_expression_;
published_table_name = published_table_name_;
}
public byte[] serialize() {
var str = new MemoryStream();
var bw = new BinaryWriter(str);
bw.write_string_cpp(stream_expression);
bw.write_string_cpp(published_table_name);
return str.ToArray();
}
public static RemoteRequest construct_from_bytes(byte[] ser_bytes) {
BinaryReader br = new BinaryReader(new MemoryStream(ser_bytes));
string stream_expression_ = br.read_string_cpp();
string published_table_name_ = br.read_string_cpp();
return new RemoteRequest(stream_expression_, published_table_name_);
}
}
}
@@ -1,54 +1,59 @@
// using BCG; using BCG;
// using bts; using System.Linq;
// using System.Linq; namespace BCG
{
public class StreamOffloadRequestor
{
public class OffloadOp
{
string name;
string stream_exp;
}
private IBlueTableSchema _schema;
private IBlueRowCollection _rows;
private PubTableBroker _broker = new PubTableBroker();
ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
public StreamOffloadRequestor()
{
_schema = new BlueTableSchema("cloud_streams",
new ColumnInfo("id", 0, typeof(int)),
new ColumnInfo("name", 1, typeof(string)),
new ColumnInfo("persist", 2, typeof(bool)),
new ColumnInfo("stream_exp", 3, typeof(string)));
_existing_offloads_vs.onChange += _on_str;
}
private void _on_str(object obj)
{
DataPacket dp = obj as DataPacket;
_rows = dp.newRows;
// this is needed so the broker knows the state of the table
_broker.createTable(_rows);
}
public void add_to_cloud(string name, string stream_exp, bool persist)
{
// check if it exists
if (_rows == null)
{
BlueTable table = new BlueTable(_schema);
IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
table.addRow(nr);
_broker.createTable(table);
// create table
}
else
{
if (exists(name))
return;
IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
_broker.addRow(nr);
}
}
// namespace BCG { bool exists(string name)
// // used on the client {
// public class StreamOffloadRequestor { return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
// public class OffloadOp { }
// string name; }
// string stream_exp; }
// }
// private IBlueTableSchema _schema;
// private IBlueRowCollection _rows;
// private PubTableBroker _broker = new PubTableBroker();
// ViewStream _existing_offloads_vs = new TableStream("cloud_streams");
// public StreamOffloadRequestor() {
// _schema = new BlueTableSchema("cloud_streams",
// new ColumnInfo("id", 0, typeof(int)),
// new ColumnInfo("name", 1, typeof(string)),
// new ColumnInfo("persist", 2, typeof(bool)),
// new ColumnInfo("stream_exp", 3, typeof(string)));
// _existing_offloads_vs.onChange += _on_str;
// }
// private void _on_str(object obj) {
// DataPacket dp = obj as DataPacket;
// _rows = dp.newRows;
// // this is needed so the broker knows the state of the table
// _broker.create_table(_rows);
// }
// public void add_to_cloud(string name, string stream_exp, bool persist) {
// // check if it exists
// if (_rows == null) {
// BlueTable table = new BlueTable(_schema);
// IBlueRow nr = new BlueRow(_schema, 0, name, persist, stream_exp);
// table.addRow(nr);
// _broker.create_table(table);
// // create table
// } else {
// if (exists(name))
// return;
// IBlueRow nr = new BlueRow(_schema, _rows.Count, name, persist, stream_exp);
// // TODO: use make_batch_changes instead
// Log.assert(false, "Fix addRow");
// //_broker.addRow(nr);
// }
// }
// bool exists(string name) {
// return _rows.to_ienum().Where(r => name == r.get<string>("name")).Count() > 0;
// }
// }
// }
@@ -1,14 +1,17 @@
using System; using System;
using System.Collections; using System.Collections;
namespace BCG { namespace BCG
internal class AddColumnsRow : IBlueRow { {
internal class AddColumnsRow : IBlueRow
{
private IBlueTableSchema mySchema; private IBlueTableSchema mySchema;
private IBlueRow myRow; private IBlueRow myRow;
//private object[] myCachedValues; //private object[] myCachedValues;
#region Constructors #region Constructors
public AddColumnsRow(IBlueTableSchema schema, IBlueRow row) { public AddColumnsRow (IBlueTableSchema schema, IBlueRow row)
{
mySchema = schema; mySchema = schema;
myRow = row; myRow = row;
//myCachedValues = new object[schema.columnNames.Length]; //myCachedValues = new object[schema.columnNames.Length];
@@ -16,78 +19,85 @@ namespace BCG {
#endregion #endregion
#region IBlueRow members #region IBlueRow members
public IBlueTableSchema schema { get { return mySchema; } } public IBlueTableSchema schema{get{return mySchema;}}
#endregion #endregion
#region ICollection members #region ICollection members
public int Count { get { return mySchema.columnNames.Length; } } public int Count {get{return mySchema.columnNames.Length;}}
public bool IsSynchronized { get { return false; } } public bool IsSynchronized {get{return false;}}
public object SyncRoot { get { return this; } } public object SyncRoot {get{return this;}}
public void CopyTo(Array array, int index) { public void CopyTo (Array array, int index)
{
int i = index; int i = index;
foreach (object o in this as IEnumerable) foreach (object o in this as IEnumerable)
array.SetValue(o, i++); array.SetValue (o, i++);
} }
#endregion #endregion
#region IDictionary members #region IDictionary members
public bool IsFixedSize { get { return true; } } public bool IsFixedSize {get{return true;}}
public bool IsReadOnly { get { return true; } } public bool IsReadOnly {get{return true;}}
/// <summary> /// <summary>
/// The key can either be the string corrsponding to the column name or /// The key can either be the string corrsponding to the column name or
/// the order of the column /// the order of the column
/// </summary> /// </summary>
public object this[object key] { public object this [object key]
get { {
get
{
if (key is string) if (key is string)
return this[(string)key]; return this[(string)key];
return this[(int)key]; return this[(int)key];
} }
set { throw new NotSupportedException("The IBlueRow is readonly"); } set {throw new NotSupportedException ("The IBlueRow is readonly");}
} }
public object this[int index] { public object this[int index]
get { {
get
{
//if (myCachedValues[index] == null) //if (myCachedValues[index] == null)
// myCachedValues[index] = mySchema[index].isConstant ? mySchema[index].colValue : myRow[index]; // myCachedValues[index] = mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
//return myCachedValues[index]; //return myCachedValues[index];
return mySchema[index].isConstant ? DBNull.Value : myRow[index]; return mySchema[index].isConstant ? mySchema[index].colValue : myRow[index];
} }
} }
public object this[string colName] { get { return this[mySchema[colName].colNo]; } } public object this[string colName] {get{return this[mySchema[colName].colNo];}}
public ICollection Keys { get { return mySchema.columnNames; } } public ICollection Keys{get{return mySchema.columnNames;}}
public ICollection Values { public ICollection Values
get { {
get
{
object[] retVal = new object[mySchema.columnNames.Length]; object[] retVal = new object[mySchema.columnNames.Length];
CopyTo(retVal, 0); CopyTo (retVal, 0);
return retVal; return retVal;
} }
} }
public void Add(object key, object value) { throw new NotSupportedException("The IBlueRow is readonly"); } public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");}
public void Clear() { throw new NotSupportedException("The IBlueRow is readonly"); } public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
public bool Contains(object key) { return mySchema[(string)key] != null; } public bool Contains (object key){return mySchema[(string)key] != null;}
/// <summary> /// <summary>
/// Note, the default enumerator is the IDictionary's enumerator, so if user needs to /// Note, the default enumerator is the IDictionary's enumerator, so if user needs to
/// iterate through the values, typecast this into an IEnumerable /// iterate through the values, typecast this into an IEnumerable
/// </summary> /// </summary>
/// <returns></returns> /// <returns></returns>
public IDictionaryEnumerator GetEnumerator() { return new BlueRowDictionaryEnumerator(this); } public IDictionaryEnumerator GetEnumerator() {return new BlueRowDictionaryEnumerator(this);}
IEnumerator IEnumerable.GetEnumerator () {return new BlueRowEnumerator(this);}
IEnumerator IEnumerable.GetEnumerator() { return new BlueRowEnumerator(this); } public void Remove (object key) {throw new NotSupportedException ("The IBlueRow is readonly");}
public void Remove(object key) { throw new NotSupportedException("The IBlueRow is readonly"); }
#endregion #endregion
} }
@@ -1,34 +1,13 @@
using System; using System;
using System.Data;
using System.Collections;
using BCG.Utilities;
using BCG.Utilities.General;
namespace BCG
{
namespace BCG { public class AddColumnsStream : MapStream
public class AddColumnsStream : ViewStream { {
public static Tuple<string, Type, object> parse_exp(string exp) { public AddColumnsStream(ViewStream vs, string columnDef) : base (vs, new ColumnAdder (columnDef)) {}
string[] toks = exp.Split(':');
string col_name = toks[0];
string col_value = toks[1];
Tuple<Type, object> tup = BlueDataUtility.guess_type(col_value);
return Tuple.Create(col_name, tup.Item1, tup.Item2);
}
private Tuple<string, Type, object> _new_col;
private IBlueTableSchema _out_schema;
public AddColumnsStream(ViewStream vs, string col_def) {
_new_col = parse_exp(col_def);
ViewStream rs = new MapStream(vs, (IBlueTableSchema s) => _get_schema(s), (IBlueRow r) => _map(r));
mySetRealSource(rs);
}
private IBlueTableSchema _get_schema(IBlueTableSchema input_schema) {
if (_out_schema == null)
_out_schema = BlueDataUtility.add_column(input_schema, _new_col.Item1, _new_col.Item2);
return _out_schema;
}
private IBlueRow _map(IBlueRow r) {
MutableBlueRow rv = new MutableBlueRow(_get_schema(r.schema), r);
rv.setValue(_new_col.Item1, _new_col.Item3);
return rv;
}
} }
} }
@@ -1,12 +0,0 @@
namespace bts {
using BCG;
using System;
using System.Collections;
public class AddConstantColumnRow {
AddConstantColumnRow(IBlueRow r, Tuple<string, Type, object>[] constant_cols) {
}
}
}
@@ -1,52 +1,73 @@
using System; // using System;
using System.Collections.Generic; // using System.Collections;
using System.Diagnostics.CodeAnalysis; // using BCG.Utilities;
using System.Linq; // using BCG.Evaluator.AddColumnsExpression;
using Bts.ColumnExpressions; // namespace BCG
// {
namespace BCG; // /// <summary>
// /// Provides methods to add columns to the schema
/// <summary> // /// </summary>
/// Provides methods to add columns to the schema // public class ColumnAdder : IStreamMapper
/// </summary> // {
/// <param name="columnsExpression"> // protected IDictionary myNewColumns;
/// String represting the columns to add to the output table // protected IBlueTableSchema mySchema;
/// </param>
public class ColumnAdder : IStreamMapper { // static syntax myParser =
// Store tuples of (column, index) to preserve order // new syntax();
protected readonly IDictionary<string, (ColumnExpression column, int index)> myNewColumns;
protected IBlueTableSchema? mySchema; // /// <summary>
// /// string represting the columns to be added to the output table.
public ColumnAdder(string columnsExpression) => // /// </summary>
myNewColumns = (ColumnExpressionInterpreter.Evaluate(columnsExpression) ?? // /// <param name="colAddExpression"></param>
throw new Exception("Invalid columnn expression: " + columnsExpression)) // public ColumnAdder (string columnDef)
.Select((c, i) => (c, i)) // {
.ToDictionary(c => c.c.Name, c => (c.c, c.i)); // AddColumnsExpression ace;
[MemberNotNull(nameof(mySchema))] // if (DataStreamLoadFileConfig.instance.useStaticParser)
public virtual IBlueRow map(IBlueRow row) { // {
// Only rebuild schema if necessary (i.e., if it's null) // lock (myParser)
mySchema ??= buildSchema(row.schema, myNewColumns); // {
return new AddColumnsRow(mySchema, row); // ace =
} // (AddColumnsExpression)myParser.Parse(columnDef);
// }
[MemberNotNull(nameof(mySchema))] // }
public IBlueTableSchema getOutputSchema(IBlueTableSchema inputSchema) => // else
// Always rebuild schema (even if it's already non-null) // {
mySchema = buildSchema(inputSchema, myNewColumns); // syntax parser = new syntax();
// ace =
[MemberNotNull(nameof(mySchema))] // (AddColumnsExpression)parser.Parse(columnDef);
static BlueTableSchema buildSchema( // }
IBlueTableSchema schema,
IDictionary<string, (ColumnExpression column, int index)> newColumns // if (ace == null)
) { // throw new Exception ("Invalid columnn expression: " + columnDef);
ColumnInfo[] cis = new ColumnInfo[schema.columnNames.Length + newColumns.Count]; // myNewColumns = ace.columns;
schema.columnInfo.CopyTo(cis, 0); // }
foreach (var (col, index) in newColumns.Values) {
var schemaIdx = schema.columnInfo.Length + index; // public virtual IBlueRow map (IBlueRow row)
cis[schemaIdx] = new ColumnInfo(col.Name, schemaIdx, col.Type, col.Value); // {
} // if (mySchema == null)
return new BlueTableSchema(cis, schema.tableName); // myConstructSchema (row.schema);
} // return new AddColumnsRow (mySchema, row);
} // }
// public IBlueTableSchema getOutputSchema (IBlueTableSchema inputSchema)
// {
// myConstructSchema (inputSchema);
// return mySchema;
// }
// private void myConstructSchema (IBlueTableSchema schema)
// {
// ColumnInfo[] cis = new ColumnInfo[schema.columnNames.Length + myNewColumns.Count];
// int ciIndex = 0;
// foreach (ColumnInfo ci in schema.columnInfo)
// cis[ciIndex++] = ci;
// foreach (DictionaryEntry de in myNewColumns)
// cis[ciIndex] = new ColumnInfo ((string)de.Key, ciIndex ++,
// ((AddColumnsExpression.ConstantInfo)de.Value).t, ((AddColumnsExpression.ConstantInfo)de.Value).v);
// mySchema = new BlueTableSchema (cis, schema.tableName);
// }
// }
// }
@@ -1,24 +1,23 @@
using bts;
using System; using System;
using System.Collections; using System.Collections;
namespace BCG { namespace BCG
{
/// <summary> /// <summary>
/// Provides a stream that with distinct rows with respect to the columns /// Provides a stream that with distinct rows with respect to the columns
/// the user specifies in the argument /// the user specifies in the argument
/// </summary> /// </summary>
[DataStreamUserRespresentation("distinct")] [DataStreamUserRespresentation ("distinct")]
public class DistinctRowStream : ViewStream { public class DistinctRowStream : ViewStream
public DistinctRowStream(ViewStream stream, params string[] cols) { {
var g_by_eval = new DistinctRowEvaluator(cols); public DistinctRowStream (ViewStream stream, params string[] cols)
ViewStream r = new GroupByStream(stream, g_by_eval); {
var proj_exp = String.Empty; ArrayList groupByCols = new ArrayList ();
foreach (var c in cols) foreach (string col in cols)
proj_exp += $"{c} "; foreach (string c in col.Split (' '))
proj_exp = proj_exp.Trim(); groupByCols.Add (c);
r = new NewProjectStream(r, proj_exp); mySetRealSource (new GroupByStream (stream,
mySetRealSource(r); new DistinctRowEvaluator (groupByCols.ToArray (typeof(string)) as string[]), false));
} }
} }
} }
@@ -0,0 +1,49 @@
// using System;
// using System.Collections.Generic;
// using System.Text;
// using System.IO;
// using System.Runtime.Serialization.Formatters.Binary;
// namespace BCG
// {
// public class DumpLoaderStream : ViewStream, ISideEffectStream
// {
// //wtf
// //public class Exception : System.Exception
// //{public Exception(string msg) : base(msg) { }}
// public DumpLoaderStream(string fileName, int secondsBetweenPolls)
// {
// mySetRealSource(
// new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls,
// ViewStreamDatabase.instance));
// }
// public DumpLoaderStream(string fileName)
// {
// if (fileName == null)
// throw new LoaderStream.Exception("File name cannot be null");
// for (int i = 0; i < 5; i++)
// try
// {
// using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
// {
// IClientDeserializer deser = new ClientUniversalDeserializer();
// byte[] the_ops;
// Array the_rows;
// deser.deserialize(new Stream[] { fs }, out the_ops, out the_rows);
// mySetRealSource(new BlueRowCollectionStaticStream((IBlueRowCollection)the_rows.GetValue(0)));
// }
// break;
// }
// catch (IOException)
// {
// System.Threading.Thread.Sleep(50);
// }
// }
// }
// }
@@ -1,25 +1,48 @@
using BCG; using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
namespace bts { using System.Runtime.Serialization.Formatters.Binary;
// This is the other side of the SaverStream, its "loads" the stream and fires once
namespace BCG
{
[DataStreamUserRespresentation("loader")] [DataStreamUserRespresentation("loader")]
public class LoaderStream : ViewStream { public class LoaderStream : ViewStream, ISideEffectStream
{
public class Exception : System.Exception
{public Exception(string msg) : base(msg) { }}
private string _filename; public LoaderStream(string fileName, int secondsBetweenPolls)
private IStreamMsgDeserilizer _deser = new StreamIODeserializationController(); {
mySetRealSource(
new PollStream(() => new LoaderStream(fileName).rows, secondsBetweenPolls,
public LoaderStream(string filename) { ViewStreamDatabase.instance));
_filename = filename;
} }
protected override void myOnFirstSubscriber() { public LoaderStream(string fileName)
byte[] ser_msg = StreamSerializationUtils.read_from_file(_filename); {
StreamIOMessage msg = _deser.deserialize(ser_msg); if (fileName == null)
Log.assert(msg.op == StreamIOMessage.StreamOp.CREATE); throw new LoaderStream.Exception("File name cannot be null");
IBlueRowCollection rows = msg.create_message.rows;
DataPacket dp = BlueDataUtility.build_dp_inc(rows, myLastValue as DataPacket); for (int i = 0; i < 5; i++)
myFireEvent(dp); try
{
using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read))
{
BinaryFormatter bf = new BinaryFormatter();
#pragma warning disable SYSLIB0011
BlueTable bt = (BlueTable)bf.Deserialize(fs);
#pragma warning restore SYSLIB0011
mySetRealSource(new BlueRowCollectionStaticStream(bt));
}
break;
}
catch (IOException)
{
System.Threading.Thread.Sleep(50);
}
} }
} }
} }
@@ -5,11 +5,9 @@ using System.Text;
using System.IO; using System.IO;
using Google.Protobuf; using Google.Protobuf;
using System.Globalization; using System.Globalization;
using BCG;
namespace bts { namespace BCG {
// Writes all the udpates to a stream in a file or multiples files dependong on the rollover size (as opposed to Saver // Writes all the udpates to a stream in a file.
// which just writes one update, as a create)
// File naming: // File naming:
// token_epoch_yyyyMMdd_HHmmss // token_epoch_yyyyMMdd_HHmmss
// File location: directory(passed)\token\ // File location: directory(passed)\token\
@@ -17,16 +15,11 @@ namespace bts {
// data: // data:
// epochtime, size, serialized data // epochtime, size, serialized data
[DataStreamUserRespresentation("pers")] [DataStreamUserRespresentation("pers")]
// Saves each udpate of a stream into a file. The token is the the unique logical name of the stream,
// which maps the name of the directory where the streams are saved. The directory (3rd arg) is the
// parent directory where the token folder is stored. Each file goes up to rolloevr_size after which a new
// file is created. The file names are token_epoch_datetime
// the first entry in a file is the complete state
public class PersistorStream : ViewStream { public class PersistorStream : ViewStream {
private ViewStream _source_stream; private ViewStream _source_stream;
private string _token; private string _token;
private FileStream _out_fs; private FileStream _out_fs;
private IStreamIOMsgSerializer _serializer;// = new UniversalStreamIOSerializer(); private IBlueDataSerializer _serializer;
private string _file_name = String.Empty; private string _file_name = String.Empty;
private long _rollover_size; private long _rollover_size;
private readonly string _directory_path; private readonly string _directory_path;
@@ -95,19 +88,20 @@ namespace bts {
void _on_change(object new_value) { void _on_change(object new_value) {
DataPacket dp = new_value as DataPacket; DataPacket dp = new_value as DataPacket;
byte[] serialized_update = null; byte[] serialized_update = null;
bool first_write = _serializer == null || _n_updates_in_current_file == 0; bool _write_create = _serializer == null || _n_updates_in_current_file == 0;
_serializer = _serializer ?? new UniversalStreamIOSerializer(); _serializer = _serializer ?? new UniversalSerializer(dp.schema);
if (first_write) { if (_write_create) {
// first file always starts with create // new files always starts with create
serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.CreateMessage(dp.newRows))); _serializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out serialized_update);
} else } else
serialized_update = _serializer.serialize(new bts.StreamIOMessage(new bts.StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows))); _serializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out serialized_update);
BinaryWriter bw = new BinaryWriter(_out_fs); BinaryWriter bw = new BinaryWriter(_out_fs);
long utc_epoch = epoch_now(); long utc_epoch = epoch_now();
//time, length of data, data //time, length of data, data
//bw.Write(utc_epoch); bw.Write(utc_epoch);
//bw.Write(serialized_update.Length); bw.Write(serialized_update.Length);
bw.Write(serialized_update); bw.Write(serialized_update);
_out_fs.Flush(); _out_fs.Flush();
myFireEvent(dp); myFireEvent(dp);
@@ -0,0 +1,190 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using System.Runtime.Serialization.Formatters.Binary;
using BCG.Utilities.General;
using bts.collections;
using System.Reflection.Metadata;
namespace BCG {
public class PlaybackStream : ViewStream, IWorker {
private readonly PlaybackType _type;
private BlueTable _current_table;
private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
// these are the packets already downloaded from the current file
private List<PlaybackPacket> _packets = new List<PlaybackPacket>();
//private IEnumerable<FileStream> _file_streams = new List<FileStream>();
private IEnumerable<string> _file_names;
private Tuple<int, FileStream> _current_fs = new Tuple<int, FileStream>(-1, null);
private DateTime _playback_t0, _playback_t1;
private bool _all_files_read = false;
SnapshotData _snapshot_data = null; // only initialized when in snapshot mode
public PlaybackStream(string datetime_start, string datetime_end, string directory_name, string token) {
_type = PlaybackType.Streaming;
_playback_t0 = PersistorStream.parse_datetime(datetime_start);
_playback_t1 = PersistorStream.parse_datetime(datetime_end);
string token_directory_name = $"{directory_name}\\{token}";
bts.Log.Gen.info($"PlaybackStream t0: {_playback_t0} t1: {_playback_t1} dir: {token_directory_name}");
List<string> files_in_range = new List<string>();
if (Directory.Exists(token_directory_name)) {
_file_names =
Directory.GetFiles(token_directory_name).Where(f => {
DateTime file_time = PersistorStream.parse_datetime_from_filename(f);
return file_time > _playback_t0 && file_time < _playback_t1;
});
}
}
protected override void myOnFirstSubscriber() {
FileStream fs = new FileStream(_snapshot_data.snapshot_file, FileMode.Open, FileAccess.Read);
_snapshot_data.packets = StreamPersistorUtils.load_packets_from_file(fs);
PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_snapshot_data.packets, _snapshot_data.snapshot_time);
if (snapshot_packet != null) {
foreach (PlaybackPacket p in _snapshot_data.packets) {
byte[] ops;
Array row_array;
_deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
for (int i = 0; i < ops.Length; i++) {
IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
byte op = ops[i];
if (op == StreamOpCode.create) {
IBlueRowCollection full_table = rows_args[0];
_current_table = new BlueTable(full_table);
} else if (op == StreamOpCode.multipleRemoveAdd) {
//IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
BlueDataUtility.absorbChanges(_current_table, dels, adds);
} else {
bts.Log.assert(false, $"Oprator: {op} is not expected");
}
if (p.idx == snapshot_packet.idx) {
myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
break;
}
}
}
}
}
private void _load_packets_from_current_file() {
int curr_idx = _current_fs.Item1;
FileStream curr_fs = _current_fs.Item2;
_packets = new List<PlaybackPacket>();
_packets.AddRange(StreamPersistorUtils.load_packets_from_file(curr_fs));
}
private bool current_file_all_replayed {
get {
bool rv = true;
_packets.ForEach(p => rv = rv && p.replayed);
return rv;
}
}
protected override void myOnLastSubscriber() {
var fs = _current_fs.Item2;
if (fs != null) {
fs.Close();
}
ViewStreamDatabase.instance.removeWorker(this);
}
// public event PlaybackStreamProgress onProgress;
// public event Action onFileExhausted;
private PlaybackPacket _locate_packet_to_replay() {
foreach (var p in _packets) {
// returns the first non replayed packet
if (!p.replayed)
return p;
}
else
{
double secondsElapsed = new BinaryReader(myFileReader).ReadDouble();
myNextPacketTime = myPlaybackStart + new TimeSpan(0, 0, 0, 0, (int)(1000 * secondsElapsed / myPlaybackSpeed));
if (onProgress != null)
onProgress(myFileReader.Position, myFileReader.Length);
}
}
#region IWorker Members
public void doWork() {
if (_all_files_read)
return;
if (current_file_all_replayed) {
_load_next_file();
}
PlaybackPacket curr_packet = _locate_packet_to_replay();
if (curr_packet.idx == 0) { // first packet in the file, fire it rightaway
byte[] ops;
Array row_array;
// what a stupid interface!
_deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
_fire_packet(ops, row_array);
curr_packet.replay_timestamp = PersistorStream.epoch_now();
} else {
PlaybackPacket prev_packet = _get_previous_packet(curr_packet);
// this is the time we need to fire (replay) this packet
long time_to_fire = prev_packet.replay_timestamp + curr_packet.offset_from_previous;
long now_epoch = PersistorStream.epoch_now();
// fire when we are past the
if (now_epoch >= time_to_fire) {
byte[] ops;
Array row_array;
_deserilializer.deserialize(new Stream[] { new MemoryStream(curr_packet.data) }, out ops, out row_array);
_fire_packet(ops, row_array);
curr_packet.replay_timestamp = PersistorStream.epoch_now();
}
}
}
private void _fire_packet(byte[] ops, Array row_array) {
for (int i = 0; i < ops.Length; i++) {
IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
byte op = ops[i];
if (op == StreamOpCode.create) {
IBlueRowCollection full_table = rows_args[0];
_current_table = new BlueTable(full_table);
//_current_table.addRows(full_table);
myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
(myLastValue as DataPacket).newRows));
} else if (op == StreamOpCode.multipleRemoveAdd) {
//IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
BlueDataUtility.absorbChanges(_current_table, dels, adds);
myFireEvent(new DataPacket(_current_table, adds, dels));
} else {
bts.Log.assert(false, $"Oprator: {op} is not expected");
}
}
}
private void _load_next_file() {
var new_idx = _current_fs.Item1 + 1;
if (new_idx >= _file_names.Count()) {
_all_files_read = true;
bts.Log.Gen.info("All files read");
return;
}
var prev_fs = _current_fs.Item2;
if (prev_fs != null)
prev_fs.Close();
var new_filename = _file_names.ElementAt(new_idx);
bts.Log.Gen.info("PlaybackStream_load_next_file new_filename=: {new_filename}");
var new_fs = new FileStream(new_filename, FileMode.Open, FileAccess.Read);
_current_fs = new Tuple<int, FileStream>(new_idx, new_fs);
_load_packets_from_current_file();
}
#endregion
}
}
@@ -0,0 +1,68 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
namespace BCG
{
public class RecorderStream : ViewStream
{
private ViewStream mySource;
private DateTime myStartTime;
private FileStream myOutFile;
private string myFileName;
private IBlueDataSerializer mySerializer;
public RecorderStream(ViewStream src, string recordFile)
{
mySource = src;
myFileName = recordFile;
myFileName = myFileName.Replace("%DATE%", DateTime.Now.ToString("yyyyMMdd_HHmmss"));
}
protected override void myOnFirstSubscriber()
{
myStartTime = DateTime.Now;
mySource.onChange += new ChangeDelegate(mySource_onChange);
myOutFile = new FileStream(myFileName, FileMode.OpenOrCreate);
}
protected override void myOnLastSubscriber()
{
mySource.onChange -= mySource_onChange;
myOutFile.Close();
}
void mySource_onChange(object newValue)
{
DataPacket dp = (DataPacket)newValue;
BlueTable toWrite = new BlueTable(dp.schema);
if (mySerializer == null)
{
mySerializer = new UniversalSerializer(dp.schema);
}
foreach (IBlueRow r in dp.newRows)
toWrite.addRow(r);
byte[] result;
if (myLastValue == null)
mySerializer.serialize(StreamOpCode.create, new IBlueRowCollection[] { dp.newRows }, out result);
else
mySerializer.serialize(StreamOpCode.multipleRemoveAdd, new IBlueRowCollection[] { dp.deletedRows, dp.addedRows }, out result);
BinaryWriter bw = new BinaryWriter(myOutFile);
//write down what time this happened
bw.Write((DateTime.Now - myStartTime).TotalSeconds);
bw.Write(result.Length);
bw.Write(result);
myOutFile.Flush();
myFireEvent(dp);
}
}
}
@@ -1,37 +1,64 @@
using System; // using System;
using System.Collections.Generic; // using System.Collections.Generic;
using System.Text; // using System.Text;
using System.IO; // using System.IO;
using BCG;
namespace bts { // using System.Runtime.Serialization.Formatters.Binary;
// this saves the updates as a Create on the same file, each update calls -save, which
// reqrites the whole data to the packet. The other side of this stream is the Loader
[DataStreamUserRespresentation("saver")]
public class SaverStream : ViewStream {
private ViewStream _stream;
private string _filename;
private IStreamIOMsgSerializer _serializer = new StreamIOSerializationController();
public SaverStream(ViewStream vs, string filename) { // namespace BCG
_stream = vs; // {
_filename = filename; // [DataStreamUserRespresentation("saver")]
} // public class SaverStream : ViewStream, ISideEffectStream
// {
// private ViewStream myStream;
// private string myFileName;
protected override void myOnFirstSubscriber() { _stream.onChange += new ChangeDelegate(_handle_changes); } // public class Exception : System.Exception
// {
// public Exception(string msg) : base(msg) { }
// }
void _handle_changes(object dp_obj) { // public SaverStream(ViewStream vs, string fileName)
DataPacket dp = dp_obj as DataPacket; // {
_save(dp); // if (fileName == null)
myFireEvent(dp_obj); // throw new SaverStream.Exception("File name cannot be null");
} // myStream = vs;
// myFileName = fileName;
protected override void myOnLastSubscriber() { _stream.onChange -= new ChangeDelegate(_handle_changes); } // }
protected void _save(DataPacket dp) { // protected override void myOnFirstSubscriber()
StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows); // { myStream.onChange += new ChangeDelegate(myHandleChanges); }
byte[] str_bytes = _serializer.serialize(new StreamIOMessage(cr_msg));
StreamSerializationUtils.write_to_file(str_bytes, _filename); // void myHandleChanges(object newValue)
} // {
} // DataPacket dp = newValue as DataPacket;
} // mySave(dp);
// myFireEvent(newValue);
// }
// protected override void myOnLastSubscriber()
// { myStream.onChange -= new ChangeDelegate(myHandleChanges); }
// protected void mySave(DataPacket dp)
// {
// for (int i = 0; i < 5 ; i++)
// try
// {
// using (FileStream fs = new FileStream(myFileName, FileMode.Create, FileAccess.Write))
// {
// BinaryFormatter bf = new BinaryFormatter();
// BlueTable t = new BlueTable(dp.schema);
// foreach (IBlueRow r in dp.newRows) // to guarantee table is serializable
// t.addRow(new BlueRow(r));
// bf.Serialize(fs, t);
// }
// break;
// }
// catch (IOException)
// {
// System.Threading.Thread.Sleep(5);
// }
// }
// }
// }
@@ -1,129 +1,115 @@
// using System; using System;
// using System.Collections.Generic; using System.Collections.Generic;
// using System.Linq; using System.Linq;
// using System.Text; using System.Text;
// using System.IO; using System.IO;
// using System.Runtime.Serialization.Formatters.Binary; using System.Runtime.Serialization.Formatters.Binary;
// using BCG.Utilities.General; using BCG.Utilities.General;
// using bts.collections; using bts.collections;
// using System.Reflection.Metadata; using System.Reflection.Metadata;
namespace BCG { namespace BCG {
public class SnapshotStream : ViewStream { public class SnapshotStream : ViewStream {
public enum SpecificSnapshot {
FirstOfDay,
LastOfDay
};
// private BlueTable _current_table;
// private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
// public DateTime _snapshot_time;
// public string _snapshot_file;
// public IEnumerable<PlaybackPacket> _packets;
// public SnapshotStream(string snapshot_date, string specific_snapshot, string directory_name, string token) {
// DateTime dt;
// var parsed = DateTime.TryParse(snapshot_date, out dt);
// dt = dt.Date;
// bts.Log.assert(parsed);
// SpecificSnapshot specific_snapshot_enum;
// parsed = Enum.TryParse<SpecificSnapshot>(specific_snapshot, out specific_snapshot_enum);
// bts.Log.assert(parsed);
// string snapshot_dt = StreamPersistorUtils.find_snapshot_file($"{directory_name}\\{token}", dt, specific_snapshot_enum);
// ViewStream rs = new SnapshotStream(snapshot_dt, directory_name, token);
// mySetRealSource(rs);
// }
// public SnapshotStream(string snapshot_dt, string directory_name, string token) { private BlueTable _current_table;
// _snapshot_time = PersistorStream.parse_datetime(snapshot_dt); private IClientDeserializer _deserilializer = new ClientUniversalDeserializer();
// string token_directory_name = $"{directory_name}\\{token}"; public DateTime _snapshot_time;
// _snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time); public string _snapshot_file;
// } public IEnumerable<PlaybackPacket> _packets;
// protected override void myOnFirstSubscriber() {
// FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read);
// _packets = StreamPersistorUtils.load_packets_from_file(fs);
// PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time);
// if (snapshot_packet != null) {
// foreach (PlaybackPacket p in _packets) {
// byte[] ops;
// Array row_array;
// _deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
// for (int i = 0; i < ops.Length; i++) {
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
// byte op = ops[i];
// if (op == StreamOpCode.create) {
// IBlueRowCollection full_table = rows_args[0];
// _current_table = new BlueTable(full_table);
// } else if (op == StreamOpCode.multipleRemoveAdd) {
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
// BlueDataUtility.absorbChanges(_current_table, dels, adds);
// } else {
// bts.Log.assert(false, $"Oprator: {op} is not expected");
// }
// if (p.idx == snapshot_packet.idx) {
// myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
// break;
// }
// }
// }
// }
// }
// private void _absorb_packet(byte[] ops, Array row_array) { public SnapshotStream(string snapshot_dt, string directory_name, string token) {
// for (int i = 0; i < ops.Length; i++) { _snapshot_time = PersistorStream.parse_datetime(snapshot_dt);
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[]; string token_directory_name = $"{directory_name}\\{token}";
// byte op = ops[i]; _snapshot_file = StreamPersistorUtils.find_snapshot_file(token_directory_name, _snapshot_time);
// if (op == StreamOpCode.create) { }
// IBlueRowCollection full_table = rows_args[0];
// _current_table = new BlueTable(full_table);
// } else if (op == StreamOpCode.multipleRemoveAdd) {
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
// BlueDataUtility.absorbChanges(_current_table, dels, adds);
// } else {
// bts.Log.assert(false, $"Operator: {op} is not expected");
// }
// }
// }
// protected override void myOnLastSubscriber() { protected override void myOnFirstSubscriber() {
// } FileStream fs = new FileStream(_snapshot_file, FileMode.Open, FileAccess.Read);
_packets = StreamPersistorUtils.load_packets_from_file(fs);
PlaybackPacket snapshot_packet = StreamPersistorUtils.find_packet_at_time(_packets, _snapshot_time);
if (snapshot_packet != null) {
foreach (PlaybackPacket p in _packets) {
byte[] ops;
Array row_array;
_deserilializer.deserialize(new Stream[] { new MemoryStream(p.data) }, out ops, out row_array);
for (int i = 0; i < ops.Length; i++) {
IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
byte op = ops[i];
if (op == StreamOpCode.create) {
IBlueRowCollection full_table = rows_args[0];
_current_table = new BlueTable(full_table);
} else if (op == StreamOpCode.multipleRemoveAdd) {
//IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
BlueDataUtility.absorbChanges(_current_table, dels, adds);
} else {
bts.Log.assert(false, $"Oprator: {op} is not expected");
}
if (p.idx == snapshot_packet.idx) {
myFireEvent(new DataPacket(_current_table, _current_table, new BlueTable(_current_table.schema)));
break;
}
}
}
}
}
// // public event SnapshotStream.Progress onProgress; private void _absorb_packet(byte[] ops, Array row_array) {
// // public event Action onFileExhausted; for (int i = 0; i < ops.Length; i++) {
// private PlaybackPacket _locate_packet_to_replay() { IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
// foreach (var p in _packets) { byte op = ops[i];
// // returns the first non replayed packet if (op == StreamOpCode.create) {
// if (!p.replayed) IBlueRowCollection full_table = rows_args[0];
// return p; _current_table = new BlueTable(full_table);
// } } else if (op == StreamOpCode.multipleRemoveAdd) {
// return null; //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
// } IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
BlueDataUtility.absorbChanges(_current_table, dels, adds);
} else {
bts.Log.assert(false, $"Operator: {op} is not expected");
}
}
}
// private void _fire_packet(byte[] ops, Array row_array) { protected override void myOnLastSubscriber() {
// for (int i = 0; i < ops.Length; i++) { }
// IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
// byte op = ops[i];
// if (op == StreamOpCode.create) {
// IBlueRowCollection full_table = rows_args[0];
// _current_table = new BlueTable(full_table);
// //_current_table.addRows(full_table);
// myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
// (myLastValue as DataPacket).newRows));
// } else if (op == StreamOpCode.multipleRemoveAdd) {
// //IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
// IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
// IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
// BlueDataUtility.absorbChanges(_current_table, dels, adds);
// myFireEvent(new DataPacket(_current_table, adds, dels));
// } else {
// bts.Log.assert(false, $"Oprator: {op} is not expected");
// }
// } // public event SnapshotStream.Progress onProgress;
// } // public event Action onFileExhausted;
private PlaybackPacket _locate_packet_to_replay() {
foreach (var p in _packets) {
// returns the first non replayed packet
if (!p.replayed)
return p;
}
return null;
}
private void _fire_packet(byte[] ops, Array row_array) {
for (int i = 0; i < ops.Length; i++) {
IBlueRowCollection[] rows_args = row_array.GetValue(i) as IBlueRowCollection[];
byte op = ops[i];
if (op == StreamOpCode.create) {
IBlueRowCollection full_table = rows_args[0];
_current_table = new BlueTable(full_table);
//_current_table.addRows(full_table);
myFireEvent(new DataPacket(full_table, full_table, myLastValue == null ? new BlueTable(full_table.schema) :
(myLastValue as DataPacket).newRows));
} else if (op == StreamOpCode.multipleRemoveAdd) {
//IBlueRowCollection[] del_added = rows_args.GetValue(0) as IBlueRowCollection[];
IBlueRowCollection dels = rows_args.GetValue(0) as IBlueRowCollection;
IBlueRowCollection adds = rows_args.GetValue(1) as IBlueRowCollection;
BlueDataUtility.absorbChanges(_current_table, dels, adds);
myFireEvent(new DataPacket(_current_table, adds, dels));
} else {
bts.Log.assert(false, $"Oprator: {op} is not expected");
}
}
}
} }
} }
@@ -3,7 +3,6 @@ using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Globalization; using System.Globalization;
using System.IO; using System.IO;
using bts;
namespace BCG { namespace BCG {
@@ -38,20 +37,6 @@ namespace BCG {
public IEnumerable<PlaybackPacket> packets = new List<PlaybackPacket>(); public IEnumerable<PlaybackPacket> packets = new List<PlaybackPacket>();
} }
public static class StreamPersistorUtils { public static class StreamPersistorUtils {
public static string find_snapshot_file(string directory_name, DateTime snapshot_date, SnapshotStream.SpecificSnapshot specific_snapshot) {
DateTime max_dt = DateTime.MinValue;
string rv = String.Empty;
string[] file_names = Directory.GetFiles(directory_name);
for (int i = 0; i < file_names.Count() - 1; i++) {
string fn = file_names[i];
var dt = parse_datetime_from_filename(fn);
if (dt > max_dt) {
max_dt = dt;
rv = fn;
}
}
return rv;
}
public static string find_snapshot_file(string directory_name, DateTime snapshot_time) { public static string find_snapshot_file(string directory_name, DateTime snapshot_time) {
string[] file_names = Directory.GetFiles(directory_name); string[] file_names = Directory.GetFiles(directory_name);
for (int i = 0; i < file_names.Count() - 1; i++) { for (int i = 0; i < file_names.Count() - 1; i++) {
@@ -1,10 +1,16 @@
using BCG;
using System;
using System.Data;
using System.Collections;
using BCG.Utilities;
using System.Text.RegularExpressions;
using Bts.DataStream; using Bts.DataStream;
using mbts;
using mbts.spark;
using mbts.config; using mbts.config;
using System;
using System.Text.RegularExpressions;
namespace BCG.StreamWorld.Addition namespace BCG.StreamWorld.Addition
{ {
[DataStreamUserRespresentation (false)] [DataStreamUserRespresentation (false)]
@@ -145,6 +151,7 @@ namespace BCG
private string myFilterString; private string myFilterString;
private BCG.StreamWorld.Addition.FilterStream myFilterStream; private BCG.StreamWorld.Addition.FilterStream myFilterStream;
private bool myUseNewFilter = Configuration.get<DataStreamConfig>().UseNewFilter; private bool myUseNewFilter = Configuration.get<DataStreamConfig>().UseNewFilter;
private HighPerformanceTimer _timer = new HighPerformanceTimer();
#region constructors #region constructors
public delegate bool OnFilter(IBlueRow r); public delegate bool OnFilter(IBlueRow r);
@@ -219,6 +226,7 @@ namespace BCG
private void myHandleCallbacks(object newValue) private void myHandleCallbacks(object newValue)
{ {
_timer.start();
DataPacket retVal; DataPacket retVal;
DataPacket arrivedPacket = newValue as DataPacket; DataPacket arrivedPacket = newValue as DataPacket;
myLastReceivedRows = arrivedPacket.newRows; myLastReceivedRows = arrivedPacket.newRows;
@@ -247,6 +255,7 @@ namespace BCG
if (myFilter.filter(r)) if (myFilter.filter(r))
myLastFiredRows.addRow(r); myLastFiredRows.addRow(r);
} }
_timer.stop();
myFireEvent(retVal); myFireEvent(retVal);
} }
@@ -1,66 +1,37 @@
using System; using System;
using System.Text.RegularExpressions; using System.Collections.Generic;
using System.Linq;
using System.Text;
using BCG.Utilities.General; using BCG.Utilities.General;
namespace BCG { namespace BCG
public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable { {
public class TimeBasedFilter : IDynamicStreamFilter, IWorker, IDisposable
public static TimeSpan parse_duration(string input) { {
int days = 0, hours = 0, minutes = 0, seconds = 0;
// Regular expression to match the pattern (e.g., "1d", "1h", "15m", "45s")
Regex regex = new Regex(@"(\d+)([dhms])");
MatchCollection matches = regex.Matches(input);
foreach (Match match in matches) {
int value = int.Parse(match.Groups[1].Value);
string unit = match.Groups[2].Value;
switch (unit) {
case "d":
days = value;
break;
case "h":
hours = value;
break;
case "m":
minutes = value;
break;
case "s":
seconds = value;
break;
}
}
return new TimeSpan(days, hours, minutes, seconds);
}
#region constructors #region constructors
public TimeBasedFilter(string col_name, string row_time_to_live)
: this(col_name, parse_duration(row_time_to_live)) {
}
public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive) public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive)
: this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) { : this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */)
{
} }
public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) public TimeBasedFilter(string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency)
: this(ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency) { : this (ViewStreamDatabase.instance, rowTimeColumnName, rowTimeToLive, filterFrequency)
{
} }
public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive) public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive)
: this(vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */) { : this (vsd, rowTimeColumnName, rowTimeToLive, new TimeSpan(0, 1, 0) /* 1 minute */)
{
} }
public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency) { public TimeBasedFilter(ViewStreamDatabase vsd, string rowTimeColumnName, TimeSpan rowTimeToLive, TimeSpan filterFrequency)
myVsd = vsd; {
_time_col = rowTimeColumnName; myVsd = vsd;
_ttl = rowTimeToLive; myRowTimeColumnName = rowTimeColumnName;
myFilterFrequency = filterFrequency; myRowTimeToLive = rowTimeToLive;
myNextFilter = DateTime.MinValue; myFilterFrequency = filterFrequency;
myNextFilter = DateTime.MinValue;
// add worker // add worker
myVsd.addWorker(this); myVsd.addWorker(this);
@@ -70,25 +41,32 @@ namespace BCG {
public event FilterChangeDelegate onFilterChange; public event FilterChangeDelegate onFilterChange;
#region IStreamFilter #region IStreamFilter
public bool filter(IBlueRow row) { public bool filter(IBlueRow row)
DateTime now = DateTime.Now; {
DateTime now = DateTime.Now;
DateTime timeStamp = (DateTime)row[myRowTimeColumnName];
DateTime row_time = row.get_datetime(_time_col, DateTime.Now); if (now > timeStamp)
if (now > row_time) { {
TimeSpan span = now - row_time; TimeSpan span = now - timeStamp;
if (span >= _ttl) {
if (span >= myRowTimeToLive)
{
return (false); return (false);
} }
} }
return (true); return (true);
} }
#endregion #endregion
#region IWorker #region IWorker
public void doWork() { public void doWork()
{
DateTime now = DateTime.Now; DateTime now = DateTime.Now;
if (now > myNextFilter) { if (now > myNextFilter)
{
// fire event // fire event
onFilterChange(); onFilterChange();
@@ -98,7 +76,8 @@ namespace BCG {
#endregion #endregion
#region IDisposable #region IDisposable
public void Dispose() { public void Dispose()
{
// remove worker // remove worker
myVsd.removeWorker(this); myVsd.removeWorker(this);
} }
@@ -106,20 +85,10 @@ namespace BCG {
#region state #region state
private ViewStreamDatabase myVsd; private ViewStreamDatabase myVsd;
private string _time_col; private string myRowTimeColumnName;
private TimeSpan _ttl; private TimeSpan myRowTimeToLive;
private TimeSpan myFilterFrequency; private TimeSpan myFilterFrequency;
private DateTime myNextFilter; private DateTime myNextFilter;
#endregion #endregion
} }
[DataStreamUserRespresentation("timefilter")]
public class TimeBasedFilterStream : ViewStream {
public TimeBasedFilterStream(ViewStream base_stream, string time_col, string time_span)
:
this(ViewStreamDatabase.instance, base_stream, time_col, TimeBasedFilter.parse_duration(time_span)) { }
public TimeBasedFilterStream(ViewStreamDatabase vsd, ViewStream base_stream, string time_col, TimeSpan time_span) {
TimeBasedFilter filter = new TimeBasedFilter(vsd, time_col, time_span);
mySetRealSource(new FilterStream(base_stream, filter));
}
}
} }
@@ -4,14 +4,18 @@ using BCG.Utilities;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
namespace BCG { namespace BCG
public class GroupByStream : MethodDrivenStream { {
private class MyFuncGroupByEvaluator : IGroupByEvaluator { public class GroupByStream : MethodDrivenStream
{
private class MyFuncGroupByEvaluator : IGroupByEvaluator
{
private string[] _groupby_cols; private string[] _groupby_cols;
private Func<DataPacket, IBlueRow> _groupby_func; private Func<DataPacket, IBlueRow> _groupby_func;
public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func) { public MyFuncGroupByEvaluator(IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
{
_groupby_cols = groupby_cols.ToArray(); _groupby_cols = groupby_cols.ToArray();
_groupby_func = groupby_func; _groupby_func = groupby_func;
} }
public IBlueRow evaluate(DataPacket packet) { return _groupby_func(packet); } public IBlueRow evaluate(DataPacket packet) { return _groupby_func(packet); }
@@ -25,7 +29,7 @@ namespace BCG {
/// Serves as a quick way to access all rows belonging to the same group /// Serves as a quick way to access all rows belonging to the same group
/// </summary> /// </summary>
private bool myIdGuard = true; private bool myIdGuard = true;
private RowDictionaryTable myBookKeepingTable; private RowDictionaryTable myBookKeepingTable ;
private IGroupByEvaluator myGroupByEvaluator; private IGroupByEvaluator myGroupByEvaluator;
private IBlueTableSchema mySchema; //input schema, as output schema can differ (IGroupByEvaluator's private IBlueTableSchema mySchema; //input schema, as output schema can differ (IGroupByEvaluator's
//can map a schema to another //can map a schema to another
@@ -33,98 +37,112 @@ namespace BCG {
private int myIdGenerator = 0; private int myIdGenerator = 0;
public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func) public GroupByStream(ViewStream stream, IEnumerable<string> groupby_cols, Func<DataPacket, IBlueRow> groupby_func)
: this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func)) { } : this(stream, new MyFuncGroupByEvaluator(groupby_cols, groupby_func))
{ }
public GroupByStream(ViewStream stream, IGroupByEvaluator groupByEvaluator) public GroupByStream (ViewStream stream, IGroupByEvaluator groupByEvaluator)
: this(stream, groupByEvaluator, true) { } : this (stream, groupByEvaluator, true)
{}
public GroupByStream(ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard) public GroupByStream (ViewStream stream, IGroupByEvaluator groupbyEvaluator, bool idGuard)
: base(stream) { : base (stream)
{
myGroupByEvaluator = groupbyEvaluator; myGroupByEvaluator = groupbyEvaluator;
myIdGuard = idGuard; myIdGuard = idGuard;
} }
protected override void myHandleFirstCall() { protected override void myHandleFirstCall ()
{
mySchema = myLastArrivedPacket.schema; mySchema = myLastArrivedPacket.schema;
myOutputSchema = myGroupByEvaluator.getOutputSchema(mySchema); myOutputSchema = myGroupByEvaluator.getOutputSchema (mySchema);
myBookKeepingTable = new RowDictionaryTable(myLastArrivedPacket.newRows, myBookKeepingTable = new RowDictionaryTable (myLastArrivedPacket.newRows,
myGroupByEvaluator.groupByColumns); myGroupByEvaluator.groupByColumns);
GroupByOutputTable temp = new GroupByOutputTable(myOutputSchema); GroupByOutputTable temp = new GroupByOutputTable (myOutputSchema);
foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups()) { foreach (IBlueRowCollection rowGroup in myBookKeepingTable.getGroups())
IBlueRow newRow = myEvaluate(new DataPacket(rowGroup, {
rowGroup, new BlueTable(mySchema))); IBlueRow newRow = myEvaluate (new DataPacket (rowGroup,
rowGroup, new BlueTable (mySchema)));
if (newRow != null) if (newRow != null)
temp.addRow(newRow, myGetGroupString(rowGroup)); temp.addRow (newRow, myGetGroupString (rowGroup));
} }
myLastFiredPacket = new DataPacket(temp, new BlueTable(temp), new BlueTable(temp.schema)); myLastFiredPacket = new DataPacket (temp, new BlueTable(temp), new BlueTable (temp.schema));
} }
/// <summary> /// <summary>
/// Adding need not to anything as deletion already takes care /// Adding need not to anything as deletion already takes care
/// of addition /// of addition
/// </summary> /// </summary>
protected override void myHandleIncrementalAdd() { } protected override void myHandleIncrementalAdd() {}
/// <summary> /// <summary>
/// Both deletes and adds are handled here to make the intermediate results not /// Both deletes and adds are handled here to make the intermediate results not
/// appear in the spitted fired DataPacket (note this is done only in GroupByStream /// appear in the spitted fired DataPacket (note this is done only in GroupByStream
/// because of the nature of it) /// because of the nature of it)
/// </summary> /// </summary>
protected override void myHandleIncrementalDelete() { protected override void myHandleIncrementalDelete()
{
myFireSwitch = false; myFireSwitch = false;
RowDictionaryTable delGroups, addGroups; RowDictionaryTable delGroups, addGroups;
//form groups for deleted and added rows //form groups for deleted and added rows
myInitIncrementalChangeData(out delGroups, out addGroups); myInitIncrementalChangeData (out delGroups, out addGroups);
ArrayList handledAddGroups = new ArrayList(); ArrayList handledAddGroups = new ArrayList ();
string groupRepString; string groupRepString;
foreach (IBlueRowCollection delGroup in delGroups.getGroups()) { foreach (IBlueRowCollection delGroup in delGroups.getGroups ())
{
// BlueEventLog.logAssert (delGroup.Count > 0); // BlueEventLog.logAssert (delGroup.Count > 0);
groupRepString = myGetGroupString(delGroup); groupRepString = myGetGroupString (delGroup);
IBlueRowCollection addGroup = addGroups.getGroup(groupRepString) == null ? IBlueRowCollection addGroup = addGroups.getGroup (groupRepString) == null ?
new BlueTable(mySchema) : addGroups.getGroup(groupRepString); new BlueTable (mySchema) : addGroups.getGroup (groupRepString);
handledAddGroups.Add(addGroup); handledAddGroups.Add (addGroup);
IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString) == null ? IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString) == null ?
new BlueTable(mySchema) : myBookKeepingTable.getGroup(groupRepString); new BlueTable (mySchema) : myBookKeepingTable.getGroup (groupRepString);
IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup, delGroup)); IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup, delGroup));
myFireSwitch |= myMergeRow(newRow, groupRepString); myFireSwitch |= myMergeRow (newRow, groupRepString);
} }
foreach (IBlueRowCollection addGroup in addGroups.getGroups()) { foreach (IBlueRowCollection addGroup in addGroups.getGroups ())
{
// BlueEventLog.logAssert (addGroup.Count > 0); // BlueEventLog.logAssert (addGroup.Count > 0);
if (!handledAddGroups.Contains(addGroup)) { if (!handledAddGroups.Contains (addGroup))
{
//this is the case when there are rows added to a group and none //this is the case when there are rows added to a group and none
//have been removed, hence it 'escapes' from being dealt with in //have been removed, hence it 'escapes' from being dealt with in
//the loop above //the loop above
groupRepString = myGetGroupString(addGroup); groupRepString = myGetGroupString (addGroup);
IBlueRowCollection aggGroup = myBookKeepingTable.getGroup(groupRepString); IBlueRowCollection aggGroup = myBookKeepingTable.getGroup (groupRepString);
// BlueEventLog.logAssert (aggGroup != null && aggGroup.Count > 0); // BlueEventLog.logAssert (aggGroup != null && aggGroup.Count > 0);
IBlueRow newRow = myEvaluate(new DataPacket(aggGroup, addGroup, IBlueRow newRow = myEvaluate (new DataPacket (aggGroup, addGroup,
new BlueTable(aggGroup.schema))); new BlueTable (aggGroup.schema)));
myFireSwitch |= myMergeRow(newRow, groupRepString); myFireSwitch |= myMergeRow (newRow, groupRepString);
} }
} }
} }
private bool myMergeRow(IBlueRow newRow, string groupRepString) { private bool myMergeRow (IBlueRow newRow, string groupRepString)
IBlueRow existingRow = ((GroupByOutputTable)myLastFiredPacket.newRows).getRow(groupRepString); {
IBlueRow existingRow = ((GroupByOutputTable) myLastFiredPacket.newRows).getRow (groupRepString);
//BlueEventLog.logAssert (newRow != null || existingRow != null); //BlueEventLog.logAssert (newRow != null || existingRow != null);
if (existingRow == null && newRow == null) if (existingRow == null && newRow == null)
return false; return false;
else if (existingRow != null && newRow != null && BlueDataUtility.compute_equality(existingRow, newRow)) else if (existingRow != null && newRow != null && BlueDataUtility.computeRowEquality (existingRow, newRow))
return false; return false;
else if (existingRow == null) { else if (existingRow == null)
{
//row added for the first time //row added for the first time
((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString); ((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString);
((BlueTable)myLastFiredPacket.addedRows).addRow(newRow); ((BlueTable) myLastFiredPacket.addedRows).addRow (newRow);
return true; return true;
} else { }
((GroupByOutputTable)myLastFiredPacket.newRows).removeRow(groupRepString); else
((BlueTable)myLastFiredPacket.deletedRows).addRow(existingRow); {
if (newRow != null) { ((GroupByOutputTable) myLastFiredPacket.newRows).removeRow (groupRepString);
((GroupByOutputTable)myLastFiredPacket.newRows).addRow(newRow, groupRepString); ((BlueTable) myLastFiredPacket.deletedRows).addRow (existingRow);
((BlueTable)myLastFiredPacket.addedRows).addRow(newRow); if (newRow != null)
{
((GroupByOutputTable) myLastFiredPacket.newRows).addRow (newRow, groupRepString);
((BlueTable) myLastFiredPacket.addedRows).addRow (newRow);
} }
return true; return true;
} }
@@ -136,68 +154,77 @@ namespace BCG {
/// </summary> /// </summary>
/// <param name="delGroups"></param> /// <param name="delGroups"></param>
/// <param name="addGroups"></param> /// <param name="addGroups"></param>
private void myInitIncrementalChangeData(out RowDictionaryTable delGroups, out RowDictionaryTable addGroups) { private void myInitIncrementalChangeData (out RowDictionaryTable delGroups, out RowDictionaryTable addGroups)
{
delGroups = delGroups =
new RowDictionaryTable(myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns); new RowDictionaryTable (myLastArrivedPacket.deletedRows, myGroupByEvaluator.groupByColumns);
addGroups = addGroups =
new RowDictionaryTable(myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns); new RowDictionaryTable (myLastArrivedPacket.addedRows, myGroupByEvaluator.groupByColumns);
foreach (IBlueRow deletedRow in myLastArrivedPacket.deletedRows) foreach (IBlueRow deletedRow in myLastArrivedPacket.deletedRows)
myBookKeepingTable.removeRow(deletedRow); myBookKeepingTable.removeRow (deletedRow);
foreach (IBlueRow addedRow in myLastArrivedPacket.addedRows) foreach (IBlueRow addedRow in myLastArrivedPacket.addedRows)
myBookKeepingTable.addRow(addedRow); myBookKeepingTable.addRow (addedRow);
myLastFiredPacket.addedRows = new BlueTable(myOutputSchema); myLastFiredPacket.addedRows = new BlueTable (myOutputSchema);
myLastFiredPacket.deletedRows = new BlueTable(myOutputSchema); myLastFiredPacket.deletedRows = new BlueTable (myOutputSchema);
} }
protected override void myOnTableClear() { protected override void myOnTableClear ()
{
if (myBookKeepingTable.Count == 0) if (myBookKeepingTable.Count == 0)
return; //ie there arent any rows, so no need to optimize the 'cleansing' process return; //ie there arent any rows, so no need to optimize the 'cleansing' process
myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows); myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows);
myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema); myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
(myLastFiredPacket.newRows as GroupByOutputTable).clear(); (myLastFiredPacket.newRows as GroupByOutputTable).clear ();
myBookKeepingTable.clear(); myBookKeepingTable.clear ();
myGroupByEvaluator.clear(); myGroupByEvaluator.clear ();
} }
private IBlueRow myEvaluate(DataPacket dp) { private IBlueRow myEvaluate (DataPacket dp)
IBlueRow evalRow = myGroupByEvaluator.evaluate(dp); {
return evalRow == null ? null : IBlueRow evalRow = myGroupByEvaluator.evaluate (dp);
return evalRow == null ? null :
myIdGuard ? myIdGuard ?
BlueDataUtility.getNewReplacedRow(evalRow, myIdGenerator++, "id") : evalRow; BlueDataUtility.getNewReplacedRow (evalRow, myIdGenerator++, "id") : evalRow;
} }
private string myGetGroupString(IBlueRowCollection group) { private string myGetGroupString (IBlueRowCollection group)
return BlueDataUtility.rowToString(BlueDataUtility.extractRow(group, 0), myGroupByEvaluator.groupByColumns); {
return BlueDataUtility.rowToString(BlueDataUtility.extractRow (group, 0), myGroupByEvaluator.groupByColumns);
} }
protected override void myOnLastSubscriber() { protected override void myOnLastSubscriber ()
{
//the packets are initialzed to null, so that if the subscription count goes to 0, and then this stream //the packets are initialzed to null, so that if the subscription count goes to 0, and then this stream
//gets resubscribed, we start the process of building data again, and treat the callback as the first //gets resubscribed, we start the process of building data again, and treat the callback as the first
//callback. //callback.
myInit(); myInit ();
myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler); myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
} }
protected override void myInit() { protected override void myInit ()
base.myInit(); {
if (myBookKeepingTable != null) { base.myInit ();
if (myBookKeepingTable != null)
{
//to avoid calling init before the first callback //to avoid calling init before the first callback
myBookKeepingTable.clear(); myBookKeepingTable.clear ();
myGroupByEvaluator.clear(); myGroupByEvaluator.clear ();
} }
} }
public static string getRepString(DataPacket dp, string[] colNames) { public static string getRepString (DataPacket dp, string[] colNames)
return BlueDataUtility.rowToString(getRepRow(dp), colNames); {
return BlueDataUtility.rowToString (getRepRow (dp), colNames);
} }
public static IBlueRow getRepRow(DataPacket dp) { public static IBlueRow getRepRow (DataPacket dp)
{
if (dp.newRows.Count > 0) if (dp.newRows.Count > 0)
return BlueDataUtility.extractRow(dp.newRows, 0); return BlueDataUtility.extractRow (dp.newRows, 0);
else if (dp.addedRows.Count > 0) else if (dp.addedRows.Count > 0)
return BlueDataUtility.extractRow(dp.addedRows, 0); return BlueDataUtility.extractRow (dp.addedRows, 0);
else if (dp.deletedRows.Count > 0) else if (dp.deletedRows.Count > 0)
return BlueDataUtility.extractRow(dp.deletedRows, 0); return BlueDataUtility.extractRow (dp.deletedRows, 0);
// BlueEventLog.logAssert (false, "Illegal Data Packet"); // BlueEventLog.logAssert (false, "Illegal Data Packet");
return null; return null;
} }
@@ -210,21 +237,22 @@ namespace BCG {
/// <param name="newValue"></param> /// <param name="newValue"></param>
/// <param name="isAdd"></param> /// <param name="isAdd"></param>
/// <returns></returns> /// <returns></returns>
public static object incrementalAdd(object lastValue, object newValue, bool isAdd) { public static object incrementalAdd (object lastValue, object newValue, bool isAdd)
if (lastValue == null || lastValue.Equals(DBNull.Value)) {
if (lastValue == null || lastValue.Equals (DBNull.Value))
return newValue; return newValue;
// BlueEventLog.logAssert (lastValue.GetType().Equals (typeof(Int32)) && newValue.GetType().Equals (typeof(Int32)) || // BlueEventLog.logAssert (lastValue.GetType().Equals (typeof(Int32)) && newValue.GetType().Equals (typeof(Int32)) ||
// lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) || // lastValue.GetType().Equals (typeof(Double)) && newValue.GetType().Equals (typeof(Double)) ||
// lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal))); // lastValue.GetType().Equals (typeof(Decimal)) && newValue.GetType().Equals (typeof(Decimal)));
Object retVal = null; Object retVal = null;
if (newValue.GetType().Equals(typeof(int))) if (newValue.GetType().Equals (typeof(int)))
retVal = isAdd ? (int)lastValue + (int)newValue : (int)lastValue - (int)newValue; retVal = isAdd ? (int)lastValue + (int)newValue : (int)lastValue - (int)newValue;
else if (newValue.GetType().Equals(typeof(Double))) else if (newValue.GetType().Equals (typeof(Double)))
retVal = isAdd ? (Double)lastValue + (Double)newValue : (Double)lastValue - (Double)newValue; retVal = isAdd ? (Double)lastValue + (Double)newValue : (Double)lastValue - (Double)newValue;
else if (lastValue.GetType().Equals(typeof(Decimal))) else if (lastValue.GetType().Equals (typeof(Decimal)))
retVal = isAdd ? (Decimal)lastValue + (Decimal)newValue : (Decimal)lastValue - (Decimal)newValue; retVal = isAdd ? (Decimal)lastValue + (Decimal)newValue : (Decimal)lastValue - (Decimal)newValue;
else { } else{}
// BlueEventLog.logAssert (false, "Illegal type"); // BlueEventLog.logAssert (false, "Illegal type");
return retVal; return retVal;
} }
@@ -1,42 +1,47 @@
using System; // using System;
using System.Collections; // using System.Collections;
// using BCG.Utilities;
using BCG.Utilities; // namespace BCG
// {
// /// <summary>
// /// Computes the max for a given column
// /// </summary>
// public class MaxGroupByEvaluator : GroupByEvaluator
// {
namespace BCG { // public MaxGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols)
/// <summary> // {}
/// Computes the max for a given column
/// </summary>
public class MaxGroupByEvaluator : GroupByEvaluator {
public MaxGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { } // public MaxGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols)
// : base (colMapEpression, outputCol, groupByCols) {}
public MaxGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) // public override IBlueRow evaluate (DataPacket dp)
: base(colMapEpression, outputCol, groupByCols) { } // {
// myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) :
public override IBlueRow evaluate(DataPacket dp) { // myOutputTable;
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : // string hashString = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
myOutputTable; // IBlueRow currMaxRow = myOutputTable.getRow (hashString);
string hashString = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols); // IBlueRow maxRow = BlueDataUtility.computeMaxRow (dp.deletedRows, myOutputColumn);
IBlueRow currMaxRow = myOutputTable.getRow(hashString); // if (maxRow != null)
IBlueRow maxRow = BlueDataUtility.computeMaxRow(dp.deletedRows, myOutputColumn); // {
if (maxRow != null) { // BlueEventLog.logAssert (currMaxRow != null);
bts.Log.assert(currMaxRow != null); // BlueEventLog.logAssert (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) <= 0);
bts.Log.assert(BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) <= 0); // if (BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) == 0)
if (BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) == 0) { // {
maxRow = BlueDataUtility.computeMaxRow(dp.newRows, myOutputColumn); // maxRow = BlueDataUtility.computeMaxRow (dp.newRows, myOutputColumn);
if (maxRow == null) // if (maxRow == null)
myOutputTable.removeRow(hashString); // myOutputTable.removeRow (hashString);
else // else
myOutputTable.addRow(maxRow, hashString); // myOutputTable.addRow (maxRow, hashString);
} // }
} // }
maxRow = BlueDataUtility.computeMaxRow(dp.addedRows, myOutputColumn); // maxRow = BlueDataUtility.computeMaxRow (dp.addedRows, myOutputColumn);
// if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true, // // if the row was replaced in the above lines, then currMaxRow < maxRow, so the if below would never return a true,
// otherwise if maxRow > currMaxRow then its replaced // // otherwise if maxRow > currMaxRow then its replaced
if (maxRow != null && (currMaxRow == null || BlueDataUtility.compare_rows(maxRow, currMaxRow, myOutputColumn) > 0)) // if (maxRow != null && (currMaxRow == null || BlueDataUtility.compareRows (maxRow, currMaxRow, myOutputColumn) > 0))
myOutputTable.addRow(maxRow, hashString); // myOutputTable.addRow (maxRow, hashString);
return myProject(myOutputTable.getRow(hashString)); // return myProject (myOutputTable.getRow (hashString));
} // }
} // }
} // }
@@ -1,15 +1,20 @@
using System; // using System;
namespace BCG { // namespace BCG
public class MaxStream : ViewStream { // {
public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' ')) { // public class MaxStream : ViewStream
} // {
public MaxStream(ViewStream stream, string outputCol, string[] inputCols) { // public MaxStream(ViewStream stream, string outputCol, string inputCols) : this(stream, outputCol, inputCols.Split(' '))
mySetRealSource(new GroupByStream(stream, // {
new MaxGroupByEvaluator(outputCol, inputCols))); // }
} // public MaxStream (ViewStream stream, string outputCol, string[] inputCols)
// {
// mySetRealSource (new GroupByStream (stream,
// new MaxGroupByEvaluator (outputCol, inputCols)));
// }
public MaxStream(ViewStream stream, string outputCol) // public MaxStream (ViewStream stream, string outputCol)
: this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { } // : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"})
} // {}
} // }
// }
@@ -1,36 +1,40 @@
using System; // using System;
// using BCG.Utilities;
using BCG.Utilities; // namespace BCG
// {
// public class MinGroupByEvaluator : MaxGroupByEvaluator
// {
// public MinGroupByEvaluator (string outputCol, string[] groupByCols)
// : base (outputCol, groupByCols) {}
namespace BCG { // public MinGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols) :
public class MinGroupByEvaluator : MaxGroupByEvaluator { // base (colMapEpression, outputCol, groupByCols) {}
public MinGroupByEvaluator(string outputCol, string[] groupByCols)
: base(outputCol, groupByCols) { }
public MinGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) : // public override IBlueRow evaluate (DataPacket dp)
base(colMapEpression, outputCol, groupByCols) { } // {
// myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable;
// string stringRep = BlueDataUtility.rowToString (GroupByStream.getRepRow (dp), myGroupByCols);
// IBlueRow currMinRow = myOutputTable.getRow (stringRep);
// IBlueRow minRow = BlueDataUtility.computeMinRow (dp.deletedRows, myOutputColumn);
// if (minRow != null)
// {
// // BlueEventLog.logAssert (currMinRow != null);
// // BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
// if (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) == 0)
// {
// minRow = BlueDataUtility.computeMinRow (dp.newRows, myOutputColumn);
// if (minRow == null)
// myOutputTable.removeRow (stringRep);
// else
// myOutputTable.addRow (minRow, stringRep);
// }
// }
// minRow = BlueDataUtility.computeMinRow (dp.addedRows, myOutputColumn);
// if (minRow != null && (currMinRow == null || BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) < 0))
// myOutputTable.addRow (minRow, stringRep);
// return myOutputTable.getRow (stringRep);
// }
public override IBlueRow evaluate(DataPacket dp) { // }
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable; // }
string stringRep = BlueDataUtility.rowToString(GroupByStream.getRepRow(dp), myGroupByCols);
IBlueRow currMinRow = myOutputTable.getRow(stringRep);
IBlueRow minRow = BlueDataUtility.computeMinRow(dp.deletedRows, myOutputColumn);
if (minRow != null) {
// BlueEventLog.logAssert (currMinRow != null);
// BlueEventLog.logAssert (BlueDataUtility.compareRows (minRow, currMinRow, myOutputColumn) >= 0);
if (BlueDataUtility.compare_rows(minRow, currMinRow, myOutputColumn) == 0) {
minRow = BlueDataUtility.computeMinRow(dp.newRows, myOutputColumn);
if (minRow == null)
myOutputTable.removeRow(stringRep);
else
myOutputTable.addRow(minRow, stringRep);
}
}
minRow = BlueDataUtility.computeMinRow(dp.addedRows, myOutputColumn);
if (minRow != null && (currMinRow == null || BlueDataUtility.compare_rows(minRow, currMinRow, myOutputColumn) < 0))
myOutputTable.addRow(minRow, stringRep);
return myOutputTable.getRow(stringRep);
}
}
}
@@ -1,13 +1,17 @@
using System; // using System;
namespace BCG { // namespace BCG
public class MinStream : ViewStream { // {
public MinStream(ViewStream stream, string outputCol, string[] inputCol) { // public class MinStream : ViewStream
mySetRealSource(new GroupByStream(stream, // {
new MinGroupByEvaluator(outputCol, inputCol))); // public MinStream (ViewStream stream, string outputCol, string[] inputCol)
} // {
// mySetRealSource (new GroupByStream (stream,
// new MinGroupByEvaluator (outputCol, inputCol)));
// }
public MinStream(ViewStream stream, string outputCol) // public MinStream (ViewStream stream, string outputCol)
: this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, new string[] { "fakeCol" }) { } // : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, new string[]{"fakeCol"})
} // {}
} // }
// }
@@ -1,8 +1,10 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using System.Linq;
using System.Text;
using bts; using System.Threading.Tasks;
using mbts;
using mbts.spark;
namespace BCG namespace BCG
{ {
@@ -12,7 +14,7 @@ namespace BCG
private INonIncGroupByEvaluator _groupby_func; private INonIncGroupByEvaluator _groupby_func;
private BlueRowHasher _row_hasher; private BlueRowHasher _row_hasher;
private string[] _groupby_cols; private string[] _groupby_cols;
private Stopwatch _timer = new(); private HighPerformanceTimer _timer = new HighPerformanceTimer();
private int _n_total_fires = 0, _n_expensive_fires = 0; private int _n_total_fires = 0, _n_expensive_fires = 0;
public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func) public NonIncGroupByStream(ViewStream main_stream, INonIncGroupByEvaluator groupby_func)
@@ -31,27 +33,21 @@ namespace BCG
{ {
DataPacket dp = nv as DataPacket; DataPacket dp = nv as DataPacket;
///grouping together ///grouping together
_timer.Restart(); _timer.start();
IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher); IDictionary<int, List<IBlueRow>> grouped_rows = BlueDataUtility.group_together (dp.newRows, _groupby_cols, ref _row_hasher);
_timer.Stop(); _timer.stop();
var groupElapsedMs = _timer.ElapsedMilliseconds; // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
if (groupElapsedMs >= 50) {
Log.DataFlow.info($"Time to group {dp.newRows.Count} rows into {grouped_rows.Count} groups: {_timer.ElapsedMilliseconds}ms");
}
// Log.log(bts.Log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to group {0} rows into {1} groups = {2}", dp.newRows.Count, grouped_rows.Count, _timer.lastIntervalTime));
/// reducing /// reducing
_timer.Restart(); _timer.start();
IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows); IBlueRowCollection total_state = _groupby_func.evaluate(grouped_rows);
_timer.Stop(); _timer.stop();
var applyElapsedMs = _timer.ElapsedMilliseconds;
_n_total_fires++; _n_total_fires++;
if (applyElapsedMs >= 50) if (_timer.lastIntervalTime > 50)
{ {
_n_expensive_fires++; _n_expensive_fires++;
Log.DataFlow.info( // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("Time to evaluate {0} groups {1} , ratio {2}/{3}", grouped_rows.Count, _timer.lastIntervalTime, _n_expensive_fires, _n_total_fires));
$"Time to evaluate {grouped_rows.Count} groups {_timer.ElapsedMilliseconds}ms, ratio {_n_expensive_fires}/{_n_total_fires}");
} }
myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows)); myFireEvent(new DataPacket(total_state, total_state, myLastValue == null ? new BlueTable(total_state.schema) : (myLastValue as DataPacket).newRows));
} }
} }
} }
@@ -1,82 +1,94 @@
using System; // using System;
namespace BCG { // namespace BCG
/// <summary> // {
/// Computes the sum for a given column // /// <summary>
/// </summary> // /// Computes the sum for a given column
public class SumGroupByEvaluator : GroupByEvaluator { // /// </summary>
public SumGroupByEvaluator(string outputCol, string[] groupByCols) : base(outputCol, groupByCols) { } // public class SumGroupByEvaluator : GroupByEvaluator
// {
// public SumGroupByEvaluator (string outputCol, string[] groupByCols) : base (outputCol, groupByCols)
// {}
public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) // public SumGroupByEvaluator (string colMapEpression, string outputCol, string[] groupByCols)
: base(colMapEpression, outputCol, groupByCols) { } // : base (colMapEpression, outputCol, groupByCols) {}
public override IBlueRow evaluate(DataPacket dp) { // public override IBlueRow evaluate (DataPacket dp)
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable; // {
// myOutputTable = myOutputTable == null ? new GroupByOutputTable (dp.schema) : myOutputTable;
IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
return sumRow; // IBlueRow sumRow = BlueDataUtility.computeSum(dp.newRows, myOutputColumn);
} // return sumRow;
} // }
// }
public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator { // public class MultiColumnSumGroupByEvaluator : IGroupByEvaluator
private string[] myOutputCols, myGroupByCols; // {
public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) { // private string[] myOutputCols, myGroupByCols;
myOutputCols = spaceSeperatedOutputCols.Split(' '); // public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols)
myGroupByCols = spaceSeperatedGroupByCols.Split(' '); // {
} // myOutputCols = spaceSeperatedOutputCols.Split(' ');
// myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
// }
public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns) { // public MultiColumnSumGroupByEvaluator(string[] output_columns, string[] groupby_columns)
myOutputCols = output_columns; // {
myGroupByCols = groupby_columns; // myOutputCols = output_columns;
} // myGroupByCols = groupby_columns;
// }
#region IGroupByEvaluator Members // #region IGroupByEvaluator Members
public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum(packet.newRows, myOutputCols); } // public IBlueRow evaluate(DataPacket packet) {return BlueDataUtility.computeSum(packet.newRows, myOutputCols);}
public string[] groupByColumns { get { return myGroupByCols; } } // public string[] groupByColumns { get { return myGroupByCols; } }
public void clear() {/*stateless*/} // public void clear() {/*stateless*/}
public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; } // public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) {return schema;}
#endregion // #endregion
} // }
public class SumGroupByEvaluator<T> : GroupByEvaluator { // public class SumGroupByEvaluator<T> : GroupByEvaluator
public SumGroupByEvaluator(string outputCol, string[] groupByCols) // {
: base(outputCol, groupByCols) { } // public SumGroupByEvaluator(string outputCol, string[] groupByCols)
// : base(outputCol, groupByCols)
// { }
public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols) // public SumGroupByEvaluator(string colMapEpression, string outputCol, string[] groupByCols)
: base(colMapEpression, outputCol, groupByCols) { } // : base(colMapEpression, outputCol, groupByCols) { }
public override IBlueRow evaluate(DataPacket dp) { // public override IBlueRow evaluate(DataPacket dp)
myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable; // {
// myOutputTable = myOutputTable == null ? new GroupByOutputTable(dp.schema) : myOutputTable;
IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn); // IBlueRow sumRow = BlueDataUtility.computeSum<T>(dp.newRows, myOutputColumn);
return sumRow; // return sumRow;
} // }
} // }
public class MultiColumnSumGroupByEvaluator<T> : IGroupByEvaluator { // public class MultiColumnSumGroupByEvaluator<T> : IGroupByEvaluator
private string[] myOutputCols, myGroupByCols; // {
public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols) { // private string[] myOutputCols, myGroupByCols;
myOutputCols = spaceSeperatedOutputCols.Split(' '); // public MultiColumnSumGroupByEvaluator(string spaceSeperatedOutputCols, string spaceSeperatedGroupByCols)
myGroupByCols = spaceSeperatedGroupByCols.Split(' '); // {
} // myOutputCols = spaceSeperatedOutputCols.Split(' ');
// myGroupByCols = spaceSeperatedGroupByCols.Split(' ');
// }
#region IGroupByEvaluator Members // #region IGroupByEvaluator Members
public IBlueRow evaluate(DataPacket packet) { return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols); } // public IBlueRow evaluate(DataPacket packet) {return BlueDataUtility.computeSum<T>(packet.newRows, myOutputCols);}
public string[] groupByColumns { get { return myGroupByCols; } } // public string[] groupByColumns { get { return myGroupByCols; } }
public void clear() {/*stateless*/} // public void clear() {/*stateless*/}
public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return schema; } // public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) {return schema;}
#endregion // #endregion
} // }
} // }
@@ -1,89 +1,107 @@
using System; // using System;
using System.Collections.Generic; // using System.Collections.Generic;
namespace BCG { // namespace BCG
public class NullMapper : IStreamMapper { // {
private Dictionary<string, bool> _valid_cols = new Dictionary<string, bool>(); // public class NullMapper : IStreamMapper
// {
// private Dictionary<string, bool> _valid_cols = new Dictionary<string, bool>();
public NullMapper(string[] valid_cols) { // public NullMapper(string[] valid_cols)
foreach (string c in valid_cols) // {
_valid_cols[c] = true; // foreach (string c in valid_cols)
_valid_cols["id"] = true; // _valid_cols[c] = true;
} // _valid_cols["id"] = true;
// }
public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; } // public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) { return input_schema; }
public IBlueRow map(IBlueRow row) { // public IBlueRow map(IBlueRow row)
MutableBlueRow out_row = new MutableBlueRow(row); // {
foreach (string c in row.schema.columnNames) // MutableBlueRow out_row = new MutableBlueRow(row);
if (!_valid_cols.ContainsKey(c)) // foreach (string c in row.schema.columnNames)
out_row.setValue(c, DBNull.Value); // if (!_valid_cols.ContainsKey(c))
return out_row; // out_row.setValue(c, DBNull.Value);
} // return out_row;
} // }
// }
public class SumStream : ViewStream { // public class SumStream : ViewStream
public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) { // {
string[] outputCols = spaceSeparatedOutputCols.Split(' '); // public SumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols)
string[] groupByCols = spaceSeparatedGroupByCols.Split(' '); // {
mySetRealSource(new GroupByStream(stream, // string[] outputCols = spaceSeparatedOutputCols.Split(' ');
outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator : // string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols))); // mySetRealSource(new GroupByStream(stream,
} // outputCols.Length == 1 ? new SumGroupByEvaluator(outputCols[0], groupByCols) as IGroupByEvaluator :
// new MultiColumnSumGroupByEvaluator(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
// }
public SumStream(ViewStream stream, string[] output_columns) { // public SumStream(ViewStream stream, string[] output_columns)
ViewStream rs = new AddColumnsStream(stream, "fake_col=0"); // {
rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false); // ViewStream rs = new AddColumnsStream(stream, "fake_col=0");
rs = new ProjectStream(rs, "!fake_col"); // rs = new SumStream(rs, output_columns, new string[] { "fake_col" }, false);
mySetRealSource(rs); // rs = new ProjectStream(rs, "!fake_col");
} // mySetRealSource(rs);
// }
public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns) { // public SumStream(ViewStream stream, string space_seperated_output_columns, string space_seperated_groupby_columns, bool force_null_into_other_columns)
string[] output_columns = space_seperated_output_columns.Split(' '); // {
string[] groupby_columns = String.IsNullOrEmpty(space_seperated_groupby_columns) ? new string[] { } : space_seperated_groupby_columns.Split(' '); // string[] output_columns = space_seperated_output_columns.Split(' ');
mySetRealSource(new SumStream(stream, output_columns, groupby_columns, force_null_into_other_columns)); // string[] groupby_columns = String.IsNullOrEmpty (space_seperated_groupby_columns) ? new string[]{} : space_seperated_groupby_columns.Split (' ');
} // mySetRealSource (new SumStream (stream, output_columns, groupby_columns, force_null_into_other_columns));
// }
public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns) { // public SumStream(ViewStream stream, string[] output_columns, string[] groupby_columns, bool force_null_into_other_columns)
ViewStream rs = null; // {
groupby_columns = groupby_columns ?? new string[] { }; // ViewStream rs = null;
if (groupby_columns.Length == 0) { // groupby_columns = groupby_columns ?? new string[] {};
// rs = new AddColumnsStream(stream, "fake_col=0"); // if (groupby_columns.Length == 0)
// rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] { "fake_col" })); // {
// rs = new ProjectStream(rs, "!fake_col"); // rs = new AddColumnsStream(stream, "fake_col=0");
} else { // rs = new GroupByStream(rs, new MultiColumnSumGroupByEvaluator(output_columns, new string[] {"fake_col"}));
rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns)); // rs = new ProjectStream(rs, "!fake_col");
} // }
if (force_null_into_other_columns) { // else
List<string> valid_cols = new List<string>(output_columns); // {
valid_cols.AddRange(groupby_columns); // rs = new GroupByStream(stream, new MultiColumnSumGroupByEvaluator(output_columns, groupby_columns));
rs = new MapStream(rs, new NullMapper(valid_cols.ToArray())); // }
} // if (force_null_into_other_columns)
mySetRealSource(rs); // {
} // List<string> valid_cols = new List<string>(output_columns);
// valid_cols.AddRange(groupby_columns);
// rs = new MapStream(rs, new NullMapper(valid_cols.ToArray()));
// }
// mySetRealSource(rs);
// }
public SumStream(ViewStream stream, string outputCol) // public SumStream (ViewStream stream, string outputCol)
: this(new AddColumnsStream(stream, "fakeCol=0"), outputCol, "fakeCol") { // : this (new AddColumnsStream (stream, "fakeCol=0"), outputCol, "fakeCol")
} // {
} // }
// }
public class IntSumStream : ViewStream { // public class IntSumStream : ViewStream
public IntSumStream(ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols) { // {
string[] outputCols = spaceSeparatedOutputCols.Split(' '); // public IntSumStream (ViewStream stream, string spaceSeparatedOutputCols, string spaceSeparatedGroupByCols)
string[] groupByCols = spaceSeparatedGroupByCols.Split(' '); // {
mySetRealSource(new GroupByStream(stream, // string[] outputCols = spaceSeparatedOutputCols.Split(' ');
outputCols.Length == 1 ? new SumGroupByEvaluator<int>(outputCols[0], groupByCols) as IGroupByEvaluator : // string[] groupByCols = spaceSeparatedGroupByCols.Split(' ');
new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols))); // mySetRealSource(new GroupByStream(stream,
} // outputCols.Length == 1 ? new SumGroupByEvaluator<int>(outputCols[0], groupByCols) as IGroupByEvaluator:
// new MultiColumnSumGroupByEvaluator<int>(spaceSeparatedOutputCols, spaceSeparatedGroupByCols)));
// }
} // }
public class SumStream<T> : ViewStream { // public class SumStream<T> : ViewStream
public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols) { // {
string[] inputCols = spaceSeparatedInputCols.Split(' '); // public SumStream(ViewStream stream, string outputCol, string spaceSeparatedInputCols)
mySetRealSource(new GroupByStream(stream, // {
new SumGroupByEvaluator<T>(outputCol, inputCols))); // string[] inputCols = spaceSeparatedInputCols.Split(' ');
} // mySetRealSource(new GroupByStream(stream,
} // new SumGroupByEvaluator<T>(outputCol, inputCols)));
} // }
// }
// }
@@ -1,79 +0,0 @@
using BCG;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using ZstdSharp.Unsafe;
namespace bts {
public class RowDiffStream : ViewStream {
private ViewStream _vs;
private readonly string _order_col, _col_1, _col_2;
private readonly int _diff;
private IBlueTableSchema _out_schema;
private readonly string _diff_col_name;
public RowDiffStream(ViewStream vs, string order_col, string col_1, string col_2, int diff, string diff_col_name) {
_vs = vs;
_order_col = order_col;
_col_1 = col_1;
_col_2 = col_2;
_diff = diff;
_diff_col_name = diff_col_name;
}
private string _diff_col_name_perc {
get {
return $"{_diff_col_name}_percentage";
}
}
protected override void myOnFirstSubscriber() {
_vs.onChange += _handle_vs;
}
protected override void myOnLastSubscriber() {
_vs.onChange -= _handle_vs;
}
private void _handle_vs(object dp_obj) {
var dp = dp_obj as DataPacket;
IBlueRowCollection nrows = dp.newRows;
_out_schema = BlueDataUtility.add_column(nrows.schema, _diff_col_name, typeof(double));
_out_schema = BlueDataUtility.add_column(_out_schema, _diff_col_name_perc, typeof(double));
List<IBlueRow> row_list = new List<IBlueRow>();
foreach (IBlueRow r in nrows)
row_list.Add(r);
//OrderBy sorts in ascedning order
var sorted_rows = row_list.OrderBy(r => r[_order_col]).ToList();
var sorted_row_array = sorted_rows.ToArray();
// reverse it to make it descending
sorted_rows.Reverse();
//sort rows so we can do the diff
OrderedBlueTable rv = new OrderedBlueTable(_out_schema);
int i = 0;
foreach (IBlueRow r in sorted_rows) {
MutableBlueRow nr = new MutableBlueRow(_out_schema, r);
//So, in the expression "x - y," where y is subtracted from x, x is the minuend, and y is the subtrahend.
// we assume x - y, and name variables like that
double x = r.get<double>(_col_1);
int y_row_idx = i + _diff;
double diff = System.Double.NaN;
double perc = System.Double.NaN;
if (y_row_idx < sorted_rows.Count()) {
var y = sorted_rows[y_row_idx].get<double>(_col_2);
diff = x - y;
perc = 100.0 * diff / y;
}
nr.setValue(_diff_col_name, diff);
nr.setValue(_diff_col_name_perc, perc);
rv.addRow(nr);
i++;
}
var dp_out = BlueDataUtility.build_dp_inc(new_rows: rv, myLastValue as DataPacket, true);
myFireEvent(dp_out);
}
}
}
@@ -27,8 +27,7 @@ namespace BCG
string prependString = isLeft ? "left" : "right"; string prependString = isLeft ? "left" : "right";
return String.Concat (prependString, colName).GetHashCode (); return String.Concat (prependString, colName).GetHashCode ();
} }
public override bool Equals (object? obj) => public override bool Equals (object obj) {return GetHashCode () == obj.GetHashCode ();}
GetHashCode() == obj?.GetHashCode();
} }
@@ -10,9 +10,9 @@ namespace BCG
{ {
public class JoinStream : ViewStream public class JoinStream : ViewStream
{ {
protected InternalJoinStream? myIjs; protected InternalJoinStream myIjs;
protected ProjectStream? myPs; protected ProjectStream myPs;
protected ColumnMap? myLeftCm, myRightCm; protected ColumnMap myLeftCm, myRightCm;
protected bool myIsProject = true; protected bool myIsProject = true;
private bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin; private bool myUseSuperJoin = DataStreamLoadFileConfig.instance.useNewJoin;
@@ -1,10 +1,10 @@
using bts; using System;
using mbts;
using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using System.Linq;
using System.Text;
using System.Threading.Tasks;
using mbts;
using mbts.spark;
namespace BCG namespace BCG
{ {
@@ -40,7 +40,8 @@ namespace BCG
public void _on_stream_cb(object dp_obj) public void _on_stream_cb(object dp_obj)
{ {
var timer = Stopwatch.StartNew(); HighPerformanceTimer timer = new HighPerformanceTimer();
timer.start();
DataPacket dp = (DataPacket)dp_obj; DataPacket dp = (DataPacket)dp_obj;
if (_schema == null) if (_schema == null)
{ {
@@ -66,12 +67,13 @@ namespace BCG
_row_map[_row_hasher.get_hash(r, _join_col_indices)] = r; _row_map[_row_hasher.get_hash(r, _join_col_indices)] = r;
} }
} }
timer.Stop(); timer.stop();
_n_total_fires++; _n_total_fires++;
if (timer.ElapsedMilliseconds >= 50) { if (timer.lastIntervalTime > 50)
{
_n_expensive_fires++; _n_expensive_fires++;
Log.Gen.info($"NonIncJoinBookkeepingData._on_stream_cb {dp.newRows.Count} rows {_join_col_indices.Length} " string log_msg = String.Format("NonIncJoinBookkeepingData._on_stream_cb {0} rows {1} key cols {2} join pos time {3} ratio {4}/{5}", dp.newRows.Count, _join_col_indices.Length, _join_position, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires);
+ $"key cols {_join_position} join pos time {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}"); // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
} }
if (on_new_data != null) if (on_new_data != null)
on_new_data(); on_new_data();
@@ -1,10 +1,10 @@
using bts; using System;
using mbts;
using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using System.Linq;
using System.Text;
using System.Threading.Tasks;
using mbts;
using mbts.spark;
namespace BCG namespace BCG
{ {
@@ -114,7 +114,8 @@ namespace BCG
if (_is_ready()) if (_is_ready())
{ {
_construct_schema(); _construct_schema();
var timer = Stopwatch.StartNew(); HighPerformanceTimer timer = new HighPerformanceTimer();
timer.start();
IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list(); IList<Pair<int, IBlueRow>> left_row_list = _book_keeping_data[0].get_list();
List<object[]> output_rows = new List<object[]>(); List<object[]> output_rows = new List<object[]>();
@@ -137,12 +138,13 @@ namespace BCG
} }
} }
IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows)); IBlueRowCollection dp_output_rows = _apply_map(new SequentialIDBlueTable(_output_schema, output_rows));
timer.Stop(); timer.stop();
_n_total_fires++; _n_total_fires++;
if (timer.ElapsedMilliseconds >= 250) { if (timer.lastIntervalTime >= 250)
{
_n_expensive_fires++; _n_expensive_fires++;
string log_msg = $"Join and map time for {_book_keeping_data.Count} " string log_msg = String.Format("Join and map time for {0} tables {1} ratio {2}/{3}", _book_keeping_data.Count, timer.lastIntervalTime, _n_expensive_fires, _n_total_fires);
+ $"tables {timer.Elapsed} ratio {_n_expensive_fires}/{_n_total_fires}"; // mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
} }
myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows)); myFireEvent(new DataPacket(dp_output_rows, dp_output_rows, myLastValue == null ? new BlueTable(_output_schema) : (myLastValue as DataPacket).newRows));
} }
@@ -152,12 +154,15 @@ namespace BCG
{ {
if (_output_mapper != null) if (_output_mapper != null)
{ {
HighPerformanceTimer timer = new HighPerformanceTimer();
IBlueRowCollection output_rows; IBlueRowCollection output_rows;
var timer = Stopwatch.StartNew(); timer.start();
output_rows = _output_mapper.map_rows(unmapped_rows); output_rows = _output_mapper.map_rows(unmapped_rows);
timer.Stop(); timer.stop();
if (timer.ElapsedMilliseconds >= 50) { if (timer.lastIntervalTime >= 50)
Log.Gen.info($"Time to map {unmapped_rows.Count} rows into {output_rows.Count} rows {timer.Elapsed}"); {
// string log_msg = String.Format("Time to map {0} rows into {1} rows {2}", unmapped_rows.Count, output_rows.Count, timer.lastIntervalTime);
// mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, log_msg);
} }
return output_rows; return output_rows;
} }
@@ -1,7 +1,11 @@
using System; using System;
using System.Collections; using System.Collections;
using System.Collections.Generic; using System.Collections.Generic;
using System.Text;
using BCG.Utilities.General;
using BCG.Utilities;
using mbts; using mbts;
using mbts.spark;
using Spark.Core.Collections; using Spark.Core.Collections;
namespace BCG namespace BCG
@@ -222,8 +226,7 @@ namespace BCG
#region IDictionary Members #region IDictionary Members
public void Add(object key, object? value) => public void Add(object key, object value) { { throw new NotSupportedException("IBlueRow is readonly"); } }
throw new NotSupportedException("IBlueRow is readonly");
public void Clear() { throw new NotSupportedException("IBlueRow is readonly"); } public void Clear() { throw new NotSupportedException("IBlueRow is readonly"); }
@@ -253,9 +256,10 @@ namespace BCG
} }
} }
public object? this[object key] { public object this[object key]
get => key is string str ? this[str] : this[(int)key]; {
set => throw new NotSupportedException("IBlueRow is readonly"); get { return key is string ? this[key.ToString()] : this[(int)key]; }
set { throw new NotSupportedException("IBlueRow is readonly"); }
} }
#endregion #endregion
@@ -290,6 +294,7 @@ namespace BCG
protected Dictionary<int, IBlueRow> myLeftUnmatchedRowsAdds = new Dictionary<int, IBlueRow>(), protected Dictionary<int, IBlueRow> myLeftUnmatchedRowsAdds = new Dictionary<int, IBlueRow>(),
myLeftUnmatchedRowsDels = new Dictionary<int, IBlueRow>(); myLeftUnmatchedRowsDels = new Dictionary<int, IBlueRow>();
private bool myManageUnmappedLeftRows = false; private bool myManageUnmappedLeftRows = false;
protected HighPerformanceTimer myTimer = new HighPerformanceTimer();
protected string myLeftProj = String.Empty, myRightProj = String.Empty; protected string myLeftProj = String.Empty, myRightProj = String.Empty;
protected IBlueTableSchema myLeftSchema, myRightSchema; protected IBlueTableSchema myLeftSchema, myRightSchema;
protected JoinProjectHelper myJoinProjectHelper; protected JoinProjectHelper myJoinProjectHelper;
@@ -356,6 +361,7 @@ namespace BCG
private void myHandleStreamChanges(DataPacket dp, bool isLeft) private void myHandleStreamChanges(DataPacket dp, bool isLeft)
{ {
myTimer.start();
myInitSchema(dp.schema, isLeft); myInitSchema(dp.schema, isLeft);
BlueTable dels = null, adds = null; BlueTable dels = null, adds = null;
if (myState != null) if (myState != null)
@@ -365,6 +371,7 @@ namespace BCG
} }
myDeleteRows(dp.deletedRows, isLeft, myState, dels); myDeleteRows(dp.deletedRows, isLeft, myState, dels);
myAddRows(dp.addedRows, isLeft, myState, adds); myAddRows(dp.addedRows, isLeft, myState, adds);
myTimer.stop();
if (myState != null) if (myState != null)
{ {
myFireEvent(new DataPacket(myState, adds, dels)); myFireEvent(new DataPacket(myState, adds, dels));
@@ -1,31 +0,0 @@
namespace BCG {
using System.Collections.Generic;
public class KeyValueToRelationalStream : ViewStream {
private readonly ViewStream _kv_str;
private readonly string _doc_id_col, _key_col, _val_col;
public KeyValueToRelationalStream(ViewStream kv_str, string doc_id_col, string key_col, string val_col) {
_kv_str = kv_str;
_doc_id_col = doc_id_col;
_key_col = key_col;
_val_col = val_col;
}
protected override void myOnFirstSubscriber() {
_kv_str.onChange += _on_str_update;
}
protected override void myOnLastSubscriber() {
_kv_str.onChange -= _on_str_update;
}
private void _on_str_update(object dp_obj) {
DataPacket dp = dp_obj as DataPacket;
var rows = dp.newRows;
var rv = BlueDataUtility.keyvalue_to_relational<string>(rows, _doc_id_col, _key_col, _val_col);
var dp_out = BlueDataUtility.build_dp_inc(rv, myLastValue as DataPacket, true);
myFireEvent(dp_out);
}
}
}
@@ -91,8 +91,7 @@ namespace BCG
} }
} }
public void Add (object key, object? value) => public void Add (object key, object value) {throw new NotSupportedException ("The IBlueRow is readonly");}
throw new NotSupportedException ("The IBlueRow is readonly");
public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");} public void Clear () {throw new NotSupportedException ("The IBlueRow is readonly");}
@@ -1,64 +1,51 @@
using System; using System;
using System.Collections; using System.Collections;
using System.Collections.Generic;
using BCG; using BCG;
using BCG.Utilities; using BCG.Utilities;
namespace BCG { namespace BCG
public class LiteralStream : ViewStream { {
public class LiteralStream : ViewStream
public static List<Tuple<ColumnInfo, object>> parse_columns(string key_value_pairs) { {
List<Tuple<ColumnInfo, object>> rv = new List<Tuple<ColumnInfo, object>>(); public LiteralStream(string tableName, string schema, string value)
string[] pairs = key_value_pairs.Split(' '); {
foreach (string p in pairs) { string [] columns = schema.Split(' ');
string[] kvp = p.Split(':');
string col_name = kvp[0];
string val = kvp[1];
var tup = BlueDataUtility.guess_type(val);
var ci = new ColumnInfo(col_name, rv.Count, tup.Item1);
rv.Add(Tuple.Create(ci, tup.Item2));
}
return rv;
}
public LiteralStream(string tableName, string schema, string value) {
string[] columns = schema.Split(' ');
ColumnInfo[] ci = new ColumnInfo[columns.Length + 1]; ColumnInfo [] ci = new ColumnInfo[columns.Length + 1];
ci[0] = new ColumnInfo("id", 0, typeof(int)); ci[0] = new ColumnInfo("id",0,typeof(int));
int n = 1; int n = 1;
foreach (string c in columns) { foreach (string c in columns)
string[] columnAndType = c.Split('/'); {
string [] columnAndType = c.Split('/');
if (columnAndType.Length == 1) if (columnAndType.Length == 1)
ci[n] = new ColumnInfo(c, n, typeof(double)); ci[n] = new ColumnInfo(c,n,typeof(double));
else else
ci[n] = new ColumnInfo(columnAndType[0], n, Type.GetType("System." + columnAndType[1], true)); ci[n] = new ColumnInfo(columnAndType[0],n,Type.GetType("System." + columnAndType[1],true));
n++; n++;
} }
BlueTable t = new BlueTable(new BlueTableSchema(tableName, ci)); BlueTable t = new BlueTable(new BlueTableSchema(tableName,ci));
string[] rows = value.Split(';'); string [] rows = value.Split(';');
n = 0; n = 0;
foreach (string r in rows) foreach (string r in rows)
try { try
if (r == string.Empty) {
if(r == string.Empty)
continue; continue;
t.addRow( t.addRow(
BlueDataUtility.createRowFromCSVFileRow(r, t.schema, ++n)); BlueDataUtility.createRowFromCSVFileRow (r, t.schema,++n));
} catch (Exception e) { }
catch (Exception e)
{
throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message)); throw (new Exception("Error occured at line: " + n.ToString() + " " + e.Message));
} }
mySetRealSource(new BlueRowCollectionStaticStream(t)); mySetRealSource(new BlueRowCollectionStaticStream(t));
} }
public LiteralStream(string schema, string value) : this("literal", schema, value) { } public LiteralStream(string schema, string value) : this ("literal",schema, value)
{}
// public LiteralStream(string value) {
// }
} }
} }
@@ -1,24 +1,24 @@
// using System; using System;
// namespace BCG namespace BCG
// { {
// public class DynamicUntypedToTypedMapper : IDynamicStreamMapper public class DynamicUntypedToTypedMapper : IDynamicStreamMapper
// { {
// IBlueTableSchema mySchema; IBlueTableSchema mySchema;
// public IBlueRow map (IBlueRow row) public IBlueRow map (IBlueRow row)
// { {
// if (mySchema == null) if (mySchema == null)
// { {
// ColumnInfo[] cis = new ColumnInfo [row.schema.columnInfo.Length]; ColumnInfo[] cis = new ColumnInfo [row.schema.columnInfo.Length];
// int i = 0; int i = 0;
// foreach (ColumnInfo ci in row.schema.columnInfo) foreach (ColumnInfo ci in row.schema.columnInfo)
// cis[i++] = ci.colType == null ? cis[i++] = ci.colType == null ?
// new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType()) : new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType()) :
// new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType(), ci.colValue); new ColumnInfo (ci.colName, ci.colNo, row[ci.colNo].GetType(), ci.colValue);
// mySchema = new BlueTableSchema (row.schema.tableName, cis); mySchema = new BlueTableSchema (row.schema.tableName, cis);
// } }
// return new BlueRow (mySchema, row); return new BlueRow (mySchema, row);
// } }
// } }
// } }
@@ -1,10 +1,15 @@
using System;
using System.Data;
using System.Collections;
using Bts.DataStream; using Bts.DataStream;
using mbts;
using mbts.spark;
using mbts.config; using mbts.config;
using System; namespace BCG
{
namespace BCG {
/// <summary> /// <summary>
@@ -16,45 +21,24 @@ namespace BCG {
[DataStreamUserRespresentation (false)] [DataStreamUserRespresentation (false)]
public class MapStream : MethodDrivenStream public class MapStream : MethodDrivenStream
{ {
private HighPerformanceTimer t = new HighPerformanceTimer ();
protected IStreamMapper myMapper; protected IStreamMapper myMapper;
protected IChangeableStreamMapper myChangeableMapper = null; protected IChangeableStreamMapper myChangeableMapper = null;
private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap; private bool myUseNewMap = Configuration.get<DataStreamConfig>().UseNewMap;
public class FuncWrapper : IStreamMapper { public MapStream(ViewStream vs, IStreamMapper mapper) : base (vs)
private Func<IBlueTableSchema, IBlueTableSchema> _schema_map_func; {
Func<IBlueRow, IBlueRow> _row_map_func;
public FuncWrapper(Func<IBlueTableSchema, IBlueTableSchema> schema_map_func,
Func<IBlueRow, IBlueRow> row_map_func) {
_schema_map_func = schema_map_func;
_row_map_func = row_map_func;
}
public IBlueTableSchema getOutputSchema(IBlueTableSchema input_schema) {
return _schema_map_func(input_schema);
}
public IBlueRow map(IBlueRow row) {
return _row_map_func(row);
}
}
public MapStream(ViewStream vs, IStreamMapper mapper) : base(vs) {
if (myUseNewMap) if (myUseNewMap)
mySetRealSource(new SuperMapStream(vs, mapper)); mySetRealSource (new SuperMapStream (vs, mapper));
else else
myMapper = mapper; myMapper = mapper;
} }
public MapStream(ViewStream vs, IChangeableStreamMapper mapper) : this (vs,(IStreamMapper)mapper)
public MapStream(ViewStream vs, Func<IBlueTableSchema, IBlueTableSchema> schema_map_func, {
Func<IBlueRow, IBlueRow> row_map_func)
:
this(vs, new FuncWrapper(schema_map_func, row_map_func)) {
}
public MapStream(ViewStream vs, IChangeableStreamMapper mapper) : this(vs, (IStreamMapper)mapper) {
if (myUseNewMap) if (myUseNewMap)
mySetRealSource(new SuperMapStream(vs, mapper)); mySetRealSource(new SuperMapStream(vs, mapper));
else { else
{
mapper.onMapperChange += new MapperChangeDelegate(myMapperChanged); mapper.onMapperChange += new MapperChangeDelegate(myMapperChanged);
myChangeableMapper = mapper; myChangeableMapper = mapper;
} }
@@ -63,68 +47,77 @@ namespace BCG {
/// <summary> /// <summary>
/// Gets or sets the value of the mapper. Setting the value will result in re /// Gets or sets the value of the mapper. Setting the value will result in re
/// </summary> /// </summary>
private void myMapperChanged() { private void myMapperChanged ()
{
if (myLastArrivedPacket != null) if (myLastArrivedPacket != null)
myChangeHandler(new DataPacket(myLastArrivedPacket.newRows, myChangeHandler (new DataPacket (myLastArrivedPacket.newRows,
myLastArrivedPacket.newRows, myLastArrivedPacket.newRows,
myLastArrivedPacket.newRows)); myLastArrivedPacket.newRows));
} }
protected override void myHandleFirstCall() { protected override void myHandleFirstCall ()
{
IBlueTableSchema outputSchema = myMapper.getOutputSchema(myLastArrivedPacket.newRows.schema);
myLastFiredPacket = new DataPacket(); IBlueTableSchema outputSchema = myMapper.getOutputSchema (myLastArrivedPacket.newRows.schema);
myLastFiredPacket.newRows = new BlueTable(outputSchema); myLastFiredPacket = new DataPacket ();
myLastFiredPacket.deletedRows = new BlueTable(outputSchema); myLastFiredPacket.newRows = new BlueTable (outputSchema);
myLastFiredPacket.deletedRows = new BlueTable (outputSchema);
foreach (IBlueRow row in myLastArrivedPacket.newRows) foreach (IBlueRow row in myLastArrivedPacket.newRows)
((BlueTable)(myLastFiredPacket.newRows)).addRow(myMapper.map(row)); ((BlueTable)(myLastFiredPacket.newRows)).addRow (myMapper.map(row));
myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows); myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows);
} }
protected override void myHandleIncrementalAdd() { protected override void myHandleIncrementalAdd()
{
//t.start (); //t.start ();
myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.newRows.schema); myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.newRows.schema);
foreach (IBlueRow row in myLastArrivedPacket.addedRows) { foreach (IBlueRow row in myLastArrivedPacket.addedRows)
IBlueRow newRow = myMapper.map(row); {
(myLastFiredPacket.newRows as BlueTable).addRow(newRow); IBlueRow newRow = myMapper.map (row);
(myLastFiredPacket.addedRows as BlueTable).addRow(newRow); (myLastFiredPacket.newRows as BlueTable).addRow (newRow);
(myLastFiredPacket.addedRows as BlueTable).addRow (newRow);
} }
//t.stop (); //t.stop ();
//System.Diagnostics.Trace.WriteLine ("adding " + myLastFiredPacket.addedRows.Count.ToString () + " : " + t.lastIntervalTime); //System.Diagnostics.Trace.WriteLine ("adding " + myLastFiredPacket.addedRows.Count.ToString () + " : " + t.lastIntervalTime);
//System.Diagnostics.Trace.WriteLine (" "); //System.Diagnostics.Trace.WriteLine (" ");
} }
protected override void myHandleIncrementalDelete() { protected override void myHandleIncrementalDelete()
{
//t.start (); //t.start ();
myLastFiredPacket.deletedRows = new BlueTable(myLastFiredPacket.newRows.schema); myLastFiredPacket.deletedRows = new BlueTable (myLastFiredPacket.newRows.schema);
foreach (IBlueRow row in myLastArrivedPacket.deletedRows) { foreach (IBlueRow row in myLastArrivedPacket.deletedRows)
{
IBlueRow delRow = (myLastFiredPacket.newRows as BlueTable)[(int)row["id"]]; IBlueRow delRow = (myLastFiredPacket.newRows as BlueTable)[(int)row["id"]];
(myLastFiredPacket.deletedRows as BlueTable).addRow(delRow); (myLastFiredPacket.deletedRows as BlueTable).addRow (delRow);
(myLastFiredPacket.newRows as BlueTable).deleteRow(row); (myLastFiredPacket.newRows as BlueTable).deleteRow (row);
} }
//t.stop (); //t.stop ();
//System.Diagnostics.Trace.WriteLine ("deleting " + myLastFiredPacket.deletedRows.Count.ToString () + " : " + t.lastIntervalTime); //System.Diagnostics.Trace.WriteLine ("deleting " + myLastFiredPacket.deletedRows.Count.ToString () + " : " + t.lastIntervalTime);
} }
protected override void myOnFirstSubscriber()
protected override void myOnFirstSubscriber() { {
myStream.onChange += new DataStream.ChangeDelegate(myChangeHandler); myStream.onChange += new DataStream.ChangeDelegate (myChangeHandler);
} }
protected override void myOnLastSubscriber() { protected override void myOnLastSubscriber ()
myInit(); {
myInit ();
if (myStream != null) if (myStream != null)
myStream.onChange -= new DataStream.ChangeDelegate(myChangeHandler); myStream.onChange -= new DataStream.ChangeDelegate (myChangeHandler);
} }
protected override void myOnTableClear() { protected override void myOnTableClear ()
{
myLastFiredPacket.deletedRows = myLastFiredPacket.newRows; myLastFiredPacket.deletedRows = myLastFiredPacket.newRows;
myLastFiredPacket.newRows = new BlueTable(myLastFiredPacket.deletedRows.schema); myLastFiredPacket.newRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
myLastFiredPacket.addedRows = new BlueTable(myLastFiredPacket.deletedRows.schema); myLastFiredPacket.addedRows = new BlueTable (myLastFiredPacket.deletedRows.schema);
} }
} }
/* /*
@@ -1,13 +1,12 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Text; using System.Text;
using BCG.Utilities; using BCG.Utilities;
using BCG.Utilities.General; using BCG.Utilities.General;
using mbts; using mbts;
namespace BCG { namespace BCG
{
/// <summary> /// <summary>
/// SuperMap gives the user the ability /// SuperMap gives the user the ability
/// to map one row to one or more rows /// to map one row to one or more rows
@@ -18,11 +17,13 @@ namespace BCG {
/// can chage itself and raise an event causing /// can chage itself and raise an event causing
/// a remapping to happen /// a remapping to happen
/// </summary> /// </summary>
public class SuperMapStream : ViewStream { public class SuperMapStream : ViewStream
{
#region super mapper that wraps around stream mapper #region super mapper that wraps around stream mapper
private class MyMappingWrapper : ISuperStreamMapper { private class MyMappingWrapper : ISuperStreamMapper
{
private IStreamMapper myMapper; private IStreamMapper myMapper;
public MyMappingWrapper(IStreamMapper mapper) { myMapper = mapper; } public MyMappingWrapper(IStreamMapper mapper) { myMapper = mapper; }
#region ISuperStreamMapper Members #region ISuperStreamMapper Members
@@ -31,7 +32,8 @@ namespace BCG {
public IBlueRow[] map(IBlueRow inputRow) { return new IBlueRow[] { myMapper.map(inputRow) }; } public IBlueRow[] map(IBlueRow inputRow) { return new IBlueRow[] { myMapper.map(inputRow) }; }
public event MapperChangeDelegate onMapperChange { public event MapperChangeDelegate onMapperChange
{
add { } add { }
remove { } remove { }
} }
@@ -40,18 +42,21 @@ namespace BCG {
} }
#endregion #endregion
private class MyFuncWrapper : ISuperStreamMapper { private class MyFuncWrapper : ISuperStreamMapper
{
private Func<IBlueTableSchema, IBlueTableSchema> _get_schema; private Func<IBlueTableSchema, IBlueTableSchema> _get_schema;
private Func<IBlueRow, IEnumerable<IBlueRow>> _map_rows; private Func<IBlueRow, IEnumerable<IBlueRow>> _map_rows;
public MyFuncWrapper(Func<IBlueTableSchema, IBlueTableSchema> get_schema, public MyFuncWrapper(Func<IBlueTableSchema, IBlueTableSchema> get_schema,
Func<IBlueRow, IEnumerable<IBlueRow>> map_rows) { Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
{
_get_schema = get_schema; _get_schema = get_schema;
_map_rows = map_rows; _map_rows = map_rows;
} }
public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return _get_schema(schema); } public IBlueTableSchema getOutputSchema(IBlueTableSchema schema) { return _get_schema(schema); }
public IBlueRow[] map(IBlueRow r) { public IBlueRow[] map(IBlueRow r)
{
IEnumerable<IBlueRow> rows = _map_rows(r); IEnumerable<IBlueRow> rows = _map_rows(r);
return rows == null ? new IBlueRow[] { } : new List<IBlueRow>(rows).ToArray(); return rows == null ? new IBlueRow[] { } : new List<IBlueRow>(rows).ToArray();
} }
@@ -71,15 +76,19 @@ namespace BCG {
public SuperMapStream(ViewStream vs, Func<IBlueTableSchema, IBlueTableSchema> get_schema, public SuperMapStream(ViewStream vs, Func<IBlueTableSchema, IBlueTableSchema> get_schema,
Func<IBlueRow, IEnumerable<IBlueRow>> map_rows) Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
: this(vs, new MyFuncWrapper(get_schema, map_rows)) { } : this(vs, new MyFuncWrapper(get_schema, map_rows))
{ }
public SuperMapStream(ViewStream vs, IBlueTableSchema out_schema, Func<IBlueRow, IEnumerable<IBlueRow>> map_rows) public SuperMapStream(ViewStream vs, IBlueTableSchema out_schema, Func<IBlueRow, IEnumerable<IBlueRow>> map_rows)
: this(vs, s => out_schema, map_rows) { } : this(vs, s => out_schema, map_rows)
{ }
public SuperMapStream(ViewStream vs, IStreamMapper mapper) public SuperMapStream(ViewStream vs, IStreamMapper mapper)
: this(vs, new MyMappingWrapper(mapper)) { } : this(vs, new MyMappingWrapper(mapper))
{ }
public SuperMapStream(ViewStream vs, ISuperStreamMapper mapper) { public SuperMapStream(ViewStream vs, ISuperStreamMapper mapper)
{
myStream = vs; myStream = vs;
myMapper = mapper; myMapper = mapper;
myMapper.onMapperChange += new MapperChangeDelegate(myHandleMapChange); myMapper.onMapperChange += new MapperChangeDelegate(myHandleMapChange);
@@ -87,7 +96,8 @@ namespace BCG {
protected override void myOnFirstSubscriber() { myStream.onChange += new ChangeDelegate(myHandleData); } protected override void myOnFirstSubscriber() { myStream.onChange += new ChangeDelegate(myHandleData); }
protected override void myOnLastSubscriber() { protected override void myOnLastSubscriber()
{
myStream.onChange -= new ChangeDelegate(myHandleData); myStream.onChange -= new ChangeDelegate(myHandleData);
myRows = null; myRows = null;
myBaseRows = null; myBaseRows = null;
@@ -95,7 +105,8 @@ namespace BCG {
//myMapper.onMapperChange -= new MapperChangeDelegate(myHandleMapChange); //myMapper.onMapperChange -= new MapperChangeDelegate(myHandleMapChange);
} }
private void myHandleData(object newValue) { private void myHandleData(object newValue)
{
DataPacket idp = newValue as DataPacket; DataPacket idp = newValue as DataPacket;
//HighPerformanceTimer timer = new HighPerformanceTimer(); //HighPerformanceTimer timer = new HighPerformanceTimer();
//timer.start(); //timer.start();
@@ -106,19 +117,24 @@ namespace BCG {
BlueTable dels = new BlueTable(myRows.schema), BlueTable dels = new BlueTable(myRows.schema),
adds = new BlueTable(myRows.schema); adds = new BlueTable(myRows.schema);
foreach (IBlueRow r in dp.deletedRows) { foreach (IBlueRow r in dp.deletedRows)
{
List<IBlueRow> mappedRows = myMap[(int)r["id"]]; List<IBlueRow> mappedRows = myMap[(int)r["id"]];
foreach (IBlueRow mr in mappedRows) { foreach (IBlueRow mr in mappedRows)
{
dels.addRow(mr); dels.addRow(mr);
myRows.deleteRow(mr); myRows.deleteRow(mr);
} }
myMap.Remove((int)r["id"]); myMap.Remove((int)r["id"]);
} }
foreach (IBlueRow r in dp.addedRows) { foreach (IBlueRow r in dp.addedRows)
{
IBlueRow[] mappedRows = myMapper.map(r); IBlueRow[] mappedRows = myMapper.map(r);
foreach (IBlueRow mr in mappedRows) { foreach (IBlueRow mr in mappedRows)
{
IBlueRow idChangedMapRow = generate_ids ? BlueDataUtility.getNewReplacedRow(mr, myID++, "id") : mr; IBlueRow idChangedMapRow = generate_ids ? BlueDataUtility.getNewReplacedRow(mr, myID++, "id") : mr;
if (!myMap.ContainsKey((int)r["id"])) { if (!myMap.ContainsKey((int)r["id"]))
{
myMap.Add((int)r["id"], new List<IBlueRow>()); myMap.Add((int)r["id"], new List<IBlueRow>());
} }
myMap[(int)r["id"]].Add(idChangedMapRow); myMap[(int)r["id"]].Add(idChangedMapRow);
@@ -129,12 +145,13 @@ namespace BCG {
//timer.stop(); //timer.stop();
// if (timer.lastIntervalTime > 20) // if (timer.lastIntervalTime > 20)
// {}//mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("SuperMapper:myHandlerData for mapper {0} took {1} msecs", myMapper.GetType().ToString(), timer.lastIntervalTime)); // {}//mbts.Logger.log(bts.log.Level.Info, bts.log.Subsystem.gen, String.Format("SuperMapper:myHandlerData for mapper {0} took {1} msecs", myMapper.GetType().ToString(), timer.lastIntervalTime));
myFireEvent(new DataPacket(myRows, adds, dels)); // myFireEvent(new DataPacket(myRows, adds, dels));
} }
private void myHandleMapChange() { private void myHandleMapChange()
{
if (myBaseRows != null) if (myBaseRows != null)
myHandleData(new DataPacket(myBaseRows, new BlueTable(myBaseRows), new BlueTable(myBaseRows))); myHandleData(new DataPacket(myBaseRows, new BlueTable (myBaseRows), new BlueTable (myBaseRows)));
} }
} }
} }
@@ -1,142 +0,0 @@
using BCG;
using System;
using System.IO;
using MongoDB.Bson;
using BCG.Utilities.General;
using System.Linq;
using System.Collections.Generic;
using System.Runtime.Intrinsics.X86;
namespace bts {
// listens to the underlying stream, and writes each seriialized packet in a
// seperate file
[DataStreamUserRespresentation("dp_recorder")]
public class DataPacketRecorderStream : ViewStream {
private ViewStream _base_stream;
private readonly string _folder;
StreamIOSerializationController _serializer = new StreamIOSerializationController();
private UInt32 _n_updates = 0;
public DataPacketRecorderStream(ViewStream base_stream, string folder) {
_base_stream = base_stream;
_folder = folder;
}
protected override void myOnFirstSubscriber() {
_base_stream.onChange += _handle_update;
}
protected override void myOnLastSubscriber() {
_base_stream.onChange -= _handle_update;
}
private void _handle_update(object dp_obj) {
DataPacket dp = dp_obj as DataPacket;
byte[] ser_data = null;
if (_n_updates == 0) {
StreamIOMessage.CreateMessage cr_msg = new StreamIOMessage.CreateMessage(dp.newRows);
ser_data = _serializer.serialize(new StreamIOMessage(cr_msg));
} else {
StreamIOMessage.MultipleRemoveAddMessage mra = new StreamIOMessage.MultipleRemoveAddMessage(dp.deletedRows, dp.addedRows);
ser_data = _serializer.serialize(new StreamIOMessage(mra));
}
Log.assert(ser_data != null);
_n_updates++;
string file_name = $"{_folder}\\{_n_updates}.bin";
_write_to_file(ser_data, file_name);
myFireEvent(dp);
}
private static void _write_to_file(byte[] data, string file_name) {
using (FileStream fs = new FileStream(file_name, FileMode.Create)) {
// Write the byte array to the file
fs.Write(data, 0, data.Length);
}
}
}
[DataStreamUserRespresentation("dp_replay")]
public class DataPacketReplayStream : ViewStream, IWorker {
private ViewStreamDatabase _vsd;
private readonly string _folder_path;
private readonly TimeSpan _fire_period;
IStreamMsgDeserilizer _deser = new StreamIODeserializationController();
private int _file_ptr = 0;
private List<Tuple<string, byte[]>> _file_data = new List<Tuple<string, byte[]>>();
private BlueTable _state = null;
public DataPacketReplayStream(string folder_path, string fire_period)
:
this(ViewStreamDatabase.instance, folder_path, TimeBasedFilter.parse_duration(fire_period)) {
}
public DataPacketReplayStream(ViewStreamDatabase vsd, string folder_path, TimeSpan fire_period) {
_vsd = vsd;
_folder_path = folder_path;
_fire_period = fire_period;
}
protected override void myOnFirstSubscriber() {
MasterWorker.getInstance(false).addWorker(this, _fire_period);
_load_all_files();
}
protected override void myOnLastSubscriber() {
MasterWorker.getInstance(false).removeWorker(this);
}
public void doWork() {
if (_file_ptr >= _file_data.Count())
// we can set _file_ptr to 0 and keep going as well
return;
var curr_d = _file_data[_file_ptr++];
var hext_txt = StreamSerializationUtils.to_hex(curr_d.Item2);
StreamIOMessage msg = _deser.deserialize(curr_d.Item2);
Log.Gen.info($"DataPacketReplayStream.doWork packet no : {_file_ptr} file : {curr_d.Item1} data: {hext_txt}");
switch (msg.op) {
case StreamIOMessage.StreamOp.CREATE: {
_state = new BlueTable(msg.create_message.rows);
break;
}
case StreamIOMessage.StreamOp.MULTIPLE_REMOVE_ADD: {
var mra = msg.multiple_remove_add_message;
foreach (IBlueRow r in mra.removed_rows)
_state.deleteRow(r);
foreach (IBlueRow r in mra.added_rows)
_state.addRow(r);
break;
}
}
var dp = BlueDataUtility.build_dp_inc(_state, myLastValue == null ? null : myLastValue as DataPacket, false);
myFireEvent(dp);
}
private void _load_all_files() {
Func<string, string> extract_file_name = (string full_path) => {
string fn = full_path.Split("\\").Last();
fn = fn.Split(".").First();
return fn;
};
string[] bin_files = Directory.GetFiles(_folder_path, "*.bin");
// Iterate over the .bin files and perform an action
foreach (string file in bin_files) {
byte[] file_data = File.ReadAllBytes(file);
_file_data.Add(Tuple.Create(file, file_data));
}
_file_data.Sort((Tuple<string, byte[]> t1, Tuple<string, byte[]> t2) => {
string fn1 = extract_file_name(t1.Item1);
string fn2 = extract_file_name(t2.Item1);
int fn_i1 = Int32.Parse(fn1);
int fn_i2 = Int32.Parse(fn2);
if (fn_i1 < fn_i2)
return -1;
else if (fn_i1 == fn_i2)
return 0;
else if (fn_i1 > fn_i2)
return 1;
return 0;
});
}
}
}
@@ -1,227 +0,0 @@
using BCG;
using BCG.Utilities.General;
using System.Collections.Generic;
//using MongoDB.Driver;
using System;
using System.IO;
using System.Linq;
namespace bts
{
public class PubSubTestingStream : ViewStream, IWorker
{
public class PayloadMessage
{
public int msg_id = 0;
public string guid;
public Int64 travel_epoch_ms = 0;
public byte[] payload = null;
public PayloadMessage(int msg_id_args, Int64 send_epoch_args, int payload_size, byte payload_byte)
{
msg_id = msg_id_args;
travel_epoch_ms = send_epoch_args;
payload = new byte[payload_size];
for (int i = 0; i < payload_size; i++)
{
payload[i] = payload_byte;
}
}
private PayloadMessage() { }
public PayloadMessage(int msg_id, int payload_size, byte payload_byte) :
this(msg_id, PubSubTestingStream.epoch_now(), payload_size, payload_byte)
{ }
public byte[] serialize()
{
var str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
bw.Write(msg_id);
bw.Write(guid);
bw.Write(travel_epoch_ms);
bw.Write(payload.Length);
bw.Write(payload);
str.Position = 0;
var bytes = str.ToArray();
return bytes;
}
public static PayloadMessage deserialize(byte[] ser_bytes)
{
var str = new MemoryStream(ser_bytes);
BinaryReader br = new BinaryReader(str);
int msg_id = br.ReadInt32();
string guid = br.ReadString();
Int64 epoch = br.ReadInt64();
int len = br.ReadInt32();
var payload = br.ReadBytes(len);
return new PayloadMessage()
{
msg_id = msg_id,
guid = guid,
travel_epoch_ms = epoch,
payload = payload
};
}
}
//public void
private readonly int _msg_size;
private readonly TimeSpan _publish_period;
private DateTime _last_publish_time = DateTime.MinValue;
private PublisherCore _pub_core = new PublisherCore();
private BlueTable _table;
private int _n_pubs = 0;
private readonly int _max_pubs;
private int _n_updates = 0;
private bool _continue_sub_thread = true;
private ViewStreamDatabase _vsd;
private bool _fire_dirty = false;
private PayloadMessage _last_sent_msg, _last_recvd_msg;
private readonly int _payload_sz;
private const byte _payload_byte = (byte)'A';
private readonly string _tag_name;
public PubSubTestingStream(int publish_period_msecs, int max_pubs, int payload_size, string tag_name, ViewStreamDatabase vsd)
{
_vsd = vsd;
_payload_sz = payload_size;
var schema = BlueDataUtility.form_schema("PubSubTest",
Tuple.Create("id", TypeCode.Int32),
Tuple.Create("n_pubs", TypeCode.Int32),
Tuple.Create("n_recvs", TypeCode.Int32),
Tuple.Create("last_sent_id", TypeCode.Int32),
Tuple.Create("last_recv_id", TypeCode.Int32),
Tuple.Create("last_sent_guid", TypeCode.String),
Tuple.Create("last_recv_guid", TypeCode.String),
Tuple.Create("match", TypeCode.Boolean),
Tuple.Create("echo_time", TypeCode.Int32));
_table = new BlueTable(schema);
_publish_period = TimeSpan.FromMilliseconds(publish_period_msecs);
_msg_size = payload_size;
_max_pubs = max_pubs;
_tag_name = tag_name;
}
public PubSubTestingStream(int publish_period_seconds, int max_pubs, int msg_size, string tag_name)
: this(publish_period_seconds, max_pubs, msg_size, tag_name, ViewStreamDatabase.instance) { }
protected override void myOnFirstSubscriber()
{
_vsd.addWorker(this);
System.Threading.Thread th = new System.Threading.Thread(_subscribe_thread_func);
th.IsBackground = true;
th.Start();
}
protected override void myOnLastSubscriber()
{
_vsd.removeWorker(this);
_continue_sub_thread = false;
}
void _log_updates(IEnumerable<PubSubServerMessage> updates)
{
var idx = 0;
foreach (var msg in updates)
{
bts.Log.Gen.info($"PubSubTesStream._subscribe_thread_func idx: {idx} msg: {msg}");
idx++;
}
}
private void _subscribe_thread_func()
{
// subscription is done in another thread
SubscriberCore sub_core = new SubscriberCore();
sub_core.subscribe(_tag_name);
while (_continue_sub_thread)
{
// TODO recv_id
var updates = sub_core.get_updates(TimeSpan.FromMilliseconds(250));
if (updates == null || updates.Count() == 0)
continue;
_log_updates(updates);
_n_updates += updates.Count();
if (updates.Count() > 0)
{
var server_msg = updates.Last();
if (server_msg.data != null && server_msg.data.Length > 0)
{
_last_recvd_msg = PayloadMessage.deserialize(server_msg.data);
_last_recvd_msg.travel_epoch_ms = epoch_now();
}
else
{
// why are we getting empty or null data ? (server is sending data regardless?)
}
_fire_dirty = true;
}
}
}
private void _publish_next_packet()
{
if (_last_sent_msg == null)
{
_last_sent_msg = new PayloadMessage(0, _payload_sz, _payload_byte)
{
guid = Guid.NewGuid().ToString()
};
}
else
{
_last_sent_msg.msg_id++;
_last_sent_msg.guid = Guid.NewGuid().ToString();
_last_sent_msg.travel_epoch_ms = epoch_now();
}
var ser_bytes = _last_sent_msg.serialize();
_n_pubs++;
_pub_core.publish("Blob", _tag_name, ser_bytes);
_fire_dirty = true;
}
private void _fire_dp()
{
object last_sent_id = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.msg_id;
object last_recv_id = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.msg_id;
object echo_time = DBNull.Value;
object last_sent_guid = _last_sent_msg == null ? (object)DBNull.Value : (object)_last_sent_msg.guid;
object last_recv_guid = _last_recvd_msg == null ? (object)DBNull.Value : (object)_last_recvd_msg.guid;
if (_last_sent_msg != null && _last_recvd_msg != null)
{
echo_time = _last_recvd_msg.travel_epoch_ms - _last_sent_msg.travel_epoch_ms;
}
var match = last_sent_id.Equals(last_recv_id) && last_sent_guid.Equals(last_recv_guid);
IBlueRow nr = new BlueRow(_table.schema, 0, _n_pubs, _n_updates,
last_sent_id, last_recv_id, last_sent_guid, last_recv_guid, match, echo_time);
BlueTable table = new BlueTable(_table.schema);
table.addRow(nr);
var dp = BlueDataUtility.build_dp_inc(table, myLastValue as DataPacket, true);
myFireEvent(dp);
}
public void doWork()
{
if (DateTime.Now - _last_publish_time > _publish_period)
if (_n_pubs <= _max_pubs)
{
_publish_next_packet();
_last_publish_time = DateTime.Now;
}
// publish
// TODO: send id
if (_fire_dirty)
{
_fire_dirty = false;
_fire_dp();
}
}
public static long epoch_now()
{
return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
}
}
}
@@ -1,227 +0,0 @@
using BCG;
using BCG.Utilities.General;
using System.Collections.Generic;
using System;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using bts.api;
namespace bts
{
[DataStreamUserRespresentation("pss_echo")]
public class PubsubEchoTestStream : ViewStream, IWorker
{
public class PayloadMessage
{
public readonly string msg_id;
public readonly Int64 timestamp = 0;
public readonly byte[] payload = null;
public static byte[] create_data(int payload_size, byte payload_byte)
{
var rv = new byte[payload_size];
for (int i = 0; i < payload_size; i++)
{
rv[i] = payload_byte;
}
return rv;
}
// public PayloadMessage(string guid_, Int64 timestamp_, int payload_size, byte payload_byte)
// {
// guid = guid_;
// timestamp = timestamp_;
// payload = new byte[payload_size];
// for (int i = 0; i < payload_size; i++)
// {
// payload[i] = payload_byte;
// }
// }
public PayloadMessage(string guid_, Int64 timestamp_, byte[] payload_)
{
msg_id = guid_;
timestamp = timestamp_;
payload = payload_;
}
private PayloadMessage() { }
public byte[] serialize()
{
var str = new MemoryStream();
BinaryWriter bw = new BinaryWriter(str);
bw.Write(msg_id);
bw.Write(timestamp);
bw.Write(payload.Length);
bw.Write(payload);
str.Position = 0;
var bytes = str.ToArray();
return bytes;
}
public static PayloadMessage deserialize(byte[] ser_bytes)
{
var str = new MemoryStream(ser_bytes);
BinaryReader br = new BinaryReader(str);
string guid = br.ReadString();
Int64 timestamp_ = br.ReadInt64();
int len = br.ReadInt32();
var payload = br.ReadBytes(len);
return new PayloadMessage(guid, timestamp_, payload);
}
}
//public void
private readonly int _msg_size;
private readonly TimeSpan _publish_period;
private DateTime _last_publish_time = DateTime.MinValue;
private PublisherCore _pub_core;
private ISubscriberCore _sub_core;
private BlueTable _table;
private int _n_pubs = 0;
private readonly int _max_pubs;
private int _n_updates = 0;
private ViewStreamDatabase _vsd;
private PayloadMessage _last_sent_msg, _last_recvd_msg;
private const byte _payload_byte = (byte)'A';
private readonly string _tag_name;
private readonly int _payload_sz;
private readonly IPubSubRouter _router;
private enum IOState
{
SEND = 0,
RECV = 1
};
IOState _state = IOState.SEND;
public PubsubEchoTestStream(string server_url, TimeSpan publish_period, int payload_size, string tag_name, ViewStreamDatabase vsd)
{
var route = new PubSubRoute(server_url, ".*");
_router = new PubSubRouter(route);
_vsd = vsd;
_payload_sz = payload_size;
var schema = BlueDataUtility.form_schema("PubSubTest",
Tuple.Create("id", TypeCode.Int32),
Tuple.Create("sent_id", TypeCode.String),
Tuple.Create("sent_time", TypeCode.Int64),
Tuple.Create("recv_id", TypeCode.String),
Tuple.Create("recv_time", TypeCode.Int64),
Tuple.Create("match", TypeCode.Boolean),
Tuple.Create("echo_time", TypeCode.Int32));
_table = new BlueTable(schema);
_publish_period = publish_period;
_tag_name = tag_name;
}
public PubsubEchoTestStream(string server_url, int publish_period_msecs, int payload_size, string tag_name)
: this(server_url, TimeSpan.FromMilliseconds(publish_period_msecs), payload_size, tag_name, ViewStreamDatabase.instance) { }
protected override void myOnFirstSubscriber()
{
_pub_core = new PublisherCore(_router);
_sub_core = new SubscriberCore(_router);
_sub_core.subscribe(_tag_name);
_vsd.addWorker(this);
}
protected override void myOnLastSubscriber()
{
_vsd.removeWorker(this);
_sub_core.unsubscribe(_tag_name);
}
private void _fire_dp_after_send()
{
// as soon as the data is published, its reflected in the stream state
Log.assert(_last_sent_msg != null);
BlueRow r = new BlueRow(_table.schema, _table.Count, _last_sent_msg.msg_id, _last_sent_msg.timestamp, DBNull.Value, DBNull.Value,
DBNull.Value, DBNull.Value);
_table.addRow(r);
var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
myFireEvent(dp_out);
}
private void _fire_dp_after_recv()
{
Log.assert(_last_recvd_msg != null);
// update the last publish row
int id = _table.Count - 1;
IBlueRow last_row = _table[id];
_table.deleteRow(last_row);
var recv_ts = epoch_now();
BlueRow nr = new BlueRow(_table.schema, id, _last_sent_msg.msg_id, _last_sent_msg.timestamp, _last_recvd_msg.msg_id, recv_ts,
String.Compare(_last_recvd_msg.msg_id, _last_sent_msg.msg_id) == 0, recv_ts - _last_recvd_msg.timestamp);
_table.addRow(nr);
var dp_out = BlueDataUtility.build_dp_inc(_table, myLastValue as DataPacket);
myFireEvent(dp_out);
}
private string _get_next_id() {
return _table.Count.ToString();
}
private string _get_next_id()
{
return _table.Count.ToString();
}
private void _publish_next_packet()
{
var bytes = PayloadMessage.create_data(_payload_sz, _payload_byte);
_last_sent_msg = new PayloadMessage(_get_next_id(), epoch_now(), bytes);
var ser_bytes = _last_sent_msg.serialize();
_n_pubs++;
_pub_core.publish("Blob", _tag_name, ser_bytes);
_last_publish_time = DateTime.Now;
}
public void doWork()
{
Func<IEnumerable<PubSubServerMessage>, PayloadMessage> find_packet = (updates) =>
{
foreach (var u in updates)
{
var recv_msg = PayloadMessage.deserialize(u.data);
if (recv_msg.msg_id == _last_sent_msg.msg_id)
{
return recv_msg;
}
}
return null;
};
switch (_state)
{
case IOState.SEND:
if (DateTime.Now - _last_publish_time >= _publish_period)
{
_publish_next_packet();
_last_publish_time = DateTime.Now;
_fire_dp_after_send();
_state = IOState.RECV;
return;
}
break;
case IOState.RECV:
var updates = _sub_core.get_updates(TimeSpan.Zero);
updates = updates.Where(u => u.tag == _tag_name);
var recv_msg = find_packet(updates);
if (recv_msg != null)
{
_last_recvd_msg = recv_msg;
_fire_dp_after_recv();
_state = IOState.SEND;
break;
}
break;
}
}
public static long epoch_now()
{
return DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
}
}
}
@@ -1,58 +0,0 @@
namespace bts {
using BCG;
using BCG.Utilities.General;
using System;
public class SimpleTCPEchoStream : ViewStream, IWorker {
private string _ip;
private int _port;
private SimpleTCPClient _tcp_client;
private IBlueTableSchema _schema = BlueDataUtility.form_schema("SimpleTCPEchoStream",
Tuple.Create("id", TypeCode.Int32),
Tuple.Create("event", TypeCode.String));
private BlueTable _table;
private SyncQueue<string> _action_queue = new SyncQueue<string>();
public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms)
: this(ip, port, msg_sz, send_wait_period_ms, Int32.MaxValue) { }
public SimpleTCPEchoStream(string ip, int port, int msg_sz, int send_wait_period_ms, int max_n_msgs) {
_ip = ip;
_port = port;
_table = new BlueTable(_schema);
_tcp_client = new SimpleTCPClient(_ip, _port, Convert.ToUInt32(msg_sz), send_wait_period_ms, max_n_msgs);
}
protected override void myOnFirstSubscriber() {
ViewStreamDatabase.instance.addWorker(this);
_tcp_client.register_listener((msg) => { _on_msg(msg); });
_tcp_client.start();
}
private void _on_msg(string msg) {
_action_queue.enqueue(msg);
}
public void doWork() {
if (_action_queue.getCount() == 0)
return;
BlueTable added_rows = new BlueTable(_schema);
var sz = _action_queue.getCount();
for (int i = 0; i < sz; i++) {
string msg = _action_queue.dequeue();
if (msg == null)
break;
IBlueRow nr = new BlueRow(_schema, _table.Count, msg);
_table.addRow(nr);
added_rows.addRow(nr);
}
if (added_rows.Count == 0)
return;
DataPacket dp = new DataPacket(_table, added_rows, new BlueTable(_schema));
myFireEvent(dp);
}
}
}
Loaded 100 of 901 files, more files were not shown because too many files have changed in this diff. Show more